# Copyright (C) 2019-2025 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information

import dataclasses
import datetime
import functools
import logging
import re
from typing import Any, Container, Dict, List, Optional, Tuple, cast

import attr
import pytest
import yaml

from swh.journal.client import EofBehavior, JournalClient
from swh.journal.serializers import kafka_to_value, key_to_kafka, value_to_kafka
from swh.journal.writer import JournalWriterInterface
from swh.model.hashutil import MultiHash, hash_to_bytes
from swh.model.model import (
    Directory,
    OriginVisit,
    OriginVisitStatus,
    Release,
    Revision,
    RevisionType,
    SkippedContent,
    Snapshot,
)
from swh.model.tests.swh_model_data import COMMITTERS, DATES, REVISIONS
from swh.model.tests.swh_model_data import TEST_OBJECTS as _TEST_OBJECTS
from swh.storage import get_storage
from swh.storage.cassandra.model import ContentRow, SkippedContentRow
from swh.storage.exc import StorageArgumentException
from swh.storage.in_memory import InMemoryStorage
from swh.storage.interface import StorageInterface
from swh.storage.replay import ModelObjectDeserializer, process_replay_objects
from swh.storage.tests.storage_data import StorageData

UTC = datetime.timezone.utc

TEST_OBJECTS = _TEST_OBJECTS.copy()
# Add a revision with metadata to check this later is dropped while being replayed
# TODO: replace this with a lower level struct (if still needed) since the
# metadata field is about to be removed from the Revision object model
TEST_OBJECTS[Revision.object_type] = list(_TEST_OBJECTS[Revision.object_type]) + [
    Revision(
        id=hash_to_bytes("51d9d94ab08d3f75512e3a9fd15132e0a7ca7928"),
        message=b"hello again",
        date=DATES[1],
        committer=COMMITTERS[1],
        author=COMMITTERS[0],
        committer_date=DATES[0],
        type=RevisionType.GIT,
        directory=b"\x03" * 20,
        synthetic=False,
        metadata={"something": "interesting"},
        parents=(REVISIONS[0].id,),
    ),
]
WRONG_ID_REG = re.compile(
    "Object has id [0-9a-f]{40}, but it should be [0-9a-f]{40}: .*"
)


def now():
    return datetime.datetime.now(datetime.UTC)


def nullify_ctime(obj: Any) -> Any:
    if isinstance(obj, (ContentRow, SkippedContentRow)):
        return dataclasses.replace(obj, ctime=None)
    else:
        return obj


@pytest.fixture()
def replayer_storage(kafka_prefix: str, kafka_server: str):
    journal_writer_config = {
        "cls": "kafka",
        "brokers": [kafka_server],
        "client_id": "kafka_writer",
        "prefix": kafka_prefix,
        "auto_flush": False,
    }
    storage_config: Dict[str, Any] = {
        "cls": "memory",
        "journal_writer": journal_writer_config,
    }
    return get_storage(**storage_config)


@pytest.fixture()
def replayer_client(kafka_prefix: str, kafka_consumer_group: str, kafka_server: str):
    deserializer = ModelObjectDeserializer()
    return JournalClient(
        brokers=kafka_server,
        group_id=kafka_consumer_group,
        prefix=kafka_prefix,
        on_eof=EofBehavior.STOP,
        value_deserializer=deserializer.convert,
    )


@pytest.fixture()
def replayer_storage_and_client(replayer_storage, replayer_client):
    # for bw compat
    return replayer_storage, replayer_client


@pytest.mark.parametrize(
    "tenacious",
    [pytest.param(True, id="tenacious"), pytest.param(False, id="non-tenacious")],
)
def test_storage_replayer(replayer_storage_and_client, caplog, tenacious):
    """Optimal replayer scenario.

    This:
    - writes objects to a source storage
    - replayer consumes objects from the topic and replays them
    - a destination storage is filled from this

    In the end, both storages should have the same content.
    """
    src, replayer = replayer_storage_and_client

    # Fill Kafka using a source storage
    nb_sent = 0
    for object_type, objects in TEST_OBJECTS.items():
        method = getattr(src, f"{object_type}_add")
        method(objects)
        nb_sent += len(objects)
    src.journal_writer.journal.flush()

    caplog.set_level(logging.ERROR, "swh.journal.replay")

    if tenacious:
        proxied_dst = get_storage("tenacious", storage={"cls": "memory"})
        dst = proxied_dst.storage
    else:
        proxied_dst = dst = get_storage(cls="memory")

    # Fill the destination storage from Kafka
    worker_fn = functools.partial(process_replay_objects, storage=proxied_dst)
    nb_inserted = replayer.process(worker_fn)
    assert nb_sent == nb_inserted

    assert isinstance(src, InMemoryStorage)  # needed to help mypy
    assert isinstance(dst, InMemoryStorage)
    check_replayed(src, dst)

    collision = 0
    for record in caplog.records:
        logtext = record.getMessage()
        if "Colliding contents:" in logtext:
            collision += 1

    assert collision == 0, "No collision should be detected"


def test_storage_replay_with_collision(
    replayer_storage_and_client: Tuple[StorageInterface, JournalClient], caplog
):
    """Another replayer scenario with collisions.

    This:
    - writes objects to the topic, including colliding contents
    - replayer consumes objects from the topic and replay them
    - This drops the colliding contents from the replay when detected

    """
    src, replayer = replayer_storage_and_client
    journal: JournalWriterInterface = src.journal_writer.journal  # type: ignore
    assert journal is not None

    # Fill Kafka using a source storage
    nb_sent = 0
    for object_type, objects in TEST_OBJECTS.items():
        method = getattr(src, f"{object_type}_add")
        method(objects)
        nb_sent += len(objects)
    journal.flush()

    # Create collision in input data
    # These should not be written in the destination
    producer = journal.producer  # type: ignore
    prefix = journal._prefix  # type: ignore
    DUPLICATE_CONTENTS = StorageData().colliding_contents["sha1"]
    for content in DUPLICATE_CONTENTS:
        topic = f"{prefix}.content"
        key = content.hashes()
        value = content.to_dict()
        value.pop("data", None)
        producer.produce(
            topic=topic,
            key=key_to_kafka(key),
            value=value_to_kafka(value),
        )
        nb_sent += 1

    producer.flush()

    caplog.set_level(logging.ERROR, "swh.journal.replay")

    # Fill the destination storage from Kafka
    dst = get_storage(cls="memory")
    worker_fn = functools.partial(process_replay_objects, storage=dst)
    nb_inserted = replayer.process(worker_fn)
    assert nb_sent == nb_inserted

    # check the logs for the collision being properly detected
    nb_collisions = 0
    for record in caplog.records:
        logtext = record.getMessage()
        if "Collision detected:" in logtext:
            nb_collisions += 1

    assert nb_collisions == 0, "No collision should be detected"

    # all objects from the src should exists in the dst storage, plus two more
    assert isinstance(src, InMemoryStorage)  # needed to help mypy
    assert isinstance(dst, InMemoryStorage)  # needed to help mypy

    check_replayed(src, dst, exclude=["contents"])

    assert (
        len(list(dst._cql_runner._contents.iter_all()))
        == len(list(src._cql_runner._contents.iter_all())) + 2
    )


def test_replay_skipped_content(replayer_storage_and_client):
    """Test the 'skipped_content' topic is properly replayed."""
    src, replayer = replayer_storage_and_client
    _check_replay_skipped_content(src, replayer, "skipped_content")


# utility functions


def check_replayed(
    src: InMemoryStorage,
    dst: InMemoryStorage,
    exclude: Optional[Container] = None,
    expected_anonymized=False,
):
    """Simple utility function to compare the content of 2 in_memory storages"""

    def fix_expected(attr, row):
        if expected_anonymized:
            if attr == "releases":
                row = dataclasses.replace(
                    row, author=row.author and row.author.anonymize()
                )
            elif attr == "revisions":
                row = dataclasses.replace(
                    row,
                    author=row.author.anonymize(),
                    committer=row.committer.anonymize(),
                )
        if attr == "revisions":
            # the replayer should now drop the metadata attribute; see
            # swh/storgae/replay.py:_insert_objects()
            row.metadata = "null"

        return row

    for attr_ in (
        "contents",
        "skipped_contents",
        "directories",
        "extid",
        "revisions",
        "releases",
        "snapshots",
        "origins",
        "origin_visits",
        "origin_visit_statuses",
        "raw_extrinsic_metadata",
    ):
        if exclude and attr_ in exclude:
            continue
        expected_objects = [
            (id, nullify_ctime(fix_expected(attr_, obj)))
            for id, obj in sorted(getattr(src._cql_runner, f"_{attr_}").iter_all())
        ]
        got_objects = [
            (id, nullify_ctime(obj))
            for id, obj in sorted(getattr(dst._cql_runner, f"_{attr_}").iter_all())
        ]
        assert got_objects == expected_objects, f"Mismatch object list for {attr_}"


def _check_replay_skipped_content(storage, replayer, topic):
    skipped_contents = _gen_skipped_contents(100)
    nb_sent = len(skipped_contents)
    producer = storage.journal_writer.journal.producer
    prefix = storage.journal_writer.journal._prefix

    for i, obj in enumerate(skipped_contents):
        content = SkippedContent.from_dict(obj)
        producer.produce(
            topic=f"{prefix}.{topic}",
            key=key_to_kafka(content.unique_key()),
            value=value_to_kafka(content.to_dict()),
        )
    producer.flush()

    dst_storage = get_storage(cls="memory")
    worker_fn = functools.partial(process_replay_objects, storage=dst_storage)
    nb_inserted = replayer.process(worker_fn)

    assert nb_sent == nb_inserted
    for content in skipped_contents:
        assert not storage.content_find({"sha1": content["sha1"]})

    # no skipped_content_find API endpoint, so use this instead
    assert not list(dst_storage.skipped_content_missing(skipped_contents))


def _updated(d1, d2):
    d1.update(d2)
    d1.pop("data", None)
    return d1


def _gen_skipped_contents(n=10):
    # we do not use the hypothesis strategy here because this does not play well with
    # pytest fixtures, and it makes test execution very slow
    now = datetime.datetime.now(tz=UTC)
    return [
        _updated(
            MultiHash.from_data(data=f"foo{i}".encode()).digest(),
            {
                "status": "absent",
                "reason": "why not",
                "origin": f"https://somewhere/{i}",
                "ctime": now,
                "length": len(f"foo{i}"),
            },
        )
        for i in range(n)
    ]


@pytest.mark.parametrize("privileged", [True, False])
def test_storage_replay_anonymized(
    kafka_prefix: str,
    kafka_consumer_group: str,
    kafka_server: str,
    privileged: bool,
):
    """Optimal replayer scenario.

    This:
    - writes objects to the topic
    - replayer consumes objects from the topic and replay them

    This tests the behavior with both a privileged and non-privileged replayer
    """
    writer_config = {
        "cls": "kafka",
        "brokers": [kafka_server],
        "client_id": "kafka_writer",
        "prefix": kafka_prefix,
        "anonymize": True,
        "auto_flush": False,
    }
    src_config: Dict[str, Any] = {"cls": "memory", "journal_writer": writer_config}

    storage = get_storage(**src_config)

    # Fill the src storage
    nb_sent = 0
    for obj_type, objs in TEST_OBJECTS.items():
        if obj_type in (
            OriginVisit.object_type,
            OriginVisitStatus.object_type,
        ):
            # these are unrelated with what we want to test here
            continue
        method = getattr(storage, f"{obj_type}_add")
        method(objs)
        nb_sent += len(objs)
    journal: JournalWriterInterface = storage.journal_writer.journal  # type:ignore
    journal.flush()

    # Fill a destination storage from Kafka, potentially using privileged topics
    dst_storage = get_storage(cls="memory")
    deserializer = ModelObjectDeserializer(
        validate=False
    )  # we cannot validate an anonymized replay
    replayer = JournalClient(
        brokers=kafka_server,
        group_id=kafka_consumer_group,
        prefix=kafka_prefix,
        stop_after_objects=nb_sent,
        privileged=privileged,
        value_deserializer=deserializer.convert,
    )
    worker_fn = functools.partial(process_replay_objects, storage=dst_storage)

    nb_inserted = replayer.process(worker_fn)
    replayer.consumer.commit()

    assert nb_sent == nb_inserted
    # Check the contents of the destination storage, and whether the anonymization was
    # properly used
    assert isinstance(storage, InMemoryStorage)  # needed to help mypy
    assert isinstance(dst_storage, InMemoryStorage)
    check_replayed(storage, dst_storage, expected_anonymized=not privileged)


def test_storage_replayer_with_validation_ok(
    replayer_storage_and_client, caplog, redisdb
):
    """Optimal replayer scenario

    with validation activated and reporter set to a redis db.

    - writes objects to a source storage
    - replayer consumes objects from the topic and replays them
    - a destination storage is filled from this
    - nothing has been reported in the redis db
    - both storages should have the same content
    """
    src, replayer = replayer_storage_and_client
    replayer.deserializer = ModelObjectDeserializer(validate=True, reporter=redisdb.set)

    # Fill Kafka using a source storage
    nb_sent = 0
    for object_type, objects in TEST_OBJECTS.items():
        method = getattr(src, f"{object_type}_add")
        method(objects)
        nb_sent += len(objects)
    src.journal_writer.journal.flush()

    # Fill the destination storage from Kafka
    dst = get_storage(cls="memory")
    worker_fn = functools.partial(process_replay_objects, storage=dst)
    nb_inserted = replayer.process(worker_fn)
    assert nb_sent == nb_inserted

    # check we do not have invalid objects reported
    invalid = 0
    for record in caplog.records:
        logtext = record.getMessage()
        if WRONG_ID_REG.match(logtext):
            invalid += 1
    assert invalid == 0, "Invalid objects should not be detected"
    assert not redisdb.keys()
    # so the dst should be the same as src storage
    check_replayed(cast(InMemoryStorage, src), cast(InMemoryStorage, dst))


def test_storage_replayer_with_validation_nok(
    replayer_storage_and_client, caplog, redisdb
):
    """Replayer scenario with invalid objects

    with validation and reporter set to a redis db.

    - writes objects to a source storage
    - replayer consumes objects from the topic and replays them
    - the destination storage is filled with only valid objects
    - the redis db contains the invalid (raw kafka mesg) objects
    """
    src, replayer = replayer_storage_and_client
    replayer.value_deserializer = ModelObjectDeserializer(
        validate=True, reporter=redisdb.set
    ).convert

    caplog.set_level(logging.ERROR, "swh.journal.replay")

    # Fill Kafka using a source storage
    nb_sent = 0
    for object_type, objects in TEST_OBJECTS.items():
        method = getattr(src, f"{object_type}_add")
        method(objects)
        nb_sent += len(objects)

    # insert invalid objects
    for object_type in (
        Revision.object_type,
        Directory.object_type,
        Release.object_type,
        Snapshot.object_type,
    ):
        method = getattr(src, f"{object_type}_add")
        method([attr.evolve(TEST_OBJECTS[object_type][0], id=b"\x00" * 20)])
        nb_sent += 1
    # also add an object that won't even be possible to instantiate; this needs
    # to be done at low kafka level (since we cannot instantiate the invalid model
    # object...)
    # we use directory[1] because it actually have some entries
    dict_repr = {
        # copy each dir entry twice
        "entries": TEST_OBJECTS[Directory.object_type][1].to_dict()["entries"] * 2,
        "id": b"\x01" * 20,
    }
    topic = f"{src.journal_writer.journal._prefix}.directory"
    src.journal_writer.journal.send(topic, dict_repr["id"], dict_repr)
    nb_sent += 1

    src.journal_writer.journal.flush()

    # Fill the destination storage from Kafka
    dst = get_storage(cls="tenacious", storage={"cls": "memory"})
    worker_fn = functools.partial(process_replay_objects, storage=dst)
    nb_inserted = replayer.process(worker_fn)
    assert nb_sent == nb_inserted

    # check we do have invalid objects reported
    invalid = 0
    for record in caplog.records:
        logtext = record.getMessage()
        if WRONG_ID_REG.match(logtext):
            invalid += 1
    assert invalid == 4, "Invalid objects should be detected"
    keys = validate_keys(redisdb.keys())

    assert {k for ts, k in keys} == {
        f"{typ}:{'0' * 40}" for typ in ("release", "revision", "snapshot", "directory")
    } | {"directory:" + "01" * 20}

    for key in redisdb.keys():
        # check the stored value looks right
        rawvalue = redisdb.get(key)
        error_struct = yaml.load(rawvalue, Loader=yaml.Loader)
        assert "kafka_message" in error_struct
        value = kafka_to_value(error_struct["kafka_message"])
        assert isinstance(value, dict)
        assert "id" in value
        assert value["id"] in (b"\x00" * 20, b"\x01" * 20)
        assert "obj" in error_struct
        assert "exc" in error_struct

    # check that invalid objects did not reach the dst storage
    for attr_ in (
        "directories",
        "revisions",
        "releases",
        "snapshots",
    ):
        for id, obj in sorted(getattr(dst._cql_runner, f"_{attr_}").iter_all()):
            assert id not in (b"\x00" * 20, b"\x01" * 20)

    # check that valid objects did reach the dst storage
    # revisions
    expected = [
        attr.evolve(rev, metadata=None) for rev in TEST_OBJECTS[Revision.object_type]
    ]
    result = dst.revision_get([obj.id for obj in TEST_OBJECTS[Revision.object_type]])
    assert result == expected
    # releases
    expected = TEST_OBJECTS[Release.object_type]
    result = dst.release_get([obj.id for obj in TEST_OBJECTS[Release.object_type]])
    assert result == expected
    # snapshot
    # result from snapshot_get is paginated, so adapt the expected to be comparable
    expected = [
        {"next_branch": None, **obj.to_dict()}
        for obj in TEST_OBJECTS[Snapshot.object_type]
    ]
    result = [dst.snapshot_get(obj.id) for obj in TEST_OBJECTS[Snapshot.object_type]]
    assert result == expected
    # directories
    for directory in TEST_OBJECTS[Directory.object_type]:
        assert set(dst.directory_get_entries(directory.id).results) == set(
            directory.entries
        )


def test_storage_replayer_with_validation_nok_with_exceptions(
    replayer_storage_and_client, caplog, redisdb
):
    """Replayer scenario with invalid objects, with exceptions

    with validation and reporter set to a redis db.

    - writes invalid objects to a source storage
    - replayer consumes objects from the topic and replays them (invalid
      objects being in the exception list)
    - the destination storage is filled with all objects
    - the redis db does not contain the invalid objects

    """
    src, replayer = replayer_storage_and_client

    caplog.set_level(logging.WARNING, "swh.journal.replay")

    # Fill Kafka using a source storage
    nb_sent = 0
    for object_type, objects in TEST_OBJECTS.items():
        method = getattr(src, f"{object_type}_add")
        method(objects)
        nb_sent += len(objects)

    # insert invalid objects
    known_invalid = []
    all_objs = {}
    for object_type in (
        Revision.object_type,
        Directory.object_type,
        Release.object_type,
        Snapshot.object_type,
    ):
        all_objs[object_type] = TEST_OBJECTS[object_type][:]
        method = getattr(src, f"{object_type}_add")
        invalid_obj = attr.evolve(TEST_OBJECTS[object_type][0], id=b"\x00" * 20)
        method([invalid_obj])
        nb_sent += 1
        all_objs[object_type].append(invalid_obj)
        known_invalid.append(
            (object_type.value, b"\x00" * 20, TEST_OBJECTS[object_type][0].id)
        )

    replayer.value_deserializer = ModelObjectDeserializer(
        validate=True,
        reporter=redisdb.set,
        known_mismatched_hashes=tuple(known_invalid),
    ).convert
    src.journal_writer.journal.flush()

    # Fill the destination storage from Kafka
    dst = get_storage(cls="memory")
    worker_fn = functools.partial(process_replay_objects, storage=dst)
    nb_inserted = replayer.process(worker_fn)
    assert nb_sent == nb_inserted

    # check we do have invalid but replicates objects reported
    invalid = 0
    for record in caplog.records:
        logtext = record.getMessage()
        if WRONG_ID_REG.match(logtext):
            assert logtext.endswith("Known exception, replicating it anyway.")
            invalid += 1
    assert invalid == 4, "Replayed invalid objects should be detected"
    assert not set(redisdb.keys())

    # check that all objects did reach the dst storage
    # revisions
    expected = [
        attr.evolve(rev, metadata=None) for rev in all_objs[Revision.object_type]
    ]
    result = dst.revision_get([obj.id for obj in all_objs[Revision.object_type]])
    assert result == expected
    # releases
    expected = all_objs[Release.object_type]
    result = dst.release_get([obj.id for obj in all_objs[Release.object_type]])
    assert result == expected
    # snapshot
    # result from snapshot_get is paginated, so adapt the expected to be comparable
    expected = [
        {"next_branch": None, **obj.to_dict()} for obj in all_objs[Snapshot.object_type]
    ]
    result = [dst.snapshot_get(obj.id) for obj in all_objs[Snapshot.object_type]]
    assert result == expected
    # directories
    for directory in all_objs[Directory.object_type]:
        assert set(dst.directory_get_entries(directory.id).results) == set(
            directory.entries
        )


def test_storage_replayer_with_validation_nok_raises(
    replayer_storage_and_client, caplog, redisdb
):
    """Replayer scenario with invalid objects

    with raise_on_error set to True

    This:
    - writes both valid & invalid objects to a source storage
    - a StorageArgumentException should be raised while replayer consumes
      objects from the topic and replays them
    """
    src, replayer = replayer_storage_and_client
    replayer.value_deserializer = ModelObjectDeserializer(
        validate=True, reporter=redisdb.set, raise_on_error=True
    ).convert

    caplog.set_level(logging.ERROR, "swh.journal.replay")

    # Fill Kafka using a source storage
    nb_sent = 0
    for object_type, objects in TEST_OBJECTS.items():
        method = getattr(src, f"{object_type}_add")
        method(objects)
        nb_sent += len(objects)

    # insert invalid objects
    for object_type in (
        Revision.object_type,
        Directory.object_type,
        Release.object_type,
        Snapshot.object_type,
    ):
        method = getattr(src, f"{object_type}_add")
        method([attr.evolve(TEST_OBJECTS[object_type][0], id=b"\x00" * 20)])
        nb_sent += 1

    # Fill the destination storage from Kafka
    dst = get_storage(cls="memory")
    worker_fn = functools.partial(process_replay_objects, storage=dst)
    with pytest.raises(StorageArgumentException):
        replayer.process(worker_fn)

    # check we do have invalid objects reported
    invalid = 0
    for record in caplog.records:
        logtext = record.getMessage()
        if WRONG_ID_REG.match(logtext):
            invalid += 1
    assert invalid == 1, "One invalid objects should be detected"
    assert len(redisdb.keys()) == 1


def test_storage_replayer_tenacious_with_hashcollisions(
    replayer_storage_and_client, caplog, redisdb
):
    """Replayer w/ tenacious proxy, filled with hash colliding content objects

    Ensure colliding objects are inserted properly
    """
    src, replayer = replayer_storage_and_client
    jwriter = src.journal_writer

    caplog.set_level(logging.ERROR, "swh.journal.replay")
    # Fill Kafka using the journal writer
    contents = [attr.evolve(c, ctime=now()) for c in TEST_OBJECTS["content"]]

    dst = get_storage(
        cls="tenacious", storage={"cls": "memory"}, error_reporter=redisdb.set
    )
    worker_fn = functools.partial(process_replay_objects, storage=dst)

    jwriter.content_add(contents)
    jwriter.journal.flush()
    nb_processed = replayer.process(worker_fn)
    assert nb_processed == len(contents)
    assert len(dst.storage._cql_runner._contents.data) == len(contents)
    assert not redisdb.keys()

    # hash collisions on sha256
    colliding_objs = [attr.evolve(c, sha256=b"\x00" * 32, data=None) for c in contents]
    jwriter.content_add(colliding_objs)
    jwriter.journal.flush()
    nb_processed = replayer.process(worker_fn)
    assert nb_processed == len(contents)
    assert sum(
        len(x) for x in dst.storage._cql_runner._contents.data.values()
    ) == 2 * len(contents)
    assert set(redisdb.keys()) == set()

    # hash collisions on blake2s256
    colliding_objs = [
        attr.evolve(c, blake2s256=b"\x00" * 32, data=None) for c in contents
    ]
    jwriter.content_add(colliding_objs)
    jwriter.journal.flush()
    nb_processed = replayer.process(worker_fn)
    assert nb_processed == len(contents)
    assert sum(
        len(x) for x in dst.storage._cql_runner._contents.data.values()
    ) == 3 * len(contents)
    assert set(redisdb.keys()) == set()

    # hash collisions on sha1
    colliding_objs = [attr.evolve(c, sha1=b"\x00" * 20, data=None) for c in contents]
    jwriter.content_add(colliding_objs)
    jwriter.journal.flush()
    nb_processed = replayer.process(worker_fn)
    assert nb_processed == len(contents)
    assert sum(
        len(x) for x in dst.storage._cql_runner._contents.data.values()
    ) == 4 * len(contents)
    assert set(redisdb.keys()) == set()

    # hash collisions on sha1_git
    colliding_objs = [
        attr.evolve(c, sha1_git=b"\x00" * 20, data=None) for c in contents
    ]
    jwriter.content_add(colliding_objs)
    jwriter.journal.flush()
    nb_processed = replayer.process(worker_fn)
    assert nb_processed == len(contents)
    assert sum(
        len(x) for x in dst.storage._cql_runner._contents.data.values()
    ) == 5 * len(contents)
    assert set(redisdb.keys()) == set()


def test_storage_replayer_tenacious_with_invalid_objects(
    replayer_storage, replayer_client, caplog, redisdb
):
    """Replayer scenario with invalid objects

    with raise_on_error set to True

    This:
    - writes both valid & invalid objects to a source storage
    - a StorageArgumentException should be raised while replayer consumes
      objects from the topic and replays them
    """
    # disable the validation made by the replayer itself since we want to test
    # errors occurring in the storage backend here
    replayer_client.value_deserializer.__self__.validate = False
    jwriter = replayer_storage.journal_writer

    caplog.set_level(logging.ERROR, "swh.journal.replay")

    n_inserted = 0
    # insert invalid objects
    for object_type in (
        Revision.object_type,
        Directory.object_type,
        Release.object_type,
        Snapshot.object_type,
    ):
        inv_obj = attr.evolve(TEST_OBJECTS[object_type][0], id=b"\x00" * 20)
        jwriter.write_additions(
            object_type,
            TEST_OBJECTS[object_type] + [inv_obj],
        )
        n_inserted += len(TEST_OBJECTS[object_type]) + 1
    jwriter.journal.flush()

    # Fill the destination storage from Kafka
    dst = get_storage(
        cls="tenacious",
        storage={"cls": "validate", "storage": {"cls": "memory"}},
        error_reporter=redisdb.set,
    )
    worker_fn = functools.partial(process_replay_objects, storage=dst)
    nb_processed = replayer_client.process(worker_fn)

    assert nb_processed == n_inserted
    assert len(redisdb.keys()) == 4

    object_types = []
    for ts, k in validate_keys(redisdb.keys()):
        object_types.append(k)

        # values are yaml serialized dicts with 2 keys: obj (as a dict) and exc
        # (dump of the exception as a list of str)
        value = redisdb.get(f"{ts}/{k}")
        # cannot use safe_load because this latter does not know how to load
        # tuples, which can be generated by some ModelObject.to_dict() implems
        error_struct = yaml.load(value, Loader=yaml.Loader)
        assert "obj" in error_struct
        assert error_struct["obj"]["id"] == b"\x00" * 20
        assert "exc" in error_struct
        assert "\n".join(error_struct["exc"]).startswith("Traceback")

    assert set(object_types) == {"revision", "directory", "release", "snapshot"}


def validate_keys(keys: List[bytes]) -> List[Tuple[str, str]]:
    """Check error reporter keys (from redis)

    Check they are correctly foratted and return them without the timestamp"""
    # keys are 'iso-date/remaiming'
    # check the timestamps for reported errors are in a reasonable range
    n = now()
    dt0 = datetime.timedelta(0)
    dt1 = datetime.timedelta(minutes=1)
    output = []
    for k in (x.decode() for x in keys):
        assert "/" in k
        tstamp, rem = k.split("/", 1)
        dt = n - datetime.datetime.fromisoformat(tstamp)
        assert dt0 < dt < dt1
        output.append((tstamp, rem))
    return output
