"""
Comprehensive Core functionality test.

Covers all Core models with CRUD operations, soft deletes, cascade deletes,
filtering, pagination, sorting, locking, error paths, object handling,
ingestion processing, digestion/extraction, and complex interactions.
Uses a function-scoped client for full isolation.
"""

import base64
import uuid

import pytest
import requests

from lqs.client import RESTClient
from lqs.common.exceptions import (
    BadRequestException,
    ConflictException,
    ForbiddenException,
    LockedException,
)
from lqs.dsm import DataStoreManager
from lqs.interface.base.models import UploadState
from lqs.interface.core import models
from lqs.interface.core.models import ProcessState
from lqs.interface.dsm.models import ProcessState as ProcessStateDSM
from lqs.transcode import Transcode


# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------

LOG_FILE_PATH = "tests/test-logs/custom_images_test.bag"
LOG_OBJECT_KEY = "log.bag"
IMAGE_TOPIC_NAME = "image"


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def uid(prefix: str = "") -> str:
    """Generate a unique name to avoid collisions."""
    return f"{prefix}_{uuid.uuid4().hex[:8]}"


def process_ingestion(
    lqs: RESTClient,
    dsm: DataStoreManager,
    ingestion: models.Ingestion,
):
    """Drive an ingestion through its full lifecycle via DSM jobs."""
    datastore_id = lqs.get_datastore_id()

    # Process the ingestion job (parses the file, creates parts)
    job = dsm.create.job(
        type="ingestion",
        event_id=None,
        datastore_id=datastore_id,
        resource_id=ingestion.id,
        process_type="ingestion",
        state="queued",
    ).data
    dsm.process.process_job(job.id, sleep_factor=0)

    ingestion = lqs.fetch.ingestion(ingestion_id=ingestion.id).data
    assert ingestion.state == ProcessState.finalizing

    # Process each ingestion part (writes records)
    parts = lqs.list.ingestion_part(ingestion_id=ingestion.id).data
    assert len(parts) > 0
    for part in parts:
        assert part.state == ProcessState.queued
        job = dsm.create.job(
            type="ingestion_part",
            event_id=None,
            datastore_id=datastore_id,
            resource_id=part.id,
            process_type="ingestion_part",
            state="queued",
        ).data
        dsm.process.process_job(job.id, sleep_factor=0)
        part = lqs.fetch.ingestion_part(
            ingestion_id=ingestion.id, ingestion_part_id=part.id
        ).data
        assert part.state == ProcessState.completed, part.error

    ingestion = lqs.fetch.ingestion(ingestion_id=ingestion.id).data
    assert ingestion.state == ProcessState.completed
    return ingestion


def run_post_ingestion(lqs: RESTClient, dsm: DataStoreManager, ingestion):
    """Run the post-ingestion job (generates auxiliary data like images)."""
    datastore_id = lqs.get_datastore_id()
    job = dsm.create.job(
        event_id=None,
        type="post_ingestion",
        datastore_id=datastore_id,
        resource_id=ingestion.id,
        process_type="ingestion",
        state=ProcessStateDSM.queued,
    ).data
    dsm.process.process_job(job.id, sleep_factor=0)
    post_job = dsm.fetch.job(job_id=job.id).data
    assert post_job.state == ProcessStateDSM.completed


def process_extraction(
    lqs: RESTClient,
    dsm: DataStoreManager,
    digestion: models.Digestion,
):
    """Drive a digestion/extraction through its full lifecycle via DSM jobs."""
    datastore_id = lqs.get_datastore_id()

    # Process digestion job
    job = dsm.create.job(
        type="digestion",
        event_id=None,
        datastore_id=datastore_id,
        resource_id=digestion.id,
        process_type="digestion",
        state="queued",
    ).data
    dsm.process.process_job(job.id, sleep_factor=0)
    digestion = lqs.fetch.digestion(digestion_id=digestion.id).data
    assert digestion.state == ProcessState.finalizing

    # Process each digestion part
    parts = lqs.list.digestion_part(digestion_id=digestion.id).data
    assert len(parts) > 0
    for part in parts:
        assert part.state == ProcessState.queued
        job = dsm.create.job(
            type="digestion_part",
            event_id=None,
            datastore_id=datastore_id,
            resource_id=part.id,
            process_type="digestion_part",
            state="queued",
        ).data
        dsm.process.process_job(job.id, sleep_factor=0)
        part = lqs.fetch.digestion_part(
            digestion_id=digestion.id, digestion_part_id=part.id
        ).data
        assert part.state == ProcessState.completed, part.error

    # Run the extraction job
    job = dsm.create.job(
        type="extraction",
        event_id=None,
        datastore_id=datastore_id,
        resource_id=digestion.id,
        process_type="digestion",
        state="queued",
    ).data
    dsm.process.process_job(job.id, sleep_factor=0)

    digestion = lqs.fetch.digestion(digestion_id=digestion.id).data
    assert digestion.state == ProcessState.completed, digestion.error_payload
    return digestion


# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------


@pytest.fixture(name="lqs")
def lqs_fixture(lqs_function_scope: RESTClient):
    """Use a function-scoped (isolated) client for every test."""
    return lqs_function_scope


# ---------------------------------------------------------------------------
# 1. Auth & Identity
# ---------------------------------------------------------------------------


class TestAuthAndIdentity:
    """User, role, and API key operations."""

    def test_fetch_me(self, lqs: RESTClient):
        me = lqs.fetch.me().data
        assert me is not None
        assert me.username is not None
        assert me.admin is True

    def test_user_crud(self, lqs: RESTClient):
        name = uid("user")
        user = lqs.create.user(username=name).data
        assert user.username == name
        assert user.admin is False

        users = lqs.list.user(username=name).data
        assert len(users) == 1

        fetched = lqs.fetch.user(user_id=user.id).data
        assert fetched.id == user.id

        updated = lqs.update.user(user_id=user.id, data={"admin": True}).data
        assert updated.admin is True

        lqs.delete.user(user_id=user.id)
        users_after = lqs.list.user(username=name).data
        assert len(users_after) == 0

    def test_role_crud(self, lqs: RESTClient):
        name = uid("role")
        policy = {
            "statement": [{"effect": "allow", "action": ["read"], "resource": ["*"]}]
        }
        role = lqs.create.role(name=name, policy=policy).data
        assert role.name == name

        roles = lqs.list.role(name=name).data
        assert len(roles) == 1

        fetched = lqs.fetch.role(role_id=role.id).data
        assert fetched.id == role.id

        updated = lqs.update.role(role_id=role.id, data={"note": "updated"}).data
        assert updated.note == "updated"

        lqs.delete.role(role_id=role.id)
        roles_after = lqs.list.role(name=name).data
        assert len(roles_after) == 0

    def test_api_key_crud(self, lqs: RESTClient):
        me = lqs.fetch.me().data
        name = uid("key")
        key = lqs.create.api_key(user_id=me.id, name=name).data
        assert key.name == name
        assert key.secret is not None  # secret returned on creation

        keys = lqs.list.api_key(name=name).data
        assert len(keys) == 1

        fetched = lqs.fetch.api_key(api_key_id=key.id).data
        assert fetched.id == key.id

        lqs.delete.api_key(api_key_id=key.id)
        keys_after = lqs.list.api_key(name=name).data
        assert len(keys_after) == 0

    def test_non_admin_permissions(self, lqs: RESTClient):
        """Non-admin user with read-only role can read but not write."""
        user = lqs.create.user(username=uid("nonadmin")).data
        api_key = lqs.create.api_key(user_id=user.id, name="na").data

        config = lqs.config.model_copy()
        config.api_key_id = api_key.id
        config.api_key_secret = api_key.secret
        na_lqs = RESTClient(config=config, http_client=lqs.http_client)

        # Can read
        groups = na_lqs.list.group().data
        assert isinstance(groups, list)

        # Cannot write
        with pytest.raises(ForbiddenException):
            na_lqs.create.group(name=uid("blocked"))


# ---------------------------------------------------------------------------
# 2. Group
# ---------------------------------------------------------------------------


class TestGroup:
    def test_group_crud(self, lqs: RESTClient):
        name = uid("grp")
        group = lqs.create.group(name=name).data
        assert group.name == name
        assert group.locked is False

        # List with count
        res = lqs.list.group(name=name)
        assert len(res.data) == 1
        assert res.count == 1

        # List without count
        res = lqs.list.group(name=name, include_count=False)
        assert len(res.data) == 1
        assert res.count is None

        # Fetch
        fetched = lqs.fetch.group(group_id=group.id).data
        assert fetched.id == group.id

        # Update
        updated = lqs.update.group(
            group_id=group.id, data={"note": "test note", "context": {"key": "val"}}
        ).data
        assert updated.note == "test note"
        assert updated.context == {"key": "val"}

    def test_group_delete_and_soft_delete(self, lqs: RESTClient):
        """Deleted empty group disappears from list results."""
        name = uid("grp_del")
        group = lqs.create.group(name=name).data
        lqs.delete.group(group_id=group.id)

        res = lqs.list.group(name=name)
        assert len(res.data) == 0

    def test_group_delete_requires_empty(self, lqs: RESTClient):
        """Cannot delete a group that still has logs."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        with pytest.raises(BadRequestException):
            lqs.delete.group(group_id=group.id)

        # Delete log first, then group succeeds
        lqs.delete.log(log_id=log.id)
        lqs.delete.group(group_id=group.id)
        assert len(lqs.list.group(name=group.name).data) == 0

    def test_group_context_field(self, lqs: RESTClient):
        ctx = {"env": "test", "version": 3}
        group = lqs.create.group(name=uid("ctx"), context=ctx).data
        assert group.context == ctx

        fetched = lqs.fetch.group(group_id=group.id).data
        assert fetched.context == ctx


# ---------------------------------------------------------------------------
# 3. Log
# ---------------------------------------------------------------------------


class TestLog:
    def test_log_crud(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        name = uid("log")
        log = lqs.create.log(name=name, group_id=group.id).data
        assert log.name == name
        assert log.group_id == group.id
        assert log.record_count == 0
        assert log.duration is None

        res = lqs.list.log(name=name, group_id=group.id)
        assert len(res.data) == 1
        assert res.count == 1

        fetched = lqs.fetch.log(log_id=log.id).data
        assert fetched.id == log.id

        updated = lqs.update.log(log_id=log.id, data={"note": "log note"}).data
        assert updated.note == "log note"

    def test_log_records_created(self, lqs: RESTClient):
        """Records created manually are accessible via list."""
        group = lqs.create.group(name=uid("dur")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        topic = lqs.create.topic(name=uid("topic"), log_id=log.id).data

        t1 = 1000000000
        t2 = 6000000000
        lqs.create.record(topic_id=topic.id, timestamp=t1)
        lqs.create.record(topic_id=topic.id, timestamp=t2)

        records = lqs.list.record(topic_id=topic.id).data
        assert len(records) == 2
        assert records[0].timestamp == t1
        assert records[1].timestamp == t2

    def test_log_delete(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        lqs.delete.log(log_id=log.id)

        logs = lqs.list.log(group_id=group.id).data
        assert len(logs) == 0

    def test_log_delete_cascades_to_topics_and_records(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        topic = lqs.create.topic(name=uid("t"), log_id=log.id).data
        lqs.create.record(topic_id=topic.id, timestamp=100)

        log_id = log.id
        lqs.delete.log(log_id=log_id)

        topics = lqs.list.topic(log_id=log_id).data
        assert len(topics) == 0

    def test_log_list_ordering(self, lqs: RESTClient):
        """Logs can be sorted by duration."""
        group = lqs.create.group(name=uid("order")).data

        for i, (t1, t2) in enumerate([(100, 200), (100, 500), (100, 300)]):
            log = lqs.create.log(name=uid(f"log{i}"), group_id=group.id).data
            topic = lqs.create.topic(name=uid("t"), log_id=log.id).data
            lqs.create.record(topic_id=topic.id, timestamp=t1)
            lqs.create.record(topic_id=topic.id, timestamp=t2)

        logs_asc = lqs.list.log(group_id=group.id, order="duration", sort="ASC").data
        durations = [log.duration for log in logs_asc if log.duration is not None]
        assert durations == sorted(durations)

        logs_desc = lqs.list.log(group_id=group.id, order="duration", sort="DESC").data
        durations_desc = [log.duration for log in logs_desc if log.duration is not None]
        assert durations_desc == sorted(durations_desc, reverse=True)


# ---------------------------------------------------------------------------
# 4. Topic
# ---------------------------------------------------------------------------


class TestTopic:
    def test_topic_crud(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        name = uid("topic")
        topic = lqs.create.topic(name=name, log_id=log.id).data
        assert topic.name == name
        assert topic.log_id == log.id

        res = lqs.list.topic(name=name, log_id=log.id)
        assert len(res.data) == 1
        assert res.count == 1

        fetched = lqs.fetch.topic(topic_id=topic.id).data
        assert fetched.id == topic.id

        updated = lqs.update.topic(topic_id=topic.id, data={"note": "topic note"}).data
        assert updated.note == "topic note"

    def test_topic_deleted_via_log_cascade(self, lqs: RESTClient):
        """Topics are deleted when their parent log is deleted."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        lqs.create.topic(name=uid("t"), log_id=log.id)

        lqs.delete.log(log_id=log.id)
        topics = lqs.list.topic(log_id=log.id).data
        assert len(topics) == 0

    def test_topic_records_listed(self, lqs: RESTClient):
        """Records created under a topic are retrievable."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        topic = lqs.create.topic(name=uid("t"), log_id=log.id).data

        t1 = 2000000000
        t2 = 8000000000
        lqs.create.record(topic_id=topic.id, timestamp=t1)
        lqs.create.record(topic_id=topic.id, timestamp=t2)

        records = lqs.list.record(topic_id=topic.id).data
        assert len(records) == 2
        assert records[0].timestamp == t1
        assert records[1].timestamp == t2

    def test_multiple_topics_per_log(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        for i in range(3):
            lqs.create.topic(name=uid(f"topic{i}"), log_id=log.id)

        listed = lqs.list.topic(log_id=log.id).data
        assert len(listed) == 3


# ---------------------------------------------------------------------------
# 5. Record
# ---------------------------------------------------------------------------


class TestRecord:
    def test_record_crud(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        topic = lqs.create.topic(name=uid("t"), log_id=log.id).data
        ts = 1672531200000000000

        record = lqs.create.record(topic_id=topic.id, timestamp=ts).data
        assert record.timestamp == ts

        res = lqs.list.record(topic_id=topic.id)
        assert len(res.data) == 1
        assert res.count == 1

        fetched = lqs.fetch.record(topic_id=topic.id, timestamp=ts).data
        assert fetched.timestamp == ts
        assert fetched.raw_data is None
        assert fetched.auxiliary_data is None

        updated = lqs.update.record(
            topic_id=topic.id, timestamp=ts, data={"note": "r note"}
        ).data
        assert updated.note == "r note"

    def test_record_delete(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        topic = lqs.create.topic(name=uid("t"), log_id=log.id).data
        ts = 100000

        lqs.create.record(topic_id=topic.id, timestamp=ts)
        lqs.delete.record(topic_id=topic.id, timestamp=ts)

        res = lqs.list.record(topic_id=topic.id)
        assert len(res.data) == 0

    def test_record_ordering(self, lqs: RESTClient):
        """Records are returned in ascending timestamp order by default."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        topic = lqs.create.topic(name=uid("t"), log_id=log.id).data

        timestamps = [3000, 1000, 5000, 2000, 4000]
        for ts in timestamps:
            lqs.create.record(topic_id=topic.id, timestamp=ts)

        records_asc = lqs.list.record(topic_id=topic.id).data
        ts_asc = [r.timestamp for r in records_asc]
        assert ts_asc == sorted(timestamps)

        records_desc = lqs.list.record(topic_id=topic.id, sort="DESC").data
        ts_desc = [r.timestamp for r in records_desc]
        assert ts_desc == sorted(timestamps, reverse=True)

    def test_record_pagination(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        topic = lqs.create.topic(name=uid("t"), log_id=log.id).data

        for ts in range(10):
            lqs.create.record(topic_id=topic.id, timestamp=ts * 1000000000)

        # Full count
        full = lqs.list.record(topic_id=topic.id)
        assert full.count == 10

        # Limit
        page = lqs.list.record(topic_id=topic.id, limit=3)
        assert len(page.data) == 3

        # Offset
        page2 = lqs.list.record(topic_id=topic.id, limit=3, offset=3)
        assert len(page2.data) == 3
        # No overlap
        ids_1 = {r.timestamp for r in page.data}
        ids_2 = {r.timestamp for r in page2.data}
        assert ids_1.isdisjoint(ids_2)

        # Offset past end
        empty = lqs.list.record(topic_id=topic.id, limit=10, offset=100)
        assert len(empty.data) == 0

    def test_record_query_data(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        topic = lqs.create.topic(name=uid("t"), log_id=log.id).data

        qd = {"height": 480, "width": 640, "format": "rgb8"}
        record = lqs.create.record(topic_id=topic.id, timestamp=100, query_data=qd).data
        assert record.query_data == qd

        fetched = lqs.fetch.record(topic_id=topic.id, timestamp=100).data
        assert fetched.query_data["height"] == 480
        assert fetched.query_data["width"] == 640

    def test_records_listed_across_topic(self, lqs: RESTClient):
        """Multiple records in a topic are all accessible."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        topic = lqs.create.topic(name=uid("t"), log_id=log.id).data

        lqs.create.record(topic_id=topic.id, timestamp=1000)
        lqs.create.record(topic_id=topic.id, timestamp=5000)

        records = lqs.list.record(topic_id=topic.id).data
        assert len(records) == 2
        assert records[0].timestamp == 1000
        assert records[1].timestamp == 5000


# ---------------------------------------------------------------------------
# 6. Label & Tag
# ---------------------------------------------------------------------------


class TestLabelAndTag:
    def test_label_crud(self, lqs: RESTClient):
        value = uid("label")
        label = lqs.create.label(value=value).data
        assert label.value == value

        labels = lqs.list.label(value=value).data
        assert len(labels) == 1

        fetched = lqs.fetch.label(label_id=label.id).data
        assert fetched.value == value

        updated = lqs.update.label(label_id=label.id, data={"note": "lab note"}).data
        assert updated.note == "lab note"

    def test_label_delete(self, lqs: RESTClient):
        label = lqs.create.label(value=uid("del")).data
        lqs.delete.label(label_id=label.id)

        labels = lqs.list.label(value=label.value).data
        assert len(labels) == 0

    def test_tag_crud(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        label = lqs.create.label(value=uid("lbl")).data

        tag = lqs.create.tag(log_id=log.id, label_id=label.id).data
        assert tag.log_id == log.id
        assert tag.label_id == label.id

        tags = lqs.list.tag(log_id=log.id).data
        assert len(tags) == 1

        fetched = lqs.fetch.tag(tag_id=tag.id).data
        assert fetched.id == tag.id

        updated = lqs.update.tag(tag_id=tag.id, data={"note": "tag note"}).data
        assert updated.note == "tag note"

    def test_tag_delete(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        label = lqs.create.label(value=uid("lbl")).data
        tag = lqs.create.tag(log_id=log.id, label_id=label.id).data

        lqs.delete.tag(tag_id=tag.id)
        tags = lqs.list.tag(log_id=log.id).data
        assert len(tags) == 0

    def test_tag_with_topic_and_time_range(self, lqs: RESTClient):
        """Tags can be associated with a specific topic and time range."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        topic = lqs.create.topic(name=uid("t"), log_id=log.id).data
        label = lqs.create.label(value=uid("lbl")).data

        tag = lqs.create.tag(
            log_id=log.id,
            label_id=label.id,
            topic_id=topic.id,
            start_time=100,
            end_time=200,
        ).data
        assert tag.topic_id == topic.id
        assert tag.start_time == 100
        assert tag.end_time == 200

    def test_list_logs_by_tag_label(self, lqs: RESTClient):
        """Logs can be filtered by tag label IDs."""
        group = lqs.create.group(name=uid("grp")).data
        log1 = lqs.create.log(name=uid("log1"), group_id=group.id).data
        log2 = lqs.create.log(name=uid("log2"), group_id=group.id).data

        label_a = lqs.create.label(value=uid("a")).data
        label_b = lqs.create.label(value=uid("b")).data

        lqs.create.tag(log_id=log1.id, label_id=label_a.id)
        lqs.create.tag(log_id=log1.id, label_id=label_b.id)
        lqs.create.tag(log_id=log2.id, label_id=label_a.id)

        # label_a on both logs
        res = lqs.list.log(tag_label_ids_includes=label_a.id)
        log_ids = {log.id for log in res.data}
        assert log1.id in log_ids
        assert log2.id in log_ids

        # label_b on only log1
        res = lqs.list.log(tag_label_ids_includes=label_b.id)
        assert len(res.data) == 1
        assert res.data[0].id == log1.id

        # Both labels — only log1 has both
        res = lqs.list.log(tag_label_ids_includes=[label_a.id, label_b.id])
        assert len(res.data) == 1
        assert res.data[0].id == log1.id

    def test_log_delete_cascades_tags(self, lqs: RESTClient):
        """Deleting a log also deletes its tags."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        label = lqs.create.label(value=uid("lbl")).data
        lqs.create.tag(log_id=log.id, label_id=label.id)

        lqs.delete.log(log_id=log.id)
        assert len(lqs.list.tag(log_id=log.id).data) == 0


# ---------------------------------------------------------------------------
# 7. Ingestion & Ingestion Part (CRUD only)
# ---------------------------------------------------------------------------


class TestIngestionCRUD:
    def test_ingestion_crud(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        name = uid("ing")

        ingestion = lqs.create.ingestion(
            log_id=log.id, name=name, state=ProcessState.ready
        ).data
        assert ingestion.name == name
        assert ingestion.state == ProcessState.ready
        assert ingestion.log_id == log.id

        res = lqs.list.ingestion(log_id=log.id, name=name)
        assert len(res.data) == 1

        fetched = lqs.fetch.ingestion(ingestion_id=ingestion.id).data
        assert fetched.id == ingestion.id

        updated = lqs.update.ingestion(
            ingestion_id=ingestion.id, data={"note": "ing note"}
        ).data
        assert updated.note == "ing note"

    def test_ingestion_delete(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        ingestion = lqs.create.ingestion(
            log_id=log.id, name=uid("ing"), state=ProcessState.ready
        ).data

        lqs.delete.ingestion(ingestion_id=ingestion.id)
        res = lqs.list.ingestion(log_id=log.id)
        assert len(res.data) == 0

    def test_ingestion_part_requires_processing_state(self, lqs: RESTClient):
        """Cannot create ingestion part unless ingestion is in 'processing' state."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        ingestion = lqs.create.ingestion(
            log_id=log.id, name=uid("ing"), state=ProcessState.ready
        ).data

        with pytest.raises(BadRequestException):
            lqs.create.ingestion_part(sequence=0, ingestion_id=ingestion.id)

        # Transition to processing, then create part
        lqs.update.ingestion(
            ingestion_id=ingestion.id, data={"state": ProcessState.processing}
        )
        part = lqs.create.ingestion_part(sequence=0, ingestion_id=ingestion.id).data
        assert part.sequence == 0

    def test_ingestion_part_crud(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        ingestion = lqs.create.ingestion(
            log_id=log.id, name=uid("ing"), state=ProcessState.processing
        ).data

        part = lqs.create.ingestion_part(sequence=0, ingestion_id=ingestion.id).data
        assert part.sequence == 0

        parts = lqs.list.ingestion_part(ingestion_id=ingestion.id).data
        assert len(parts) == 1

        fetched = lqs.fetch.ingestion_part(
            ingestion_id=ingestion.id, ingestion_part_id=part.id
        ).data
        assert fetched.id == part.id

        updated = lqs.update.ingestion_part(
            ingestion_id=ingestion.id,
            ingestion_part_id=part.id,
            data={"sequence": 1},
        ).data
        assert updated.sequence == 1

    def test_log_delete_cascades_ingestion(self, lqs: RESTClient):
        """Deleting a log cascades to its ingestions."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        lqs.create.ingestion(log_id=log.id, name=uid("ing"), state=ProcessState.ready)

        lqs.delete.log(log_id=log.id)
        assert len(lqs.list.ingestion(log_id=log.id).data) == 0


# ---------------------------------------------------------------------------
# 8. Digestion, Digestion Part, Digestion Topic (CRUD only)
# ---------------------------------------------------------------------------


class TestDigestionCRUD:
    def test_digestion_crud(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        name = uid("dig")

        digestion = lqs.create.digestion(
            log_id=log.id, name=name, state=ProcessState.ready
        ).data
        assert digestion.name == name
        assert digestion.state == ProcessState.ready

        res = lqs.list.digestion(log_id=log.id, name=name)
        assert len(res.data) == 1

        fetched = lqs.fetch.digestion(digestion_id=digestion.id).data
        assert fetched.id == digestion.id

        updated = lqs.update.digestion(
            digestion_id=digestion.id, data={"note": "dig note"}
        ).data
        assert updated.note == "dig note"

    def test_digestion_delete(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        digestion = lqs.create.digestion(
            log_id=log.id, name=uid("dig"), state=ProcessState.ready
        ).data

        lqs.delete.digestion(digestion_id=digestion.id)
        res = lqs.list.digestion(log_id=log.id)
        assert len(res.data) == 0

    def test_digestion_part_requires_processing_state(self, lqs: RESTClient):
        """Cannot create digestion part unless digestion is in 'processing' state."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        digestion = lqs.create.digestion(
            log_id=log.id, name=uid("dig"), state=ProcessState.ready
        ).data

        with pytest.raises(BadRequestException):
            lqs.create.digestion_part(sequence=0, digestion_id=digestion.id)

        lqs.update.digestion(
            digestion_id=digestion.id, data={"state": ProcessState.processing}
        )
        part = lqs.create.digestion_part(sequence=0, digestion_id=digestion.id).data
        assert part.sequence == 0

    def test_digestion_part_crud(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        digestion = lqs.create.digestion(
            log_id=log.id, name=uid("dig"), state=ProcessState.processing
        ).data

        part = lqs.create.digestion_part(sequence=0, digestion_id=digestion.id).data
        parts = lqs.list.digestion_part(digestion_id=digestion.id).data
        assert len(parts) == 1

        fetched = lqs.fetch.digestion_part(
            digestion_id=digestion.id, digestion_part_id=part.id
        ).data
        assert fetched.id == part.id

        updated = lqs.update.digestion_part(
            digestion_id=digestion.id,
            digestion_part_id=part.id,
            data={"sequence": 1},
        ).data
        assert updated.sequence == 1

    def test_digestion_topic_requires_non_processing_state(self, lqs: RESTClient):
        """Cannot create digestion topic if digestion is in 'processing' state."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        topic = lqs.create.topic(name=uid("t"), log_id=log.id).data
        digestion = lqs.create.digestion(
            log_id=log.id, name=uid("dig"), state=ProcessState.processing
        ).data

        with pytest.raises(BadRequestException):
            lqs.create.digestion_topic(digestion_id=digestion.id, topic_id=topic.id)

        lqs.update.digestion(
            digestion_id=digestion.id, data={"state": ProcessState.ready}
        )
        dt = lqs.create.digestion_topic(
            digestion_id=digestion.id, topic_id=topic.id
        ).data
        assert dt.topic_id == topic.id

    def test_digestion_topic_crud(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        topic = lqs.create.topic(name=uid("t"), log_id=log.id).data
        digestion = lqs.create.digestion(
            log_id=log.id, name=uid("dig"), state=ProcessState.ready
        ).data

        dt = lqs.create.digestion_topic(
            digestion_id=digestion.id, topic_id=topic.id
        ).data
        assert dt.topic_id == topic.id

        dts = lqs.list.digestion_topic(digestion_id=digestion.id).data
        assert len(dts) == 1

        fetched = lqs.fetch.digestion_topic(
            digestion_id=digestion.id, digestion_topic_id=dt.id
        ).data
        assert fetched.id == dt.id

        updated = lqs.update.digestion_topic(
            digestion_id=digestion.id,
            digestion_topic_id=dt.id,
            data={"frequency": 2},
        ).data
        assert updated.frequency == 2

    def test_log_delete_cascades_digestion(self, lqs: RESTClient):
        """Deleting a log cascades to its digestions."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        lqs.create.digestion(log_id=log.id, name=uid("dig"), state=ProcessState.ready)

        lqs.delete.log(log_id=log.id)
        assert len(lqs.list.digestion(log_id=log.id).data) == 0


# ---------------------------------------------------------------------------
# 9. Workflow & Hook
# ---------------------------------------------------------------------------


class TestWorkflowAndHook:
    def test_workflow_crud(self, lqs: RESTClient):
        name = uid("wf")
        workflow = lqs.create.workflow(name=name, process_type="ingestion").data
        assert workflow.name == name
        assert workflow.process_type == "ingestion"

        res = lqs.list.workflow(name=name)
        assert len(res.data) == 1

        fetched = lqs.fetch.workflow(workflow_id=workflow.id).data
        assert fetched.id == workflow.id

        updated = lqs.update.workflow(
            workflow_id=workflow.id, data={"note": "wf note"}
        ).data
        assert updated.note == "wf note"

    def test_workflow_delete(self, lqs: RESTClient):
        wf = lqs.create.workflow(name=uid("wf"), process_type="digestion").data
        lqs.delete.workflow(workflow_id=wf.id)

        res = lqs.list.workflow(name=wf.name)
        assert len(res.data) == 0

    def test_workflow_soft_delete_preserves_hooks(self, lqs: RESTClient):
        """Soft-deleting a workflow does not cascade-delete hooks."""
        wf = lqs.create.workflow(name=uid("wf"), process_type="ingestion").data
        lqs.create.hook(
            workflow_id=wf.id,
            trigger_process="ingestion",
            trigger_state="failed",
            name=uid("hook"),
        )

        lqs.delete.workflow(workflow_id=wf.id)
        # Soft delete does not cascade — hooks remain
        hooks = lqs.list.hook(workflow_id=wf.id).data
        assert len(hooks) == 1

    def test_hook_crud(self, lqs: RESTClient):
        wf = lqs.create.workflow(name=uid("wf"), process_type="ingestion").data
        name = uid("hook")
        hook = lqs.create.hook(
            workflow_id=wf.id,
            trigger_process="ingestion",
            trigger_state="completed",
            name=name,
        ).data
        assert hook.name == name
        assert hook.trigger_process == "ingestion"
        assert hook.trigger_state == "completed"

        hooks = lqs.list.hook(workflow_id=wf.id, name=name).data
        assert len(hooks) == 1

        fetched = lqs.fetch.hook(workflow_id=wf.id, hook_id=hook.id).data
        assert fetched.id == hook.id

        updated = lqs.update.hook(
            workflow_id=wf.id, hook_id=hook.id, data={"note": "hook note"}
        ).data
        assert updated.note == "hook note"

    def test_hook_delete(self, lqs: RESTClient):
        wf = lqs.create.workflow(name=uid("wf"), process_type="ingestion").data
        hook = lqs.create.hook(
            workflow_id=wf.id,
            trigger_process="ingestion",
            trigger_state="failed",
            name=uid("hook"),
        ).data

        lqs.delete.hook(hook_id=hook.id, workflow_id=wf.id)
        hooks = lqs.list.hook(workflow_id=wf.id).data
        assert len(hooks) == 0

    def test_workflow_with_multiple_hooks(self, lqs: RESTClient):
        """Workflow can have multiple hooks with different triggers."""
        wf = lqs.create.workflow(name=uid("wf"), process_type="ingestion").data

        h1 = lqs.create.hook(
            workflow_id=wf.id,
            trigger_process="ingestion",
            trigger_state="completed",
            name=uid("h1"),
        ).data

        h2 = lqs.create.hook(
            workflow_id=wf.id,
            trigger_process="ingestion",
            trigger_state="failed",
            name=uid("h2"),
        ).data

        hooks = lqs.list.hook(workflow_id=wf.id).data
        assert len(hooks) == 2

        # Delete one hook — the other persists
        lqs.delete.hook(hook_id=h1.id, workflow_id=wf.id)
        hooks = lqs.list.hook(workflow_id=wf.id).data
        assert len(hooks) == 1
        assert hooks[0].id == h2.id


# ---------------------------------------------------------------------------
# 10. Callback
# ---------------------------------------------------------------------------


class TestCallback:
    def test_callback_crud(self, lqs: RESTClient):
        name = uid("cb")
        cb = lqs.create.callback(name=name).data
        assert cb.name == name

        res = lqs.list.callback(name=name)
        assert len(res.data) == 1

        fetched = lqs.fetch.callback(callback_id=cb.id).data
        assert fetched.id == cb.id

        updated = lqs.update.callback(callback_id=cb.id, data={"note": "cb note"}).data
        assert updated.note == "cb note"

    def test_callback_delete(self, lqs: RESTClient):
        cb = lqs.create.callback(name=uid("cb")).data
        lqs.delete.callback(callback_id=cb.id)

        res = lqs.list.callback(name=cb.name)
        assert len(res.data) == 0


# ---------------------------------------------------------------------------
# 11. Object Store
# ---------------------------------------------------------------------------


class TestObjectStore:
    def test_object_store_crud(self, lqs: RESTClient):
        name = uid("os")
        obj_store = lqs.create.object_store(
            name=name,
            bucket_name=name,
            access_key_id="minioadmin",
            secret_access_key="minioadmin",
            region_name="us-east-1",
            endpoint_url="http://localhost:9000",
        ).data
        assert obj_store.name == name
        assert obj_store.bucket_name == name

        res = lqs.list.object_store(bucket_name=name)
        assert len(res.data) == 1

        fetched = lqs.fetch.object_store(object_store_id=obj_store.id).data
        assert fetched.id == obj_store.id

        updated = lqs.update.object_store(
            object_store_id=obj_store.id, data={"note": "os note"}
        ).data
        assert updated.note == "os note"

    def test_object_store_delete(self, lqs: RESTClient):
        name = uid("os")
        obj_store = lqs.create.object_store(
            name=name,
            bucket_name=name,
            access_key_id="minioadmin",
            secret_access_key="minioadmin",
            region_name="us-east-1",
            endpoint_url="http://localhost:9000",
        ).data

        lqs.delete.object_store(object_store_id=obj_store.id)
        res = lqs.list.object_store(name=name)
        assert len(res.data) == 0

    def test_object_store_name_like_filter(self, lqs: RESTClient):
        prefix = uid("osfilt")
        lqs.create.object_store(
            name=prefix,
            bucket_name=prefix,
            access_key_id="minioadmin",
            secret_access_key="minioadmin",
            region_name="us-east-1",
            endpoint_url="http://localhost:9000",
        )

        res = lqs.list.object_store(name_like=prefix)
        assert len(res.data) == 1
        assert res.data[0].name == prefix


# ---------------------------------------------------------------------------
# 12. Query
# ---------------------------------------------------------------------------


class TestQuery:
    def test_query_crud(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        name = uid("q")

        query = lqs.create.query(log_id=log.id, name=name, statement="SELECT 1").data
        assert query.name == name
        assert query.statement == "SELECT 1"

        res = lqs.list.query(log_id=log.id, name=name)
        assert len(res.data) == 1

        fetched = lqs.fetch.query(log_id=log.id, query_id=query.id).data
        assert fetched.id == query.id

        updated = lqs.update.query(
            log_id=log.id, query_id=query.id, data={"note": "q note"}
        ).data
        assert updated.note == "q note"

    def test_query_deleted_via_log_cascade(self, lqs: RESTClient):
        """Queries are deleted when their parent log is deleted."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        lqs.create.query(log_id=log.id, name=uid("q"), statement="SELECT 1")

        lqs.delete.log(log_id=log.id)
        res = lqs.list.query(log_id=log.id)
        assert len(res.data) == 0


# ---------------------------------------------------------------------------
# 13. Locking
# ---------------------------------------------------------------------------


class TestLocking:
    def test_lock_group(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp"), locked=False).data
        assert group.locked is False
        assert group.lock_token is None

        locked = lqs.update.group(group_id=group.id, data={"locked": True}).data
        assert locked.locked is True
        assert locked.lock_token is not None

    def test_create_locked(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp"), locked=True).data
        assert group.locked is True
        assert group.lock_token is not None

    def test_locked_parent_blocks_child_creation(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp"), locked=True).data
        with pytest.raises(LockedException):
            lqs.create.log(group_id=group.id, name=uid("log"))

    def test_locked_parent_blocks_child_delete(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(group_id=group.id, name=uid("log")).data

        locked_group = lqs.update.group(group_id=group.id, data={"locked": True}).data

        with pytest.raises(LockedException):
            lqs.delete.log(log_id=log.id)

        # But works with lock token
        lqs.delete.log(log_id=log.id, lock_token=locked_group.lock_token)

    def test_locked_child_blocks_parent_delete(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(group_id=group.id, name=uid("log")).data
        topic = lqs.create.topic(log_id=log.id, name=uid("t"), locked=True).data

        with pytest.raises(LockedException):
            lqs.delete.log(log_id=log.id)

        lqs.delete.log(log_id=log.id, lock_token=topic.lock_token)

    def test_lock_log(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(group_id=group.id, name=uid("log"), locked=True).data
        assert log.locked is True

        with pytest.raises(LockedException):
            lqs.update.log(log_id=log.id, data={"locked": False})

    def test_lock_topic(self, lqs: RESTClient):
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(group_id=group.id, name=uid("log")).data
        topic = lqs.create.topic(log_id=log.id, name=uid("t"), locked=True).data
        assert topic.locked is True
        assert topic.lock_token is not None


# ---------------------------------------------------------------------------
# 14. Log Object Upload & Download
# ---------------------------------------------------------------------------


class TestLogObjects:
    def test_upload_and_fetch_object(self, lqs: RESTClient):
        """Upload a file via the utils helper and fetch it back."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        (log_object, log_object_parts) = lqs.utils.upload_log_object(
            log_id=log.id,
            file_path=LOG_FILE_PATH,
            object_key=LOG_OBJECT_KEY,
        )
        assert log_object.key.endswith(LOG_OBJECT_KEY)
        assert len(log_object_parts) > 0

        # Fetch the object and verify it's a valid ROS bag
        fetched = lqs.fetch.log_object(
            log_id=log.id,
            object_key=LOG_OBJECT_KEY,
            redirect=False,
            offset=0,
            length=13,
        ).data
        headers = {"Range": "bytes=0-12"}
        object_data = requests.get(fetched.presigned_url, headers=headers).content
        assert object_data == b"#ROSBAG V2.0\n"

    def test_upload_conflict(self, lqs: RESTClient):
        """Re-uploading the same object key without overwrite raises ConflictException."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        lqs.utils.upload_log_object(
            log_id=log.id,
            file_path=LOG_FILE_PATH,
            object_key=LOG_OBJECT_KEY,
        )

        with pytest.raises(ConflictException):
            lqs.utils.upload_log_object(
                log_id=log.id,
                file_path=LOG_FILE_PATH,
                object_key=LOG_OBJECT_KEY,
            )

    def test_upload_overwrite(self, lqs: RESTClient):
        """Overwrite flag allows re-uploading to the same key."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        lqs.utils.upload_log_object(
            log_id=log.id,
            file_path=LOG_FILE_PATH,
            object_key=LOG_OBJECT_KEY,
        )

        # Should succeed with overwrite=True
        lqs.utils.upload_log_object(
            log_id=log.id,
            file_path=LOG_FILE_PATH,
            object_key=LOG_OBJECT_KEY,
            continue_upload=False,
            overwrite=True,
        )

    def test_list_log_objects(self, lqs: RESTClient):
        """Uploaded objects appear in list results."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        lqs.utils.upload_log_object(
            log_id=log.id,
            file_path=LOG_FILE_PATH,
            object_key=LOG_OBJECT_KEY,
        )

        objects = lqs.list.log_objects(log_id=log.id, prefix=LOG_OBJECT_KEY).data
        assert len(objects) >= 1
        assert objects[0].upload_state == UploadState.complete

    def test_delete_log_object(self, lqs: RESTClient):
        """Log objects can be deleted."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        (log_object, _) = lqs.utils.upload_log_object(
            log_id=log.id,
            file_path=LOG_FILE_PATH,
            object_key=LOG_OBJECT_KEY,
        )

        lqs.delete.log_object(log_id=log.id, object_key=log_object.key)

        objects = lqs.list.log_objects(log_id=log.id, prefix=LOG_OBJECT_KEY).data
        assert len(objects) == 0


# ---------------------------------------------------------------------------
# 15. Ingestion Processing (E2E)
# ---------------------------------------------------------------------------


class TestIngestionProcessing:
    """End-to-end ingestion: upload → ingest → verify records."""

    def test_ingest_ros_bag(self, lqs: RESTClient, dsm: DataStoreManager):
        """Upload a ROS bag, run ingestion, verify topics and records are created."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        # Upload the bag file
        lqs.utils.upload_log_object(
            log_id=log.id,
            file_path=LOG_FILE_PATH,
            object_key=LOG_OBJECT_KEY,
        )

        # Create and process the ingestion
        ingestion = lqs.create.ingestion(
            log_id=log.id,
            object_key=LOG_OBJECT_KEY,
            state=ProcessState.queued,
        ).data
        ingestion = process_ingestion(lqs, dsm, ingestion)

        assert ingestion.state == ProcessState.completed
        assert ingestion.error_payload is None, ingestion.error_payload

        # Verify topics were created
        topics = lqs.list.topic(log_id=log.id).data
        assert len(topics) == 1  # "image" topic from the bag
        image_topic = topics[0]
        assert image_topic.name == IMAGE_TOPIC_NAME
        assert image_topic.record_count == 10

        # Verify log-level stats
        log = lqs.fetch.log(log_id=log.id).data
        assert log.record_count == 10
        assert log.record_size > 0

        # Verify records are accessible and ordered
        records = lqs.list.record(topic_id=image_topic.id, limit=10).data
        assert len(records) == 10
        for i in range(len(records) - 1):
            assert records[i].timestamp < records[i + 1].timestamp

        # Verify query_data was extracted
        for record in records:
            assert record.query_data is not None
            assert record.query_data["height"] == 32
            assert record.query_data["width"] == 32

    def test_ingest_and_read_raw_data(self, lqs: RESTClient, dsm: DataStoreManager):
        """After ingestion, raw data can be fetched and deserialized."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        lqs.utils.upload_log_object(
            log_id=log.id,
            file_path=LOG_FILE_PATH,
            object_key=LOG_OBJECT_KEY,
        )

        ingestion = lqs.create.ingestion(
            log_id=log.id,
            object_key=LOG_OBJECT_KEY,
            state=ProcessState.queued,
        ).data
        process_ingestion(lqs, dsm, ingestion)

        image_topic = lqs.list.topic(log_id=log.id, name=IMAGE_TOPIC_NAME).data[0]

        # Fetch with raw data
        record = lqs.list.record(
            topic_id=image_topic.id, limit=1, include_raw_data=True
        ).data[0]
        assert record.raw_data is not None

        # Deserialize the raw data
        message_bytes = base64.b64decode(record.raw_data)
        transcoder = Transcode()
        record_data = transcoder.deserialize(
            type_encoding=image_topic.type_encoding,
            type_name=image_topic.type_name,
            type_data=image_topic.type_data,
            message_bytes=message_bytes,
        )
        assert record_data["height"] == 32
        assert record_data["width"] == 32
        assert len(record_data["data"]) == 32 * 32 * 3

    def test_ingest_and_post_ingestion(self, lqs: RESTClient, dsm: DataStoreManager):
        """Post-ingestion generates auxiliary data (e.g., image thumbnails)."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        lqs.utils.upload_log_object(
            log_id=log.id,
            file_path=LOG_FILE_PATH,
            object_key=LOG_OBJECT_KEY,
        )

        ingestion = lqs.create.ingestion(
            log_id=log.id,
            object_key=LOG_OBJECT_KEY,
            state=ProcessState.queued,
        ).data
        ingestion = process_ingestion(lqs, dsm, ingestion)
        run_post_ingestion(lqs, dsm, ingestion)

        # After post-ingestion, records should have auxiliary_data
        image_topic = lqs.list.topic(log_id=log.id, name=IMAGE_TOPIC_NAME).data[0]

        record = lqs.fetch.record(
            topic_id=image_topic.id,
            timestamp=0,
            include_auxiliary_data=True,
        ).data
        assert record.auxiliary_data is not None
        assert "image" in record.auxiliary_data

    def test_delete_ingestion_part_removes_records(
        self, lqs: RESTClient, dsm: DataStoreManager
    ):
        """Deleting an ingestion part removes its records and updates counts."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        lqs.utils.upload_log_object(
            log_id=log.id,
            file_path=LOG_FILE_PATH,
            object_key=LOG_OBJECT_KEY,
        )

        ingestion = lqs.create.ingestion(
            log_id=log.id,
            object_key=LOG_OBJECT_KEY,
            state=ProcessState.queued,
        ).data
        ingestion = process_ingestion(lqs, dsm, ingestion)

        # Before delete: 10 records
        image_topic = lqs.list.topic(log_id=log.id, name=IMAGE_TOPIC_NAME).data[0]
        assert image_topic.record_count == 10

        # Delete the ingestion part
        parts = lqs.list.ingestion_part(ingestion_id=ingestion.id).data
        assert len(parts) == 1
        lqs.delete.ingestion_part(
            ingestion_part_id=parts[0].id, ingestion_id=ingestion.id
        )

        # After delete: records removed, topic count updated
        image_topic = lqs.fetch.topic(topic_id=image_topic.id).data
        assert image_topic.record_count == 0

        log = lqs.fetch.log(log_id=log.id).data
        assert log.record_count == 0

    def test_usage_record_created(self, lqs: RESTClient, dsm: DataStoreManager):
        """Ingestion creates a usage record for billing/tracking."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        lqs.utils.upload_log_object(
            log_id=log.id,
            file_path=LOG_FILE_PATH,
            object_key=LOG_OBJECT_KEY,
        )

        ingestion = lqs.create.ingestion(
            log_id=log.id,
            object_key=LOG_OBJECT_KEY,
            state=ProcessState.queued,
        ).data
        process_ingestion(lqs, dsm, ingestion)

        datastore_id = lqs.get_datastore_id()
        usage = dsm.list.usage_record(
            datastore_id=datastore_id,
            category="ingestion_processing",
            sort="DESC",
            limit=1,
        ).data
        assert len(usage) > 0
        latest = usage[0]
        assert latest.category == "ingestion_processing"
        assert latest.usage_data["ingestion_id"] == str(ingestion.id)
        assert latest.usage_data["record_count"] > 0


# ---------------------------------------------------------------------------
# 16. Digestion / Extraction Processing (E2E)
# ---------------------------------------------------------------------------


class TestDigestionProcessing:
    """End-to-end digestion: ingest → create digestion → extract → verify output."""

    def test_ros_bag_extraction(self, lqs: RESTClient, dsm: DataStoreManager):
        """Ingest a ROS bag, then extract selected topics to a new bag."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        # Upload and ingest
        lqs.utils.upload_log_object(
            log_id=log.id,
            file_path=LOG_FILE_PATH,
            object_key=LOG_OBJECT_KEY,
        )
        ingestion = lqs.create.ingestion(
            log_id=log.id,
            object_key=LOG_OBJECT_KEY,
            state=ProcessState.queued,
        ).data
        process_ingestion(lqs, dsm, ingestion)

        # Set up digestion with a topic
        topics = lqs.list.topic(log_id=log.id).data
        assert len(topics) > 0

        digestion = lqs.create.digestion(
            log_id=log.id,
            name=uid("dig"),
            state=ProcessState.ready,
        ).data
        lqs.create.digestion_topic(
            digestion_id=digestion.id,
            topic_id=topics[0].id,
        )
        digestion = lqs.update.digestion(
            digestion_id=digestion.id,
            data={"state": ProcessState.queued},
        ).data

        # Run the extraction
        digestion = process_extraction(lqs, dsm, digestion)
        assert digestion.state == ProcessState.completed
        assert digestion.error_payload is None, digestion.error_payload

        # Verify the extracted bag exists and is a valid ROS bag
        extracted_key = f"digestions/{digestion.id}/extraction_{digestion.id}.bag"
        extracted_obj = lqs.fetch.log_object(
            log_id=log.id,
            object_key=extracted_key,
            redirect=False,
            offset=0,
            length=13,
        ).data
        headers = {"Range": "bytes=0-12"}
        data = requests.get(extracted_obj.presigned_url, headers=headers).content
        assert data == b"#ROSBAG V2.0\n"

    def test_digestion_with_workflow_context(
        self, lqs: RESTClient, dsm: DataStoreManager
    ):
        """Digestion respects workflow_context for configuration."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        lqs.utils.upload_log_object(
            log_id=log.id,
            file_path=LOG_FILE_PATH,
            object_key=LOG_OBJECT_KEY,
        )
        ingestion = lqs.create.ingestion(
            log_id=log.id,
            object_key=LOG_OBJECT_KEY,
            state=ProcessState.queued,
        ).data
        process_ingestion(lqs, dsm, ingestion)

        topics = lqs.list.topic(log_id=log.id).data
        assert len(topics) > 0

        digestion = lqs.create.digestion(
            log_id=log.id,
            name=uid("dig"),
            state=ProcessState.ready,
            workflow_context={"version": 2},
        ).data
        assert digestion.workflow_context == {"version": 2}

        lqs.create.digestion_topic(
            digestion_id=digestion.id,
            topic_id=topics[0].id,
        )
        digestion = lqs.update.digestion(
            digestion_id=digestion.id,
            data={"state": ProcessState.queued},
        ).data

        digestion = process_extraction(lqs, dsm, digestion)
        assert digestion.state == ProcessState.completed


# ---------------------------------------------------------------------------
# 17. Complex Interactions
# ---------------------------------------------------------------------------


class TestComplexInteractions:
    def test_full_hierarchy_lifecycle(self, lqs: RESTClient):
        """Create a full hierarchy, verify stats, delete from leaf to root."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        # Multiple topics with records
        topic1 = lqs.create.topic(name=uid("t1"), log_id=log.id).data
        topic2 = lqs.create.topic(name=uid("t2"), log_id=log.id).data

        for ts in [100, 200, 300]:
            lqs.create.record(topic_id=topic1.id, timestamp=ts)
        for ts in [150, 250]:
            lqs.create.record(topic_id=topic2.id, timestamp=ts)

        # Verify records are accessible
        records_t1 = lqs.list.record(topic_id=topic1.id).data
        assert len(records_t1) == 3
        records_t2 = lqs.list.record(topic_id=topic2.id).data
        assert len(records_t2) == 2

        # Attach ancillary resources
        label = lqs.create.label(value=uid("lbl")).data
        lqs.create.tag(log_id=log.id, label_id=label.id)
        lqs.create.query(log_id=log.id, name=uid("q"), statement="SELECT 1")
        lqs.create.ingestion(log_id=log.id, name=uid("ing"), state=ProcessState.ready)
        lqs.create.digestion(log_id=log.id, name=uid("dig"), state=ProcessState.ready)

        # Delete log — everything under it cascades
        lqs.delete.log(log_id=log.id)
        assert len(lqs.list.log(group_id=group.id).data) == 0
        assert len(lqs.list.tag(log_id=log.id).data) == 0
        assert len(lqs.list.query(log_id=log.id).data) == 0
        assert len(lqs.list.ingestion(log_id=log.id).data) == 0
        assert len(lqs.list.digestion(log_id=log.id).data) == 0

        # Now the group is empty and can be deleted
        lqs.delete.group(group_id=group.id)
        assert len(lqs.list.group(name=group.name).data) == 0

    def test_multiple_groups_with_logs(self, lqs: RESTClient):
        """Verify resources are properly scoped to their parents."""
        g1 = lqs.create.group(name=uid("g1")).data
        g2 = lqs.create.group(name=uid("g2")).data

        l1 = lqs.create.log(name=uid("l1"), group_id=g1.id).data
        l2 = lqs.create.log(name=uid("l2"), group_id=g2.id).data

        logs_g1 = lqs.list.log(group_id=g1.id).data
        logs_g2 = lqs.list.log(group_id=g2.id).data
        assert len(logs_g1) == 1
        assert len(logs_g2) == 1
        assert logs_g1[0].id == l1.id
        assert logs_g2[0].id == l2.id

    def test_group_id_in_filter(self, lqs: RESTClient):
        """Filter logs by multiple group IDs."""
        g1 = lqs.create.group(name=uid("g1")).data
        g2 = lqs.create.group(name=uid("g2")).data
        g3 = lqs.create.group(name=uid("g3")).data

        lqs.create.log(name=uid("l1"), group_id=g1.id)
        lqs.create.log(name=uid("l2"), group_id=g2.id)
        lqs.create.log(name=uid("l3"), group_id=g3.id)

        res = lqs.list.log(group_id_in=[g1.id, g2.id])
        assert res.count == 2
        group_ids = {log.group_id for log in res.data}
        assert group_ids == {g1.id, g2.id}

    def test_ingestion_state_lifecycle(self, lqs: RESTClient):
        """Full ingestion state progression: ready → processing → parts → finalize."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        ingestion = lqs.create.ingestion(
            log_id=log.id, name=uid("ing"), state=ProcessState.ready
        ).data
        assert ingestion.state == ProcessState.ready

        # Transition to processing
        ingestion = lqs.update.ingestion(
            ingestion_id=ingestion.id, data={"state": ProcessState.processing}
        ).data
        assert ingestion.state == ProcessState.processing

        # Create parts
        lqs.create.ingestion_part(sequence=0, ingestion_id=ingestion.id)
        lqs.create.ingestion_part(sequence=1, ingestion_id=ingestion.id)

        parts = lqs.list.ingestion_part(ingestion_id=ingestion.id).data
        assert len(parts) == 2

        # Transition to finalizing
        ingestion = lqs.update.ingestion(
            ingestion_id=ingestion.id, data={"state": ProcessState.finalizing}
        ).data
        assert ingestion.state == ProcessState.finalizing

    def test_digestion_setup_lifecycle(self, lqs: RESTClient):
        """Full digestion setup: create, add topics, transition states."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        topic1 = lqs.create.topic(name=uid("t1"), log_id=log.id).data
        topic2 = lqs.create.topic(name=uid("t2"), log_id=log.id).data

        digestion = lqs.create.digestion(
            log_id=log.id, name=uid("dig"), state=ProcessState.ready
        ).data

        # Add topics with different frequencies
        lqs.create.digestion_topic(
            digestion_id=digestion.id, topic_id=topic1.id, frequency=1
        )
        lqs.create.digestion_topic(
            digestion_id=digestion.id, topic_id=topic2.id, frequency=2
        )

        dts = lqs.list.digestion_topic(digestion_id=digestion.id).data
        assert len(dts) == 2

        # Transition to processing and create parts
        digestion = lqs.update.digestion(
            digestion_id=digestion.id, data={"state": ProcessState.processing}
        ).data
        assert digestion.state == ProcessState.processing

        part = lqs.create.digestion_part(sequence=0, digestion_id=digestion.id).data
        assert part.sequence == 0

    def test_context_and_note_on_all_resources(self, lqs: RESTClient):
        """Verify context (JSON) and note (string) fields work across resources."""
        ctx = {"environment": "test", "version": 1}
        note = "Test note"

        group = lqs.create.group(name=uid("grp"), context=ctx, note=note).data
        assert group.context == ctx
        assert group.note == note

        log = lqs.create.log(
            name=uid("log"), group_id=group.id, context=ctx, note=note
        ).data
        assert log.context == ctx
        assert log.note == note

        topic = lqs.create.topic(
            name=uid("t"), log_id=log.id, context=ctx, note=note
        ).data
        assert topic.context == ctx
        assert topic.note == note

    def test_include_count_parameter(self, lqs: RESTClient):
        """Verify include_count=False returns None for count across resources."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        topic = lqs.create.topic(name=uid("t"), log_id=log.id).data
        lqs.create.record(topic_id=topic.id, timestamp=100)

        assert lqs.list.group(name=group.name, include_count=False).count is None
        assert lqs.list.log(name=log.name, include_count=False).count is None
        assert lqs.list.topic(name=topic.name, include_count=False).count is None
        assert lqs.list.record(topic_id=topic.id, include_count=False).count is None

        # With count
        assert lqs.list.group(name=group.name).count == 1
        assert lqs.list.log(name=log.name).count == 1
        assert lqs.list.topic(name=topic.name, log_id=log.id).count == 1
        assert lqs.list.record(topic_id=topic.id).count == 1

    def test_list_by_id_filter(self, lqs: RESTClient):
        """All resources support listing by id= filter."""
        group = lqs.create.group(name=uid("grp")).data
        res = lqs.list.group(id=group.id)
        assert len(res.data) == 1
        assert res.data[0].id == group.id

        log = lqs.create.log(name=uid("log"), group_id=group.id).data
        res = lqs.list.log(id=log.id)
        assert len(res.data) == 1
        assert res.data[0].id == log.id

        topic = lqs.create.topic(name=uid("t"), log_id=log.id).data
        res = lqs.list.topic(id=topic.id)
        assert len(res.data) == 1
        assert res.data[0].id == topic.id

    def test_multiple_records_across_topics(self, lqs: RESTClient):
        """Records across multiple topics are independently accessible."""
        group = lqs.create.group(name=uid("grp")).data
        log = lqs.create.log(name=uid("log"), group_id=group.id).data

        t1 = lqs.create.topic(name=uid("t1"), log_id=log.id).data
        t2 = lqs.create.topic(name=uid("t2"), log_id=log.id).data

        lqs.create.record(topic_id=t1.id, timestamp=1000)
        lqs.create.record(topic_id=t1.id, timestamp=3000)
        lqs.create.record(topic_id=t2.id, timestamp=500)
        lqs.create.record(topic_id=t2.id, timestamp=4000)

        records_t1 = lqs.list.record(topic_id=t1.id).data
        assert len(records_t1) == 2
        assert records_t1[0].timestamp == 1000
        assert records_t1[1].timestamp == 3000

        records_t2 = lqs.list.record(topic_id=t2.id).data
        assert len(records_t2) == 2
        assert records_t2[0].timestamp == 500
        assert records_t2[1].timestamp == 4000
