#!/usr/bin/env python3
# pylint: disable=too-many-lines

"""Handle data processing DB interactions."""

import logging

from sqlalchemy import sql, select, update, and_, or_

from autowisp.multiprocessing_util import (
    setup_process,
    get_log_outerr_filenames,
)
from autowisp.database.processing import ProcessingManager
from autowisp.database.interface import start_db_session, get_project_home
from autowisp import processing_steps
from autowisp.database.user_interface import get_processing_sequence
from autowisp.data_reduction.data_reduction_file import DataReductionFile

# False positive due to unusual importing
# pylint: disable=no-name-in-module
from autowisp.database.data_model import (
    StepDependencies,
    ImageProcessingProgress,
    ProcessedImages,
    Step,
    Image,
    ObservingSession,
    MasterType,
    InputMasterTypes,
    Condition,
    ConditionExpression,
)
from autowisp.database.data_model.provenance import (
    Camera,
    CameraChannel,
    CameraType,
)

# pylint: enable=no-name-in-module


class NoMasterError(ValueError):
    """Raised when no suitable master can be found for a batch of frames."""


# Intended to be used as simple callable
# pylint: disable=too-few-public-methods
class ExpressionMatcher:
    """
    Compare condition expressions for an image/channel to a target.

    Usually check if matched expressions and master expression values are
    identical, but also handles special case of calibrate step.
    """

    def _get_master_values(self, image_id, channel):
        """Return ready to compare masster expression values."""

        if channel is None:
            return tuple(
                self._get_master_values(image_id, channel)
                for channel in sorted(
                    filter(None, self._evaluated_expressions[image_id].keys())
                )
            )
        self._logger.debug(
            "Getting master expression values for expression ids %s, "
            "image %d, channel %s",
            repr(self._master_expression_ids),
            image_id,
            channel,
        )
        return tuple(
            self._evaluated_expressions[image_id][channel]["values"][
                expression_id
            ]
            for expression_id in self._master_expression_ids
        )

    def __init__(  # pylint: disable=too-many-arguments
        self,
        evaluated_expressions,
        ref_image_id,
        ref_channel,
        master_expression_ids,
        *,
        masters_only=False,
    ):
        """
        Set up comparison to the given evaluated expressions.

        """

        self._logger = logging.getLogger(__name__)
        self._evaluated_expressions = evaluated_expressions
        self._master_expression_ids = master_expression_ids
        reference_evaluated = evaluated_expressions[ref_image_id][ref_channel]
        self._ref_matched = reference_evaluated["matched"]
        self.ref_master_values = self._get_master_values(
            ref_image_id, ref_channel
        )
        self._masters_only = masters_only
        self._logger.debug(
            "Finding images matching expressions %s and values %s",
            repr(self._ref_matched),
            repr(self.ref_master_values),
        )

    def __call__(self, image_id, channel):
        """True iff the expressions for the given image/channel match."""

        image_evaluated = self._evaluated_expressions[image_id][channel]
        image_master_values = self._get_master_values(image_id, channel)

        self._logger.debug(
            "Comparing %s to %s and %s to %s",
            repr(image_evaluated["matched"]),
            repr(self._ref_matched),
            repr(image_master_values),
            repr(self.ref_master_values),
        )
        return (
            self._masters_only
            or image_evaluated["matched"] == self._ref_matched
        ) and image_master_values == self.ref_master_values


# pylint: enable=too-few-public-methods


def get_master_expression_ids(step_id, image_type_id, db_session):
    """
    List all condition expression IDs determining input or output masters.

    Args:
        step_id(int):    The ID of the step for which to return the master
            expression IDs.

        image_type_id(int):     The type of images being processed by the step
            for which to return the master expression IDs.

    Returns:
        [int]:
            The combined expression IDs reqired to determine which required
            masters can be used for the given step or which masters will be
            created by it.
    """

    return sorted(
        set(
            db_session.scalars(
                select(ConditionExpression.id)
                .select_from(InputMasterTypes)
                .join(MasterType)
                .join(
                    Condition,
                    # False positive
                    # pylint: disable=no-member
                    MasterType.condition_id == Condition.id,
                    # pylint: enable=no-member
                )
                .join(ConditionExpression)
                .where(InputMasterTypes.step_id == step_id)
                .where(InputMasterTypes.image_type_id == image_type_id)
                .group_by(ConditionExpression.id)
            ).all()
            + db_session.scalars(
                select(ConditionExpression.id)
                .select_from(MasterType)
                .join(
                    Condition,
                    or_(
                        # False positive
                        # pylint: disable=no-member
                        MasterType.condition_id == Condition.id,
                        (
                            MasterType.maker_image_split_condition_id
                            == Condition.id
                        ),
                        # pylint: enable-no-member
                    ),
                )
                .join(ConditionExpression)
                .where(MasterType.maker_step_id == step_id)
                .where(MasterType.maker_image_type_id == image_type_id)
            ).all()
        )
    )


def remove_failed_prerequisite(
    pending, pending_image_type_id, prereq_step_id, db_session
):
    """Remove from pending any entries that failed the prerequisite step."""

    prereq_statuses = [
        db_session.execute(
            select(ProcessedImages.status)
            .outerjoin(ImageProcessingProgress)
            .where(
                (ProcessedImages.image_id == image.id),
                ProcessedImages.channel == channel,
                ImageProcessingProgress.step_id == prereq_step_id,
                (
                    ImageProcessingProgress.image_type_id
                    == pending_image_type_id
                ),
            )
        ).scalar_one_or_none()
        for image, channel, _ in pending
    ]
    dropped = []
    for i in range(len(pending) - 1, -1, -1):
        if prereq_statuses[i] and prereq_statuses[i] < 0:
            dropped.append(pending.pop(i))

    return dropped


# pylint: disable=too-many-instance-attributes
class ImageProcessingManager(ProcessingManager):
    """
    Read configuration and record processing progress in the database.

    Attrs:
        See `ProcessingManager`.

        pending(dict):    Indexed by step ID, and image type ID list of
            (Image, channel name, status) tuples listing all the images of the
            given type that have not been processed by the currently selected
            version of the step in the key and their status if previous
            processing by that step was interrupted or None if not.

        _failed_dependencies(dict):    Dictionary with keys (step, image_type)
            that contains the list of images and channels that failed the given
            step.
    """

    def _set_calibration_config(self, config, first_image):
        """Retrun the specially formatted argument for the calibration step."""

        config["split_channels"] = self._get_split_channels(first_image)
        config["extra_header"] = self._get_extra_header(first_image)
        result = {
            (
                "split_channels",
                "".join(
                    repr(c)
                    for c in first_image.observing_session.camera.channels
                ),
            ),
            ("observing_session", config["extra_header"]["OBSSSNID"]),
        }
        self._logger.debug(
            "Calibration step configuration:\n%s",
            "\n\t".join((f"{k}: {v!r}" for k, v in config.items())),
        )
        return result

    def _split_by_master(self, batch, input_master_type):
        """Split the given list of images by the best master of given type."""

        result = {}

        for image, channel, status in batch:
            if channel is None:
                best_master = tuple(
                    (
                        channel,
                        self._evaluated_expressions[image.id][channel][
                            "masters"
                        ][input_master_type.master_type.name],
                    )
                    for channel in sorted(
                        filter(
                            None, self._evaluated_expressions[image.id].keys()
                        )
                    )
                )
            else:
                best_master = self._evaluated_expressions[image.id][channel][
                    "masters"
                ][input_master_type.master_type.name]

            if best_master in result:
                result[best_master].append((image, channel, status))
            else:
                result[best_master] = [(image, channel, status)]
        return result

    # Could not find good way to simplify
    # pylint: disable=too-many-locals
    def _get_batch_config(
        self, batch, master_expression_values, step, db_session
    ):
        """
        Split given batch of images by configuration for given step.

        The batch must already be split by all relevant condition expressions.
        Only splits batches by the best master for each image.

        Args:
            batch([Image, channel, status]):    List of database image instances
                and for channels which to find the configuration(s). The channel
                should be ``None`` for the ``calibrate`` step

            master_expression_values(tuple):    The values the expressions
                required to select input masters or to guarantee a unique output
                master. Should be provided in consistent order for all batches
                processed by the same step.

            step(Step):    The database step instance to configure.

            db_session:    Database session to use for queries.

        Returns:
            dict:
                keys:    guaranteed to match iff configuration, output master
                    conditions, and all best input master(s) match. In other
                    words, if this function is called separately on multiple
                    batches, it is safe to combine and process together those
                    that end up with the same key.

                values:
                    dict:    The configuration to use for the given (sub-)batch.

                    [Image]:     The (sub-)batch of images to process with given
                        configuration.
        """

        self._logger.debug("Finding configuration for batch: %s", repr(batch))
        first_image_expressions = self._evaluated_expressions[batch[0][0].id]
        config, config_key = self.get_config(
            first_image_expressions[batch[0][1]]["matched"],
            db_session,
            db_step=step,
        )
        config_key |= {master_expression_values}
        if step.name == "calibrate":
            config_key |= self._set_calibration_config(config, batch[0][0])
        config["processing_step"] = step.name
        config["image_type"] = batch[0][0].image_type.name

        result = {config_key: (config, batch)}
        for input_master_type in db_session.scalars(
            select(InputMasterTypes).filter_by(
                step_id=step.id, image_type_id=batch[0][0].image_type_id
            )
        ).all():
            for config_key, (config, sub_batch) in list(result.items()):
                del result[config_key]
                splits = self._split_by_master(sub_batch, input_master_type)

                for best_master, sub_batch in splits.items():
                    if best_master is None:
                        if input_master_type.optional:
                            assert config_key not in result
                            result[config_key] = (config, sub_batch)
                        else:
                            result[None] = (
                                "No master "
                                + input_master_type.master_type.name
                                + " found!",
                                sub_batch,
                            )

                    else:
                        new_config = dict(config)
                        new_config[
                            input_master_type.config_name.replace("-", "_")
                        ] = (
                            best_master
                            if isinstance(best_master, str)
                            else dict(best_master)
                        )
                        key_extra = {
                            (input_master_type.config_name, best_master)
                        }
                        result[config_key | key_extra] = (new_config, sub_batch)

        return result

    # pylint: enable=too-many-locals

    def _clean_pending_per_dependencies(
        self, db_session, from_step_id=None, from_image_type_id=None
    ):
        """Remove pending images from steps if they failed a required step."""

        dropped = {}
        for (step_id, image_type_id), pending in self.pending.items():
            if (
                from_image_type_id is not None
                and image_type_id != from_image_type_id
            ):
                continue
            for prereq_step_id in db_session.scalars(
                select(StepDependencies.blocking_step_id).where(
                    StepDependencies.blocked_step_id == step_id,
                    StepDependencies.blocked_image_type_id == image_type_id,
                    StepDependencies.blocking_image_type_id == image_type_id,
                )
            ):
                if from_step_id is not None and prereq_step_id != from_step_id:
                    continue
                if (step_id, image_type_id) not in dropped:
                    dropped[(step_id, image_type_id)] = []

                failed_prereq = remove_failed_prerequisite(
                    pending, image_type_id, prereq_step_id, db_session
                )
                self.pending[(step_id, image_type_id)] = pending

                dropped[(step_id, image_type_id)].extend(failed_prereq)

                self._logger.info(
                    "The following image/channel combinations failed %s. "
                    "Excluding from %s:\n\t%s",
                    db_session.scalar(
                        select(Step.name).filter_by(id=prereq_step_id)
                    ),
                    db_session.scalar(select(Step.name).filter_by(id=step_id)),
                    "\n\t".join(
                        image.raw_fname + ":" + channel
                        for image, channel in failed_prereq
                    ),
                )

        return dropped

    def _check_ready(self, step, image_type, db_session):
        """
        Check if the given type of images is ready to process with given step.

        Args:
            step(Step):    The step to check for readiness.

            image_type(ImageType):    The type of images to check for readiness.

            db_session(Session):    The database session to use.

        Returns:
            bool:    Whether all requirements for the specified processing are
                satisfied.
        """

        for requirement in db_session.execute(
            select(
                StepDependencies.blocking_step_id,
                StepDependencies.blocking_image_type_id,
            )
            .where(StepDependencies.blocked_step_id == step.id)
            .where(StepDependencies.blocked_image_type_id == image_type.id)
        ).all():
            if self.pending[requirement]:
                self._logger.debug(
                    "Not ready for %s of %d %s frames because of %d pending %s "
                    "type ID images for step ID %s:\n\t%s",
                    step.name,
                    len(self.pending[(step.id, image_type.id)]),
                    image_type.name,
                    len(self.pending[requirement]),
                    requirement[1],
                    requirement[0],
                    "\n\t".join(
                        f"{e[0]!r}: {e[1]!r}" for e in self.pending[requirement]
                    ),
                )
                return False
        return True

    def _get_interrupted(self, need_cleanup, db_session):
        """Return list of interrupted files and configuration for cleanup."""

        self.current_step = need_cleanup[0][2]
        self._current_processing = db_session.scalar(
            select(ImageProcessingProgress).where(
                ImageProcessingProgress.id == need_cleanup[0][1].progress_id
            )
        )
        input_type = getattr(
            processing_steps, self.current_step.name
        ).input_type

        for entry in need_cleanup:
            assert entry[2] == self.current_step

        pending = [
            (
                image,
                None if input_type == "raw" else processed.channel,
                processed.status,
            )
            for image, processed, _ in need_cleanup
        ]

        for image, _, __ in need_cleanup:
            if image.id not in self._evaluated_expressions:
                self.evaluate_expressions_image(image, db_session)

        cleanup_batches = self._get_config_batches(
            pending, input_type, db_session
        )
        result = {}
        for (config_key, status), (config, batch) in cleanup_batches.items():
            if config_key not in result:
                result[config_key] = (config, [])
            result[config_key][1].extend([(fname, status) for fname in batch])

        return list(result.values())

    def _cleanup_interrupted(self, db_session):
        """Cleanup previously interrupted processing for the current step."""

        need_cleanup = db_session.execute(
            select(Image, ProcessedImages, Step)
            .join(ProcessedImages)
            .join(ImageProcessingProgress)
            .join(Step)
            .where(~ProcessedImages.final)
            .order_by(Step.name)
        ).all()

        if not need_cleanup:
            return

        step_module = getattr(processing_steps, need_cleanup[0][2].name)

        for config, interrupted in self._get_interrupted(
            need_cleanup, db_session
        ):
            self._logger.warning(
                "Cleaning up interrupted %s processing of %d images:\n"
                "%s\n"
                "config: %s",
                need_cleanup[0][2],
                len(interrupted),
                repr(interrupted),
                repr(config),
            )
            new_status = step_module.cleanup_interrupted(interrupted, config)
            for _, processed, _ in need_cleanup:
                assert new_status >= -1
                assert new_status <= processed.status
                if new_status == -1:
                    db_session.delete(processed)
                else:
                    processed.status = new_status

    def _init_processed_ids(self, image, channels, step_input_type):
        """Prepare to record processing of the given image by current step."""

        if channels == [None]:
            channels = self._evaluated_expressions[image.id].keys()

        for channel_name in channels:
            if channel_name is None:
                continue

            step_input_fname = self.get_step_input(
                image, channel_name, step_input_type
            )

            if step_input_fname not in self._processed_ids:
                self._processed_ids[step_input_fname] = []
            self._processed_ids[step_input_fname].append(
                {"image_id": image.id, "channel": channel_name}
            )

    def _start_step(self, step, image_type, db_session):
        """
        Record the start of a processing step and return the images to process.

        Args:
            step(Step):    The database step to start.

            image_type(ImageType):    The database type of image to start
                processing.

            db_session:    Active session for database queries.

        Returns:
            [(Image, str)]:
                The list of images and channels to process.

            str:
                The type of input expected by the current step.
        """

        self._create_current_processing(
            step, ("image_type", image_type.id), db_session
        )

        pending_images = self.pending[(step.id, image_type.id)].copy()
        for image, channel, status in self._failed_dependencies.get(
            (step.id, image_type.id), []
        ):
            self._logger.info(
                "Prerequisite failed for %s of %s", step.name, image
            )
            db_session.add(
                ProcessedImages(
                    image_id=image.id,
                    channel=channel,
                    progress_id=self._current_processing.id,
                    status=-1,
                    final=True,
                )
            )
            self._some_failed = True

        self._processed_ids = {}
        step_input_type = getattr(processing_steps, step.name).input_type

        if step_input_type == "raw":
            added = set()
            new_pending = []
            for image, _, status in pending_images:
                if image.id not in added:
                    added.add(image.id)
                    new_pending.append((image, None, status))
            pending_images = new_pending

        for image, channel_name, _ in pending_images:
            self.evaluate_expressions_image(image, db_session)
            self._init_processed_ids(image, [channel_name], step_input_type)

        self._logger.info(
            "Starting %s step for %d %s images",
            self.current_step.name,
            len(pending_images),
            image_type.name,
        )

        return pending_images, step_input_type

    def _process_batch(  # pylint: disable=too-many-arguments
        self, batch, *, start_status, config, step_name, image_type_name
    ):
        """Run the current step for a batch of images given configuration."""

        step_module = getattr(processing_steps, step_name)

        new_masters = getattr(step_module, step_name)(
            batch,
            start_status,
            config,
            self._start_processing,
            self._end_processing,
        )
        if new_masters:
            self.add_masters(new_masters, step_name, image_type_name)

    def _start_processing(self, input_fname, status=0):
        """
        Mark in the database that processing the given file has begun.

        Args:
            input_fname:    The filename of the input (DR or FITS) that is about
                to begin processing.

        Returns:
            None
        """

        assert self.current_step is not None
        assert self._current_processing is not None
        self._logger.debug(
            "Starting processing IDs: %s",
            repr(self._processed_ids[input_fname]),
        )
        with start_db_session() as db_session:
            for starting_id in self._processed_ids[input_fname]:
                db_session.add(
                    ProcessedImages(
                        **starting_id,
                        progress_id=self._current_processing.id,
                        status=status,
                        final=False,
                    )
                )

    def _end_processing(self, input_fname, status=1, final=True):
        """
        Record that the current step has finished processing the given file.

        Args:
            input_fname:    The filename of the input (DR or FITS) that was
                processed.

        Returns:
            None
        """

        assert self.current_step is not None
        assert self._current_processing is not None
        assert status != -1

        if status < 0:
            self._some_failed = True
        self._logger.debug(
            "Finished processing %s", repr(self._processed_ids[input_fname])
        )
        with start_db_session() as db_session:
            for finished_id in self._processed_ids[input_fname]:
                db_session.execute(
                    update(ProcessedImages)
                    .where(ProcessedImages.image_id == finished_id["image_id"])
                    .where(ProcessedImages.channel == finished_id["channel"])
                    .where(
                        ProcessedImages.progress_id
                        == self._current_processing.id
                    )
                    .values(status=status, final=final)
                )

    # No good way to simplify
    # pylint: disable=too-many-locals
    def _get_config_batches(self, pending_images, step_input_type, db_session):
        """Return the batches of images to process with identical config."""

        result = {}
        check_image_type_id = pending_images[0][0].image_type_id
        for (
            by_condition,
            master_expression_values,
        ) in self.group_pending_by_conditions(
            pending_images,
            db_session,
            match_observing_session=self.current_step.name == "calibrate",
        ):
            for config_key, (config, batch) in self._get_batch_config(
                by_condition,
                master_expression_values,
                self.current_step,
                db_session,
            ).items():
                if config_key is None:
                    self._logger.warning(
                        "Excluding the following images from %s:\n\t%s",
                        config,
                        "\n\t".join(
                            [
                                self.get_step_input(
                                    image, channel, step_input_type
                                )
                                for image, channel, _ in batch
                            ]
                        ),
                    )
                    continue
                for image, channel, status in batch:
                    assert image.image_type_id == check_image_type_id

                    if (config_key, status) not in result:
                        result[config_key, status] = (config, [])
                    result[config_key, status][1].append(
                        self.get_step_input(image, channel, step_input_type)
                    )

        return result

    # pylint: enable=too-many-locals

    def _prepare_processing(self, step, image_type, limit_to_steps):
        """Prepare for processing images of given type by a calibration step."""

        with start_db_session() as db_session:
            setup_process(
                task="main",
                parent_pid="",
                processing_step=step.name,
                image_type=image_type.name,
                **self._processing_config,
            )
            step = db_session.merge(step)
            image_type = db_session.merge(image_type)

            self.set_pending(db_session, [(step, image_type)])

            if limit_to_steps is not None and step.name not in limit_to_steps:
                self._logger.debug(
                    "Skipping disabled %s for %s frames",
                    step.name,
                    image_type.name,
                )
                return step.name, image_type.name, None

            if not self._check_ready(step, image_type, db_session):
                return step.name, image_type.name, None

            pending_images, step_input_type = self._start_step(
                step, image_type, db_session
            )
            if not pending_images:
                return step.name, image_type.name, None

            return (
                step.name,
                image_type.name,
                self._get_config_batches(
                    pending_images, step_input_type, db_session
                ),
            )

    def _finalize_processing(self):
        """Update database and instance after processing."""

        with start_db_session() as db_session:
            self._current_processing = db_session.merge(
                self._current_processing
            )
            self._current_processing.finished = (
                # False positive
                # pylint: disable=not-callable
                sql.func.now()
                # pylint: enable=not-callable
            )
            pending = self.pending[
                (
                    self._current_processing.step_id,
                    self._current_processing.image_type_id,
                )
            ]

            self._logger.info(
                "Removing from pending all successful images for "
                "progress: %s",
                self._current_processing,
            )
            for finished_image_id, finished_channel in db_session.execute(
                select(ProcessedImages.image_id, ProcessedImages.channel)
                .where(
                    ProcessedImages.progress_id == self._current_processing.id
                )
                .where(
                    # pylint: disable=singleton-comparison
                    ProcessedImages.final
                    == True
                    # pylint: enable=singleton-comparison
                )
                .where(
                    or_(ProcessedImages.status > 0, ProcessedImages.status < -1)
                )
            ).all():
                found = False
                for i, (image, channel, _) in enumerate(pending):
                    if (
                        image.id == finished_image_id
                        and channel == finished_channel
                    ):
                        assert not found
                        del pending[i]
                        found = True
                        break
                if not found:
                    self._logger.error(
                        "Completed image ID %d, channel %s not found in "
                        "pending for step ID %d, image type ID %d:\n\t%s",
                        finished_image_id,
                        finished_channel,
                        self._current_processing.step_id,
                        self._current_processing.image_type_id,
                        "\n\t".join(f"{e[0]!r}: {e[1]!r}" for e in pending),
                    )
                    raise RuntimeError("Finished non-pending image!")

                self.pending[
                    (
                        self._current_processing.step_id,
                        self._current_processing.image_type_id,
                    )
                ] = pending

    #            if self._some_failed:
    #                dropped = self._clean_pending_per_dependencies(
    #                    db_session,
    #                    self._current_processing.step_id,
    #                    self._current_processing.image_type_id
    #                )
    #                for step_imtype, dropped_images in dropped.items():
    #                    if step_imtype in self._failed_dependencies:
    #                        self._failed_dependencies[
    #                            step_imtype
    #                        ].extend(
    #                            dropped_images
    #                        )
    #                    else:
    #                        self._failed_dependencies[step_imtype] = (
    #                            dropped_images
    #                        )

    def __init__(self, *args, **kwargs):
        """Initialize self._failed_dependencies in addition to normali init."""

        self._failed_dependencies = {}
        super().__init__(*args, **kwargs)

    def get_step_input(self, image, channel_name, step_input_type):
        """Return the name of the file required by the current step."""

        if step_input_type == "raw":
            return image.raw_fname

        if step_input_type.startswith("calibrated"):
            return self._evaluated_expressions[image.id][channel_name][
                "calibrated"
            ]

        if step_input_type == "dr":
            return self._evaluated_expressions[image.id][channel_name]["dr"]

        raise ValueError(f"Invalid step input type {step_input_type}")

    def set_pending(self, db_session, steps_imtypes=None, invert=False):
        """
        Set the unprocessed images and channels split by step and image type.

        Set the self.pending attribute to a dictionary with format ``{(step.id,
        image_type.id): (Image, str)}``, containing the images and channels of
        the specified type for which the specified step has not applied with the
        current configuration.

        Args:
            db_session(Session):    The database session to use.

            steps_imtypes(Step, ImageType):    The step image type combinations
                to determine pending images for. If unspecified, the full
                processing sequence defined in the database is used.

            invert(bool):    If True, returns successfully completed (not
                failed) instead of pending.


        Returns:
            None
        """

        status_select = (
            select(
                ProcessedImages.image_id,
                ProcessedImages.channel,
                sql.func.max(ProcessedImages.status).label("status"),
            )
            .join(ImageProcessingProgress)
            .where(ProcessedImages.status > 0)
            .where(ProcessedImages.final == 0)
            .group_by(ProcessedImages.image_id, ProcessedImages.channel)
        )

        for step, image_type in steps_imtypes or get_processing_sequence(
            db_session, True
        ):
            failed_prereq_subquery = (
                select(ProcessedImages.image_id, ProcessedImages.channel)
                .select_from(StepDependencies)
                .join(
                    ImageProcessingProgress,
                    and_(
                        StepDependencies.blocking_step_id
                        == ImageProcessingProgress.step_id,
                        StepDependencies.blocking_image_type_id
                        == ImageProcessingProgress.image_type_id,
                    ),
                )
                .join(ProcessedImages)
                .where(StepDependencies.blocked_step_id == step.id)
                .where(StepDependencies.blocked_image_type_id == image_type.id)
                .where(ProcessedImages.status < 0)
                .group_by(ProcessedImages.image_id, ProcessedImages.channel)
                .subquery()
            )

            processed_subquery = (
                select(ProcessedImages.image_id, ProcessedImages.channel)
                .join(ImageProcessingProgress)
                .where(ImageProcessingProgress.step_id == step.id)
                .where(ImageProcessingProgress.image_type_id == image_type.id)
                .where(
                    ImageProcessingProgress.configuration_version
                    == self.step_version[step.name]
                )
                .where(ProcessedImages.final)
            )

            status_subquery = (
                status_select.where(ImageProcessingProgress.step_id == step.id)
                .where(ImageProcessingProgress.image_type_id == image_type.id)
                .where(
                    ImageProcessingProgress.configuration_version
                    == self.step_version[step.name]
                )
                .subquery()
            )

            if invert:
                processed_subquery = processed_subquery.where(
                    ProcessedImages.status > 0
                )
            processed_subquery = processed_subquery.subquery()

            query = (
                select(Image, CameraChannel.name, status_subquery.c.status)
                .join(
                    ObservingSession,
                )
                .join(Camera)
                .join(CameraType)
                .join(CameraChannel)
                .outerjoin(
                    processed_subquery,
                    # False positive
                    # pylint: disable=no-member
                    and_(
                        Image.id == processed_subquery.c.image_id,
                        CameraChannel.name == processed_subquery.c.channel,
                    ),
                    # pylint: enable=no-member
                )
                .outerjoin(
                    failed_prereq_subquery,
                    and_(
                        Image.id  # pylint: disable=no-member
                        == failed_prereq_subquery.c.image_id,
                        CameraChannel.name == failed_prereq_subquery.c.channel,
                    ),
                )
                .outerjoin(
                    status_subquery,
                    and_(
                        Image.id  # pylint: disable=no-member
                        == status_subquery.c.image_id,
                        CameraChannel.name == status_subquery.c.channel,
                    ),
                )
                .where(
                    Image.image_type_id  # pylint: disable=no-member
                    == image_type.id
                )
            )
            # This is how NULL comparison is done in SQLAlchemy
            # pylint: disable=singleton-comparison
            if invert:
                query = query.where(processed_subquery.c.image_id != None)
            else:
                query = query.where(processed_subquery.c.image_id == None)

            self.pending[(step.id, image_type.id)] = db_session.execute(
                query.where(failed_prereq_subquery.c.image_id == None)
            ).all()

            self._failed_dependencies[(step.id, image_type.id)] = (
                db_session.execute(
                    query.where(failed_prereq_subquery.c.image_id != None)
                ).all()
            )
            # pylint: enable=singleton-comparison

            self._logger.debug(
                "%s is pending for %d and failed dependencies for %d %s images",
                step.name,
                len(self.pending[(step.id, image_type.id)]),
                len(self._failed_dependencies[(step.id, image_type.id)]),
                image_type.name,
            )

        self._logger.debug("Pending: %s", repr(self.pending))

    def group_pending_by_conditions(  # pylint: disable=too-many-arguments
        self,
        pending_images,
        db_session,
        *,
        match_observing_session=False,
        step_id=None,
        masters_only=False,
    ):
        """
        Group pendig_images by condition expression values.

        Args:
            pending_images([Image, str]):    A list of the images (instance of
                Image DB class) and channels to group.

            db_session:    Database session to use for querries.

            match_observing_session:    Whether each group of images needs to
                be from the same observing session.

            step_id(int):    The ID of the step for which to group the pending
                images. If not specified, defaults to the current step.

            masters_only:    If True, grouping is done only by the values
                expressions required to determine the input or output masters
                for the current step.

        Returns:
            [([Image, str], tuple)]:
                Each entry is contains a list of the image/channel combinations
                matching a unique set of conditions and the second entry is the
                master expression values for all images in the list.
        """

        image_type_id = pending_images[0][0].image_type_id
        result = []
        master_expression_ids = get_master_expression_ids(
            step_id or self.current_step.id, image_type_id, db_session
        )
        while pending_images:
            self._logger.debug(
                "Finding images matching the same expressions as image id %d, "
                "channel %s",
                pending_images[-1][0].id,
                pending_images[-1][1],
            )
            batch = []
            match_expressions = ExpressionMatcher(
                self._evaluated_expressions,
                pending_images[-1][0].id,
                pending_images[-1][1],
                master_expression_ids,
                masters_only=masters_only,
            )
            observing_session_id = pending_images[-1][0].observing_session_id

            for i in range(len(pending_images) - 1, -1, -1):
                if (
                    not match_observing_session
                    or pending_images[i][0].observing_session_id
                    == observing_session_id
                ) and match_expressions(
                    pending_images[i][0].id, pending_images[i][1]
                ):
                    batch.append(pending_images.pop(i))
                else:
                    self._logger.debug("Not a match")

            self._logger.debug(
                "Image batch:\n\t%s",
                "\n\t".join(
                    f"{image.raw_fname}: {channel} status {status}"
                    for image, channel, status in batch
                ),
            )

            result.append((batch, match_expressions.ref_master_values))
        return result

    def find_processing_outputs(self, processing_progress, db_session=None):
        """Return all logging and output filenames for given processing ID."""

        if db_session is None:
            # False positivie
            # pylint: disable=redefined-argument-from-local
            with start_db_session() as db_session:
                # pylint: enable=redefined-argument-from-local
                return self.find_processing_outputs(
                    processing_progress, db_session
                )

        if not isinstance(processing_progress, ImageProcessingProgress):
            return self.find_processing_outputs(
                db_session.scalar(
                    select(ImageProcessingProgress).filter_by(
                        id=processing_progress
                    )
                ),
                db_session,
            )

        main_fnames = get_log_outerr_filenames(
            existing_pid=processing_progress.run.process_id,
            task="*",
            parent_pid="",
            processing_step=processing_progress.step.name,
            image_type=processing_progress.image_type.name,
            **self._processing_config,
        )
        logging.info("Main fnames: %s", repr(main_fnames))
        assert len(main_fnames[0]) == len(main_fnames[1]) == 1

        return (
            tuple(fname[0] for fname in main_fnames),
            get_log_outerr_filenames(
                existing_pid="*",
                task="*",
                parent_pid=processing_progress.run.process_id,
                processing_step=processing_progress.step.name,
                image_type=processing_progress.image_type.name,
                **self._processing_config,
            ),
        )

    def __call__(self, limit_to_steps=None):
        """Perform all the processing for the given steps (all if None)."""

        with start_db_session() as db_session:
            processing_sequence = get_processing_sequence(db_session, True)

        DataReductionFile.get_file_structure()

        for step, image_type in processing_sequence:
            (step_name, image_type_name, processing_batches) = (
                self._prepare_processing(step, image_type, limit_to_steps)
            )
            self._logger.debug(
                "At start of %s step for %s images, project home %s pending:\n\t%s",
                step_name,
                image_type_name,
                get_project_home(),
                "\n\t".join(
                    f"{key!r}: {len(val)}" for key, val in self.pending.items()
                ),
            )
            if processing_batches is None:
                continue

            self._finalize_processing()
            for (_, start_status), (
                config,
                batch,
            ) in processing_batches.items():
                with start_db_session() as db_session:
                    self._create_current_processing(
                        step, ("image_type", image_type.id), db_session
                    )

                self._logger.debug(
                    "Starting %s for a batch of %d %s images from status %s "
                    "with config:\n%s",
                    step_name,
                    len(batch),
                    image_type_name,
                    start_status,
                    repr(config),
                )

                self._process_batch(
                    batch,
                    start_status=start_status,
                    config=config,
                    step_name=step_name,
                    image_type_name=image_type_name,
                )
                self._logger.debug(
                    "Processed %s batch of %d images.", step_name, len(batch)
                )
                self._finalize_processing()
                self._logger.debug(
                    "After processing batch, pending:\n\t%s",
                    "\n\t".join(
                        f"{key!r}: {len(val)}"
                        for key, val in self.pending.items()
                    ),
                )

                self._some_failed = False

    def add_raw_images(self, image_collection):
        """Add the given RAW images to the database for processing."""

        with start_db_session() as db_session:
            default_expression_id = db_session.scalar(
                select(ConditionExpression.id).where(
                    ConditionExpression.notes == "Default expression"
                )
            )
            configuration = self.get_config(
                {default_expression_id},
                db_session,
                step_name="add_images_to_db",
            )[0]
        processing_steps.add_images_to_db.add_images_to_db(
            image_collection, configuration
        )


# pylint: enable=too-many-instance-attributes
