"""
Created on 2025-02-25

@author: wf
"""
import subprocess
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass, field
import datetime
import gc
import logging
import os
from pathlib import Path
import shutil
import sys
import tarfile
import tempfile
from typing import Generator, List, Optional, Tuple
from PIL import Image
import cairo
import djvu.decode
from djvuviewer.djvu_core import DjVuImage, DjVuFile, DjVuPage
from djvuviewer.djvu_config import DjVuConfig
from ngwidgets.profiler import Profiler
import numpy


if sys.platform != "win32":
    import resource

@dataclass
class ImageJob:
    """
    Represents a processed DjVu page, including document, page, page job, and image data.
    """

    djvu_path: str  # fully qualifying path of the container DjVu document
    document: djvu.decode.Document
    page: djvu.decode.Page
    page_index: int  # Added page_index to track position
    relurl: str  # Added relurl for context
    pagejob: Optional[djvu.decode.PageJob] = field(default=None)
    image: Optional[DjVuImage] = field(default=None)
    # flags
    verbose: bool = False
    debug: bool = False
    # fields for database records
    iso_date: Optional[str] = field(default=None)
    filesize: Optional[int] = field(default=None)
    error: Optional[Exception] = field(default=None)

    def __post_init__(self):
        """Initialize profiler if not provided"""
        self.profiler = Profiler(
            f"Image Job {self.relurl}#{self.page_index:04d}",
            profile=self.verbose or self.debug,
        )
        self.profiler.start()

    def log(self, msg):
        if self.verbose or self.debug:
            self.profiler.time(" " + msg)

    def get_size(self) -> Tuple[int, int]:
        """Get the width and height of the page if pagejob is available"""
        if self.pagejob:
            return self.pagejob.size
        return (0, 0)

    @staticmethod
    def get_fileinfo(filepath: str):
        filesize = None
        iso_date = None
        if os.path.exists(filepath):
            # Set file size in bytes
            filesize = os.path.getsize(filepath)

            # Get file modification time and convert to UTC ISO format with second precision
            mtime = os.path.getmtime(filepath)
            datetime_obj = datetime.datetime.fromtimestamp(
                mtime, tz=datetime.timezone.utc
            )
            iso_date = datetime_obj.isoformat(timespec="seconds")
        return iso_date, filesize

    def set_fileinfo(self, filepath: str):
        """
        Set filesize and ISO date with sec prec for
        the given filepath

        Args:
            filepath (str): Path to the file
        """
        self.iso_date, self.filesize = self.get_fileinfo(filepath)

    @staticmethod
    def get_prefix(relurl: str):
        prefix = os.path.splitext(os.path.basename(relurl))[0]
        return prefix

    @staticmethod
    def get_relative_image_path(relurl: str):
        image_rel_dir = os.path.dirname(relurl)
        # Ensure image_rel_dir is treated as a relative path
        if image_rel_dir.startswith(
            os.sep
        ):  # os.sep is '/' on Unix and '\\' on Windows
            image_rel_dir = image_rel_dir.lstrip(os.sep)
        return image_rel_dir

    @property
    def prefix(self) -> str:
        prefix = ImageJob.get_prefix(relurl=self.relurl)
        return prefix

    @property
    def filename(self) -> str:
        try:
            # Attempt to safely decode the file name
            filename = self.page.file.name.encode("utf-8", errors="replace").decode(
                "utf-8"
            )
        except Exception as e:
            if self.debug:
                logging.warn(
                    f"Failed to decode filename for page {self.page_index}: {e}"
                )
            filename = f"page_{self.page_index:04d}.djvu"
        return filename

    @property
    def dirname(self) -> str:
        dirname = os.path.dirname(self.djvu_path)
        return dirname

    @property
    def filepath(self) -> str:
        filepath = os.path.join(self.dirname, self.filename)
        return filepath


class DjVuContext(djvu.decode.Context):
    """
    A lightweight wrapper around djvu.decode.Context to handle messages.
    """

    def __init__(self):
        super().__init__()
        self.message_handler = None

    def handle_message(self, message):
        """
        Handles messages from the DjVu decoding context.
        """
        if self.message_handler:
            self.message_handler(message)


class DjVuProcessor:
    """
    Processes DjVu files and converts pages to image buffers.

    see https://raw.githubusercontent.com/jwilk-archive/python-djvulibre/refs/heads/master/examples/djvu2png
    with Copyright © 2010-2021 Jakub Wilk <jwilk@jwilk.net> and GNU General Public License version 2
    """

    def __init__(
        self,
        tar: bool = True,
        verbose: bool = False,
        debug: bool = False,
        batch_size: int = 100,
        limit_gb: int = 16,
        max_workers: int = None,
        clean_temp: bool=True,
    ):
        """
        Initializes the DjVuProcessor.

        Args:
            tar(bool,optional): Enable tarball creation (default: True).
            verbose (bool, optional): Enable verbose output (default: False).
            debug (bool, optional): Enable debug logging (default: False).
            batch_size (int, optional): Number of pages to process in each batch (default: 100).
            limit_gb(int): maximum amount of memory to be used in GB
            max_workers (int, optional): Maximum number of worker threads (default: min(CPU count, 8)).
            clean_temp(bool): if True remove files from temp directories when tar done
        """
        self.tar = tar
        self.verbose = verbose
        self.debug = debug
        self.batch_size = batch_size
        self.limit_gb = limit_gb

        # Set a reasonable default for max_workers if not specified
        if max_workers is None:
            self.max_workers = os.cpu_count() * 4
        else:
            self.max_workers = max_workers
        self.clean_temp=clean_temp
        self.context = DjVuContext()  # delegate context instance
        self.context.message_handler = self.handle_message
        self.cairo_pixel_format = cairo.FORMAT_ARGB32
        self.djvu_pixel_format = djvu.decode.PixelFormatRgbMask(
            0xFF0000, 0xFF00, 0xFF, bpp=32
        )
        self.djvu_pixel_format.rows_top_to_bottom = 1
        self.djvu_pixel_format.y_top_to_bottom = 0

    def create_tarball(
        self, source_dir: str, output_tar: str, include_ext: Optional[List[str]] = None
    ):
        """
        Creates a tar archive from the given source directory, including only specific file types.

        Args:
            source_dir (str): Directory containing files to archive.
            output_tar (str): Path to the output tar file.
            include_ext (Optional[List[str]]): List of file extensions to include.
                - "yaml": Includes metadata files.
                - "png": Includes lossless original images.
                - "jpg": Includes compressed thumbnails.
                Defaults to ["yaml", "png", "jpg"].
        """
        if include_ext is None:
            include_ext = [
                "yaml",
                "png",
                "jpg",
            ]  # yaml metadata, png lossless original, jpg thumbnails
        with tarfile.open(output_tar, "w") as tar:
            for file in os.listdir(source_dir):
                if any(file.lower().endswith(ext) for ext in include_ext):
                    tar.add(os.path.join(source_dir, file), arcname=file)

    def handle_message(self, message):
        if isinstance(message, djvu.decode.ErrorMessage):
            raise Exception(message)

    def check_memory_usage(self):
        """Check if memory usage exceeds the given limit in GB"""
        if sys.platform == "win32":
            # On Windows, we'll just do a GC and return False (no check)
            gc.collect()
            return False, 0
        else:
            # Get current memory usage in bytes
            usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
            # Convert to GB (note: on some systems this is KB, others it's bytes)
            if sys.platform == "darwin":  # macOS reports in bytes
                usage_gb = usage / (1024 * 1024 * 1024)
            else:  # Linux reports in KB
                usage_gb = usage / (1024 * 1024)

            return usage_gb >= self.limit_gb, usage_gb

    def save_image_to_png_via_cairo(
        self, image_job: ImageJob, output_path: str, free_buffer: bool = False
    ):
        """
        Saves the rendered DjVu page as a PNG file.

        Args:
            image_job (ImageJob): The processed image job containing buffer data
            output_path (str): The path where the PNG file should be saved.
            free_buffer(bool): if True free the image buffer
        """
        if not image_job.image or image_job.image._buffer is None:
            raise ValueError("Image buffer not available in ImageJob")

        width, height = image_job.get_size()
        surface = cairo.ImageSurface.create_for_data(
            image_job.image._buffer, cairo.Format.ARGB32, width, height
        )
        surface.write_to_png(output_path)
        surface.flush()
        surface.finish()
        surface = None  # Explicitly free Cairo surface
        if free_buffer:
            image_job.image._buffer = None


    def save_image_to_png(self, image_job: ImageJob, output_path: str, free_buffer: bool = True) -> None:
        """
        Save a decoded DjVu image to PNG using PIL.

        This method converts the ARGB32 buffer stored in the ImageJob to RGB format
        and saves it as a PNG file. The buffer is expected to be in Cairo's ARGB32 format
        (0xAARRGGBB as uint32), which is produced by the render_page method.

        Args:
            image_job (ImageJob): The image job containing the decoded image buffer.
                The buffer should be stored in image_job.image._buffer as a tuple
                containing a numpy array with dtype uint32 in ARGB32 format.
            output_path (str): The file path where the PNG image should be saved.
            free_buffer (bool, optional): If True, frees the image buffer after saving
                to reduce memory usage. Defaults to True.

        Raises:
            ValueError: If the image or buffer is not available in the ImageJob.

        Note:
            The buffer format is Cairo ARGB32, where each pixel is a 32-bit integer
            with the layout 0xAARRGGBB (alpha, red, green, blue from MSB to LSB).
            This is converted to RGB24 for PNG output. The buffer may have stride
            padding for alignment, which is handled automatically.

        See Also:
            https://github.com/Piolie/DjvuRleImagePlugin
        """
        if not image_job.image or image_job.image._buffer is None:
            raise ValueError("Image buffer not available in ImageJob")

        try:
            image = image_job.image
            width = image.width
            height = image.height

            # Extract buffer from tuple
            buffer_data = image_job.image._buffer
            if isinstance(buffer_data, tuple):
                buffer_data = buffer_data[0]

            # Convert from uint32 ARGB to RGB bytes
            # The buffer is in ARGB32 format (0xAARRGGBB as uint32)
            # Buffer shape is (height, stride_width) where stride_width >= width due to alignment
            rgb_array = numpy.zeros((height, width, 3), dtype=numpy.uint8)

            # Extract RGB channels from ARGB32, handling possible stride padding
            # Red channel: bits 16-23
            rgb_array[:, :, 0] = (buffer_data[:, :width] >> 16) & 0xFF
            # Green channel: bits 8-15
            rgb_array[:, :, 1] = (buffer_data[:, :width] >> 8) & 0xFF
            # Blue channel: bits 0-7
            rgb_array[:, :, 2] = buffer_data[:, :width] & 0xFF

            # Create PIL Image from RGB array
            img = Image.fromarray(rgb_array, mode='RGB')

            # Save as PNG
            img.save(output_path, 'PNG')

        finally:
            # Free buffer if requested
            if free_buffer and image_job.image:
                image_job.image._buffer = None
                # Also explicitly delete the image object to help with memory cleanup
                del image_job.image
                image_job.image = None

    def save_as_png(
        self, image_job: ImageJob, output_dir: str, free_buffer: bool
    ) -> str:
        """
        Save an image job as PNG in the specified directory

        Args:
            image_job: The image job to save
            output_dir: Directory to save to
            free_buffer(bool): if True free the buffer

        Returns:
            Path to the saved PNG file
        """
        output_path = os.path.join(
            output_dir, f"{image_job.prefix}_page_{image_job.page_index:04d}.png"
        )
        image_job.log("save png start")
        # Save PNG
        self.save_image_to_png(image_job, output_path, free_buffer)
        image_job.log("save png done")
        return output_path

    def render_pagejob_to_buffer(self, image_job: ImageJob, mode: int) -> numpy.ndarray:
        """
        Renders a DjVu page job to a color buffer.

        Args:
            image_job (ImageJob): The job containing the pagejob to render
            mode (int): Rendering mode.

        Returns:
            numpy.ndarray: The rendered color buffer.
        """
        if not image_job.pagejob:
            raise ValueError("PageJob not available")

        width, height = image_job.get_size()
        rect = (0, 0, width, height)

        bytes_per_line = cairo.ImageSurface.format_stride_for_width(
            self.cairo_pixel_format, width
        )
        assert bytes_per_line % 4 == 0

        color_buffer = numpy.zeros((height, bytes_per_line // 4), dtype=numpy.uint32)
        image_job.pagejob.render(
            mode,
            rect,
            rect,
            self.djvu_pixel_format,
            row_alignment=bytes_per_line,
            buffer=color_buffer,
        )

        if mode == djvu.decode.RENDER_FOREGROUND:
            mask_buffer = numpy.zeros_like(color_buffer)
            image_job.pagejob.render(
                djvu.decode.RENDER_MASK_ONLY,
                rect,
                rect,
                self.djvu_pixel_format,
                row_alignment=bytes_per_line,
                buffer=mask_buffer,
            )
            color_buffer |= mask_buffer << 24

        color_buffer ^= 0xFF000000  # Apply transparency
        return color_buffer

    def ensure_file_exists(self, path: str):
        """
        check that the given file exists
        """
        if not os.path.isfile(path):
            msg = f"file {path} not found"
            raise ValueError(msg)

    def get_djvu_file(self, djvu_path: str,config:DjVuConfig) -> DjVuFile:
        """
        Efficiently retrieves DjVu file metadata and page structure.

        This method parses the document structure and page headers to construct
        a DjVuFile object with DjVuPage children without fully decoding
        the pixel data of the images.

        Args:
            djvu_path (str): The file system path to the .djvu file.
            config(DjVuConfig): the config to use for path handling

        Returns:
            DjVuFile: The structured representation of the DjVu file.
        """
        # 1. Get container file metadata
        self.ensure_file_exists(djvu_path)
        iso_date, filesize = ImageJob.get_fileinfo(djvu_path)

        pages: List[DjVuPage] = []
        is_bundled = False
        dir_pages = 0
        doc_info_captured = False

        # 2. Iterate pages using existing generator
        # document is the same reference in every iteration
        for document, page in self.yield_pages(djvu_path):

            # Capture document-level flags on first pass
            if not doc_info_captured:
                # document.type: 0=Single, 1=Indirect, 2=Bundled
                is_bundled = (document.type == 2)
                # document.files contains the directory of included files
                dir_pages = len(document.files)
                doc_info_captured = True

            page_index = len(pages) + 1

            # 3. Retrieve Page Metadata (Lightweight)
            # get_info waits for IFF headers (dimensions, dpi) but not pixel decoding
            try:
                page.get_info(wait=True)
                valid_page = True
                width = page.width
                height = page.height
                dpi = page.dpi
                error_msg = None
            except Exception as e:
                valid_page = False
                width = 0
                height = 0
                dpi = 0
                error_msg = str(e)
                if self.debug:
                    logging.warning(f"Failed to get info for page {page_index} in {djvu_path}: {e}")

            # 4. Determine Filename and File-specific stats
            try:
                # Handle bytes vs str filename encoding issues typical in djvulibre bindings
                page_filename = page.file.name
                if isinstance(page_filename, bytes):
                    page_filename = page_filename.decode("utf-8", errors="replace")
            except Exception:
                page_filename = f"page_{page_index:04d}.djvu"

            # Default to container stats
            page_iso = iso_date
            page_size = filesize

            # If not bundled (Indirect), the page is likely a separate file on disk
            if not is_bundled:
                dirname = os.path.dirname(djvu_path)
                component_path = os.path.join(dirname, page_filename)
                # Only override if the component file physically exists
                if os.path.exists(component_path):
                    page_iso, page_size = ImageJob.get_fileinfo(component_path)

            relpath=config.djvu_relpath(djvu_path)
            # 5. Construct Page Object
            djvu_page = DjVuPage(
                path=page_filename,
                page_index=page_index,
                valid=valid_page,
                iso_date=page_iso,
                filesize=page_size,
                width=width,
                height=height,
                dpi=dpi,
                djvu_path=relpath,
                error_msg=error_msg
            )
            pages.append(djvu_page)

        # 6. Construct and return File Object
        djvu_file = DjVuFile(
            path=djvu_path,
            page_count=len(pages),
            bundled=is_bundled,
            iso_date=iso_date,
            filesize=filesize,
            # tar_filesize/date are not calculated by the processor, typically 0 or None initially
            tar_filesize=0,
            dir_pages=dir_pages,
            pages=pages
        )

        return djvu_file

    def yield_pages(self, djvu_path: str):
        """
        yield the pages for the given djvu_path
        """
        self.ensure_file_exists(djvu_path)
        document = self.context.new_document(djvu.decode.FileURI(djvu_path))
        document.decoding_job.wait()
        for page in document.pages:
            yield document, page

    def create_image_jobs(self, djvu_path: str, relurl: str) -> List[ImageJob]:
        """
        Create initial image jobs for all pages in the document

        Args:
            djvu_path (str): Path to the DjVu file
            relurl (str): Relative URL

        Returns:
            List[ImageJob]: List of initialized image jobs
        """
        image_jobs = []
        page_index = 0
        for document, page in self.yield_pages(djvu_path):
            page_index += 1
            job = ImageJob(
                djvu_path=djvu_path,
                document=document,
                page=page,
                page_index=page_index,
                relurl=relurl,
                debug=self.debug,
                verbose=self.verbose,
            )
            image_jobs.append(job)

        return image_jobs

    def decode_page(self, image_job: ImageJob, wait: bool = True) -> ImageJob:
        """
        Decodes a single page and updates the ImageJob

        Args:
            image_job (ImageJob): The job to process
            wait (bool): Whether to wait for decoding to complete

        Returns:
            ImageJob: Updated image job with pagejob
        """
        try:
            file_size_msg = ""
            filepath = image_job.filepath
            # check whether the document is bundled or not
            if image_job.document.type != 2:
                # we need to check the file is external
                self.ensure_file_exists(filepath)
                image_job.set_fileinfo(filepath)
                file_size_msg = f"{filepath}:{image_job.filesize} bytes "
            else:
                # For bundled files, get the container metadata
                container_path = image_job.djvu_path
                if os.path.exists(container_path):
                    image_job.set_fileinfo(container_path)
            image_job.log(f" page.decode {file_size_msg} start")
            pagejob = image_job.page.decode(wait=wait)
            image_job.log(" page.decode done")
            # Update the image job with the decoded page job
            image_job.pagejob = pagejob
        except Exception as e:
            # Store exception but don't raise
            image_job.error = e
        finally:
            gc.collect()
        return image_job

    def render_page(
        self, image_job: ImageJob, mode: int = djvu.decode.RENDER_COLOR
    ) -> ImageJob:
        """
        Renders a page and updates the ImageJob with the rendered image

        Args:
            image_job (ImageJob): The job to process
            mode (int): Rendering mode

        Returns:
            ImageJob: Updated image job with rendered image
        """
        try:
            if image_job.pagejob:
                image_job.log(" render start")

                width, height = image_job.get_size()
                color_buffer = self.render_pagejob_to_buffer(image_job, mode)

                image = DjVuImage(
                    width=width,
                    height=height,
                    dpi=image_job.pagejob.dpi,
                    iso_date=image_job.iso_date,
                    filesize=image_job.filesize,
                    page_index=image_job.page_index,
                    djvu_path=image_job.relurl,
                    path=image_job.filename,
                )
                image._buffer = (color_buffer,)

                # Update the image job with the rendered image
                image_job.image = image
                image_job.log(" render done")
        except Exception as e:
            # Store exception but don't raise
            image_job.error = e
        return image_job

    def prepare(self, output_path: str, relurl: str):
        """
        Prepares the output directory and sets up temporary storage if tarball creation is enabled.

        Args:
            output_path (str): The final destination path for output files.
            relurl(str): the relative url to process

        Attributes:
            final_output_path (str): The actual output path where the final files will be stored.
            temp_dir (Optional[str]): A temporary directory for intermediate storage if tarball creation is enabled.
            output_path (str): The working output path (either temporary or final).
            profiler (Profiler): Profiler instance for tracking processing time.
        """
        self.final_output_path = output_path
        self.profiler = Profiler(
            f"processing {relurl}", profile=self.verbose or self.debug
        )
        # process without any output
        if output_path is None:
            return
        if self.tar:
            # Use a temporary directory for intermediate PNG storage
            self.temp_dir = tempfile.mkdtemp()
            self.output_path = self.temp_dir
        else:
            self.output_path = output_path
        # Prepare output directory if needed
        os.makedirs(self.final_output_path, exist_ok=True)

    def wrap_as_tarball(self, djvu_path: str):
        """
        Wraps processed output files into a tarball

        Args:
            djvu_path (str): The path to the original DjVu file.

        """
        tarball_path = os.path.join(
            self.final_output_path, f"{Path(djvu_path).stem}.tar"
        )
        self.create_tarball(self.output_path, tarball_path)
        if self.clean_temp:
            shutil.rmtree(self.temp_dir)

    def process(
        self,
        djvu_path: str,
        relurl: str,
        mode: int = djvu.decode.RENDER_COLOR,
        wait: bool = True,
        save_png: bool = False,
        free_buffer: bool = True,
        output_path: str = None,
    ) -> Generator[ImageJob, None, None]:
        """
        Converts a DjVu URL to image buffers with sequential decoding and rendering.
        """
        self.prepare(output_path=output_path, relurl=relurl)
        # Step 1: Create image jobs for all pages
        image_jobs = self.create_image_jobs(djvu_path, relurl)
        self.profiler.time(f" create image jobs")

        # Process each page sequentially
        for job in image_jobs:
            # Step 2: Decode the page
            decoded_job = self.decode_page(job, wait)

            # Step 3: Render the page
            rendered_job = self.render_page(decoded_job, mode)
            self.profiler.time(f" process page {rendered_job.page_index:4d}")

            # Step 4: Optionally save to PNG
            if save_png:
                self.save_as_png(rendered_job, self.output_path, free_buffer)

            yield rendered_job

    def process_batch(
        self,
        image_jobs: List[ImageJob],
        mode: int = djvu.decode.RENDER_COLOR,
        wait: bool = True,
        save_png: bool = False,
        free_buffer: bool = True,
    ) -> Generator[ImageJob, None, None]:
        """
        Process a batch of image jobs with parallel execution.
        To be called from within process_parallel.
        """
        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            # Step 2: Decode all pages in parallel
            decode_futures = [
                executor.submit(self.decode_page, job, wait) for job in image_jobs
            ]

            # Step 3 & 4: Render and save all pages in parallel
            render_futures: List[Future] = []
            save_futures: List[Future] = []

            # Submit rendering jobs as decoding completes
            for future in decode_futures:
                decoded_job = future.result()
                render_futures.append(
                    executor.submit(self.render_page, decoded_job, mode)
                )

            # Process rendered jobs as they become available
            for future in render_futures:
                # check memory before processing
                exceeds_limit, usage = self.check_memory_usage()
                if exceeds_limit:
                    msg = f"Memory usage {usage} GB exceeds {self.limit_gb} GB limit"
                    raise Exception(msg)
                rendered_job = future.result()
                self.profiler.time(f" process page {rendered_job.page_index:4d}")

                # Optionally save to PNG in parallel
                if save_png:
                    save_future=executor.submit(
                        self.save_as_png, rendered_job, self.output_path, free_buffer
                    )
                    save_futures.append(save_future)

                yield rendered_job
            # Wait for all PNG saves
            for future in save_futures:
                future.result()

        # Clean up memory after processing the batch
        gc.collect()
        self.profiler.time(" memory cleaned after batch")

    def process_parallel(
        self,
        djvu_path: str,
        relurl: str,
        mode: int = djvu.decode.RENDER_COLOR,
        wait: bool = True,
        save_png: bool = False,
        output_path: str = None,
    ) -> Generator[ImageJob, None, None]:
        """
        Converts a DjVu URL to image buffers with fully parallel decoding and rendering.

        Args:
            djvu_path (str): Path to the DjVu file.
            relurl (str): Relative URL for referencing the file.
            mode (int, optional): Rendering mode (default: djvu.decode.RENDER_COLOR).
            wait (bool, optional): Whether to wait for processing to complete (default: True).
            save_png (bool, optional): Whether to save output as PNG files (default: False).
            output_path (str, optional): Directory path to save PNG files (default: None).

        Yields:
            Generator[ImageJob, None, None]: A generator yielding image jobs.
        """
        self.prepare(output_path=output_path, relurl=relurl)

        # Step 1: Create image jobs for all pages
        image_jobs = self.create_image_jobs(djvu_path, relurl)
        total_pages = len(image_jobs)
        self.profiler.time(f" create {total_pages} image jobs")

        # Process Steps 2 to 5 in batches
        for batch_start in range(0, total_pages, self.batch_size):
            batch_end = min(batch_start + self.batch_size, total_pages)
            batch = image_jobs[batch_start:batch_end]

            self.profiler.time(
                f" processing batch {batch_start//self.batch_size + 1}: pages {batch_start+1}-{batch_end}"
            )

            # Process this batch
            for rendered_job in self.process_batch(
                batch, mode, wait, save_png=save_png
            ):
                yield rendered_job
