"""
Video import service module.

Provides high-level functions for importing and anonymizing video files,
combining VideoFile creation with frame-level anonymization.

Changelog:
    October 14, 2025: Added file locking mechanism to prevent race conditions
                      during concurrent video imports (matches PDF import pattern)
"""

import logging
import os
import random
import shutil
import sys
import time
from contextlib import contextmanager
from datetime import date
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union

from django.db import transaction
from django.db.models.fields.files import FieldFile
from lx_anonymizer import FrameCleaner
from moviepy import video

from endoreg_db.models import EndoscopyProcessor, SensitiveMeta, VideoFile
from endoreg_db.models.media.video.video_file_anonymize import _cleanup_raw_assets
from endoreg_db.utils.hashs import get_video_hash
from endoreg_db.utils.paths import ANONYM_VIDEO_DIR, STORAGE_DIR, VIDEO_DIR

# File lock configuration (matches PDF import)
STALE_LOCK_SECONDS = 6000  # 100 minutes - reclaim locks older than this
MAX_LOCK_WAIT_SECONDS = (
    90  # New: wait up to 90s for a non-stale lock to clear before skipping
)

logger = logging.getLogger(__name__)


class VideoImportService:
    """
    Service for importing and anonymizing video files.
    Uses a central video instance pattern for cleaner state management.

    Features (October 14, 2025):
        - File locking to prevent concurrent processing of the same video
        - Stale lock detection and reclamation (600s timeout)
        - Hash-based duplicate detection
        - Graceful fallback processing without lx_anonymizer
    """

    def __init__(self, project_root: Optional[Path] = None):
        # Set up project root path
        if project_root:
            self.project_root = Path(project_root)
        else:
            self.project_root = Path(__file__).parent.parent.parent.parent

        # Track processed files to prevent duplicates
        try:
            # Ensure anonym_video directory exists before listing files
            anonym_video_dir = Path(ANONYM_VIDEO_DIR)
            if anonym_video_dir.exists():
                self.processed_files = set(
                    str(anonym_video_dir / file)
                    for file in os.listdir(ANONYM_VIDEO_DIR)
                )
            else:
                logger.info(f"Creating anonym_videos directory: {anonym_video_dir}")
                anonym_video_dir.mkdir(parents=True, exist_ok=True)
                self.processed_files = set()
        except Exception as e:
            logger.warning(f"Failed to initialize processed files tracking: {e}")
            self.processed_files = set()

        # Central video instance and processing context
        self.current_video: Optional[VideoFile] = None
        self.processing_context: Dict[str, Any] = {}

        self.delete_source = True

        self.logger = logging.getLogger(__name__)

        self.cleaner = (
            None  # This gets instantiated in the perform_frame_cleaning method
        )

    def _require_current_video(self) -> VideoFile:
        """Return the current VideoFile or raise if it has not been initialized."""
        if self.current_video is None:
            raise RuntimeError("Current video instance is not set")
        return self.current_video

    @contextmanager
    def _file_lock(self, path: Path):
        """
        Create a file lock to prevent duplicate processing of the same video.

        This context manager creates a .lock file alongside the video file.
        If the lock file already exists, it checks if it's stale (older than
        STALE_LOCK_SECONDS) and reclaims it if necessary. If it's not stale,
        we now WAIT (up to MAX_LOCK_WAIT_SECONDS) instead of failing immediately.
        """
        lock_path = Path(str(path) + ".lock")
        fd = None
        try:
            deadline = time.time() + MAX_LOCK_WAIT_SECONDS
            while True:
                try:
                    # Atomic create; fail if exists
                    fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
                    break  # acquired
                except FileExistsError:
                    # Check for stale lock
                    age = None
                    try:
                        st = os.stat(lock_path)
                        age = time.time() - st.st_mtime
                    except FileNotFoundError:
                        # Race: lock removed between exists and stat; retry acquire in next loop
                        age = None

                    if age is not None and age > STALE_LOCK_SECONDS:
                        try:
                            logger.warning(
                                "Stale lock detected for %s (age %.0fs). Reclaiming lock...",
                                path,
                                age,
                            )
                            lock_path.unlink()
                        except Exception as e:
                            logger.warning(
                                "Failed to remove stale lock %s: %s", lock_path, e
                            )
                        # Loop continues and retries acquire immediately
                        continue

                    # Not stale: wait until deadline, then give up gracefully
                    if time.time() >= deadline:
                        raise ValueError(f"File already being processed: {path}")
                    time.sleep(1.0)

            os.write(fd, b"lock")
            os.close(fd)
            fd = None
            yield
        finally:
            try:
                if fd is not None:
                    os.close(fd)
                if lock_path.exists():
                    lock_path.unlink()
            except OSError:
                pass

    def processed(self) -> bool:
        """Indicates if the current file has already been processed."""
        return getattr(self, "_processed", False)

    def import_and_anonymize(
        self,
        file_path: Union[Path, str],
        center_name: str,
        processor_name: str,
        save_video: bool = True,
        delete_source: bool = True,
    ) -> "VideoFile|None":
        """
        High-level helper that orchestrates the complete video import and anonymization process.
        Uses the central video instance pattern for improved state management.
        """
        # DEFENSIVE: Initialize processing_context immediately to prevent KeyError crashes
        self.processing_context = {"file_path": Path(file_path)}

        try:
            # Initialize processing context
            self._initialize_processing_context(
                file_path, center_name, processor_name, save_video, delete_source
            )

            # Validate and prepare file (may raise ValueError if another worker holds a non-stale lock)
            try:
                self._validate_and_prepare_file()
            except ValueError as ve:
                # Relaxed behavior: if another process is working on this file, skip cleanly
                if "already being processed" in str(ve):
                    self.logger.info(f"Skipping {file_path}: {ve}")
                    return None
                raise

            # Create or retrieve video instance
            self._create_or_retrieve_video_instance()

            # Create sensitive meta file, ensure raw is moved out of processing folder watched by file watcher.
            self._create_sensitive_file()

            # Setup processing environment
            self._setup_processing_environment()

            # Process frames and metadata
            self._process_frames_and_metadata()

            # Finalize processing
            self._finalize_processing()

            # Move files and cleanup
            self._cleanup_and_archive()

            return self.current_video

        except Exception as e:
            # Safe file path access - handles cases where processing_context wasn't initialized
            safe_file_path = getattr(self, "processing_context", {}).get(
                "file_path", file_path
            )
            # Debug: Log context state for troubleshooting
            context_keys = list(getattr(self, "processing_context", {}).keys())
            self.logger.debug(f"Context keys during error: {context_keys}")
            self.logger.error(
                f"Video import and anonymization failed for {safe_file_path}: {e}"
            )
            self._cleanup_on_error()
            raise
        finally:
            self._cleanup_processing_context()

    def _initialize_processing_context(
        self,
        file_path: Union[Path, str],
        center_name: str,
        processor_name: str,
        save_video: bool,
        delete_source: bool,
    ):
        """Initialize the processing context for the current video import."""
        self.processing_context = {
            "file_path": Path(file_path),
            "center_name": center_name,
            "processor_name": processor_name,
            "save_video": save_video,
            "delete_source": delete_source,
            "processing_started": False,
            "frames_extracted": False,
            "anonymization_completed": False,
            "error_reason": None,
        }

        self.logger.info(f"Initialized processing context for: {file_path}")

    def _validate_and_prepare_file(self):
        """
        Validate the video file and prepare for processing.

        Uses file locking to prevent concurrent processing of the same video file.
        This prevents race conditions where multiple workers might try to process
        the same video simultaneously.

        The lock is acquired here and held for the entire import process.
        See _file_lock() for lock reclamation logic.
        """
        file_path = self.processing_context["file_path"]

        # Acquire file lock to prevent concurrent processing
        # Lock will be held until finally block in import_and_anonymize()
        try:
            self.processing_context["_lock_context"] = self._file_lock(file_path)
            self.processing_context["_lock_context"].__enter__()
        except Exception:
            self._cleanup_processing_context()
            raise

        self.logger.info("Acquired file lock for: %s", file_path)

        # Check if already processed (memory-based check)
        if str(file_path) in self.processed_files:
            self.logger.info("File %s already processed, skipping", file_path)
            self._processed = True
            raise ValueError(f"File already processed: {file_path}")

        # Check file exists
        if not file_path.exists():
            raise FileNotFoundError(f"Video file not found: {file_path}")

        self.logger.info("File validation completed for: %s", file_path)

    def _create_or_retrieve_video_instance(self):
        """Create or retrieve the VideoFile instance and move to final storage."""

        self.logger.info("Creating VideoFile instance...")

        self.current_video = VideoFile.create_from_file_initialized(
            file_path=self.processing_context["file_path"],
            center_name=self.processing_context["center_name"],
            processor_name=self.processing_context["processor_name"],
            delete_source=self.processing_context["delete_source"],
            save_video_file=self.processing_context["save_video"],
        )

        if not self.current_video:
            raise RuntimeError("Failed to create VideoFile instance")

        # Immediately move to final storage locations
        self._move_to_final_storage()

        self.logger.info("Created VideoFile with UUID: %s", self.current_video.uuid)

        # Get and mark processing state
        state = VideoFile.get_or_create_state(self.current_video)
        if not state:
            raise RuntimeError("Failed to create VideoFile state")

        state.mark_processing_started(save=True)
        self.processing_context["processing_started"] = True

    def _move_to_final_storage(self):
        """
        Move video from raw_videos to final storage locations.
        - Raw video → /data/videos (raw_file_path)
        - Processed video will later → /data/anonym_videos (file_path)
        """
        from endoreg_db.utils import data_paths

        source_path = Path(self.processing_context["file_path"])
        _current_video = self._require_current_video()
        videos_dir = Path(data_paths["video"])
        storage_root = Path(data_paths["storage"])

        videos_dir.mkdir(parents=True, exist_ok=True)

        # --- Derive stored_raw_path safely ---
        stored_raw_path = None
        try:
            if hasattr(_current_video, "get_raw_file_path"):
                candidate = _current_video.get_raw_file_path()
                if candidate:
                    candidate_path = Path(candidate)
                    # Accept only if under storage_root
                    try:
                        candidate_path.relative_to(storage_root)
                        stored_raw_path = candidate_path
                    except ValueError:
                        # outside storage_root, reset
                        stored_raw_path = None
        except Exception:
            stored_raw_path = None

        # Fallback: derive from UUID + suffix - ALWAYS use UUID for consistency
        if not stored_raw_path:
            suffix = source_path.suffix or ".mp4"
            uuid_str = getattr(_current_video, "uuid", None)
            if uuid_str:
                filename = f"{uuid_str}{suffix}"
            else:
                # Emergency fallback with timestamp to avoid conflicts
                import time

                timestamp = int(time.time())
                filename = f"video_{timestamp}{suffix}"
                self.logger.warning(
                    "No UUID available, using timestamp-based filename: %s", filename
                )
            stored_raw_path = videos_dir / filename
            self.logger.debug("Using UUID-based raw filename: %s", filename)

        delete_source = bool(self.processing_context.get("delete_source", True))
        stored_raw_path.parent.mkdir(parents=True, exist_ok=True)

        # --- Move or copy raw video ---
        try:
            if delete_source:
                # Try atomic move first, fallback to copy+unlink
                try:
                    os.replace(source_path, stored_raw_path)
                    self.logger.info("Moved raw video to: %s", stored_raw_path)
                except Exception:
                    shutil.copy2(source_path, stored_raw_path)
                    os.remove(source_path)
                    self.logger.info(
                        "Copied & removed raw video to: %s", stored_raw_path
                    )
            else:
                shutil.copy2(source_path, stored_raw_path)
                self.logger.info("Copied raw video to: %s", stored_raw_path)
        except Exception as e:
            self.logger.error("Failed to move/copy video to final storage: %s", e)
            raise

        # --- Ensure DB raw_file is relative to storage root ---
        try:
            rel_path = stored_raw_path.relative_to(storage_root)
        except Exception:
            rel_path = Path("videos") / stored_raw_path.name

        if _current_video.raw_file.name != rel_path.as_posix():
            _current_video.raw_file.name = rel_path.as_posix()
            _current_video.save(update_fields=["raw_file"])
            self.logger.info("Updated raw_file path to: %s", rel_path.as_posix())

        # --- Store for later stages ---
        self.processing_context["raw_video_path"] = stored_raw_path
        self.processing_context["video_filename"] = stored_raw_path.name

    def _setup_processing_environment(self):
        """Setup the processing environment without file movement."""
        video = self._require_current_video()

        # Initialize video specifications
        video.initialize_video_specs()



        # Extract frames BEFORE processing to prevent pipeline 1 conflicts
        self.logger.info("Pre-extracting frames to avoid pipeline conflicts...")
        try:
            frames_extracted = video.extract_frames(overwrite=False)
            if frames_extracted:
                self.processing_context["frames_extracted"] = True
                self.logger.info("Frame extraction completed successfully")
                        # Initialize frame objects in database
                video.initialize_frames(video.get_frame_paths())

                # CRITICAL: Immediately save the frames_extracted state to database
                # to prevent refresh_from_db() in pipeline 1 from overriding it
                state = video.get_or_create_state()
                if not state.frames_extracted:
                    state.frames_extracted = True
                    state.save(update_fields=["frames_extracted"])
                    self.logger.info("Persisted frames_extracted=True to database")
            else:
                self.logger.warning("Frame extraction failed, but continuing...")
                self.processing_context["frames_extracted"] = False
        except Exception as e:
            self.logger.warning(
                f"Frame extraction failed during setup: {e}, but continuing..."
            )
            self.processing_context["frames_extracted"] = False

        # Ensure default patient data
        self._ensure_default_patient_data(video_instance=video)

        self.logger.info("Processing environment setup completed")

    def _process_frames_and_metadata(self):
        """Process frames and extract metadata with anonymization."""
        # Check frame cleaning availability
        frame_cleaning_available, frame_cleaner = (
            self._ensure_frame_cleaning_available()
        )
        video = self._require_current_video()

        raw_file_field = video.raw_file
        has_raw_file = isinstance(raw_file_field, FieldFile) and bool(
            raw_file_field.name
        )

        if not (frame_cleaning_available and has_raw_file):
            self.logger.warning(
                "Frame cleaning not available or conditions not met, using fallback anonymization."
            )
            self._fallback_anonymize_video()
            return

        try:
            self.logger.info(
                "Starting frame-level anonymization with processor ROI masking..."
            )

            # Get processor ROI information
            endoscope_data_roi_nested, endoscope_image_roi = (
                self._get_processor_roi_info()
            )

            # Perform frame cleaning with timeout to prevent blocking
            from concurrent.futures import ThreadPoolExecutor
            from concurrent.futures import TimeoutError as FutureTimeoutError

            with ThreadPoolExecutor(max_workers=1) as executor:
                future = executor.submit(
                    self._perform_frame_cleaning,
                    endoscope_data_roi_nested,
                    endoscope_image_roi,
                )
                try:
                    # Increased timeout to better accommodate ffmpeg + OCR
                    future.result(timeout=50000)
                    self.processing_context["anonymization_completed"] = True
                    self.logger.info(
                        "Frame cleaning completed successfully within timeout"
                    )
                except FutureTimeoutError:
                    self.logger.warning(
                        "Frame cleaning timed out; entering grace period check for cleaned output"
                    )
                    # Grace period: detect if cleaned file appears shortly after timeout
                    raw_video_path = self.processing_context.get("raw_video_path")
                    video_filename = self.processing_context.get(
                        "video_filename",
                        Path(raw_video_path).name if raw_video_path else "video.mp4",
                    )
                    grace_seconds = 60
                    expected_cleaned_path: Optional[Path] = None
                    processed_field = video.processed_file
                    if isinstance(processed_field, FieldFile) and processed_field.name:
                        try:
                            expected_cleaned_path = Path(processed_field.path)
                        except (NotImplementedError, TypeError, ValueError):
                            expected_cleaned_path = None
                    found = False
                    if expected_cleaned_path is not None:
                        for _ in range(grace_seconds):
                            if expected_cleaned_path.exists():
                                self.processing_context["cleaned_video_path"] = (
                                    expected_cleaned_path
                                )
                                self.processing_context["anonymization_completed"] = (
                                    True
                                )
                                self.logger.info(
                                    "Detected cleaned video during grace period: %s",
                                    expected_cleaned_path,
                                )
                                found = True
                                break
                            time.sleep(1)
                    else:
                        self._fallback_anonymize_video()
                    if not found:
                        raise TimeoutError(
                            "Frame cleaning operation timed out - likely Ollama connection issue"
                        )

        except Exception as e:
            self.logger.warning(
                "Frame cleaning failed (reason: %s), falling back to simple copy", e
            )
            # Try fallback anonymization when frame cleaning fails
            try:
                self._fallback_anonymize_video()
            except Exception as fallback_error:
                self.logger.error(
                    "Fallback anonymization also failed: %s", fallback_error
                )
                # If even fallback fails, mark as not anonymized but continue import
                self.processing_context["anonymization_completed"] = False
                self.processing_context["error_reason"] = (
                    f"Frame cleaning failed: {e}, Fallback failed: {fallback_error}"
                )

    def _save_anonymized_video(self):
        original_raw_file_path_to_delete = None
        original_raw_frame_dir_to_delete = None
        video = self._require_current_video()
        anonymized_video_path = video.get_target_anonymized_video_path()

        if not anonymized_video_path.exists():
            raise RuntimeError(
                f"Processed video file not found after assembly for {video.uuid}: {anonymized_video_path}"
            )

        new_processed_hash = get_video_hash(anonymized_video_path)
        if (
            video.__class__.objects.filter(processed_video_hash=new_processed_hash)
            .exclude(pk=video.pk)
            .exists()
        ):
            raise ValueError(
                f"Processed video hash {new_processed_hash} already exists for another video (Video: {video.uuid})."
            )

        video.processed_video_hash = new_processed_hash
        video.processed_file.name = anonymized_video_path.relative_to(
            STORAGE_DIR
        ).as_posix()

        update_fields = [
            "processed_video_hash",
            "processed_file",
            "frame_dir",
        ]

        if self.delete_source:
            original_raw_file_path_to_delete = video.get_raw_file_path()
            original_raw_frame_dir_to_delete = video.get_frame_dir_path()

            video.raw_file.name = None  # type: ignore[assignment]

            update_fields.extend(["raw_file", "video_hash"])

            transaction.on_commit(
                lambda: _cleanup_raw_assets(
                    video_uuid=video.uuid,
                    raw_file_path=original_raw_file_path_to_delete,
                    raw_frame_dir=original_raw_frame_dir_to_delete,
                )
            )

        video.save(update_fields=update_fields)
        video.state.mark_anonymized(save=True)
        video.refresh_from_db()
        self.current_video = video
        return True

    def _fallback_anonymize_video(self):
        """
        Fallback to create anonymized video if lx_anonymizer is not available.
        """
        try:
            self.logger.info("Attempting fallback video anonymization...")
            video = self.current_video
            if video is None:
                self.logger.warning(
                    "No VideoFile instance available for fallback anonymization"
                )

            # Strategy 2: Simple copy (no processing, just copy raw to processed)
            self.logger.info(
                "Using simple copy fallback (raw video will be used as 'processed' video)"
            )
            self.processing_context["anonymization_completed"] = False
            self.processing_context["use_raw_as_processed"] = True
            self.logger.warning(
                "Fallback: Video will be imported without anonymization (raw copy used)"
            )
        except Exception as e:
            self.logger.error(
                f"Error during fallback anonymization: {e}", exc_info=True
            )
            self.processing_context["anonymization_completed"] = False
            self.processing_context["error_reason"] = str(e)

    def _finalize_processing(self):
        """Finalize processing and update video state."""
        self.logger.info("Updating video processing state...")

        with transaction.atomic():
            video = self._require_current_video()
            try:
                video.refresh_from_db()
            except Exception as refresh_error:
                self.logger.warning(
                    "Could not refresh VideoFile %s from DB: %s",
                    video.uuid,
                    refresh_error,
                )

            state = video.get_or_create_state()

            # Only mark frames as extracted if they were successfully extracted
            if self.processing_context.get("frames_extracted", False):
                state.frames_extracted = True
                self.logger.info("Marked frames as extracted in state")
            else:
                self.logger.warning("Frames were not extracted, not updating state")

            # Always mark these as true (metadata extraction attempts were made)
            state.frames_initialized = True
            state.video_meta_extracted = True
            state.text_meta_extracted = True

            # ✅ FIX: Only mark as processed if anonymization actually completed
            anonymization_completed = self.processing_context.get(
                "anonymization_completed", False
            )
            if anonymization_completed:
                state.mark_sensitive_meta_processed(save=False)
                self.logger.info(
                    "Anonymization completed - marking sensitive meta as processed"
                )
            else:
                self.logger.warning(
                    f"Anonymization NOT completed - NOT marking as processed. Reason: {self.processing_context.get('error_reason', 'Unknown')}"
                )
                # Explicitly mark as NOT processed
                state.sensitive_meta_processed = False

            # Save all state changes
            state.save()
            self.logger.info("Video processing state updated")

        # Signal completion
        self._signal_completion()

    def _cleanup_and_archive(self):
        """Move processed video to anonym_videos and cleanup."""
        from endoreg_db.utils import data_paths

        anonym_videos_dir = data_paths["anonym_video"]  # /data/anonym_videos
        anonym_videos_dir.mkdir(parents=True, exist_ok=True)

        video = self._require_current_video()

        processed_video_path = None
        if "cleaned_video_path" in self.processing_context:
            processed_video_path = self.processing_context["cleaned_video_path"]
        else:
            raw_video_path = self.processing_context.get("raw_video_path")
            if raw_video_path and Path(raw_video_path).exists():
                # Use UUID-based naming to avoid conflicts
                suffix = Path(raw_video_path).suffix or ".mp4"
                processed_filename = f"processed_{video.uuid}{suffix}"
                processed_video_path = Path(raw_video_path).parent / processed_filename
                try:
                    shutil.copy2(str(raw_video_path), str(processed_video_path))
                    self.logger.info(
                        "Copied raw video for processing: %s", processed_video_path
                    )
                except Exception as exc:
                    self.logger.error("Failed to copy raw video: %s", exc)
                    processed_video_path = None

        if processed_video_path and Path(processed_video_path).exists():
            try:
                ext = Path(processed_video_path).suffix or ".mp4"
                anonym_video_filename = f"anonym_{video.uuid}{ext}"
                anonym_target_path = anonym_videos_dir / anonym_video_filename

                shutil.move(str(processed_video_path), str(anonym_target_path))
                self.logger.info("Moved processed video to: %s", anonym_target_path)

                if anonym_target_path.exists():
                    try:
                        storage_root = data_paths["storage"]
                        relative_path = anonym_target_path.relative_to(storage_root)
                        video.processed_file.name = str(relative_path)
                        video.save(update_fields=["processed_file"])
                        self.logger.info(
                            "Updated processed_file path to: %s", relative_path
                        )
                    except Exception as exc:
                        self.logger.error(
                            "Failed to update processed_file path: %s", exc
                        )
                        video.processed_file.name = (
                            f"anonym_videos/{anonym_video_filename}"
                        )
                        video.save(update_fields=["processed_file"])
                        self.logger.info(
                            "Updated processed_file path using fallback: %s",
                            f"anonym_videos/{anonym_video_filename}",
                        )

                    self.processing_context["anonymization_completed"] = True
                else:
                    self.logger.warning(
                        "Processed video file not found after move: %s",
                        anonym_target_path,
                    )
            except Exception as exc:
                self.logger.error(
                    "Failed to move processed video to anonym_videos: %s", exc
                )
        else:
            self.logger.warning(
                "No processed video available - processed_file will remain empty"
            )

        try:
            from endoreg_db.utils.paths import RAW_FRAME_DIR

            shutil.rmtree(RAW_FRAME_DIR, ignore_errors=True)
            self.logger.debug(
                "Cleaned up temporary frames directory: %s", RAW_FRAME_DIR
            )
        except Exception as exc:
            self.logger.warning("Failed to remove directory %s: %s", RAW_FRAME_DIR, exc)

        source_path = self.processing_context["file_path"]
        if self.processing_context["delete_source"] and Path(source_path).exists():
            try:
                os.remove(source_path)
                self.logger.info("Removed remaining source file: %s", source_path)
            except Exception as exc:
                self.logger.warning(
                    "Failed to remove source file %s: %s", source_path, exc
                )

        if not video.processed_file or not Path(video.processed_file.path).exists():
            self.logger.warning(
                "No processed_file found after cleanup - video will be unprocessed"
            )
            try:
                video.anonymize(delete_original_raw=self.delete_source)
                video.save(update_fields=["processed_file"])
                self.logger.info("Late-stage anonymization succeeded")
            except Exception as e:
                self.logger.error("Late-stage anonymization failed: %s", e)
                self.processing_context["anonymization_completed"] = False

        self.logger.info("Cleanup and archiving completed")

        self.processed_files.add(str(self.processing_context["file_path"]))

        with transaction.atomic():
            video.refresh_from_db()
            if hasattr(video, "state") and self.processing_context.get(
                "anonymization_completed"
            ):
                video.state.mark_sensitive_meta_processed(save=True)

        self.logger.info(
            "Import and anonymization completed for VideoFile UUID: %s", video.uuid
        )
        self.logger.info("Raw video stored in: /data/videos")
        self.logger.info("Processed video stored in: /data/anonym_videos")

    def _create_sensitive_file(
        self,
        video_instance: VideoFile | None = None,
        file_path: Path | str | None = None,
    ) -> Path:
        """Create or move a sensitive copy of the raw video file inside storage."""

        video = video_instance or self._require_current_video()

        raw_field: FieldFile | None = getattr(video, "raw_file", None)
        source_path: Path | None = None
        try:
            if raw_field and raw_field.path:
                source_path = Path(raw_field.path)
        except Exception:
            source_path = None

        if source_path is None and file_path is not None:
            source_path = Path(file_path)

        if source_path is None:
            raise ValueError("No file path available for creating sensitive file")
        if not raw_field:
            raise ValueError(
                "VideoFile must have a raw_file to create a sensitive file"
            )

        target_dir = VIDEO_DIR / "sensitive"
        if not target_dir.exists():
            self.logger.info("Creating sensitive file directory: %s", target_dir)
            os.makedirs(target_dir, exist_ok=True)

        target_file_path = target_dir / source_path.name
        try:
            shutil.move(str(source_path), str(target_file_path))
            self.logger.info(
                "Moved raw file to sensitive directory: %s", target_file_path
            )
        except Exception as exc:
            self.logger.warning(
                "Failed to move raw file to sensitive dir, copying instead: %s", exc
            )
            shutil.copy(str(source_path), str(target_file_path))
            try:
                os.remove(source_path)
            except FileNotFoundError:
                pass

        try:
            from endoreg_db.utils import data_paths

            storage_root = data_paths["storage"]
            relative_path = target_file_path.relative_to(storage_root)
            video.raw_file.name = str(relative_path)
            video.save(update_fields=["raw_file"])
            self.logger.info(
                "Updated video.raw_file to point to sensitive location: %s",
                relative_path,
            )
        except Exception as exc:
            self.logger.warning("Failed to set relative path, using fallback: %s", exc)
            video.raw_file.name = f"videos/sensitive/{target_file_path.name}"
            video.save(update_fields=["raw_file"])
            self.logger.info(
                "Updated video.raw_file using fallback method: videos/sensitive/%s",
                target_file_path.name,
            )

        self.processing_context["raw_video_path"] = target_file_path
        self.processing_context["video_filename"] = target_file_path.name

        self.logger.info(
            "Created sensitive file for %s at %s", video.uuid, target_file_path
        )
        return target_file_path

    def _get_processor_roi_info(
        self,
    ) -> Tuple[Optional[List[List[Dict[str, Any]]]], Optional[Dict[str, Any]]]:
        """Get processor ROI information for masking."""
        endoscope_data_roi_nested = None
        endoscope_image_roi = None

        video = self._require_current_video()

        try:
            video_meta = getattr(video, "video_meta", None)
            processor = getattr(video_meta, "processor", None) if video_meta else None
            if processor:
                assert isinstance(processor, EndoscopyProcessor), (
                    "Processor is not of type EndoscopyProcessor"
                )
                endoscope_image_roi = processor.get_roi_endoscope_image()
                endoscope_data_roi_nested = processor.get_sensitive_rois()
                self.logger.info(
                    "Retrieved processor ROI information: endoscope_image_roi=%s",
                    endoscope_image_roi,
                )
            else:
                self.logger.warning(
                    "No processor found for video %s, proceeding without ROI masking",
                    video.uuid,
                )
        except Exception as exc:
            self.logger.error("Failed to retrieve processor ROI information: %s", exc)

        # Convert dict to nested list if necessary to match return type
        if isinstance(endoscope_data_roi_nested, dict):
            # Convert dict[str, dict[str, int | None] | None] to List[List[Dict[str, Any]]]
            converted_roi = []
            for key, value in endoscope_data_roi_nested.items():
                if isinstance(value, dict):
                    converted_roi.append([value])
                elif value is None:
                    converted_roi.append([])
            endoscope_data_roi_nested = converted_roi

        return endoscope_data_roi_nested, endoscope_image_roi

    def _ensure_default_patient_data(
        self, video_instance: VideoFile | None = None
    ) -> None:
        """Ensure minimum patient data is present on the video's SensitiveMeta."""

        video = video_instance or self._require_current_video()

        sensitive_meta = getattr(video, "sensitive_meta", None)
        if not sensitive_meta:
            self.logger.info(
                "No SensitiveMeta found for video %s, creating default", video.uuid
            )
            default_data = {
                "patient_first_name": "Patient",
                "patient_last_name": "Unknown",
                "patient_dob": date(1990, 1, 1),
                "examination_date": date.today(),
                "center_name": video.center.name
                if video.center
                else "university_hospital_wuerzburg",
            }
            try:
                sensitive_meta = SensitiveMeta.create_from_dict(default_data)
                video.sensitive_meta = sensitive_meta
                video.save(update_fields=["sensitive_meta"])
                self.logger.info(
                    "Created default SensitiveMeta for video %s", video.uuid
                )
            except Exception as exc:
                self.logger.error(
                    "Failed to create default SensitiveMeta for video %s: %s",
                    video.uuid,
                    exc,
                )
                return
        else:
            update_data: Dict[str, Any] = {}
            if not sensitive_meta.patient_first_name:
                update_data["patient_first_name"] = "Patient"
            if not sensitive_meta.patient_last_name:
                update_data["patient_last_name"] = "Unknown"
            if not sensitive_meta.patient_dob:
                update_data["patient_dob"] = date(1990, 1, 1)
            if not sensitive_meta.examination_date:
                update_data["examination_date"] = date.today()

            if update_data:
                try:
                    sensitive_meta.update_from_dict(update_data)
                    state = video.get_or_create_state()
                    state.mark_sensitive_meta_processed(save=True)
                    self.logger.info(
                        "Updated missing SensitiveMeta fields for video %s: %s",
                        video.uuid,
                        list(update_data.keys()),
                    )
                except Exception as exc:
                    self.logger.error(
                        "Failed to update SensitiveMeta for video %s: %s",
                        video.uuid,
                        exc,
                    )

    def _ensure_frame_cleaning_available(self):
        """
        Ensure frame cleaning modules are available by adding lx-anonymizer to path.

        Returns:
            Tuple of (availability_flag, FrameCleaner_class, ReportReader_class)
        """
        try:
            # Check if we can find lx-anonymizer
            from lx_anonymizer import FrameCleaner  # type: ignore[import]

            if FrameCleaner:
                return True, FrameCleaner()

        except Exception as e:
            self.logger.warning(
                f"Frame cleaning not available: {e} Please install or update lx_anonymizer."
            )

        return False, None

    def _perform_frame_cleaning(self, endoscope_data_roi_nested, endoscope_image_roi):
        """Perform frame cleaning and anonymization."""
        # Instantiate frame cleaner
        is_available, frame_cleaner = self._ensure_frame_cleaning_available()

        if not is_available:
            raise RuntimeError("Frame cleaning not available")

        # Prepare parameters for frame cleaning
        raw_video_path = self.processing_context.get("raw_video_path")

        if not raw_video_path or not Path(raw_video_path).exists():
            try:
                self.current_video = self._require_current_video()
                raw_video_path = self.current_video.get_raw_file_path()
            except Exception:
                raise RuntimeError(f"Raw video path not found: {raw_video_path}")

        # Create temporary output path for cleaned video using UUID to avoid naming conflicts
        video = self._require_current_video()
        # Ensure raw_video_path is not None
        if not raw_video_path:
            raise RuntimeError(
                "raw_video_path is None, cannot construct cleaned_video_path"
            )
        suffix = Path(raw_video_path).suffix or ".mp4"
        cleaned_filename = f"cleaned_{video.uuid}{suffix}"
        cleaned_video_path = Path(raw_video_path).parent / cleaned_filename
        self.logger.debug("Using UUID-based cleaned filename: %s", cleaned_filename)

        # Clean video with ROI masking (heavy I/O operation)
        actual_cleaned_path, extracted_metadata = frame_cleaner.clean_video(
            video_path=Path(raw_video_path),
            endoscope_image_roi=endoscope_image_roi,
            endoscope_data_roi_nested=endoscope_data_roi_nested,
            output_path=cleaned_video_path,
            technique="mask_overlay",
        )

        # Store cleaned video path for later use in _cleanup_and_archive
        self.processing_context["cleaned_video_path"] = actual_cleaned_path
        self.processing_context["extracted_metadata"] = extracted_metadata

        # Update sensitive metadata with extracted information
        self._update_sensitive_metadata(extracted_metadata)
        self.logger.info(
            f"Extracted metadata from frame cleaning: {extracted_metadata}"
        )

        self.logger.info(
            f"Frame cleaning with ROI masking completed: {actual_cleaned_path}"
        )
        self.logger.info("Cleaned video will be moved to anonym_videos during cleanup")

    def _update_sensitive_metadata(self, extracted_metadata: Dict[str, Any]):
        """
        Update sensitive metadata with extracted information.
        Args:
            extracted_metadata (Dict[str, Any]): Extracted metadata to update.
        """
        video = self._require_current_video()
        sensitive_meta = getattr(video, "sensitive_meta", None)

        if not (sensitive_meta and extracted_metadata):
            return

        sm = sensitive_meta
        updated_fields = []

        # Ensure center is set from video.center if not in extracted_metadata
        metadata_to_update = extracted_metadata.copy()

        # FIX: Set center object instead of center_name string
        if not hasattr(sm, "center") or not sm.center:
            if video.center:
                metadata_to_update["center"] = video.center
                self.logger.debug(
                    "Added center object '%s' to metadata for SensitiveMeta update",
                    video.center.name,
                )
            else:
                center_name = metadata_to_update.get("center_name")
                if center_name:
                    try:
                        from ..models.administration import Center

                        center_obj = Center.objects.get(name=center_name)
                        metadata_to_update["center"] = center_obj
                        self.logger.debug(
                            "Loaded center object '%s' from center_name", center_name
                        )
                        metadata_to_update.pop("center_name", None)
                    except Center.DoesNotExist:
                        self.logger.error(
                            "Center '%s' not found in database", center_name
                        )
                        return

        try:
            sm.update_from_dict(metadata_to_update)
            updated_fields = list(
                extracted_metadata.keys()
            )  # Only log originally extracted fields
        except KeyError as e:
            self.logger.warning(f"Failed to update SensitiveMeta field {e}")
            return

        if updated_fields:
            try:
                sm.save()  # Remove update_fields to allow all necessary fields to be saved
                self.logger.info(
                    "Updated SensitiveMeta fields for video %s: %s",
                    video.uuid,
                    updated_fields,
                )

                state = video.get_or_create_state()
                state.mark_sensitive_meta_processed(save=True)
                self.logger.info(
                    "Marked sensitive metadata as processed for video %s", video.uuid
                )
            except Exception as e:
                self.logger.error(f"Failed to save SensitiveMeta: {e}")
                raise  # Re-raise to trigger fallback in calling method
        else:
            self.logger.info(
                "No SensitiveMeta fields updated for video %s - all existing values preserved",
                video.uuid,
            )

    def _signal_completion(self):
        """Signal completion to the tracking system."""
        try:
            video = self._require_current_video()

            raw_field: FieldFile | None = getattr(video, "raw_file", None)
            raw_exists = False
            if raw_field and getattr(raw_field, "path", None):
                try:
                    raw_exists = Path(raw_field.path).exists()
                except (ValueError, OSError):
                    raw_exists = False

            video_processing_complete = (
                video.sensitive_meta is not None
                and video.video_meta is not None
                and raw_exists
            )

            if video_processing_complete:
                self.logger.info(
                    "Video %s processing completed successfully - ready for validation",
                    video.uuid,
                )

                # Update completion flags if they exist
                completion_fields = []
                for field_name in [
                    "import_completed",
                    "processing_complete",
                    "ready_for_validation",
                ]:
                    if hasattr(video, field_name):
                        setattr(video, field_name, True)
                        completion_fields.append(field_name)

                if completion_fields:
                    video.save(update_fields=completion_fields)
                    self.logger.info("Updated completion flags: %s", completion_fields)
            else:
                self.logger.warning(
                    "Video %s processing incomplete - missing required components",
                    video.uuid,
                )

        except Exception as e:
            self.logger.warning(f"Failed to signal completion status: {e}")

    def _cleanup_on_error(self):
        """Cleanup processing context on error."""
        if self.current_video and hasattr(self.current_video, "state"):
            try:
                if self.processing_context.get("processing_started"):
                    self.current_video.state.frames_extracted = False
                    self.current_video.state.frames_initialized = False
                    self.current_video.state.video_meta_extracted = False
                    self.current_video.state.text_meta_extracted = False
                    self.current_video.state.save()
            except Exception as e:
                self.logger.warning(f"Error during cleanup: {e}")

    def _cleanup_processing_context(self):
        """
        Cleanup processing context and release file lock.

        This method is always called in the finally block of import_and_anonymize()
        to ensure the file lock is released even if processing fails.
        """
        # DEFENSIVE: Ensure processing_context exists before accessing it
        if not hasattr(self, "processing_context"):
            self.processing_context = {}

        try:
            # Release file lock if it was acquired
            lock_context = self.processing_context.get("_lock_context")
            if lock_context is not None:
                try:
                    lock_context.__exit__(None, None, None)
                    self.logger.info("Released file lock")
                except Exception as e:
                    self.logger.warning(f"Error releasing file lock: {e}")

            # Remove file from processed set if processing failed
            file_path = self.processing_context.get("file_path")
            if file_path and not self.processing_context.get("anonymization_completed"):
                file_path_str = str(file_path)
                if file_path_str in self.processed_files:
                    self.processed_files.remove(file_path_str)
                    self.logger.info(
                        f"Removed {file_path_str} from processed files (failed processing)"
                    )

        except Exception as e:
            self.logger.warning(f"Error during context cleanup: {e}")
        finally:
            # Reset context
            self.current_video = None
            self.processing_context = {}


# Convenience function for callers/tests that expect a module-level import_and_anonymize
def import_and_anonymize(
    file_path,
    center_name: str,
    processor_name: str,
    save_video: bool = True,
    delete_source: bool = True,
) -> VideoFile | None:
    """Module-level helper that instantiates VideoImportService and runs import_and_anonymize.
    Kept for backward compatibility with callers that import this function directly.
    """
    service = VideoImportService()
    return service.import_and_anonymize(
        file_path=file_path,
        center_name=center_name,
        processor_name=processor_name,
        save_video=save_video,
        delete_source=delete_source,
    )
