"""
Scheduler Service with DB locking to prevent double-fire.
Designed to run in multiple workers without conflicts.
"""

import asyncio
import hashlib
import logging
import json
import re
from datetime import date, datetime, timedelta
from typing import Optional, List, Dict, Any, Tuple
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from croniter import croniter
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, and_, update, or_
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.exc import IntegrityError
import random

from ..db import AsyncSessionLocal

log = logging.getLogger(__name__)


_SOQL_SELECT_BLOCK_RE = re.compile(r"(?is)\bselect\s+([\s\S]+?)\s+\bfrom\b")
_SOQL_ALIAS_RE = re.compile(r"(?is)\s+as\s+([A-Za-z0-9_.]+)\s*$")
_SOQL_AGG_RE = re.compile(
    r"(?is)^(sum|avg|count|min|max|count_distinct)\s*\(\s*(.*?)\s*\)\s*$"
)

_PRESET_SCHEDULE_JITTER_MAX_SECONDS = {
    "15min": 120,
    "hourly": 300,
    "daily": 900,
    "weekly": 1800,
}


def _schedule_jitter_seconds(schedule: str, job_id: Optional[str]) -> int:
    if not job_id:
        return 0
    normalized = str(schedule or "").strip().lower()
    max_seconds = _PRESET_SCHEDULE_JITTER_MAX_SECONDS.get(normalized, 0)
    if max_seconds <= 0:
        return 0
    digest = hashlib.sha256(f"{job_id}:{normalized}".encode()).hexdigest()
    return int(digest[:8], 16) % (max_seconds + 1)


def _split_soql_select_fields(soql: str) -> List[str]:
    match = _SOQL_SELECT_BLOCK_RE.search(str(soql or ""))
    if not match:
        return []
    block = match.group(1) or ""
    fields: List[str] = []
    current: List[str] = []
    depth = 0
    in_single = False
    in_double = False

    for ch in block:
        if ch == "'" and not in_double:
            in_single = not in_single
        elif ch == '"' and not in_single:
            in_double = not in_double
        elif not in_single and not in_double:
            if ch == "(":
                depth += 1
            elif ch == ")" and depth > 0:
                depth -= 1
            elif ch == "," and depth == 0:
                field = "".join(current).strip()
                if field:
                    fields.append(field)
                current = []
                continue
        current.append(ch)

    tail = "".join(current).strip()
    if tail:
        fields.append(tail)
    return fields


def _prettify_field_name(raw: str) -> str:
    text = str(raw or "").strip()
    if not text:
        return text
    leaf = text.split(".")[-1]
    spaced = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", leaf.replace("_", " "))
    words = [w for w in re.split(r"\s+", spaced.strip()) if w]
    if not words:
        return leaf
    return " ".join(word[:1].upper() + word[1:] for word in words)


def _friendly_aggregate_label(func: str, inner: str) -> str:
    fn = str(func or "").upper()
    arg = str(inner or "").strip()
    if fn == "COUNT" and (arg == "*" or arg.lower().endswith(".id") or arg.lower() == "id"):
        return "Count"
    label = "Records" if arg == "*" else _prettify_field_name(arg)
    if fn == "SUM":
        return f"Total {label}"
    if fn == "AVG":
        return f"Avg {label}"
    if fn == "COUNT":
        return f"Count {label}"
    if fn == "COUNT_DISTINCT":
        return f"Distinct {label}"
    if fn == "MIN":
        return f"Min {label}"
    if fn == "MAX":
        return f"Max {label}"
    return f"{fn} {label}".strip()


def _infer_expr_header_labels_from_soql(soql: Optional[str]) -> Dict[str, str]:
    fields = _split_soql_select_fields(str(soql or ""))
    if not fields:
        return {}

    mapping: Dict[str, str] = {}
    aggregate_index = 0
    for field in fields:
        alias_match = _SOQL_ALIAS_RE.search(field)
        base_expr = field[: alias_match.start()].strip() if alias_match else field.strip()
        agg_match = _SOQL_AGG_RE.match(base_expr)
        if not agg_match:
            continue
        # If the planner/compiler emitted aliases, Salesforce won't emit exprN for these.
        if alias_match:
            continue
        func, inner = agg_match.group(1), agg_match.group(2)
        mapping[f"expr{aggregate_index}"] = _friendly_aggregate_label(func, inner)
        aggregate_index += 1
    return mapping


def _apply_header_aliases(headers: List[str], alias_map: Dict[str, str]) -> List[str]:
    if not headers or not alias_map:
        return headers
    out: List[str] = []
    seen: Dict[str, int] = {}
    for header in headers:
        candidate = str(alias_map.get(header, header))
        count = seen.get(candidate, 0) + 1
        seen[candidate] = count
        if count > 1:
            out.append(f"{candidate} ({count})")
        else:
            out.append(candidate)
    return out


def _normalize_sheet_cell(value: Any) -> Any:
    """Normalize Salesforce values so they are safe for Google Sheets writes."""
    if value is None:
        return ""
    if isinstance(value, (str, int, float, bool)):
        return value
    if isinstance(value, (datetime, date)):
        return value.isoformat()
    if isinstance(value, dict):
        # Strip Salesforce record metadata payloads.
        cleaned = {k: v for k, v in value.items() if k != "attributes"}
        if not cleaned:
            return ""
        if "Name" in cleaned and cleaned["Name"] not in (None, ""):
            return _normalize_sheet_cell(cleaned["Name"])
        if "name" in cleaned and cleaned["name"] not in (None, ""):
            return _normalize_sheet_cell(cleaned["name"])
        return json.dumps(cleaned, ensure_ascii=False, sort_keys=True)
    if isinstance(value, (list, tuple, set)):
        return json.dumps(list(value), ensure_ascii=False)
    return str(value)


def _records_to_sheet_table(
    records: List[Dict[str, Any]], soql: Optional[str] = None
) -> Tuple[List[str], List[List[Any]]]:
    """Convert Salesforce records into tabular headers + rows for Sheets."""
    headers: List[str] = []
    seen: set[str] = set()
    normalized_records: List[Dict[str, Any]] = []

    for raw_record in records or []:
        if isinstance(raw_record, dict):
            record = raw_record
        else:
            record = {"value": raw_record}
        normalized_records.append(record)
        for key in record.keys():
            key_text = str(key)
            if key_text == "attributes":
                continue
            if key_text not in seen:
                seen.add(key_text)
                headers.append(key_text)

    rows = [
        [_normalize_sheet_cell(record.get(header, "")) for header in headers]
        for record in normalized_records
    ]
    alias_map = _infer_expr_header_labels_from_soql(soql)
    headers = _apply_header_aliases(headers, alias_map)
    return headers, rows


class SchedulerService:
    """
    Background scheduler that picks up due jobs with DB locking.
    Only one worker will execute each job (no double-fire).
    """

    def __init__(self):
        self.scheduler = AsyncIOScheduler(timezone="UTC")
        self.is_leader = False
        self.lock_acquired = False

    async def start(self, db: Optional[AsyncSession] = None):
        """Start the scheduler and become leader if possible."""
        try:
            # Session argument intentionally ignored; each scheduled execution
            # uses an isolated session created from AsyncSessionLocal.
            _ = db

            # Try to become leader (atomic lock acquisition)
            await self._try_acquire_leader_lock()

            if self.is_leader:
                log.info("This worker is the scheduler leader")
                # Schedule recurring job picker
                self.scheduler.add_job(
                    self._pick_and_enqueue_due_jobs,
                    trigger="interval",
                    seconds=30,  # Check every 30s for due jobs
                    id="job_picker",
                    max_instances=1,
                    replace_existing=True,
                )

                # Catch up on missed runs at startup
                await self._catchup_missed_runs()

                self.scheduler.start()
                log.info("Scheduler started successfully")
            else:
                log.info("Another worker is the scheduler leader, staying passive")

        except Exception as e:
            log.error(f"Failed to start scheduler: {e}", exc_info=True)

    async def _try_acquire_leader_lock(self):
        """Acquire scheduler leader lock atomically."""
        try:
            from ..models import SystemConfig
        except Exception as import_err:
            self.is_leader = True
            self.lock_acquired = False
            log.warning(
                "SystemConfig unavailable; using lockless scheduler mode: %s",
                import_err,
            )
            return

        try:
            worker_id = f"worker-{random.randint(1000, 9999)}"
            now = datetime.utcnow()
            expires_at = now + timedelta(minutes=5)
            self.is_leader = False
            self.lock_acquired = False

            async with AsyncSessionLocal() as db:
                takeover_stmt = (
                    update(SystemConfig)
                    .where(SystemConfig.key == "scheduler_leader")
                    .where(
                        or_(
                            SystemConfig.expires_at.is_(None),
                            SystemConfig.expires_at < now,
                        )
                    )
                    .values(value=worker_id, expires_at=expires_at, updated_at=now)
                )
                takeover = await db.execute(takeover_stmt)

                if (takeover.rowcount or 0) == 1:
                    await db.commit()
                    self.is_leader = True
                    self.lock_acquired = True
                    log.info("Acquired leader lock (took over from expired leader)")
                else:
                    try:
                        db.add(
                            SystemConfig(
                                key="scheduler_leader",
                                value=worker_id,
                                expires_at=expires_at,
                                created_at=now,
                                updated_at=now,
                            )
                        )
                        await db.commit()
                        self.is_leader = True
                        self.lock_acquired = True
                        log.info("Acquired leader lock (first leader)")
                    except IntegrityError:
                        await db.rollback()
                        self.is_leader = False
                        self.lock_acquired = False

            if self.is_leader and self.lock_acquired:
                self.scheduler.add_job(
                    self._refresh_leader_lock,
                    trigger="interval",
                    seconds=120,
                    args=[worker_id],
                    id="leader_lock_refresh",
                    max_instances=1,
                    replace_existing=True,
                )

        except Exception as e:
            self.is_leader = True
            self.lock_acquired = False
            log.warning(
                "Leader lock unavailable; continuing in lockless scheduler mode: %s",
                e,
            )

    async def _refresh_leader_lock(self, worker_id: str):
        """Refresh leader lock to keep leadership."""
        from ..models import SystemConfig

        try:
            now = datetime.utcnow()
            async with AsyncSessionLocal() as db:
                refreshed = await db.execute(
                    update(SystemConfig)
                    .where(SystemConfig.key == "scheduler_leader")
                    .where(SystemConfig.value == worker_id)
                    .values(
                        expires_at=now + timedelta(minutes=5),
                        updated_at=now,
                    )
                )
                await db.commit()
                if (refreshed.rowcount or 0) == 1:
                    log.debug("Refreshed leader lock")
                else:
                    self.is_leader = False
                    log.warning("Lost scheduler leader lock for %s", worker_id)

        except Exception as e:
            log.error(f"Failed to refresh leader lock: {e}")

    async def _pick_and_enqueue_due_jobs(self):
        """
        Pick due jobs and enqueue them (with DB locking to prevent double-fire).
        Uses UPDATE...WHERE next_run_at<=now to atomically claim jobs.
        """
        from ..models import ScheduledJob

        try:
            now = datetime.utcnow()
            due_job_ids: List[str] = []

            async with AsyncSessionLocal() as db:
                result = await db.execute(
                    select(ScheduledJob)
                    .where(
                        and_(
                            ScheduledJob.is_active == True,
                            ScheduledJob.paused_by_system == False,
                            ScheduledJob.next_run_at <= now,
                        )
                    )
                    .with_for_update(skip_locked=True)
                )

                due_jobs = result.scalars().all()
                if not due_jobs:
                    return

                log.info(f"Found {len(due_jobs)} due jobs to execute")

                for job in due_jobs:
                    try:
                        next_run = self._calculate_next_run(
                            job.schedule, now, job_id=job.id
                        )
                        job.next_run_at = next_run
                        job.last_run_at = now
                        due_job_ids.append(job.id)

                    except Exception as e:
                        log.error(f"Failed to enqueue job {job.id}: {e}", exc_info=True)
                        job.error_count = (job.error_count or 0) + 1

                        error_str = str(e).lower()
                        is_google_auth_error = "google_auth_missing" in error_str or (
                            "google" in error_str
                            and any(
                                x in error_str
                                for x in [
                                    "oauth",
                                    "auth",
                                    "token",
                                    "credential",
                                    "invalid_grant",
                                ]
                            )
                        )
                        is_auth_error = is_google_auth_error or any(
                            x in error_str
                            for x in [
                                "refresh_token",
                                "expired",
                                "unauthorized",
                                "authentication",
                                "invalid_grant",
                                "token",
                                "credentials",
                            ]
                        )

                        if job.error_count >= 5:
                            job.paused_by_system = True

                            if is_auth_error:
                                if is_google_auth_error:
                                    job.pause_reason = (
                                        "Google Sheets authorization expired. Please reconnect."
                                    )
                                    await self._notify_auth_failure(
                                        job, db, provider="Google Sheets"
                                    )
                                else:
                                    job.pause_reason = (
                                        "Salesforce credentials expired. Please reconnect."
                                    )
                                    await self._notify_auth_failure(
                                        job, db, provider="Salesforce"
                                    )
                            else:
                                job.pause_reason = f"Too many errors: {str(e)[:200]}"
                                await self._notify_schedule_paused(job, str(e), db)

                            log.warning(
                                f"Paused job {job.id} after 5 errors (auth_error={is_auth_error})"
                            )

                        elif job.error_count == 3 and is_auth_error:
                            await self._notify_auth_failure(
                                job,
                                db,
                                provider="Google Sheets"
                                if is_google_auth_error
                                else "Salesforce",
                            )

                await db.commit()

            for due_job_id in due_job_ids:
                await self._execute_job_by_id(due_job_id)

        except Exception as e:
            log.error(f"Error in job picker: {e}", exc_info=True)

    async def _execute_job_by_id(self, job_id: str) -> None:
        """Load and execute a job with an isolated DB session."""
        from ..models import ScheduledJob

        async with AsyncSessionLocal() as session:
            result = await session.execute(
                select(ScheduledJob).where(ScheduledJob.id == job_id)
            )
            job = result.scalar_one_or_none()
            if not job:
                log.warning("Scheduled job %s was not found at execution time", job_id)
                return
            await self._execute_job(job, session)

    def _calculate_next_run(
        self, schedule: str, from_time: datetime, job_id: Optional[str] = None
    ) -> datetime:
        """Calculate next run time from schedule (cron or preset)."""
        normalized = str(schedule or "").strip().lower()
        jitter_seconds = _schedule_jitter_seconds(normalized, job_id)

        if normalized == "15min":
            next_run = from_time + timedelta(minutes=15)
            return next_run + timedelta(seconds=jitter_seconds)
        if normalized == "hourly":
            next_run = from_time + timedelta(hours=1)
            return next_run + timedelta(seconds=jitter_seconds)
        if normalized == "daily":
            next_run = from_time + timedelta(days=1)
            return next_run + timedelta(seconds=jitter_seconds)
        if normalized == "weekly":
            next_run = from_time + timedelta(weeks=1)
            return next_run + timedelta(seconds=jitter_seconds)

        # Parse as cron
        try:
            cron = croniter(schedule, from_time)
            return cron.get_next(datetime)
        except Exception as e:
            log.error(f"Invalid cron schedule '{schedule}': {e}")
            # Default to daily
            return from_time + timedelta(days=1)

    async def _execute_job(self, job, db: AsyncSession) -> Dict[str, Any]:
        """
        Execute a scheduled job with idempotency.
        Creates job_run record and delegates to worker.
        """
        from ..models import JobRun
        import json
        from sqlalchemy import func

        run = None
        try:
            config_data = (
                json.loads(job.config)
                if isinstance(job.config, str)
                else (job.config or {})
            )
            config_hash_source = json.dumps(config_data, sort_keys=True)
            window_basis = job.last_run_at or datetime.utcnow()
            window_start = window_basis.replace(second=0, microsecond=0)
            idempotency_key = hashlib.sha256(
                f"{job.account_id}:{job.sheet_id}:{config_hash_source}:{window_start.isoformat()}".encode()
            ).hexdigest()

            # Enforce per-job rate limits if configured
            rate_limit = config_data.get("rate_limit_per_hour")
            if rate_limit:
                window_cutoff = datetime.utcnow() - timedelta(hours=1)
                count_result = await db.execute(
                    select(func.count(JobRun.id)).where(
                        JobRun.job_id == job.id,
                        JobRun.started_at >= window_cutoff,
                        JobRun.status.in_(("running", "success", "failed")),
                    )
                )
                executions_last_hour = count_result.scalar() or 0

                if executions_last_hour >= rate_limit:
                    skip_run = JobRun(
                        id=f"run_{datetime.utcnow().timestamp()}",
                        job_id=job.id,
                        idempotency_key=f"{idempotency_key}_skip",
                        status="skipped",
                        started_at=datetime.utcnow(),
                        completed_at=datetime.utcnow(),
                        duration_ms=0,
                        rows_returned=0,
                        error_message=f"Skipped due to rate limit ({rate_limit} per hour)",
                    )
                    db.add(skip_run)
                    await db.commit()
                    await self._record_job_metric(
                        job, "skipped", 0, 0, "rate_limit_exceeded"
                    )
                    log.info(
                        f"Job {job.id} skipped: rate limit {executions_last_hour}/{rate_limit} reached"
                    )
                    return {
                        "ok": True,
                        "status": "skipped",
                        "run_id": skip_run.id,
                        "error": skip_run.error_message,
                    }

            # Create job run record atomically. Duplicate idempotency keys are no-ops.
            run_id = f"run_{datetime.utcnow().timestamp()}"
            started_at = datetime.utcnow()
            inserted_run_id: Optional[str] = None
            try:
                insert_stmt = (
                    pg_insert(JobRun)
                    .values(
                        id=run_id,
                        job_id=job.id,
                        idempotency_key=idempotency_key,
                        status="pending",
                        started_at=started_at,
                    )
                    .on_conflict_do_nothing(index_elements=[JobRun.idempotency_key])
                    .returning(JobRun.id)
                )
                inserted = await db.execute(insert_stmt)
                inserted_run_id = inserted.scalar_one_or_none()
                await db.commit()
            except Exception:
                await db.rollback()
                run = JobRun(
                    id=run_id,
                    job_id=job.id,
                    idempotency_key=idempotency_key,
                    status="pending",
                    started_at=started_at,
                )
                db.add(run)
                try:
                    await db.commit()
                    inserted_run_id = run.id
                except IntegrityError:
                    await db.rollback()
                    inserted_run_id = None

            if not inserted_run_id:
                log.info(
                    f"Job {job.id} already executed (idempotency key: {idempotency_key[:8]}...)"
                )
                return {
                    "ok": True,
                    "status": "skipped",
                    "run_id": None,
                    "error": "Duplicate execution skipped by idempotency guard",
                }

            run_result = await db.execute(select(JobRun).where(JobRun.id == inserted_run_id))
            run = run_result.scalar_one_or_none()
            if not run:
                raise RuntimeError(f"Failed to load inserted job run {inserted_run_id}")

            # Keep schedule list state accurate for both manual runs and cron runs.
            job.last_run_at = started_at
            job.updated_at = datetime.utcnow()
            await db.commit()

            # Enqueue to worker (RQ or direct execution)
            if job.job_type == "soql_query":
                await self._execute_soql_job(job, run, db, config_override=config_data)
            elif job.job_type == "report_export":
                await self._execute_report_job(
                    job, run, db, config_override=config_data
                )
            else:
                raise ValueError(f"Unknown job type: {job.job_type}")

            final_status = str(getattr(run, "status", "") or "").strip().lower()
            return {
                "ok": final_status == "success",
                "status": final_status or "unknown",
                "run_id": run.id if run is not None else None,
                "error": getattr(run, "error_message", None),
            }

        except Exception as e:
            log.error(f"Failed to execute job {job.id}: {e}", exc_info=True)
            if run is not None and run.status == "pending":
                run.status = "failed"
                run.completed_at = datetime.utcnow()
                run.duration_ms = int(
                    (run.completed_at - run.started_at).total_seconds() * 1000
                )
                run.rows_returned = run.rows_returned or 0
                run.error_message = str(e)[:500]
                job.last_write_status = "failed"
                job.last_write_at = datetime.utcnow()
                job.last_write_error = str(e)[:500]
                job.updated_at = datetime.utcnow()
                await db.commit()
            # Record in DLQ
            await self._record_to_dlq(job.id, str(e), db)
            return {
                "ok": False,
                "status": str(getattr(run, "status", "failed") or "failed").lower(),
                "run_id": run.id if run is not None else None,
                "error": str(e)[:500],
            }

    async def _execute_soql_job(self, job, run, db: AsyncSession, config_override=None):
        """Execute SOQL query job."""
        from .salesforce_gateway import SalesforceGateway
        from .direct_sheets_writer import (
            DirectSheetsWriter,
            DirectSheetsWriterError,
            RateLimitedError,
        )
        from .google_sheets_oauth import ensure_google_access_token
        from ..security.crypto import decrypt_str
        from .. import settings as saas_settings
        import json

        try:
            run.status = "running"
            await db.commit()

            access_token, integration = await ensure_google_access_token(
                job.account_id, db
            )
            if not access_token:
                raise RuntimeError("google_auth_missing")

            refresh_token = None
            if integration and integration.refresh_token:
                refresh_token = (
                    decrypt_str(integration.refresh_token)
                    if saas_settings.ENCRYPTION_KEY
                    else integration.refresh_token
                )

            config = config_override or (
                json.loads(job.config) if isinstance(job.config, str) else job.config
            )
            soql = config.get("soql")

            # Get gateway
            gateway = await SalesforceGateway.for_tenant(job.account_id, db)

            # Execute query
            records = []
            async for batch in gateway.paginate_soql(soql):
                records.extend(batch)

                # Enforce row limit based on tier
                from ..middleware.quota_checker import get_quota

                quota = await get_quota(job.account_id, db)
                row_limit = quota.get_limits()["rows_per_run"]

                if len(records) >= row_limit:
                    records = records[:row_limit]
                    log.warning(
                        f"Truncated query results to {row_limit} rows for job {job.id}"
                    )
                    break

            if not records:
                log.info(f"Job {job.id} returned no records")
                run.status = "success"
                run.rows_returned = 0
                run.completed_at = datetime.utcnow()
                run.duration_ms = int(
                    (run.completed_at - run.started_at).total_seconds() * 1000
                )
                job.last_write_status = "success"
                job.last_write_at = datetime.utcnow()
                job.last_write_error = None
                job.updated_at = datetime.utcnow()
                await db.commit()
                return

            sheet_name = (
                config.get("target_sheet")
                or config.get("sheet_name")
                or "Scheduled Data"
            )

            # Convert raw Salesforce payloads (including AggregateResult) into sheet-safe values.
            headers, rows = _records_to_sheet_table(records, soql=soql)

            sheet_gid = (
                job.sheet_gid
                or config.get("sheet_gid")
                or config.get("sheetGid")
                or config.get("target_sheet_gid")
            )
            cached_title = None
            if hasattr(job, "cached_sheet_title"):
                cached_title = job.cached_sheet_title
            if isinstance(config, dict):
                cached_title = cached_title or config.get("cached_sheet_title")
                cached_title = cached_title or config.get("target_sheet")
                cached_title = cached_title or config.get("sheet_name")
            writer = DirectSheetsWriter(
                access_token=access_token,
                refresh_token=refresh_token,
                expires_at=integration.token_expires_at if integration else None,
                account_id=job.account_id,
            )
            try:
                write_result = await asyncio.to_thread(
                    writer.write_to_sheet,
                    spreadsheet_id=job.sheet_id,
                    sheet_name=sheet_name,
                    headers=headers,
                    rows=rows,
                    sheet_gid=sheet_gid,
                    cached_title=cached_title,
                )
            except RateLimitedError as rate_err:
                raise rate_err
            except DirectSheetsWriterError as write_err:
                raise RuntimeError(f"Google Sheets write failed: {write_err}") from write_err
            except Exception as write_err:
                raise RuntimeError(f"Google Sheets write failed: {write_err}") from write_err

            log.info(
                f"Job {job.id} completed: {len(records)} rows, write_result: {write_result}"
            )

            run.status = "success"
            run.rows_returned = len(records)
            run.completed_at = datetime.utcnow()
            run.duration_ms = int(
                (run.completed_at - run.started_at).total_seconds() * 1000
            )
            run.output_location = None
            job.last_write_status = "success"
            job.last_write_at = datetime.utcnow()
            job.last_write_error = None
            job.updated_at = datetime.utcnow()

            await db.commit()

            await self._record_job_metric(
                job, "success", len(records), run.duration_ms, None
            )

        except Exception as e:
            log.error(f"SOQL job {job.id} failed: {e}", exc_info=True)
            run.status = "failed"
            run.completed_at = datetime.utcnow()
            run.duration_ms = int(
                (run.completed_at - run.started_at).total_seconds() * 1000
            )
            if not run.rows_returned:
                run.rows_returned = 0
            error_str = str(e).lower()
            rate_limited = isinstance(e, RateLimitedError) or "rate limit" in error_str
            auth_missing = not rate_limited and (
                "google_auth_missing" in error_str
                or (
                    "google" in error_str
                    and any(
                        x in error_str
                        for x in ["oauth", "auth", "token", "credential", "invalid_grant"]
                    )
                )
            )
            error_message = str(e)[:500]
            if auth_missing and "google_auth_missing" in error_message.lower():
                error_message = (
                    "Google Sheets authorization required. Reconnect in Schedule Manager."
                )
            if rate_limited:
                jitter_seconds = random.randint(60, 300)
                job.next_run_at = datetime.utcnow() + timedelta(seconds=jitter_seconds)
                error_message = "Google Sheets rate limited. Retrying soon."
            run.error_message = error_message
            if rate_limited:
                job.last_write_status = "rate_limited"
            else:
                job.last_write_status = "auth_missing" if auth_missing else "failed"
            job.last_write_at = datetime.utcnow()
            job.last_write_error = error_message
            job.updated_at = datetime.utcnow()
            await db.commit()
            await self._record_job_metric(
                job, "failed", run.rows_returned or 0, run.duration_ms, str(e)
            )
            raise

    async def _execute_report_job(
        self, job, run, db: AsyncSession, config_override=None
    ):
        """Execute report export job."""
        from .salesforce_gateway import SalesforceGateway
        from .direct_sheets_writer import (
            DirectSheetsWriter,
            DirectSheetsWriterError,
            RateLimitedError,
        )
        from .google_sheets_oauth import ensure_google_access_token
        from ..middleware.quota_checker import get_quota
        from ..security.crypto import decrypt_str
        from .. import settings as saas_settings
        import json
        import csv
        import io

        try:
            run.status = "running"
            await db.commit()

            access_token, integration = await ensure_google_access_token(
                job.account_id, db
            )
            if not access_token:
                raise RuntimeError("google_auth_missing")

            refresh_token = None
            if integration and integration.refresh_token:
                refresh_token = (
                    decrypt_str(integration.refresh_token)
                    if saas_settings.ENCRYPTION_KEY
                    else integration.refresh_token
                )

            config = config_override or (
                json.loads(job.config)
                if isinstance(job.config, str)
                else (job.config or {})
            )
            report_id = config.get("report_id")
            if not report_id:
                raise ValueError("Report ID is required for report export jobs")

            target_sheet = (
                config.get("target_sheet")
                or config.get("sheet_name")
                or config.get("name")
                or "Scheduled Report"
            )

            gateway = await SalesforceGateway.for_tenant(job.account_id, db)
            export_result = await gateway.export_report(report_id)

            if not export_result or not export_result.get("success"):
                error_message = (
                    export_result.get("error") if export_result else "Unknown error"
                )
                raise ValueError(f"Report export failed: {error_message}")

            csv_content = export_result.get("csv") or ""
            headers: List[str] = []
            rows: List[List[str]] = []

            if csv_content:
                reader = csv.reader(io.StringIO(csv_content))
                for index, row_values in enumerate(reader):
                    if index == 0:
                        headers = row_values
                    else:
                        rows.append(row_values)

            quota = await get_quota(job.account_id, db)
            limits = quota.get_limits() if quota else {}
            row_limit = limits.get("rows_per_run")
            if row_limit and len(rows) > row_limit:
                rows = rows[:row_limit]
                log.warning(
                    f"Truncated report results to {row_limit} rows for job {job.id}"
                )

            if headers:
                normalized_rows = []
                header_count = len(headers)
                for row_values in rows:
                    if len(row_values) > header_count:
                        normalized_rows.append(row_values[:header_count])
                    elif len(row_values) < header_count:
                        normalized_rows.append(
                            row_values + [""] * (header_count - len(row_values))
                        )
                    else:
                        normalized_rows.append(row_values)
                rows = normalized_rows

            sheet_gid = (
                job.sheet_gid
                or config.get("sheet_gid")
                or config.get("sheetGid")
                or config.get("target_sheet_gid")
            )
            cached_title = None
            if hasattr(job, "cached_sheet_title"):
                cached_title = job.cached_sheet_title
            if isinstance(config, dict):
                cached_title = cached_title or config.get("cached_sheet_title")
                cached_title = cached_title or config.get("target_sheet")
                cached_title = cached_title or config.get("sheet_name")
            if headers and rows:
                writer = DirectSheetsWriter(
                    access_token=access_token,
                    refresh_token=refresh_token,
                    expires_at=integration.token_expires_at if integration else None,
                    account_id=job.account_id,
                )
                try:
                    write_result = await asyncio.to_thread(
                        writer.write_to_sheet,
                        spreadsheet_id=job.sheet_id,
                        sheet_name=target_sheet,
                        headers=headers,
                        rows=rows,
                        sheet_gid=sheet_gid,
                        cached_title=cached_title,
                    )
                except RateLimitedError as rate_err:
                    raise rate_err
                except DirectSheetsWriterError as write_err:
                    raise RuntimeError(
                        f"Google Sheets write failed: {write_err}"
                    ) from write_err
                except Exception as write_err:
                    raise RuntimeError(
                        f"Google Sheets write failed: {write_err}"
                    ) from write_err
            else:
                write_result = {"ok": True, "message": "No rows returned"}

            run.status = "success"
            run.rows_returned = len(rows)
            run.completed_at = datetime.utcnow()
            run.duration_ms = int(
                (run.completed_at - run.started_at).total_seconds() * 1000
            )
            run.output_location = (
                write_result.get("storage_key")
                if isinstance(write_result, dict)
                else None
            )
            job.last_write_status = "success"
            job.last_write_at = datetime.utcnow()
            job.last_write_error = None
            job.updated_at = datetime.utcnow()

            await db.commit()

            await self._record_job_metric(
                job, "success", len(rows), run.duration_ms, None
            )

        except Exception as e:
            log.error(f"Report job {job.id} failed: {e}", exc_info=True)
            run.status = "failed"
            run.completed_at = datetime.utcnow()
            run.duration_ms = int(
                (run.completed_at - run.started_at).total_seconds() * 1000
            )
            if not run.rows_returned:
                run.rows_returned = len(rows) if "rows" in locals() else 0
            error_str = str(e).lower()
            rate_limited = isinstance(e, RateLimitedError) or "rate limit" in error_str
            auth_missing = not rate_limited and (
                "google_auth_missing" in error_str
                or (
                    "google" in error_str
                    and any(
                        x in error_str
                        for x in ["oauth", "auth", "token", "credential", "invalid_grant"]
                    )
                )
            )
            error_message = str(e)[:500]
            if auth_missing and "google_auth_missing" in error_message.lower():
                error_message = (
                    "Google Sheets authorization required. Reconnect in Schedule Manager."
                )
            if rate_limited:
                jitter_seconds = random.randint(60, 300)
                job.next_run_at = datetime.utcnow() + timedelta(seconds=jitter_seconds)
                error_message = "Google Sheets rate limited. Retrying soon."
            run.error_message = error_message
            if rate_limited:
                job.last_write_status = "rate_limited"
            else:
                job.last_write_status = "auth_missing" if auth_missing else "failed"
            job.last_write_at = datetime.utcnow()
            job.last_write_error = error_message
            job.updated_at = datetime.utcnow()
            await db.commit()
            await self._record_job_metric(
                job, "failed", run.rows_returned or 0, run.duration_ms, str(e)
            )
            raise

    async def _record_job_metric(
        self, job, status: str, rows: int, duration_ms: int, error: str = None
    ):
        """Record job execution metrics for monitoring dashboards."""
        from ..monitoring.scheduler_metrics import record_job_run

        try:
            await record_job_run(
                account_id=job.account_id,
                job_id=job.id,
                status=status,
                rows=rows,
                duration_ms=duration_ms or 0,
                error=error[:200] if error else None,
            )
        except Exception as exc:
            log.error(f"Failed to record scheduler metric for {job.id}: {exc}")

    async def _notify_auth_failure(
        self, job, db: AsyncSession, provider: str = "Salesforce"
    ):
        """Notify user that auth credentials have expired."""
        from ..services.email_notifications import send_auth_failure_notification
        from ..models import User
        from sqlalchemy import select as sa_select

        try:
            # Get user email
            result = await db.execute(
                sa_select(User).where(User.account_id == job.account_id).limit(1)
            )
            user = result.scalar_one_or_none()

            if user and user.email:
                await send_auth_failure_notification(
                    user_email=user.email,
                    integration_provider=provider,
                    account_id=job.account_id,
                    failure_reason=job.pause_reason,
                )
        except Exception as e:
            log.error(f"Failed to send auth failure notification: {e}")

    async def _notify_schedule_paused(self, job, error_message: str, db: AsyncSession):
        """Notify user that a schedule has been paused due to errors."""
        from ..services.email_notifications import send_schedule_paused_notification
        from ..models import User
        from sqlalchemy import select as sa_select

        try:
            # Get user email
            result = await db.execute(
                sa_select(User).where(User.account_id == job.account_id).limit(1)
            )
            user = result.scalar_one_or_none()

            if user and user.email:
                schedule_name = (
                    job.config.get("name", job.job_type)
                    if isinstance(job.config, dict)
                    else job.job_type
                )

                await send_schedule_paused_notification(
                    user_email=user.email,
                    schedule_name=schedule_name,
                    job_id=job.id,
                    error_message=error_message,
                    account_id=job.account_id,
                )
        except Exception as e:
            log.error(f"Failed to send schedule paused notification: {e}")

    async def _catchup_missed_runs(self):
        """Catch up on jobs that were missed during downtime."""
        from ..models import ScheduledJob

        try:
            now = datetime.utcnow()
            tolerance = timedelta(hours=1)  # Only catch up runs missed within 1 hour

            async with AsyncSessionLocal() as db:
                result = await db.execute(
                    select(ScheduledJob).where(
                        and_(
                            ScheduledJob.is_active == True,
                            ScheduledJob.paused_by_system == False,
                            ScheduledJob.last_run_at < now - tolerance,
                            ScheduledJob.next_run_at < now,
                        )
                    )
                )
                missed_jobs = result.scalars().all()
                missed_job_ids = [job.id for job in missed_jobs]

            if missed_job_ids:
                log.info(f"Catching up on {len(missed_job_ids)} missed jobs")
                for job_id in missed_job_ids:
                    await self._execute_job_by_id(job_id)

        except Exception as e:
            log.error(f"Failed to catch up on missed runs: {e}", exc_info=True)

    async def _record_to_dlq(self, job_id: str, error: str, db: AsyncSession):
        """Record failed job to Dead Letter Queue."""
        from ..models import JobRunDLQ

        try:
            dlq_entry = JobRunDLQ(
                id=f"dlq_{datetime.utcnow().timestamp()}",
                job_id=job_id,
                run_id="",  # May not have run_id if job creation failed
                error_count=1,
                last_error=error[:500],
                last_error_at=datetime.utcnow(),
                created_at=datetime.utcnow(),
            )
            db.add(dlq_entry)
            await db.commit()
            log.info(f"Recorded job {job_id} to DLQ")

        except Exception as e:
            log.error(f"Failed to record to DLQ: {e}")

    async def stop(self):
        """Stop the scheduler gracefully."""
        if self.scheduler.running:
            self.scheduler.shutdown(wait=True)
            log.info("Scheduler stopped")


# Singleton instance
_scheduler_instance: Optional[SchedulerService] = None


async def get_scheduler() -> SchedulerService:
    """Get or create scheduler singleton."""
    global _scheduler_instance
    if _scheduler_instance is None:
        _scheduler_instance = SchedulerService()
    return _scheduler_instance
