"""Rule-based matching engine: checks if a task is already satisfied by existing code.

This module is used during codebase-aware decomposition to identify tasks
that are redundant (because the code already exists) or trivially small
(and should be merged into a parent task).  It works in concert with
``loom.scanner.CodebaseSummary`` to compare a proposed task graph against
the actual project source.

Three matching rules are applied against a ``CodebaseSummary``:

  1. **FileExistenceRule** — exact or stem-based path matching in the task
     title or context fields.  Confidence: 1.0 for exact path, 0.75 for
     stem match.
  2. **SymbolExistenceRule** — function/class name matching with a fuzzy
     ``difflib.SequenceMatcher`` fallback.  Confidence: 0.90 for exact
     token match, scaled ratio for fuzzy matches.
  3. **KeywordPatternRule** — maps creation verbs (create, add, implement,
     etc.) to artifact types and checks token overlap with existing
     symbols.

The aggregator ``match_task_to_codebase`` runs all rules and returns the
highest-confidence ``MatchResult``.  ``classify_match`` converts a
``MatchResult`` into a ``ValidationStatus`` (pending / uncertain /
pre_completed) using configurable confidence thresholds.

The ``validate_task_graph`` function applies these rules across an entire
decomposed task graph, pruning pre-completed tasks, folding trivial tasks
into their parents, and repairing dependency edges.

The ``merge_trivial_tasks`` function folds tasks whose complexity score
falls below a configurable threshold into their parent or dependent tasks,
using reverse-topological ordering to handle cascading chains.
"""

from __future__ import annotations

import copy
import difflib
import logging
import re
from dataclasses import dataclass, field
from enum import StrEnum
from typing import Protocol

from pydantic import BaseModel, Field

from loom.complexity import COMPLEXITY_THRESHOLD, compute_complexity_score
from loom.scanner import CodebaseSummary

log = logging.getLogger(__name__)


# ── String normalisation ─────────────────────────────────────────────

_CAMEL_SPLIT = re.compile(r"(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")

STOPWORDS: frozenset[str] = frozenset(
    {
        "a", "an", "the", "to", "for", "of", "in", "on", "and", "or",
        "with", "from", "by", "is", "are", "was", "were", "be", "been",
        "it", "its", "this", "that", "as", "at", "do", "does", "did",
        "has", "have", "had", "not", "no", "but", "if", "so", "up",
    }
)


def normalise(text: str) -> list[str]:
    """Lower-case, split camelCase, remove stopwords, return tokens.

    >>> normalise("createUserProfile for the API")
    ['create', 'user', 'profile', 'api']
    """
    # Split camelCase / PascalCase first, then lower
    expanded = _CAMEL_SPLIT.sub(" ", text).lower()
    # Split on non-alphanumeric characters
    tokens = re.split(r"[^a-z0-9]+", expanded)
    return [t for t in tokens if t and t not in STOPWORDS]


# ── Data models ──────────────────────────────────────────────────────


class ValidationStatus(StrEnum):
    """Overall validation verdict for a task."""

    PENDING = "pending"
    PRE_COMPLETED = "pre_completed"
    UNCERTAIN = "uncertain"


@dataclass(frozen=True)
class MatchResult:
    """Result of a single matching rule evaluation."""

    rule_name: str
    confidence: float
    matched_artifact: str
    rationale: str


@dataclass
class ValidatorConfig:
    """Configurable confidence thresholds."""

    high_threshold: float = 0.85
    uncertain_threshold: float = 0.65


# ── Rule protocol ────────────────────────────────────────────────────


class MatchingRule(Protocol):
    """Interface for matching rules."""

    name: str

    def evaluate(
        self,
        task_tokens: list[str],
        task_dict: dict,
        summary: CodebaseSummary,
    ) -> MatchResult | None:
        """Return a MatchResult if the rule fires, else None."""
        ...


# ── Rule implementations ─────────────────────────────────────────────


class FileExistenceRule:
    """Match when a file path (or its stem) mentioned in the task already exists."""

    name: str = "file_existence"

    def evaluate(
        self,
        task_tokens: list[str],
        task_dict: dict,
        summary: CodebaseSummary,
    ) -> MatchResult | None:
        if not summary.files:
            return None

        # Collect searchable text from the task
        search_text = (task_dict.get("title", "") + " " + _context_text(task_dict)).lower()

        best: MatchResult | None = None
        for fs in summary.files:
            path_lower = fs.path.lower()
            stem = path_lower.rsplit("/", 1)[-1].rsplit(".", 1)[0]  # e.g. "store"

            # Skip very short stems that would false-positive on common words
            if len(stem) < 3:
                continue

            # Exact path match (highest confidence)
            if path_lower in search_text:
                return MatchResult(
                    rule_name=self.name,
                    confidence=1.0,
                    matched_artifact=fs.path,
                    rationale=f"Exact file path '{fs.path}' found in task text",
                )

            # Stem match (lower confidence)
            if stem in search_text:
                candidate = MatchResult(
                    rule_name=self.name,
                    confidence=0.75,
                    matched_artifact=fs.path,
                    rationale=f"File stem '{stem}' found in task text",
                )
                if best is None or candidate.confidence > best.confidence:
                    best = candidate

        return best


class SymbolExistenceRule:
    """Match when a function/class name from the codebase appears in the task."""

    name: str = "symbol_existence"

    # Minimum symbol length to avoid matching on tiny common names like 'f' or 'id'
    MIN_SYMBOL_LEN = 4

    def evaluate(
        self,
        task_tokens: list[str],
        task_dict: dict,
        summary: CodebaseSummary,
    ) -> MatchResult | None:
        if not summary.files:
            return None

        # Collect all symbols from the codebase (normalised)
        symbol_map: dict[str, tuple[str, str]] = {}  # normalised -> (original, file_path)
        for fs in summary.files:
            for sym in fs.functions + fs.classes:
                if len(sym) < self.MIN_SYMBOL_LEN:
                    continue
                norm_tokens = normalise(sym)
                norm_key = "_".join(norm_tokens) if norm_tokens else sym.lower()
                symbol_map[norm_key] = (sym, fs.path)

        if not symbol_map:
            return None

        # Build normalised task text for matching
        task_text_norm = "_".join(task_tokens)

        best: MatchResult | None = None

        for norm_key, (original, file_path) in symbol_map.items():
            # Exact normalised match
            if norm_key in task_text_norm or all(
                tok in task_tokens for tok in norm_key.split("_")
            ):
                candidate = MatchResult(
                    rule_name=self.name,
                    confidence=0.90,
                    matched_artifact=f"{file_path}::{original}",
                    rationale=f"Symbol '{original}' matches task tokens",
                )
                if best is None or candidate.confidence > best.confidence:
                    best = candidate
                continue

            # Fuzzy fallback using difflib
            ratio = difflib.SequenceMatcher(None, norm_key, task_text_norm).ratio()
            if ratio >= 0.70:
                candidate = MatchResult(
                    rule_name=self.name,
                    confidence=round(ratio * 0.85, 2),  # scale down fuzzy matches
                    matched_artifact=f"{file_path}::{original}",
                    rationale=f"Fuzzy match '{original}' (ratio={ratio:.2f})",
                )
                if best is None or candidate.confidence > best.confidence:
                    best = candidate

        return best


# ── Keyword → artifact mapping ────────────────────────────────────────

# Action verbs that indicate a task is about *creating* something.
# Mapped to the artifact type that should already exist.
_VERB_ARTIFACT_MAP: dict[str, str] = {
    "create": "file",
    "add": "function",
    "implement": "function",
    "write": "function",
    "build": "class",
    "define": "class",
    "setup": "file",
    "configure": "file",
    "scaffold": "file",
}


class KeywordPatternRule:
    """Match when a creation-verb + artifact name suggests existing code covers the task."""

    name: str = "keyword_pattern"

    def evaluate(
        self,
        task_tokens: list[str],
        task_dict: dict,
        summary: CodebaseSummary,
    ) -> MatchResult | None:
        if not summary.files or not task_tokens:
            return None

        # Check if a creation verb leads the title tokens
        title_tokens = normalise(task_dict.get("title", ""))
        if not title_tokens:
            return None

        verb = title_tokens[0]
        if verb not in _VERB_ARTIFACT_MAP:
            return None

        artifact_type = _VERB_ARTIFACT_MAP[verb]
        # The remaining tokens are the "what" (object being created)
        object_tokens = [t for t in title_tokens[1:] if t not in STOPWORDS]
        if not object_tokens:
            return None

        # Search for the object in the codebase
        best: MatchResult | None = None

        for fs in summary.files:
            candidates: list[str] = []
            if artifact_type in ("file", "function", "class"):
                candidates.extend(fs.functions)
                candidates.extend(fs.classes)
                # Also check the file stem
                stem = fs.path.rsplit("/", 1)[-1].rsplit(".", 1)[0]
                candidates.append(stem)

            for cand in candidates:
                cand_tokens = normalise(cand)
                if not cand_tokens:
                    continue

                # Check token overlap
                overlap = set(object_tokens) & set(cand_tokens)
                if not overlap:
                    continue

                score = len(overlap) / max(len(object_tokens), 1)
                if score >= 0.5:
                    candidate = MatchResult(
                        rule_name=self.name,
                        confidence=round(min(score, 0.95), 2),
                        matched_artifact=f"{fs.path}::{cand}",
                        rationale=(
                            f"Verb '{verb}' + object tokens {object_tokens} "
                            f"overlap with '{cand}' (score={score:.2f})"
                        ),
                    )
                    if best is None or candidate.confidence > best.confidence:
                        best = candidate

        return best


# ── Aggregator ────────────────────────────────────────────────────────

# Default rule instances
_DEFAULT_RULES: list[MatchingRule] = [
    FileExistenceRule(),
    SymbolExistenceRule(),
    KeywordPatternRule(),
]


def match_task_to_codebase(
    task_dict: dict,
    summary: CodebaseSummary,
    rules: list[MatchingRule] | None = None,
    config: ValidatorConfig | None = None,
) -> MatchResult:
    """Run all rules against a task and return the highest-confidence result.

    Returns a MatchResult with confidence=0.0 if no rule fires.
    """
    if rules is None:
        rules = _DEFAULT_RULES
    if config is None:
        config = ValidatorConfig()

    task_tokens = normalise(task_dict.get("title", ""))

    results: list[MatchResult] = []
    for rule in rules:
        result = rule.evaluate(task_tokens, task_dict, summary)
        if result is not None:
            results.append(result)

    if not results:
        return MatchResult(
            rule_name="none",
            confidence=0.0,
            matched_artifact="",
            rationale="No matching rule fired",
        )

    # Return the highest-confidence match
    return max(results, key=lambda r: r.confidence)


def classify_match(result: MatchResult, config: ValidatorConfig | None = None) -> ValidationStatus:
    """Classify a MatchResult into a ValidationStatus."""
    if config is None:
        config = ValidatorConfig()

    if result.confidence >= config.high_threshold:
        return ValidationStatus.PRE_COMPLETED
    elif result.confidence >= config.uncertain_threshold:
        return ValidationStatus.UNCERTAIN
    else:
        return ValidationStatus.PENDING


# ── Helpers ──────────────────────────────────────────────────────────


def _context_text(task_dict: dict) -> str:
    """Extract searchable text from a task's context field."""
    ctx = task_dict.get("context", {})
    if isinstance(ctx, str):
        return ctx
    parts: list[str] = []
    if isinstance(ctx, dict):
        for key in ("description", "notes", "files"):
            val = ctx.get(key)
            if isinstance(val, str):
                parts.append(val)
            elif isinstance(val, list):
                parts.extend(str(v) for v in val)
    return " ".join(parts)


# ── Graph pruning models ─────────────────────────────────────────────


class PruningDecision(BaseModel):
    """Record of a single pruning decision for a task."""

    task_id: str
    action: str  # "pre_completed" | "folded" | "kept"
    reason: str
    confidence: float


class ValidationResult(BaseModel):
    """Result of validating and pruning a decomposed task graph."""

    tasks: list[dict] = Field(default_factory=list)
    pruned: list[PruningDecision] = Field(default_factory=list)
    folded: list[PruningDecision] = Field(default_factory=list)
    stats: dict = Field(default_factory=dict)


# ── Complexity scoring ────────────────────────────────────────────────


def _task_complexity(task: dict) -> float:
    """Estimate task complexity using ``loom.complexity.compute_complexity_score``.

    Returns a normalised value in [0.0, 1.0] for backward compatibility
    with ``validate_task_graph`` which uses a 0.2 threshold.  The new
    module scores on [1.0, 10.0], so we map: ``(score - 1) / 9``.

    The original implementation counted tokens from ``title`` + context text,
    so we build a synthetic description from those fields before delegating.
    """
    # Build a description string from the fields the old function used.
    title = task.get("title", "")
    ctx_text = _context_text(task)
    synth_desc = f"{title} {ctx_text}".strip() if (title or ctx_text) else ""

    # Build a task dict that compute_complexity_score understands.
    enriched = {**task, "description": synth_desc}
    raw = compute_complexity_score(enriched)
    return (raw - 1.0) / 9.0


# ── Cycle detection ──────────────────────────────────────────────────


class _CycleError(Exception):
    """Raised when a dependency cycle is detected in the task graph."""


def _detect_cycles(tasks: list[dict]) -> None:
    """DFS-based cycle detection on a list of task dicts.

    Raises ``_CycleError`` if a cycle is found.
    """
    adj: dict[str, list[str]] = {}
    all_ids: set[str] = set()
    for t in tasks:
        tid = t.get("id", "")
        all_ids.add(tid)
        adj[tid] = list(t.get("depends_on", []))

    WHITE, GRAY, BLACK = 0, 1, 2
    color: dict[str, int] = {tid: WHITE for tid in all_ids}

    def dfs(node: str, path: list[str]) -> None:
        color[node] = GRAY
        path.append(node)
        for neighbor in adj.get(node, []):
            if neighbor not in all_ids:
                continue
            if color[neighbor] == GRAY:
                cycle_start = path.index(neighbor)
                raise _CycleError(
                    f"Dependency cycle: {' -> '.join(path[cycle_start:] + [neighbor])}"
                )
            if color[neighbor] == WHITE:
                dfs(neighbor, path)
        path.pop()
        color[node] = BLACK

    for tid in all_ids:
        if color[tid] == WHITE:
            dfs(tid, [])


# ── Graph pruning ────────────────────────────────────────────────────


def validate_task_graph(
    tasks: list[dict],
    summary: CodebaseSummary,
    *,
    complexity_threshold: float = 0.2,
    mode: str = "mark",
) -> ValidationResult:
    """Prune a decomposed task graph by removing tasks already satisfied by code.

    Parameters
    ----------
    tasks:
        List of task dicts (keys: id, title, context, depends_on, parent_id).
    summary:
        Codebase summary from ``scan_codebase()``.
    complexity_threshold:
        Tasks with complexity below this threshold **and** a parent_id are
        candidates for folding into their parent.
    mode:
        ``"mark"`` — keep pruned tasks in the output but annotate them with
        ``context.pruned = True``; ``"delete"`` — remove them entirely.

    Returns
    -------
    ValidationResult
        Contains the (possibly reduced) task list, lists of pruning and
        folding decisions, and aggregate stats.
    """
    # Detect cycles before processing
    _detect_cycles(tasks)

    # Deep copy so we don't mutate the caller's data
    tasks = copy.deepcopy(tasks)

    # Index tasks by id
    task_map: dict[str, dict] = {t["id"]: t for t in tasks}

    pruned: list[PruningDecision] = []
    folded: list[PruningDecision] = []
    kept: list[PruningDecision] = []

    pruned_ids: set[str] = set()
    folded_ids: set[str] = set()

    # ── Phase 1: identify pre-completed tasks ────────────────────────
    for task in tasks:
        match = match_task_to_codebase(task, summary)
        status = classify_match(match)

        if status == ValidationStatus.PRE_COMPLETED:
            decision = PruningDecision(
                task_id=task["id"],
                action="pre_completed",
                reason=match.rationale,
                confidence=match.confidence,
            )
            pruned.append(decision)
            pruned_ids.add(task["id"])
            log.info(
                "PRUNE pre_completed: %s (%.2f) — %s",
                task["id"],
                match.confidence,
                match.rationale,
            )

    # ── Phase 2: fold trivially small tasks into parent ──────────────
    for task in tasks:
        tid = task["id"]
        if tid in pruned_ids:
            continue
        parent_id = task.get("parent_id")
        if not parent_id or parent_id not in task_map:
            continue
        if parent_id in pruned_ids or parent_id in folded_ids:
            continue

        complexity = _task_complexity(task)
        if complexity < complexity_threshold:
            decision = PruningDecision(
                task_id=tid,
                action="folded",
                reason=f"Complexity {complexity:.2f} below threshold {complexity_threshold}; folded into parent {parent_id}",
                confidence=complexity,
            )
            folded.append(decision)
            folded_ids.add(tid)
            log.info(
                "FOLD %s into parent %s (complexity=%.2f)",
                tid,
                parent_id,
                complexity,
            )

    # ── Phase 3: dependency repair ───────────────────────────────────
    removed_ids = pruned_ids | folded_ids

    # Build a map: folded task id -> that task's depends_on (for hoisting)
    folded_deps: dict[str, list[str]] = {}
    for tid in folded_ids:
        folded_deps[tid] = list(task_map[tid].get("depends_on", []))

    for task in tasks:
        tid = task["id"]
        if tid in removed_ids:
            continue

        old_deps: list[str] = list(task.get("depends_on", []))
        new_deps: list[str] = []

        for dep_id in old_deps:
            if dep_id in pruned_ids:
                # Pre-completed: just remove the dependency (it's done)
                log.info(
                    "DEP REPAIR: removed pruned dep %s from %s",
                    dep_id,
                    tid,
                )
                continue
            elif dep_id in folded_ids:
                # Folded: hoist the folded task's deps into this task
                hoisted = [
                    d for d in folded_deps.get(dep_id, [])
                    if d not in removed_ids and d != tid
                ]
                new_deps.extend(hoisted)
                log.info(
                    "DEP REPAIR: replaced folded dep %s with hoisted deps %s for %s",
                    dep_id,
                    hoisted,
                    tid,
                )
            else:
                new_deps.append(dep_id)

        # Deduplicate while preserving order
        seen: set[str] = set()
        deduped: list[str] = []
        for d in new_deps:
            if d not in seen:
                seen.add(d)
                deduped.append(d)
        task["depends_on"] = deduped

    # ── Phase 4: apply mode (mark or delete) ─────────────────────────
    if mode == "delete":
        remaining_tasks = [t for t in tasks if t["id"] not in removed_ids]
    else:
        remaining_tasks = []
        for t in tasks:
            if t["id"] in pruned_ids:
                ctx = t.get("context", {})
                if not isinstance(ctx, dict):
                    ctx = {}
                ctx["pruned"] = True
                ctx["pruned_reason"] = "pre_completed"
                t["context"] = ctx
            elif t["id"] in folded_ids:
                ctx = t.get("context", {})
                if not isinstance(ctx, dict):
                    ctx = {}
                ctx["pruned"] = True
                ctx["pruned_reason"] = "folded"
                t["context"] = ctx
            remaining_tasks.append(t)

    # Record kept decisions
    for t in remaining_tasks:
        tid = t["id"]
        if tid not in removed_ids:
            kept.append(PruningDecision(
                task_id=tid,
                action="kept",
                reason="No matching rule fired or below threshold",
                confidence=0.0,
            ))

    stats = {
        "total_input": len(tasks),
        "pre_completed_count": len(pruned),
        "folded_count": len(folded),
        "remaining_count": len([t for t in remaining_tasks if t["id"] not in removed_ids]),
    }

    return ValidationResult(
        tasks=remaining_tasks,
        pruned=pruned,
        folded=folded,
        stats=stats,
    )


# ── Trivial task merger ──────────────────────────────────────────────


def _find_cycle_participants(tasks: list[dict], dep_key: str = "dependencies") -> set[str]:
    """Return the set of task IDs that participate in any dependency cycle.

    Uses Tarjan-style DFS colouring.  Any node still GRAY when we revisit
    it is part of a cycle — we collect the entire cycle path.
    """
    adj: dict[str, list[str]] = {}
    all_ids: set[str] = set()
    for t in tasks:
        tid = t.get("id", "")
        all_ids.add(tid)
        adj[tid] = list(t.get(dep_key, []))

    WHITE, GRAY, BLACK = 0, 1, 2
    color: dict[str, int] = {tid: WHITE for tid in all_ids}
    cyclic: set[str] = set()

    def dfs(node: str, path: list[str]) -> None:
        color[node] = GRAY
        path.append(node)
        for neighbor in adj.get(node, []):
            if neighbor not in all_ids:
                continue
            if color[neighbor] == GRAY:
                # Found a cycle — collect all nodes in the cycle path
                cycle_start = path.index(neighbor)
                cyclic.update(path[cycle_start:])
            elif color[neighbor] == WHITE:
                dfs(neighbor, path)
        path.pop()
        color[node] = BLACK

    for tid in all_ids:
        if color[tid] == WHITE:
            dfs(tid, [])

    return cyclic


def _reverse_topo_order(tasks: list[dict], skip_ids: set[str], dep_key: str = "dependencies") -> list[str]:
    """Return task IDs in reverse topological order (leaves first).

    Tasks in *skip_ids* are excluded.  If cycles exist among the
    remaining tasks this still returns a best-effort ordering (the
    cycle participants should already be filtered out via *skip_ids*).
    """
    all_ids: set[str] = set()
    adj: dict[str, list[str]] = {}
    in_degree: dict[str, int] = {}

    for t in tasks:
        tid = t.get("id", "")
        if tid in skip_ids:
            continue
        all_ids.add(tid)
        adj[tid] = []
        in_degree[tid] = 0

    for t in tasks:
        tid = t.get("id", "")
        if tid in skip_ids:
            continue
        for dep in t.get(dep_key, []):
            if dep in all_ids:
                adj[dep].append(tid)
                in_degree[tid] = in_degree.get(tid, 0) + 1

    # Kahn's algorithm
    queue: list[str] = [tid for tid in all_ids if in_degree.get(tid, 0) == 0]
    order: list[str] = []
    while queue:
        node = queue.pop(0)
        order.append(node)
        for neighbor in adj.get(node, []):
            in_degree[neighbor] -= 1
            if in_degree[neighbor] == 0:
                queue.append(neighbor)

    # Reverse so leaves come first
    order.reverse()
    return order


def merge_trivial_tasks(
    tasks: list[dict],
    threshold: float | None = None,
    dep_key: str = "dependencies",
) -> tuple[list[dict], list[dict]]:
    """Merge trivial tasks into their parent or dependent tasks.

    A trivial task is one whose ``compute_complexity_score`` falls below
    *threshold* (defaults to ``COMPLEXITY_THRESHOLD``).  Epics are never
    merged.

    The merge runs in a loop (max 10 iterations) until no further merges
    occur, handling cascading chains of trivial tasks.

    Parameters
    ----------
    tasks:
        List of task dicts.  Expected keys: ``id``, ``title``,
        ``description``, ``files``, ``subtasks``, ``dependencies``
        (list of task IDs), ``type``, ``parent_id``, ``notes``.
    threshold:
        Complexity score below which a task is considered trivial.
        Defaults to ``COMPLEXITY_THRESHOLD`` from ``loom.complexity``.

    Returns
    -------
    tuple[list[dict], list[dict]]
        ``(remaining_tasks, merge_report)`` where *merge_report* is a
        list of dicts recording each fold operation.
    """
    if not tasks:
        return ([], [])

    if threshold is None:
        threshold = COMPLEXITY_THRESHOLD

    # Deep-copy to avoid mutating caller data
    tasks = copy.deepcopy(tasks)

    merge_report: list[dict] = []

    # Identify cycle participants once (they are never merged)
    cyclic_ids = _find_cycle_participants(tasks, dep_key=dep_key)
    if cyclic_ids:
        log.warning(
            "Skipping merge for tasks in dependency cycles: %s",
            cyclic_ids,
        )

    MAX_PASSES = 10

    for pass_num in range(MAX_PASSES):
        merged_any = False

        # Build lookup maps for this pass
        task_map: dict[str, dict] = {t["id"]: t for t in tasks}

        # Build reverse-dependency map: task_id -> list of tasks that depend on it
        dependents_map: dict[str, list[str]] = {t["id"]: [] for t in tasks}
        for t in tasks:
            for dep_id in t.get(dep_key, []):
                if dep_id in dependents_map:
                    dependents_map[dep_id].append(t["id"])

        # Identify trivial tasks for this pass
        trivial_ids: set[str] = set()
        for t in tasks:
            tid = t["id"]
            # Never merge epics
            if t.get("status") == "epic" or t.get("type") == "epic":
                continue
            # Never merge tasks in cycles
            if tid in cyclic_ids:
                continue
            score = compute_complexity_score(t)
            if score < threshold:
                trivial_ids.add(tid)

        if not trivial_ids:
            break

        # Process in reverse topological order (leaves first)
        order = _reverse_topo_order(tasks, cyclic_ids, dep_key=dep_key)
        # Filter to only trivial tasks, preserving the reverse-topo order
        ordered_trivial = [tid for tid in order if tid in trivial_ids]

        removed_in_pass: set[str] = set()

        for tid in ordered_trivial:
            if tid in removed_in_pass:
                continue
            task = task_map.get(tid)
            if task is None:
                continue

            destination_id: str | None = None
            reason: str = ""

            parent_id = task.get("parent_id")

            # Strategy 1: fold into parent if parent exists and is still in the graph
            if parent_id and parent_id in task_map and parent_id not in removed_in_pass:
                destination_id = parent_id
                reason = "trivial task folded into parent"
            else:
                # Strategy 2: fold into first dependent (a task that depends on this one)
                deps_of_this = dependents_map.get(tid, [])
                for dep_tid in deps_of_this:
                    if dep_tid not in removed_in_pass and dep_tid in task_map:
                        destination_id = dep_tid
                        reason = "trivial task folded into dependent"
                        break

            if destination_id is None:
                # No parent, no dependents — keep the task
                continue

            # Fold: append to destination's notes
            dest_task = task_map[destination_id]
            existing_notes = dest_task.get("notes") or ""
            fold_note = f"[Folded from: {task.get('title', tid)}] {task.get('description', '') or ''}"
            if existing_notes:
                dest_task["notes"] = existing_notes + "\n" + fold_note
            else:
                dest_task["notes"] = fold_note

            # Rewire dependencies: replace this task's ID in other tasks'
            # dependency lists with this task's own dependencies (transitive)
            task_deps = list(task.get(dep_key, []))
            for other_task in tasks:
                other_id = other_task["id"]
                if other_id in removed_in_pass:
                    continue
                other_deps = other_task.get(dep_key, [])
                if tid in other_deps:
                    # Replace tid with task's own dependencies
                    new_deps: list[str] = []
                    for d in other_deps:
                        if d == tid:
                            # Substitute with the removed task's deps,
                            # excluding self-references and already-removed tasks
                            for sub_dep in task_deps:
                                if sub_dep != other_id and sub_dep not in removed_in_pass:
                                    new_deps.append(sub_dep)
                        else:
                            new_deps.append(d)
                    # Deduplicate preserving order
                    seen: set[str] = set()
                    deduped: list[str] = []
                    for d in new_deps:
                        if d not in seen:
                            seen.add(d)
                            deduped.append(d)
                    other_task[dep_key] = deduped

            removed_in_pass.add(tid)
            merged_any = True

            merge_report.append({
                "task_id": tid,
                "title": task.get("title", ""),
                "destination_id": destination_id,
                "reason": reason,
            })

            log.info(
                "MERGE pass=%d: folded %s (%s) into %s — %s",
                pass_num,
                tid,
                task.get("title", ""),
                destination_id,
                reason,
            )

        # Remove merged tasks from the list
        if removed_in_pass:
            tasks = [t for t in tasks if t["id"] not in removed_in_pass]

        if not merged_any:
            break

    return (tasks, merge_report)


# ── File-affinity merging ─────────────────────────────────────────────


def compute_file_overlap(files_a: list[str], files_b: list[str]) -> float:
    """Jaccard similarity of two file lists.

    Returns 0.0 when either list is empty, otherwise
    ``|intersection| / |union|`` in the range [0.0, 1.0].
    """
    if not files_a or not files_b:
        return 0.0
    set_a = set(files_a)
    set_b = set(files_b)
    intersection = set_a & set_b
    union = set_a | set_b
    return len(intersection) / len(union) if union else 0.0


def merge_by_file_affinity(
    tasks: list[dict],
    overlap_threshold: float = 0.5,
    max_complexity: float = 4.0,
    dep_key: str = "depends_on",
) -> tuple[list[dict], list[dict]]:
    """Merge tasks sharing significant file overlap.

    Pairs of non-epic tasks with ``context.files`` Jaccard similarity ≥
    *overlap_threshold* are candidates for merging.  The lower-complexity
    task is folded into the higher-complexity one, provided the source
    complexity is below *max_complexity*.

    Safety guards:
    - Epics are never merged.
    - Cycle participants are never merged.
    - If the destination depends on the source (directly or transitively),
      the merge is skipped to avoid creating a self-dependency.

    Returns ``(remaining_tasks, merge_report)``.
    """
    if not tasks:
        return ([], [])

    tasks = copy.deepcopy(tasks)
    merge_report: list[dict] = []

    # Identify cycle participants (never merge these)
    cyclic_ids = _find_cycle_participants(tasks, dep_key=dep_key)

    task_map: dict[str, dict] = {t["id"]: t for t in tasks}

    # Extract file lists for non-epic tasks
    file_map: dict[str, list[str]] = {}
    for t in tasks:
        tid = t["id"]
        if t.get("status") == "epic" or t.get("type") == "epic":
            continue
        if tid in cyclic_ids:
            continue
        ctx = t.get("context") or {}
        files = ctx.get("files") if isinstance(ctx, dict) else None
        if isinstance(files, list) and files:
            file_map[tid] = files

    # Build all candidate pairs sorted by overlap descending
    candidates: list[tuple[float, str, str]] = []
    ids_with_files = list(file_map.keys())
    for i, id_a in enumerate(ids_with_files):
        for id_b in ids_with_files[i + 1:]:
            overlap = compute_file_overlap(file_map[id_a], file_map[id_b])
            if overlap >= overlap_threshold:
                candidates.append((overlap, id_a, id_b))

    candidates.sort(key=lambda x: x[0], reverse=True)

    # Build dependency lookup for safety checks
    def _depends_on(src: str, dst: str) -> bool:
        """Check if dst depends on src (directly)."""
        return src in task_map.get(dst, {}).get(dep_key, [])

    removed: set[str] = set()

    for overlap, id_a, id_b in candidates:
        if id_a in removed or id_b in removed:
            continue
        if id_a not in task_map or id_b not in task_map:
            continue

        task_a = task_map[id_a]
        task_b = task_map[id_b]

        # Determine merge direction: lower complexity → higher complexity
        score_a = compute_complexity_score(task_a)
        score_b = compute_complexity_score(task_b)

        if score_a <= score_b:
            source, dest = task_a, task_b
            source_id, dest_id = id_a, id_b
            source_score = score_a
        else:
            source, dest = task_b, task_a
            source_id, dest_id = id_b, id_a
            source_score = score_b

        # Only merge if the source is below max_complexity
        if source_score >= max_complexity:
            continue

        # Safety: skip if dest depends on source (would create self-dep)
        if _depends_on(source_id, dest_id):
            continue
        # Safety: skip if source depends on dest (would lose ordering)
        if _depends_on(dest_id, source_id):
            continue

        # Merge: union file lists
        dest_ctx = dest.get("context") or {}
        if not isinstance(dest_ctx, dict):
            dest_ctx = {}
        source_ctx = source.get("context") or {}
        if not isinstance(source_ctx, dict):
            source_ctx = {}

        dest_files = set(dest_ctx.get("files") or [])
        source_files = set(source_ctx.get("files") or [])
        dest_ctx["files"] = sorted(dest_files | source_files)
        dest["context"] = dest_ctx

        # Append description/notes
        src_desc = source_ctx.get("description", "")
        if src_desc:
            existing = dest_ctx.get("description", "")
            dest_ctx["description"] = (existing + "\n" + f"[Merged from: {source.get('title', source_id)}] {src_desc}").strip()

        existing_notes = dest.get("notes") or ""
        fold_note = f"[File-affinity merge from: {source.get('title', source_id)}]"
        dest["notes"] = (existing_notes + "\n" + fold_note).strip() if existing_notes else fold_note

        # Rewire dependencies: tasks depending on source now depend on dest
        source_deps = list(source.get(dep_key, []))
        for t in tasks:
            if t["id"] in removed or t["id"] == dest_id:
                continue
            t_deps = t.get(dep_key, [])
            if source_id in t_deps:
                new_deps = []
                for d in t_deps:
                    if d == source_id:
                        # Replace with source's own deps + dest
                        for sub in source_deps:
                            if sub != t["id"] and sub not in removed:
                                new_deps.append(sub)
                        if dest_id != t["id"]:
                            new_deps.append(dest_id)
                    else:
                        new_deps.append(d)
                # Deduplicate
                seen: set[str] = set()
                deduped: list[str] = []
                for d in new_deps:
                    if d not in seen:
                        seen.add(d)
                        deduped.append(d)
                t[dep_key] = deduped

        # Add source's deps to dest (if not already present)
        dest_deps = list(dest.get(dep_key, []))
        for d in source_deps:
            if d != dest_id and d not in dest_deps and d not in removed:
                dest_deps.append(d)
        dest[dep_key] = dest_deps

        removed.add(source_id)
        merge_report.append({
            "source_id": source_id,
            "source_title": source.get("title", ""),
            "dest_id": dest_id,
            "dest_title": dest.get("title", ""),
            "overlap": round(overlap, 3),
            "reason": "file-affinity merge",
        })

        log.info(
            "FILE-AFFINITY MERGE: folded %s (%s) into %s (%s) — overlap=%.2f",
            source_id, source.get("title", ""),
            dest_id, dest.get("title", ""),
            overlap,
        )

    # Remove merged tasks
    remaining = [t for t in tasks if t["id"] not in removed]
    return (remaining, merge_report)
