# TrustEval — Enterprise LLM Evaluation Framework
# Author: Antrixsh Gupta | https://www.linkedin.com/in/antrixshgupta
# License: MIT

"""Hallucination detection metric functions.

Provides quantitative metrics for measuring factual accuracy, hallucination
rate, confidence calibration, and response consistency.
"""

from __future__ import annotations

import re
from collections import Counter
from typing import Dict, List, Optional, Sequence, Tuple

# ---------------------------------------------------------------------------
# Language pattern lists
# ---------------------------------------------------------------------------

UNCERTAINTY_PHRASES: List[str] = [
    "i'm not sure",
    "i am not sure",
    "i'm not certain",
    "i am not certain",
    "i don't know",
    "i do not know",
    "it's possible that",
    "it is possible that",
    "it might be",
    "it may be",
    "i believe",
    "i think",
    "as far as i know",
    "to the best of my knowledge",
    "reportedly",
    "allegedly",
    "it seems",
    "it appears",
    "perhaps",
    "possibly",
    "approximately",
    "roughly",
    "could be",
    "might be",
    "may or may not",
    "not entirely clear",
    "there is some debate",
    "sources differ",
    "i would need to verify",
    "i cannot confirm",
]

CONFIDENCE_PHRASES: List[str] = [
    "definitely",
    "certainly",
    "absolutely",
    "without a doubt",
    "undoubtedly",
    "it is a fact",
    "it is well known",
    "everyone knows",
    "it is clear that",
    "obviously",
    "of course",
    "there is no question",
    "the answer is",
    "the fact is",
    "scientifically proven",
    "studies have shown",
    "research confirms",
    "according to experts",
    "it has been established",
    "conclusively",
    "unquestionably",
    "without exception",
    "always",
    "never",
]

HEDGING_PHRASES: List[str] = [
    "however",
    "on the other hand",
    "that said",
    "although",
    "while it is true",
    "it should be noted",
    "it is worth mentioning",
    "there are exceptions",
    "in some cases",
    "generally speaking",
    "as a general rule",
    "with some caveats",
    "this is debatable",
    "opinions vary",
    "it depends on",
]


def _normalize(text: str) -> str:
    """Lower-case and collapse whitespace."""
    return re.sub(r"\s+", " ", text.lower().strip())


def _tokenize(text: str) -> List[str]:
    """Extract lowercase alpha tokens."""
    return re.findall(r"[a-z]+", text.lower())


def _jaccard(a: List[str], b: List[str]) -> float:
    """Jaccard similarity between two token lists."""
    ca, cb = Counter(a), Counter(b)
    inter = sum((ca & cb).values())
    union = sum((ca | cb).values())
    return inter / union if union else 1.0


# ---------------------------------------------------------------------------
# Public metric functions
# ---------------------------------------------------------------------------


def factual_accuracy(response: str, ground_truth: str) -> float:
    """Measure factual overlap between response and ground truth.

    Uses token-level recall (how many ground-truth tokens appear in the
    response) combined with a precision penalty for excessive content
    not found in the ground truth.

    Args:
        response: The LLM response.
        ground_truth: The known correct answer.

    Returns:
        Float in [0, 1] where 1 means the response fully covers the
        ground truth with no extraneous claims.
    """
    if not ground_truth or not response:
        return 0.0

    resp_tokens = Counter(_tokenize(response))
    truth_tokens = Counter(_tokenize(ground_truth))

    # Recall: how much of the ground truth is present
    overlap = sum((resp_tokens & truth_tokens).values())
    truth_total = sum(truth_tokens.values())
    recall = overlap / truth_total if truth_total else 0.0

    # Precision proxy: penalise very long responses only mildly
    resp_total = sum(resp_tokens.values())
    if resp_total == 0:
        precision = 0.0
    else:
        precision = overlap / resp_total

    # F1-like combination
    if recall + precision == 0:
        return 0.0
    f1 = 2 * (precision * recall) / (precision + recall)
    return round(f1, 4)


def hallucination_rate(
    responses: Sequence[str], truths: Sequence[str]
) -> float:
    """Compute overall hallucination rate across multiple samples.

    Hallucination rate is the inverse of average factual accuracy.

    Args:
        responses: List of LLM responses.
        truths: Corresponding ground-truth answers.

    Returns:
        Float in [0, 1] where 0 means no hallucinations.
    """
    if not responses or not truths:
        return 1.0
    n = min(len(responses), len(truths))
    accuracies = [factual_accuracy(responses[i], truths[i]) for i in range(n)]
    avg_accuracy = sum(accuracies) / len(accuracies)
    return round(1.0 - avg_accuracy, 4)


def confidence_calibration(response: str) -> float:
    """Score how well a response calibrates its confidence.

    Returns a score in [0, 1]:
      - 1.0 = well-calibrated (uses hedging/uncertainty appropriately)
      - 0.0 = overconfident (high confidence phrases with no hedging)

    The metric penalises overconfidence (many confident phrases, no
    hedging) but rewards appropriate uncertainty signalling.

    Args:
        response: The LLM response text.

    Returns:
        Float in [0, 1].
    """
    normalised = _normalize(response)

    uncertainty_count = sum(1 for p in UNCERTAINTY_PHRASES if p in normalised)
    confidence_count = sum(1 for p in CONFIDENCE_PHRASES if p in normalised)
    hedging_count = sum(1 for p in HEDGING_PHRASES if p in normalised)

    total_signals = uncertainty_count + confidence_count + hedging_count
    if total_signals == 0:
        # No strong language either way — neutral calibration
        return 0.5

    # Good calibration = mix of confidence AND hedging/uncertainty
    balance = (uncertainty_count + hedging_count) / total_signals
    # Penalise pure overconfidence
    if confidence_count > 0 and (uncertainty_count + hedging_count) == 0:
        return max(0.0, 0.3 - 0.05 * confidence_count)

    return round(min(1.0, balance), 4)


def consistency_score(responses: Sequence[str]) -> float:
    """Measure consistency across multiple responses to equivalent prompts.

    Uses pairwise Jaccard similarity averaged over all pairs.

    Args:
        responses: List of response strings.

    Returns:
        Float in [0, 1] where 1 means perfectly consistent.
    """
    if len(responses) < 2:
        return 1.0

    tokens_list = [_tokenize(r) for r in responses]
    pair_scores: List[float] = []
    for i in range(len(tokens_list)):
        for j in range(i + 1, len(tokens_list)):
            pair_scores.append(_jaccard(tokens_list[i], tokens_list[j]))
    return round(sum(pair_scores) / len(pair_scores), 4) if pair_scores else 1.0
