"""Smart ingest preprocessor for GoldenMatch.

Intelligent detection of delimiters, headers, encoding, and structure
for messy/unstructured files.  Returns clean Polars DataFrames.
"""

from __future__ import annotations

import re
import statistics
from pathlib import Path

import polars as pl


# ---------------------------------------------------------------------------
# Encoding Detection
# ---------------------------------------------------------------------------

def detect_encoding(path: Path) -> str:
    """Detect file encoding by checking BOM and attempting decodes."""
    with open(path, "rb") as f:
        raw = f.read(8192)

    # BOM checks
    if raw[:3] == b"\xef\xbb\xbf":
        return "utf8"
    if raw[:2] in (b"\xff\xfe", b"\xfe\xff"):
        return "utf-16"

    # Try utf-8
    try:
        raw.decode("utf-8")
        return "utf8"
    except UnicodeDecodeError:
        pass

    # Try latin-1 and look for mojibake indicating utf-8 source
    text = raw.decode("latin-1")
    mojibake_patterns = ["Ã©", "Ã¨", "Ã¼", "Ã¶", "Ã¤", "Ã±", "Ã§"]
    for pat in mojibake_patterns:
        if pat in text:
            return "utf8"

    return "utf8"


# ---------------------------------------------------------------------------
# Delimiter Sniffing
# ---------------------------------------------------------------------------

_CANDIDATE_DELIMITERS = [",", "\t", "|", ";"]


def detect_delimiter(lines: list[str]) -> str:
    """Pick the delimiter with the most consistent count per line."""
    non_empty = [ln for ln in lines if ln.strip()][:20]
    if not non_empty:
        return ","

    best_delim = ","
    best_variance = float("inf")
    best_mean = 0.0

    for delim in _CANDIDATE_DELIMITERS:
        counts = [ln.count(delim) for ln in non_empty]
        if all(c == 0 for c in counts):
            continue
        if any(c == 0 for c in counts):
            continue  # need at least 1 occurrence per line
        mean = statistics.mean(counts)
        var = statistics.pvariance(counts)
        if var < best_variance or (var == best_variance and mean > best_mean):
            best_variance = var
            best_mean = mean
            best_delim = delim

    return best_delim


# ---------------------------------------------------------------------------
# Header Detection
# ---------------------------------------------------------------------------

_DIGIT_HEAVY_RE = re.compile(r"\d")


def _header_score(values: list[str]) -> float:
    """Score a row as a potential header (higher = more header-like)."""
    if not values:
        return 0.0
    score = 0.0
    for v in values:
        v = v.strip()
        if not v:
            continue
        # Non-numeric bonus
        try:
            float(v)
        except ValueError:
            score += 1.0
        # Identifier-like bonus (short, lowercase/underscores)
        if re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_ ]*", v) and len(v) < 40:
            score += 0.5
        # Penalise heavy digit content
        digit_ratio = len(_DIGIT_HEAVY_RE.findall(v)) / max(len(v), 1)
        if digit_ratio > 0.5:
            score -= 0.5

    # Uniqueness bonus
    stripped = [v.strip() for v in values if v.strip()]
    if stripped and len(set(stripped)) == len(stripped):
        score += 1.0

    return score


def detect_header_row(lines: list[str], delimiter: str) -> tuple[int, bool]:
    """Return (header_row_index, has_header)."""
    candidates = lines[:20]
    if not candidates:
        return (0, False)

    best_idx = 0
    best_score = -1.0
    for i, line in enumerate(candidates):
        parts = line.split(delimiter)
        s = _header_score(parts)
        if s > best_score:
            best_score = s
            best_idx = i

    # Threshold: if best score is very low, assume no header
    if best_score < 1.5:
        return (0, False)
    return (best_idx, True)


# ---------------------------------------------------------------------------
# Junk Row Detection
# ---------------------------------------------------------------------------

_JUNK_PATTERNS = re.compile(
    r"^\s*(generated by|report:|page \d|total:|end of file|---|===)",
    re.IGNORECASE,
)


def detect_skip_rows(
    lines: list[str], delimiter: str, header_row: int
) -> list[int]:
    """Return indices of junk rows to skip."""
    if not lines:
        return []

    header_parts = lines[header_row].split(delimiter) if header_row < len(lines) else []
    expected_cols = len(header_parts)

    skip: list[int] = []
    for i, line in enumerate(lines):
        if i == header_row:
            continue
        stripped = line.strip()
        # Completely empty
        if not stripped:
            skip.append(i)
            continue
        # Separator lines (dashes / equals)
        if re.fullmatch(r"[-=\s]+", stripped):
            skip.append(i)
            continue
        # Common junk patterns
        if _JUNK_PATTERNS.match(stripped):
            skip.append(i)
            continue
        # Significantly fewer columns than header
        if expected_cols > 1:
            parts = stripped.split(delimiter)
            if len(parts) < expected_cols * 0.5:
                skip.append(i)
                continue

    return skip


# ---------------------------------------------------------------------------
# Fixed-Width Detection
# ---------------------------------------------------------------------------

def detect_fixed_width(lines: list[str]) -> list[tuple[int, int]] | None:
    """Detect column boundaries for fixed-width data."""
    data_lines = [ln for ln in lines if ln.strip()][:30]
    if len(data_lines) < 3:
        return None

    # Find max line length
    max_len = max(len(ln) for ln in data_lines)
    if max_len < 5:
        return None

    # Count space positions across lines
    space_counts: dict[int, int] = {}
    for ln in data_lines:
        padded = ln.ljust(max_len)
        for pos in range(max_len):
            if padded[pos] == " ":
                space_counts[pos] = space_counts.get(pos, 0) + 1

    threshold = len(data_lines) * 0.8

    # Find positions that are spaces in almost all lines
    boundary_positions: list[int] = []
    in_boundary = False
    for pos in range(max_len):
        if space_counts.get(pos, 0) >= threshold:
            if not in_boundary:
                boundary_positions.append(pos)
                in_boundary = True
        else:
            in_boundary = False

    if len(boundary_positions) < 1:
        return None

    # Build column boundaries
    boundaries: list[tuple[int, int]] = []
    start = 0
    for bp in boundary_positions:
        if bp > start:
            boundaries.append((start, bp))
        # Skip the space gap — find next non-space-boundary
        start = bp + 1
        # Advance past consecutive boundary spaces
        while start < max_len and space_counts.get(start, 0) >= threshold:
            start += 1

    if start < max_len:
        boundaries.append((start, max_len))

    if len(boundaries) < 2:
        return None

    return boundaries


def parse_fixed_width(
    lines: list[str],
    boundaries: list[tuple[int, int]],
    has_header: bool,
) -> pl.DataFrame:
    """Parse fixed-width data using column boundaries."""
    data_lines = [ln for ln in lines if ln.strip()]
    if not data_lines:
        return pl.DataFrame()

    if has_header:
        header_line = data_lines[0]
        col_names = [header_line[s:e].strip() for s, e in boundaries]
        # Deduplicate / clean empty names
        col_names = [n if n else f"column_{i+1}" for i, n in enumerate(col_names)]
        data_lines = data_lines[1:]
    else:
        col_names = [f"column_{i+1}" for i in range(len(boundaries))]

    rows: list[dict[str, str]] = []
    for ln in data_lines:
        row = {}
        for ci, (s, e) in enumerate(boundaries):
            val = ln[s:e].strip() if s < len(ln) else ""
            row[col_names[ci]] = val
        rows.append(row)

    return pl.DataFrame(rows)


# ---------------------------------------------------------------------------
# Key-Value Detection
# ---------------------------------------------------------------------------

_KV_RE_COLON = re.compile(r"^([^:]+):(.+)$")
_KV_RE_EQUALS = re.compile(r"^([^=]+)=(.+)$")


def detect_key_value(lines: list[str]) -> tuple[str, list[str]] | None:
    """Detect if file is key:value or key=value format."""
    non_empty = [ln for ln in lines if ln.strip()]
    if not non_empty:
        return None

    for sep, pattern in [(":", _KV_RE_COLON), ("=", _KV_RE_EQUALS)]:
        matches = 0
        keys: list[str] = []
        for ln in non_empty:
            m = pattern.match(ln.strip())
            if m:
                matches += 1
                keys.append(m.group(1).strip())
        if matches / len(non_empty) > 0.6:
            unique_keys = list(dict.fromkeys(keys))  # preserve order, dedupe
            return (sep, unique_keys)

    return None


def parse_key_value(text: str, separator: str, keys: list[str]) -> pl.DataFrame:
    """Parse key-value text into a DataFrame of records."""
    if separator == ":":
        pattern = re.compile(r"^([^:]+):(.*)$")
    else:
        pattern = re.compile(r"^([^=]+)=(.*)$")

    records: list[dict[str, str]] = []
    current: dict[str, str] = {}
    last_key: str | None = None
    first_key = keys[0] if keys else None

    for line in text.splitlines():
        stripped = line.strip()
        if not stripped:
            continue
        m = pattern.match(stripped)
        if m:
            key = m.group(1).strip()
            val = m.group(2).strip()
            # New record starts when the first key repeats
            if key == first_key and current:
                records.append(current)
                current = {}
            current[key] = val
            last_key = key
        elif last_key is not None:
            # Continuation line
            current[last_key] = current.get(last_key, "") + " " + stripped

    if current:
        records.append(current)

    if not records:
        return pl.DataFrame()

    # Normalise so all records have same keys
    all_keys = list(dict.fromkeys(k for r in records for k in r))
    rows = [{k: r.get(k, "") for k in all_keys} for r in records]
    return pl.DataFrame(rows)


# ---------------------------------------------------------------------------
# Block Detection
# ---------------------------------------------------------------------------

def detect_blocks(lines: list[str]) -> list[list[str]] | None:
    """Detect records separated by blank lines."""
    blocks: list[list[str]] = []
    current_block: list[str] = []

    for line in lines:
        if line.strip():
            current_block.append(line)
        else:
            if current_block:
                blocks.append(current_block)
                current_block = []
    if current_block:
        blocks.append(current_block)

    if len(blocks) < 2:
        return None

    # Check for consistent structure (similar line count per block)
    lengths = [len(b) for b in blocks]
    if len(set(lengths)) <= 2:  # at most 2 different block sizes
        return blocks

    # Allow some variance
    avg = statistics.mean(lengths)
    if all(abs(l - avg) / max(avg, 1) < 0.5 for l in lengths):
        return blocks

    return None


def parse_blocks(blocks: list[list[str]]) -> pl.DataFrame:
    """Parse blank-line-separated blocks into a DataFrame."""
    if not blocks:
        return pl.DataFrame()

    # Try key-value within each block
    first_block = blocks[0]
    kv = detect_key_value(first_block)
    if kv:
        sep, keys = kv
        records: list[dict[str, str]] = []
        for block in blocks:
            text = "\n".join(block)
            df = parse_key_value(text, sep, keys)
            if len(df) > 0:
                records.append(df.row(0, named=True))
        if records:
            all_keys = list(dict.fromkeys(k for r in records for k in r))
            rows = [{k: r.get(k, "") for k in all_keys} for r in records]
            return pl.DataFrame(rows)

    # Positional: each line in a block is a field
    num_fields = max(len(b) for b in blocks)
    col_names = [f"column_{i+1}" for i in range(num_fields)]
    rows_list: list[dict[str, str]] = []
    for block in blocks:
        row = {}
        for i, line in enumerate(block):
            if i < num_fields:
                row[col_names[i]] = line.strip()
        # Fill missing
        for i in range(len(block), num_fields):
            row[col_names[i]] = ""
        rows_list.append(row)

    return pl.DataFrame(rows_list)


# ---------------------------------------------------------------------------
# Entity Extraction
# ---------------------------------------------------------------------------

_EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}")
_PHONE_RE = re.compile(
    r"(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}"
)
_ZIP_RE = re.compile(r"\b\d{5}(?:-\d{4})?\b")
_NAME_RE = re.compile(r"\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,3})\b")
_ADDR_RE = re.compile(
    r"\b\d+\s+[A-Z][a-zA-Z]+(?:\s+[A-Z]?[a-zA-Z]+)*"
    r"\s+(?:St|Street|Ave|Avenue|Blvd|Boulevard|Dr|Drive|Rd|Road|Ln|Lane|Way|Ct|Court|Pl|Place)\b",
    re.IGNORECASE,
)


def extract_entities(text: str) -> pl.DataFrame:
    """Extract recognisable entities from unstructured text."""
    lines = text.splitlines()

    # Collect entities with their line numbers
    entity_lines: list[tuple[int, str, str]] = []  # (line_no, type, value)

    for i, line in enumerate(lines):
        for m in _EMAIL_RE.finditer(line):
            entity_lines.append((i, "email", m.group()))
        for m in _PHONE_RE.finditer(line):
            entity_lines.append((i, "phone", m.group()))
        for m in _ZIP_RE.finditer(line):
            entity_lines.append((i, "zip", m.group()))
        for m in _NAME_RE.finditer(line):
            entity_lines.append((i, "name", m.group()))
        for m in _ADDR_RE.finditer(line):
            entity_lines.append((i, "address", m.group()))

    if not entity_lines:
        return pl.DataFrame()

    # Proximity-based grouping: entities within 3 lines of each other
    entity_lines.sort(key=lambda x: x[0])
    groups: list[list[tuple[int, str, str]]] = []
    current_group: list[tuple[int, str, str]] = [entity_lines[0]]

    for ent in entity_lines[1:]:
        if ent[0] - current_group[-1][0] <= 3:
            current_group.append(ent)
        else:
            groups.append(current_group)
            current_group = [ent]
    groups.append(current_group)

    # Convert groups to records
    records: list[dict[str, str]] = []
    for group in groups:
        record: dict[str, str] = {}
        for _, etype, value in group:
            if etype not in record:
                record[etype] = value
        records.append(record)

    # Build DataFrame with consistent columns
    all_types = list(dict.fromkeys(t for r in records for t in r))
    rows = [{t: r.get(t, "") for t in all_types} for r in records]
    return pl.DataFrame(rows)


# ---------------------------------------------------------------------------
# Master Smart Load
# ---------------------------------------------------------------------------

_TEXT_SUFFIXES = {".csv", ".txt", ".tsv", ".dat", ".tab", ".psv", ".log", ".asc"}


def smart_load(
    path: Path,
    parse_mode: str = "auto",
    delimiter: str | None = None,
    encoding: str | None = None,
    header_row: int | None = None,
    has_header: bool | None = None,
    skip_rows: list[int] | None = None,
) -> tuple[pl.DataFrame, dict]:
    """Intelligently load a messy text file into a Polars DataFrame.

    Returns (dataframe, metadata_dict).
    """
    path = Path(path)
    log: list[str] = []

    # 1. Encoding
    enc = encoding or detect_encoding(path)
    log.append(f"encoding={enc}")

    # 2. Read raw text
    try:
        text = path.read_text(encoding=enc, errors="replace")
    except Exception:
        text = path.read_text(encoding="utf8-lossy", errors="replace")
        enc = "utf8"
        log.append("encoding fallback to utf8")

    lines = text.splitlines()

    meta: dict = {
        "encoding": enc,
        "delimiter": delimiter or "",
        "header_row": header_row if header_row is not None else 0,
        "has_header": has_header if has_header is not None else True,
        "skip_rows": skip_rows or [],
        "parse_mode": parse_mode,
        "detection_log": log,
    }

    # --- Explicit parse modes ---
    if parse_mode == "fixed_width":
        boundaries = detect_fixed_width(lines)
        if boundaries is None:
            log.append("fixed_width detection failed, falling back to empty")
            return pl.DataFrame(), meta
        hdr = has_header if has_header is not None else True
        log.append(f"fixed_width columns={len(boundaries)}")
        df = parse_fixed_width(lines, boundaries, hdr)
        meta["parse_mode"] = "fixed_width"
        return df, meta

    if parse_mode == "key_value":
        kv = detect_key_value(lines)
        if kv is None:
            log.append("key_value detection failed")
            return pl.DataFrame(), meta
        sep, keys = kv
        log.append(f"key_value sep={sep!r} keys={len(keys)}")
        df = parse_key_value(text, sep, keys)
        meta["parse_mode"] = "key_value"
        return df, meta

    if parse_mode == "block":
        blocks = detect_blocks(lines)
        if blocks is None:
            log.append("block detection failed")
            return pl.DataFrame(), meta
        log.append(f"blocks={len(blocks)}")
        df = parse_blocks(blocks)
        meta["parse_mode"] = "block"
        return df, meta

    if parse_mode == "entity_extract":
        log.append("entity extraction")
        df = extract_entities(text)
        meta["parse_mode"] = "entity_extract"
        return df, meta

    if parse_mode == "delimited":
        delim = delimiter or detect_delimiter(lines)
        meta["delimiter"] = delim
        hr_idx, hdr = (header_row, True) if header_row is not None else detect_header_row(lines, delim)
        if has_header is not None:
            hdr = has_header
        meta["header_row"] = hr_idx
        meta["has_header"] = hdr
        sr = skip_rows if skip_rows is not None else detect_skip_rows(lines, delim, hr_idx)
        meta["skip_rows"] = sr
        log.append(f"delimited delim={delim!r} header_row={hr_idx} skip={sr}")
        df = _parse_delimited(lines, delim, hr_idx, hdr, sr)
        meta["parse_mode"] = "delimited"
        return df, meta

    # --- Auto mode ---
    delim = delimiter or detect_delimiter(lines)
    meta["delimiter"] = delim
    log.append(f"auto: detected delimiter={delim!r}")

    # Check delimiter quality: count columns for consistency
    non_empty = [ln for ln in lines if ln.strip()][:20]
    col_counts = [len(ln.split(delim)) for ln in non_empty] if non_empty else [1]
    consistent = len(set(col_counts)) <= 2 and max(col_counts) > 1

    if consistent:
        hr_idx, hdr = (header_row, True) if header_row is not None else detect_header_row(lines, delim)
        if has_header is not None:
            hdr = has_header
        sr = skip_rows if skip_rows is not None else detect_skip_rows(lines, delim, hr_idx)
        meta["header_row"] = hr_idx
        meta["has_header"] = hdr
        meta["skip_rows"] = sr
        log.append(f"auto: delimited mode hr={hr_idx} hdr={hdr} skip={sr}")
        df = _parse_delimited(lines, delim, hr_idx, hdr, sr)
        if df.width > 1:
            meta["parse_mode"] = "delimited"
            return df, meta
        log.append("auto: delimited produced single column, trying alternatives")

    # Try fixed-width
    boundaries = detect_fixed_width(lines)
    if boundaries and len(boundaries) >= 2:
        hdr = has_header if has_header is not None else True
        log.append(f"auto: trying fixed_width cols={len(boundaries)}")
        df = parse_fixed_width(lines, boundaries, hdr)
        if df.width > 1:
            meta["parse_mode"] = "fixed_width"
            return df, meta

    # Try key-value
    kv = detect_key_value(lines)
    if kv:
        sep, keys = kv
        log.append(f"auto: key_value sep={sep!r}")
        df = parse_key_value(text, sep, keys)
        if len(df) > 0:
            meta["parse_mode"] = "key_value"
            return df, meta

    # Try blocks
    blocks = detect_blocks(lines)
    if blocks:
        log.append(f"auto: blocks={len(blocks)}")
        df = parse_blocks(blocks)
        if len(df) > 0:
            meta["parse_mode"] = "block"
            return df, meta

    # Entity extraction as last resort
    log.append("auto: entity extraction fallback")
    df = extract_entities(text)
    if len(df) > 0:
        meta["parse_mode"] = "entity_extract"
        return df, meta

    # Nothing worked — return single-column text
    log.append("auto: returning raw lines")
    meta["parse_mode"] = "raw"
    df = pl.DataFrame({"text": [ln for ln in lines if ln.strip()]})
    return df, meta


def _parse_delimited(
    lines: list[str],
    delimiter: str,
    header_row: int,
    has_header: bool,
    skip_rows: list[int],
) -> pl.DataFrame:
    """Parse lines as delimited text into a DataFrame."""
    # Filter out skip rows
    keep = [
        (i, ln)
        for i, ln in enumerate(lines)
        if i not in set(skip_rows) and ln.strip()
    ]
    if not keep:
        return pl.DataFrame()

    if has_header:
        # Find the header line
        header_line = None
        data_lines: list[str] = []
        for i, ln in keep:
            if i == header_row:
                header_line = ln
            elif i > header_row:
                data_lines.append(ln)
        if header_line is None:
            # Fallback: first kept line is header
            header_line = keep[0][1]
            data_lines = [ln for _, ln in keep[1:]]
        col_names = [c.strip() for c in header_line.split(delimiter)]
    else:
        data_lines = [ln for _, ln in keep]
        num_cols = max(len(ln.split(delimiter)) for ln in data_lines) if data_lines else 0
        col_names = [f"column_{i+1}" for i in range(num_cols)]

    # Deduplicate column names
    seen: dict[str, int] = {}
    deduped: list[str] = []
    for name in col_names:
        if not name:
            name = "unnamed"
        if name in seen:
            seen[name] += 1
            deduped.append(f"{name}_{seen[name]}")
        else:
            seen[name] = 0
            deduped.append(name)
    col_names = deduped

    rows: list[dict[str, str]] = []
    for ln in data_lines:
        parts = [p.strip() for p in ln.split(delimiter)]
        row = {}
        for ci, name in enumerate(col_names):
            row[name] = parts[ci] if ci < len(parts) else ""
        rows.append(row)

    if not rows:
        return pl.DataFrame({c: [] for c in col_names})

    return pl.DataFrame(rows)
