"""Admin endpoints for managing the Wikidata enrichment pipeline."""

from __future__ import annotations

import json
import sys
import threading
import uuid
import os
import logging
from datetime import datetime, timezone
from pathlib import Path

import duckdb
from typing import Any, Dict, List, Optional

from fastapi import APIRouter, Body, Depends, HTTPException, Query
from sqlalchemy.ext.asyncio import AsyncSession

from ...auth_security import AdminContext, require_admin
from ...db import get_session
from ...services.admin_audit import log_admin_action

logger = logging.getLogger(__name__)

_FILE_PATH = Path(__file__).resolve()
REPO_ROOT = None

for candidate in _FILE_PATH.parents:
    scripts_dir = candidate / "scripts"
    config_dir = candidate / "config"
    if (
        (scripts_dir / "wikidata_pipeline.py").exists()
        and (config_dir / "wikidata_pipeline.yaml").exists()
    ):
        REPO_ROOT = candidate
        break

if REPO_ROOT is None:
    for candidate in _FILE_PATH.parents:
        if (candidate / "scripts").exists() and (candidate / "config").exists():
            REPO_ROOT = candidate
            break

if REPO_ROOT is None:
    REPO_ROOT = _FILE_PATH.parents[4]

SCRIPTS_DIR = REPO_ROOT / "scripts"
if str(REPO_ROOT) not in sys.path:
    sys.path.insert(0, str(REPO_ROOT))
if str(SCRIPTS_DIR) not in sys.path:
    sys.path.insert(0, str(SCRIPTS_DIR))

# Import the pipeline module with a fallback loader so we can run inside
# containerized deployments even when the scripts directory isn't a package.
PIPELINE_IMPORT_ERROR: Optional[str] = None
try:  # noqa: WPS433 - runtime import so CLI + API share the same codepath
    from scripts import wikidata_pipeline as pipeline  # type: ignore
except Exception as exc:  # pragma: no cover - hard failure during startup
    pipeline = None
    fallback_exc = exc
    try:
        import importlib.util

        pipeline_path = SCRIPTS_DIR / "wikidata_pipeline.py"
        if pipeline_path.exists():
            spec = importlib.util.spec_from_file_location(
                "wikidata_pipeline",
                pipeline_path,
            )
            if not spec or not spec.loader:  # pragma: no cover - defensive
                raise RuntimeError("Wikidata pipeline spec loader unavailable")
            module = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(module)
            pipeline = module
        else:
            raise RuntimeError(
                f"Wikidata pipeline file not found at {pipeline_path}"
            ) from fallback_exc
    except Exception as exc2:
        PIPELINE_IMPORT_ERROR = str(exc2)
        logger.exception("Wikidata pipeline import failed: %s", exc2)

router = APIRouter(tags=["Admin Wikidata"])


def _require_pipeline():  # type: ignore[override]
    if pipeline is None:
        detail = "Wikidata pipeline unavailable"
        if PIPELINE_IMPORT_ERROR:
            detail = f"{detail}: {PIPELINE_IMPORT_ERROR}"
        raise HTTPException(status_code=503, detail=detail)
    return pipeline


CONFIG_FILENAME = "wikidata_pipeline.yaml"
FALLBACK_CONFIG_JSON = "wikidata_pipeline.json"
RUN_STATE_FILE = REPO_ROOT / "logs" / "wikidata_pipeline_status.json"
QUERIES_FILE = REPO_ROOT / "config" / "wikidata_queries.json"

RUN_STATE_LOCK = threading.Lock()
QUERIES_LOCK = threading.Lock()
RUN_THREAD: Optional[threading.Thread] = None

SOURCE_LABELS = {
    "wikidata": "Wikidata",
    "sec_edgar": "SEC EDGAR",
    "companies_house": "Companies House",
    "dbpedia": "DBpedia",
}

_BQ_CLIENT = None
_BQ_CLIENT_LOCK = threading.Lock()


def _bq_project_id() -> str:
    return (
        os.getenv("FM_FOUNDRYGRAPH_PROJECT_ID")
        or os.getenv("GOOGLE_CLOUD_PROJECT")
        or ""
    ).strip()


def _bq_dataset() -> str:
    return (os.getenv("FM_FOUNDRYGRAPH_DATASET") or "foundrygraph_gold").strip()


def _bq_table(table_name: str) -> str:
    project = _bq_project_id()
    dataset = _bq_dataset()
    if project:
        return f"{project}.{dataset}.{table_name}"
    return f"{dataset}.{table_name}"


def _get_bq_client():
    global _BQ_CLIENT
    if _BQ_CLIENT is not None:
        return _BQ_CLIENT
    with _BQ_CLIENT_LOCK:
        if _BQ_CLIENT is not None:
            return _BQ_CLIENT
        try:
            from google.cloud import bigquery
        except Exception as exc:  # pragma: no cover
            raise RuntimeError("google-cloud-bigquery not installed") from exc
        project = _bq_project_id() or None
        _BQ_CLIENT = bigquery.Client(project=project)
        return _BQ_CLIENT


def _duckdb_available() -> bool:
    try:
        return _duckdb_path().exists()
    except Exception:
        return False


def _parse_first_label(value: Any) -> Optional[str]:
    if value is None:
        return None
    if isinstance(value, (list, tuple)):
        if not value:
            return None
        first = value[0]
        if isinstance(first, dict):
            return first.get("label") or first.get("name")
        if isinstance(first, str):
            return first
    if isinstance(value, dict):
        return value.get("label") or value.get("name")
    if isinstance(value, str):
        try:
            parsed = json.loads(value)
        except Exception:
            return None
        return _parse_first_label(parsed)
    return None


def _parse_numeric_field(value: Any, key: str) -> Optional[float]:
    if value is None:
        return None
    if isinstance(value, (int, float)):
        return float(value)
    if isinstance(value, dict):
        raw = value.get(key)
        try:
            return float(raw) if raw is not None else None
        except Exception:
            return None
    if isinstance(value, str):
        try:
            parsed = json.loads(value)
        except Exception:
            return None
        return _parse_numeric_field(parsed, key)
    return None


def _bq_count(table_name: str, where_clause: Optional[str] = None) -> int:
    table_ref = _bq_table(table_name)
    query = f"SELECT COUNT(*) AS cnt FROM `{table_ref}`"
    if where_clause:
        query = f"{query} WHERE {where_clause}"
    try:
        client = _get_bq_client()
        rows = list(client.query(query, project=_bq_project_id() or None).result())
        return int(rows[0].cnt) if rows else 0
    except Exception as exc:
        logger.warning("BQ count failed for %s: %s", table_name, exc)
        return 0


def _bq_count_distinct(table_name: str, field: str, where_clause: Optional[str] = None) -> int:
    table_ref = _bq_table(table_name)
    query = f"SELECT COUNT(DISTINCT {field}) AS cnt FROM `{table_ref}`"
    if where_clause:
        query = f"{query} WHERE {where_clause}"
    try:
        client = _get_bq_client()
        rows = list(client.query(query, project=_bq_project_id() or None).result())
        return int(rows[0].cnt) if rows else 0
    except Exception as exc:
        logger.warning("BQ distinct count failed for %s.%s: %s", table_name, field, exc)
        return 0


def _empty_coverage_metrics() -> Dict[str, Dict[str, int]]:
    return {
        "totals": {
            "companies": 0,
            "domains": 0,
            "relationships": 0,
            "external_ids": 0,
            "brands": 0,
        },
        "coverage": {
            "countries": 0,
            "industries": 0,
            "headquarters": 0,
            "formation_places": 0,
            "employees": 0,
            "revenue": 0,
        },
        "families": {
            "links": 0,
            "unique_companies": 0,
            "unique_families": 0,
        },
    }


def _compute_bq_coverage_metrics() -> Dict[str, Dict[str, int]]:
    companies_table = _bq_table("companies_gold")
    query = f"""
        SELECT
            COUNT(*) AS companies,
            COUNTIF(domain IS NOT NULL AND domain != '') AS domains,
            COUNTIF(countries IS NOT NULL) AS countries,
            COUNTIF(industries IS NOT NULL) AS industries,
            COUNTIF(headquarters IS NOT NULL) AS headquarters,
            COUNTIF(employees IS NOT NULL) AS employees,
            COUNTIF(revenue IS NOT NULL) AS revenue
        FROM `{companies_table}`
    """
    try:
        client = _get_bq_client()
        rows = list(client.query(query, project=_bq_project_id() or None).result())
        if not rows:
            return _empty_coverage_metrics()
        row = rows[0]
    except Exception as exc:
        logger.warning("BQ coverage query failed: %s", exc)
        return _empty_coverage_metrics()

    relationships = _bq_count("company_relationships")
    external_ids = _bq_count("company_external_ids")
    brands = _bq_count("company_brands")
    family_links = _bq_count("company_family_links", "has_family = TRUE")
    family_companies = _bq_count_distinct(
        "company_family_links", "wikidata_id", "has_family = TRUE"
    )
    family_ids = _bq_count_distinct(
        "company_family_links", "family_id", "has_family = TRUE"
    )

    return {
        "totals": {
            "companies": int(getattr(row, "companies", 0) or 0),
            "domains": int(getattr(row, "domains", 0) or 0),
            "relationships": relationships,
            "external_ids": external_ids,
            "brands": brands,
        },
        "coverage": {
            "countries": int(getattr(row, "countries", 0) or 0),
            "industries": int(getattr(row, "industries", 0) or 0),
            "headquarters": int(getattr(row, "headquarters", 0) or 0),
            "formation_places": 0,
            "employees": int(getattr(row, "employees", 0) or 0),
            "revenue": int(getattr(row, "revenue", 0) or 0),
        },
        "families": {
            "links": family_links,
            "unique_companies": family_companies,
            "unique_families": family_ids,
        },
    }


def _bq_top_labels(field: str, limit: int = 15) -> List[Dict[str, Any]]:
    table = _bq_table("companies_gold")
    query = f"""
        SELECT
            COALESCE(JSON_VALUE(item, '$.label'), JSON_VALUE(item, '$'), 'Unknown') AS label,
            COUNT(*) AS cnt
        FROM `{table}`,
        UNNEST(JSON_QUERY_ARRAY({field})) AS item
        WHERE {field} IS NOT NULL
        GROUP BY label
        ORDER BY cnt DESC
        LIMIT {limit}
    """
    try:
        client = _get_bq_client()
        rows = list(client.query(query, project=_bq_project_id() or None).result())
        return [{"label": row.label, "count": int(row.cnt)} for row in rows]
    except Exception as exc:
        logger.warning("BQ top labels failed for %s: %s", field, exc)
        return []


def _bq_analytics() -> Dict[str, Any]:
    metrics = _compute_bq_coverage_metrics()
    total_companies = metrics["totals"]["companies"]
    coverage = {
        "with_revenue": metrics["coverage"]["revenue"],
        "with_employees": metrics["coverage"]["employees"],
        "with_headquarters": metrics["coverage"]["headquarters"],
        "with_industry": metrics["coverage"]["industries"],
        "with_website": metrics["totals"]["domains"],
        "with_external_ids": metrics["totals"]["external_ids"],
    }

    by_country_raw = _bq_top_labels("countries", limit=15)
    by_industry_raw = _bq_top_labels("industries", limit=15)

    by_country = [{"country": row["label"], "count": row["count"]} for row in by_country_raw]
    by_industry = [{"industry": row["label"], "count": row["count"]} for row in by_industry_raw]

    by_source: List[Dict[str, Any]] = []
    try:
        table = _bq_table("company_external_ids")
        query = f"""
            SELECT source, COUNT(DISTINCT wikidata_id) AS cnt
            FROM `{table}`
            GROUP BY source
            ORDER BY cnt DESC
        """
        client = _get_bq_client()
        rows = list(client.query(query, project=_bq_project_id() or None).result())
        for row in rows:
            source_id = row.source or "unknown"
            by_source.append(
                {
                    "source": SOURCE_LABELS.get(source_id, source_id.replace("_", " ").title()),
                    "count": int(row.cnt),
                }
            )
    except Exception as exc:
        logger.warning("BQ by_source failed: %s", exc)

    data_completeness = [
        {"field": "Revenue", "percentage": _percent(coverage["with_revenue"], total_companies)},
        {"field": "Employees", "percentage": _percent(coverage["with_employees"], total_companies)},
        {"field": "Headquarters", "percentage": _percent(coverage["with_headquarters"], total_companies)},
        {"field": "Industry", "percentage": _percent(coverage["with_industry"], total_companies)},
        {"field": "Website", "percentage": _percent(coverage["with_website"], total_companies)},
        {"field": "External IDs", "percentage": _percent(coverage["with_external_ids"], total_companies)},
    ]

    return {
        "total_companies": int(total_companies),
        "coverage": coverage,
        "by_country": by_country,
        "by_industry": by_industry,
        "by_source": by_source,
        "data_completeness": data_completeness,
    }


def _bq_search_companies(query: str) -> List[Dict[str, Any]]:
    results: Dict[str, Dict[str, Any]] = {}
    like_param = f"%{query.lower()}%"
    ticker_param = query.upper()
    is_wikidata_id = query.upper().startswith("Q")

    try:
        client = _get_bq_client()
        from google.cloud import bigquery

        companies_table = _bq_table("companies_gold")

        label_query = f"""
            SELECT wikidata_id, label, countries, industries
            FROM `{companies_table}`
            WHERE LOWER(label) LIKE @like_param
            LIMIT 25
        """
        label_config = bigquery.QueryJobConfig(
            query_parameters=[
                bigquery.ScalarQueryParameter("like_param", "STRING", like_param)
            ]
        )
        for row in client.query(label_query, job_config=label_config, project=_bq_project_id() or None).result():
            results.setdefault(
                row.wikidata_id,
                {
                    "wikidata_id": row.wikidata_id,
                    "name": row.label,
                    "country": _parse_first_label(getattr(row, "countries", None)),
                    "industry": _parse_first_label(getattr(row, "industries", None)),
                },
            )

        if is_wikidata_id:
            id_query = f"""
                SELECT wikidata_id, label, countries, industries
                FROM `{companies_table}`
                WHERE wikidata_id = @wikidata_id
                LIMIT 1
            """
            id_config = bigquery.QueryJobConfig(
                query_parameters=[
                    bigquery.ScalarQueryParameter("wikidata_id", "STRING", query.upper())
                ]
            )
            for row in client.query(id_query, job_config=id_config, project=_bq_project_id() or None).result():
                results.setdefault(
                    row.wikidata_id,
                    {
                        "wikidata_id": row.wikidata_id,
                        "name": row.label,
                        "country": _parse_first_label(getattr(row, "countries", None)),
                        "industry": _parse_first_label(getattr(row, "industries", None)),
                    },
                )

        external_ids_table = _bq_table("company_external_ids")
        ticker_query = f"""
            SELECT cp.wikidata_id, cp.label, cp.countries, cp.industries
            FROM `{companies_table}` cp
            JOIN `{external_ids_table}` cei
              ON cp.wikidata_id = cei.wikidata_id
            WHERE cei.id_type = 'ticker' AND UPPER(cei.id_value) = @ticker
            LIMIT 25
        """
        ticker_config = bigquery.QueryJobConfig(
            query_parameters=[
                bigquery.ScalarQueryParameter("ticker", "STRING", ticker_param)
            ]
        )
        for row in client.query(ticker_query, job_config=ticker_config, project=_bq_project_id() or None).result():
            results.setdefault(
                row.wikidata_id,
                {
                    "wikidata_id": row.wikidata_id,
                    "name": row.label,
                    "country": _parse_first_label(getattr(row, "countries", None)),
                    "industry": _parse_first_label(getattr(row, "industries", None)),
                },
            )
    except Exception as exc:
        logger.warning("BQ wikidata search failed: %s", exc)

    return list(results.values())[:25]


def _bq_company_detail(wikidata_id: str) -> Dict[str, Any]:
    try:
        client = _get_bq_client()
        from google.cloud import bigquery

        companies_table = _bq_table("companies_gold")
        detail_query = f"""
            SELECT label, website, industries, countries, headquarters, revenue, employees, updated_at
            FROM `{companies_table}`
            WHERE wikidata_id = @wikidata_id
            LIMIT 1
        """
        detail_config = bigquery.QueryJobConfig(
            query_parameters=[
                bigquery.ScalarQueryParameter("wikidata_id", "STRING", wikidata_id)
            ]
        )
        rows = list(client.query(detail_query, job_config=detail_config, project=_bq_project_id() or None).result())
        if not rows:
            raise HTTPException(status_code=404, detail="Company not found")
        row = rows[0]
    except HTTPException:
        raise
    except Exception as exc:
        logger.warning("BQ company detail failed: %s", exc)
        raise HTTPException(status_code=502, detail="Failed to fetch company detail from BigQuery") from exc

    label = getattr(row, "label", None)
    website = getattr(row, "website", None)
    industries = getattr(row, "industries", None)
    countries = getattr(row, "countries", None)
    headquarters = getattr(row, "headquarters", None)
    revenue = getattr(row, "revenue", None)
    employees = getattr(row, "employees", None)
    updated_at = getattr(row, "updated_at", None)

    country_label = _parse_first_label(countries)
    industry_label = _parse_first_label(industries)
    hq_label = _parse_first_label(headquarters)
    revenue_amount = _parse_numeric_field(revenue, "amount")
    employees_count = _parse_numeric_field(employees, "count")
    last_updated = updated_at.isoformat() if isinstance(updated_at, datetime) else None

    stock_ticker = None
    data_sources: List[str] = []
    try:
        external_ids_table = _bq_table("company_external_ids")
        ticker_query = f"""
            SELECT id_value
            FROM `{external_ids_table}`
            WHERE wikidata_id = @wikidata_id AND id_type = 'ticker'
            ORDER BY last_refreshed DESC
            LIMIT 1
        """
        ticker_config = bigquery.QueryJobConfig(
            query_parameters=[
                bigquery.ScalarQueryParameter("wikidata_id", "STRING", wikidata_id)
            ]
        )
        ticker_rows = list(
            client.query(ticker_query, job_config=ticker_config, project=_bq_project_id() or None).result()
        )
        if ticker_rows:
            stock_ticker = getattr(ticker_rows[0], "id_value", None)

        source_query = f"""
            SELECT DISTINCT source
            FROM `{external_ids_table}`
            WHERE wikidata_id = @wikidata_id
        """
        source_rows = list(
            client.query(source_query, job_config=ticker_config, project=_bq_project_id() or None).result()
        )
        for source_row in source_rows:
            source_id = getattr(source_row, "source", None) or "unknown"
            data_sources.append(
                SOURCE_LABELS.get(source_id, source_id.replace("_", " ").title())
            )
    except Exception as exc:
        logger.warning("BQ external id lookup failed: %s", exc)

    if not data_sources:
        data_sources = [SOURCE_LABELS.get("wikidata", "Wikidata")]

    return {
        "wikidata_id": wikidata_id,
        "name": label,
        "aliases": [],
        "description": "",
        "website": website,
        "headquarters": hq_label,
        "industry": industry_label,
        "country": country_label,
        "revenue": revenue_amount,
        "employees": employees_count,
        "stock_ticker": stock_ticker,
        "founded_year": None,
        "data_sources": data_sources,
        "last_updated": last_updated,
    }


def _config_path() -> Path:
    yaml_path = REPO_ROOT / "config" / CONFIG_FILENAME
    if yaml_path.exists():
        return yaml_path
    json_path = REPO_ROOT / "config" / FALLBACK_CONFIG_JSON
    if json_path.exists():
        return json_path
    return yaml_path


def _resolve_path(raw_path: str) -> Path:
    path = Path(raw_path)
    if path.is_absolute():
        return path
    primary = (REPO_ROOT / path).resolve()
    if primary.exists():
        return primary
    fallback = (REPO_ROOT.parent / path).resolve()
    if fallback.exists():
        return fallback
    return primary


def _load_config() -> Dict[str, Any]:
    cfg_path = _config_path()
    pipeline_mod = _require_pipeline()
    if cfg_path.exists():
        return pipeline_mod.load_config(cfg_path)
    return pipeline_mod.load_config(None)


def _duckdb_path() -> Path:
    config = _load_config()
    try:
        raw = config["paths"]["output_db"]
    except KeyError as exc:
        raise HTTPException(
            status_code=500, detail="Pipeline config missing paths.output_db"
        ) from exc

    primary = _resolve_path(raw)
    if primary.exists():
        return primary

    fallback = (REPO_ROOT.parent / raw).resolve()
    if fallback.exists():
        return fallback

    return primary


def _open_duckdb(read_only: bool = True) -> duckdb.DuckDBPyConnection:
    db_path = _duckdb_path()
    if not db_path.exists():
        raise HTTPException(
            status_code=404, detail="Wikidata company database not found"
        )

    def _try_load(conn: duckdb.DuckDBPyConnection) -> bool:
        try:
            conn.execute("LOAD json")
            return True
        except duckdb.Error:
            return False

    def _install_json_extension() -> None:
        with duckdb.connect(str(db_path), read_only=False) as rw_conn:
            try:
                rw_conn.execute("INSTALL json")
            except duckdb.Error:
                pass
            try:
                rw_conn.execute("LOAD json")
            except duckdb.Error:
                pass

    conn = duckdb.connect(str(db_path), read_only=read_only)
    if _try_load(conn):
        return conn

    conn.close()
    _install_json_extension()
    conn = duckdb.connect(str(db_path), read_only=read_only)
    try:
        conn.execute("LOAD json")
    except duckdb.Error:
        pass
    return conn


def _percent(part: int, total: int) -> float:
    if total <= 0:
        return 0.0
    return round((part / total) * 100, 2)


def _validate_updates(
    updates: Dict[str, Any], reference: Dict[str, Any], *, path: str = ""
) -> None:
    for key, value in updates.items():
        if key not in reference:
            raise HTTPException(
                status_code=400, detail=f"Unknown config key: {path + key}"
            )
        ref_value = reference[key]
        if isinstance(value, dict):
            if not isinstance(ref_value, dict):
                raise HTTPException(
                    status_code=400, detail=f"Key {path + key} is not a nested mapping"
                )
            _validate_updates(value, ref_value, path=f"{path + key}.")


def _write_config(config: Dict[str, Any]) -> Path:
    cfg_path = _config_path()
    cfg_path.parent.mkdir(parents=True, exist_ok=True)
    pipeline_mod = _require_pipeline()
    if cfg_path.suffix in {".yaml", ".yml"}:
        if pipeline_mod.yaml is None:
            raise HTTPException(
                status_code=500, detail="PyYAML is required to write YAML configs"
            )
        cfg_path.write_text(
            pipeline_mod.yaml.safe_dump(config, sort_keys=False), encoding="utf-8"
        )
    else:
        cfg_path.write_text(json.dumps(config, indent=2), encoding="utf-8")
    return cfg_path


def _read_history(path: Path, limit: int) -> List[Dict[str, Any]]:
    if not path.exists():
        return []
    entries: List[Dict[str, Any]] = []
    with path.open("r", encoding="utf-8") as handle:
        for line in handle:
            line = line.strip()
            if not line:
                continue
            try:
                entries.append(json.loads(line))
            except json.JSONDecodeError:
                continue
    entries.sort(key=lambda item: item.get("started_at", ""), reverse=True)
    return entries[:limit]


def _load_run_state() -> Dict[str, Any]:
    if RUN_STATE_FILE.exists():
        try:
            data = json.loads(RUN_STATE_FILE.read_text(encoding="utf-8"))
            if isinstance(data, dict):
                return data
        except json.JSONDecodeError:
            pass
    return {
        "status": "idle",
        "phase": None,
        "run_id": None,
        "started_at": None,
        "finished_at": None,
        "last_update": None,
        "progress": None,
        "options": None,
        "error": None,
        "result": None,
    }


def _persist_run_state(state: Dict[str, Any]) -> None:
    RUN_STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
    RUN_STATE_FILE.write_text(
        json.dumps(state, indent=2, default=str), encoding="utf-8"
    )


RUN_STATE = _load_run_state()


def _update_run_state(**updates: Any) -> Dict[str, Any]:
    with RUN_STATE_LOCK:
        RUN_STATE.update(updates)
        RUN_STATE["last_update"] = datetime.now(timezone.utc).isoformat()
        _persist_run_state(RUN_STATE)
        return RUN_STATE.copy()


def _load_queries() -> Dict[str, Any]:
    if QUERIES_FILE.exists():
        try:
            data = json.loads(QUERIES_FILE.read_text(encoding="utf-8"))
            if isinstance(data, dict) and "queries" in data:
                return data
        except json.JSONDecodeError:
            pass
    return {"queries": []}


def _save_queries(payload: Dict[str, Any]) -> None:
    QUERIES_FILE.parent.mkdir(parents=True, exist_ok=True)
    QUERIES_FILE.write_text(
        json.dumps(payload, indent=2, default=str), encoding="utf-8"
    )


def _compute_trends(entries: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    points: List[Dict[str, Any]] = []
    for entry in entries:
        started = entry.get("started_at")
        build_counts = entry.get("steps", {}).get("build", {}).get("counts", {})
        summary_metrics = entry.get("steps", {}).get("summary", {}).get("metrics", {})
        coverage = summary_metrics.get("coverage", {})
        families = summary_metrics.get("families", {})
        points.append(
            {
                "started_at": started,
                "profiles": build_counts.get("company_profiles"),
                "relationships": build_counts.get("company_relationships"),
                "external_ids": build_counts.get("company_external_ids"),
                "family_links": build_counts.get("company_family_links"),
                "countries": coverage.get("countries"),
                "industries": coverage.get("industries"),
                "family_companies": families.get("unique_companies"),
            }
        )
    points.sort(key=lambda item: item.get("started_at") or "")
    return points


def _launch_pipeline_run(options: Dict[str, Any]) -> Dict[str, Any]:
    global RUN_THREAD  # noqa: WPS420

    with RUN_STATE_LOCK:
        if RUN_THREAD and RUN_THREAD.is_alive():
            raise HTTPException(
                status_code=409, detail="Pipeline run already in progress"
            )

        run_id = str(uuid.uuid4())
        state = _update_run_state(
            status="running",
            phase="initializing",
            run_id=run_id,
            started_at=datetime.now(timezone.utc).isoformat(),
            finished_at=None,
            progress=None,
            options=options,
            error=None,
            result=None,
        )

        def runner() -> None:
            global RUN_THREAD  # noqa: WPS420
            try:
                config = _load_config()
                pipeline_mod = _require_pipeline()
                pipeline_instance = pipeline_mod.WikidataPipeline(config)

                def progress_cb(stage: str, payload: Dict[str, Any]) -> None:
                    _update_run_state(
                        phase=stage, progress={"stage": stage, "payload": payload}
                    )

                result = pipeline_instance.run_full(
                    skip_build=options.get("skip_build", False),
                    skip_reconcile=options.get("skip_reconcile", False),
                    skip_summary=options.get("skip_summary", False),
                    disable_backup=options.get("disable_backup", False),
                    reconcile_dry_run=options.get("reconcile_dry_run"),
                    report_override=options.get("report_override"),
                    build_limit=options.get("limit"),
                    progress_cb=progress_cb,
                )
                _update_run_state(
                    status="succeeded",
                    phase="completed",
                    finished_at=datetime.now(timezone.utc).isoformat(),
                    result=result,
                )
            except Exception as exc:  # pragma: no cover - defensive
                _update_run_state(
                    status="failed",
                    phase="error",
                    finished_at=datetime.now(timezone.utc).isoformat(),
                    error=str(exc),
                )
            finally:
                RUN_THREAD = None

        RUN_THREAD = threading.Thread(
            target=runner, name="wikidata-pipeline", daemon=True
        )
        RUN_THREAD.start()
        return state


def _launch_summary_run() -> Dict[str, Any]:
    global RUN_THREAD  # noqa: WPS420

    with RUN_STATE_LOCK:
        if RUN_THREAD and RUN_THREAD.is_alive():
            raise HTTPException(
                status_code=409, detail="Pipeline run already in progress"
            )

        run_id = str(uuid.uuid4())
        state = _update_run_state(
            status="running",
            phase="summary",
            run_id=run_id,
            started_at=datetime.now(timezone.utc).isoformat(),
            finished_at=None,
            progress=None,
            options={"summary_only": True},
            error=None,
            result=None,
        )

        def runner() -> None:
            global RUN_THREAD  # noqa: WPS420
            try:
                config = _load_config()
                pipeline_mod = _require_pipeline()
                pipeline_instance = pipeline_mod.WikidataPipeline(config)
                result = pipeline_instance.run_summary()
                _update_run_state(
                    status="succeeded",
                    phase="completed",
                    finished_at=datetime.now(timezone.utc).isoformat(),
                    result=result.detail,
                )
            except Exception as exc:  # pragma: no cover - defensive
                _update_run_state(
                    status="failed",
                    phase="error",
                    finished_at=datetime.now(timezone.utc).isoformat(),
                    error=str(exc),
                )
            finally:
                RUN_THREAD = None

        RUN_THREAD = threading.Thread(
            target=runner, name="wikidata-summary", daemon=True
        )
        RUN_THREAD.start()
        return state


@router.get("/config")
async def get_pipeline_config(
    admin_ctx: AdminContext = Depends(require_admin),
) -> Dict[str, Any]:  # pragma: no mutate - read-only endpoint
    """Return the effective pipeline configuration (defaults + overrides)."""

    pipeline_mod = _require_pipeline()
    config = _load_config()
    cfg_path = _config_path()
    last_modified: Optional[str] = None
    if cfg_path.exists():
        mtime = datetime.fromtimestamp(cfg_path.stat().st_mtime, tz=timezone.utc)
        last_modified = mtime.isoformat()

    return {
        "config": config,
        "source_path": str(cfg_path),
        "yaml_available": pipeline_mod.yaml is not None,
        "last_modified": last_modified,
    }


@router.put("/config")
async def update_pipeline_config(
    payload: Dict[str, Any] = Body(..., description="Partial config overrides"),
    admin_ctx: AdminContext = Depends(require_admin),
    db: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    """Merge overrides into the pipeline configuration and persist to disk."""

    pipeline_mod = _require_pipeline()
    updates = payload.get("updates", payload)
    if not isinstance(updates, dict):
        raise HTTPException(
            status_code=400,
            detail="Payload must be an object or contain an 'updates' object",
        )

    _validate_updates(updates, pipeline_mod.DEFAULT_CONFIG)

    current = _load_config()
    merged = pipeline_mod.deep_update(current, updates)
    cfg_path = _write_config(merged)

    await log_admin_action(
        admin_ctx.user_id,
        admin_ctx.email,
        "wikidata_config_update",
        db,
        details={"updates": updates},
    )

    return {
        "config": merged,
        "source_path": str(cfg_path),
    }


@router.post("/run")
async def trigger_pipeline_run(
    payload: Optional[Dict[str, Any]] = Body(
        default=None, description="Override options for this run"
    ),
    admin_ctx: AdminContext = Depends(require_admin),
    db: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    """Kick off a pipeline run in the background."""

    options = payload or {}
    state = _launch_pipeline_run(options)

    await log_admin_action(
        admin_ctx.user_id,
        admin_ctx.email,
        "wikidata_run_trigger",
        db,
        details={"options": options},
    )

    return state


@router.get("/status")
async def get_run_status(
    admin_ctx: AdminContext = Depends(require_admin),
) -> Dict[str, Any]:
    """Return the latest pipeline run status."""

    with RUN_STATE_LOCK:
        return RUN_STATE.copy()


@router.get("/coverage")
async def get_coverage_snapshot(
    admin_ctx: AdminContext = Depends(require_admin),
) -> Dict[str, Any]:
    """Return the latest coverage metrics (and compute if missing)."""

    config = _load_config()
    coverage_path = _resolve_path(config["paths"]["coverage_json"])
    metrics: Dict[str, Any]
    if coverage_path.exists():
        metrics = json.loads(coverage_path.read_text(encoding="utf-8"))
        last_updated = datetime.fromtimestamp(
            coverage_path.stat().st_mtime, tz=timezone.utc
        )
    else:
        if _duckdb_available():
            output_db = _resolve_path(config["paths"]["output_db"])
            try:
                pipeline_mod = _require_pipeline()
                metrics = pipeline_mod.coverage_mod.compute_metrics(str(output_db))
            except Exception as exc:
                return {
                    "metrics": _empty_coverage_metrics(),
                    "source_path": str(coverage_path),
                    "last_updated": None,
                    "error": f"Coverage computation failed: {exc}",
                }
            last_updated = datetime.now(timezone.utc)
        else:
            metrics = _compute_bq_coverage_metrics()
            last_updated = datetime.now(timezone.utc)
    return {
        "metrics": metrics,
        "source_path": str(coverage_path),
        "last_updated": last_updated.isoformat() if last_updated else None,
    }


@router.get("/analytics")
async def get_wikidata_analytics(
    admin_ctx: AdminContext = Depends(require_admin),
) -> Dict[str, Any]:
    """Return aggregate analytics for the Wikidata company dataset."""

    if not _duckdb_available():
        return _bq_analytics()

    with _open_duckdb(read_only=True) as con:
        total_companies = con.execute(
            "SELECT COUNT(*) FROM company_profiles"
        ).fetchone()[0]

        coverage = {
            "with_revenue": con.execute(
                "SELECT COUNT(*) FROM company_profiles WHERE revenue IS NOT NULL"
            ).fetchone()[0],
            "with_employees": con.execute(
                "SELECT COUNT(*) FROM company_profiles WHERE employees IS NOT NULL"
            ).fetchone()[0],
            "with_headquarters": con.execute(
                "SELECT COUNT(*) FROM company_profiles WHERE headquarters IS NOT NULL AND json_array_length(headquarters) > 0"
            ).fetchone()[0],
            "with_industry": con.execute(
                "SELECT COUNT(*) FROM company_profiles WHERE industries IS NOT NULL AND json_array_length(industries) > 0"
            ).fetchone()[0],
            "with_website": con.execute(
                "SELECT COUNT(*) FROM company_profiles WHERE website IS NOT NULL AND length(trim(website)) > 0"
            ).fetchone()[0],
            "with_external_ids": con.execute(
                "SELECT COUNT(DISTINCT wikidata_id) FROM company_external_ids"
            ).fetchone()[0],
        }

        by_country_rows = con.execute(
            """
            SELECT
                COALESCE(json_extract_string(country_label, '$'), 'Unknown') AS country,
                COUNT(*) AS cnt
            FROM company_profiles
            CROSS JOIN UNNEST(json_extract(countries, '$[*].label')) AS t(country_label)
            WHERE countries IS NOT NULL AND json_type(countries) = 'ARRAY'
            GROUP BY 1
            ORDER BY cnt DESC
            LIMIT 15
            """
        ).fetchall()
        by_industry_rows = con.execute(
            """
            SELECT
                COALESCE(json_extract_string(industry_label, '$'), 'Unknown') AS industry,
                COUNT(*) AS cnt
            FROM company_profiles
            CROSS JOIN UNNEST(json_extract(industries, '$[*].label')) AS t(industry_label)
            WHERE industries IS NOT NULL AND json_type(industries) = 'ARRAY'
            GROUP BY 1
            ORDER BY cnt DESC
            LIMIT 15
            """
        ).fetchall()
        by_source_rows = con.execute(
            """
            SELECT COALESCE(source, 'unknown') AS source, COUNT(DISTINCT wikidata_id) AS cnt
            FROM company_external_ids
            GROUP BY 1
            ORDER BY cnt DESC
            """
        ).fetchall()

    by_country = [{"country": row[0], "count": int(row[1])} for row in by_country_rows]
    by_industry = [
        {"industry": row[0], "count": int(row[1])} for row in by_industry_rows
    ]
    by_source = [
        {
            "source": SOURCE_LABELS.get(row[0], row[0].replace("_", " ").title()),
            "count": int(row[1]),
        }
        for row in by_source_rows
    ]

    data_completeness = [
        {
            "field": "Revenue",
            "percentage": _percent(coverage["with_revenue"], total_companies),
        },
        {
            "field": "Employees",
            "percentage": _percent(coverage["with_employees"], total_companies),
        },
        {
            "field": "Headquarters",
            "percentage": _percent(coverage["with_headquarters"], total_companies),
        },
        {
            "field": "Industry",
            "percentage": _percent(coverage["with_industry"], total_companies),
        },
        {
            "field": "Website",
            "percentage": _percent(coverage["with_website"], total_companies),
        },
        {
            "field": "External IDs",
            "percentage": _percent(coverage["with_external_ids"], total_companies),
        },
    ]

    return {
        "total_companies": int(total_companies),
        "coverage": coverage,
        "by_country": by_country,
        "by_industry": by_industry,
        "by_source": by_source,
        "data_completeness": data_completeness,
    }


@router.get("/search")
async def search_wikidata_companies(
    q: str = Query(..., min_length=2),
    admin_ctx: AdminContext = Depends(require_admin),
) -> List[Dict[str, Any]]:
    """Search companies by label, ticker, or Wikidata identifier."""

    query = q.strip()
    if not query:
        return []

    if not _duckdb_available():
        return _bq_search_companies(query)

    results: Dict[str, Dict[str, Any]] = {}
    like_param = f"%{query.lower()}%"
    ticker_param = query.upper()
    is_wikidata_id = query.upper().startswith("Q")

    with _open_duckdb(read_only=True) as con:
        label_rows = con.execute(
            """
            SELECT
                wikidata_id,
                label,
                COALESCE(json_extract_string(countries, '$[0].label'), '') AS country,
                COALESCE(json_extract_string(industries, '$[0].label'), '') AS industry
            FROM company_profiles
            WHERE lower(label) LIKE ?
            ORDER BY label
            LIMIT 25
            """,
            [like_param],
        ).fetchall()
        for wid, name, country, industry in label_rows:
            results.setdefault(
                wid,
                {
                    "wikidata_id": wid,
                    "name": name,
                    "country": country or None,
                    "industry": industry or None,
                },
            )

        ticker_rows = con.execute(
            """
            SELECT
                cp.wikidata_id,
                cp.label,
                COALESCE(json_extract_string(cp.countries, '$[0].label'), '') AS country,
                COALESCE(json_extract_string(cp.industries, '$[0].label'), '') AS industry
            FROM company_profiles cp
            JOIN company_external_ids cei
              ON cp.wikidata_id = cei.wikidata_id
            WHERE cei.id_type = 'ticker' AND upper(cei.id_value) = ?
            LIMIT 25
            """,
            [ticker_param],
        ).fetchall()
        for wid, name, country, industry in ticker_rows:
            results.setdefault(
                wid,
                {
                    "wikidata_id": wid,
                    "name": name,
                    "country": country or None,
                    "industry": industry or None,
                },
            )

        if is_wikidata_id:
            id_rows = con.execute(
                """
                SELECT
                    wikidata_id,
                    label,
                    COALESCE(json_extract_string(countries, '$[0].label'), '') AS country,
                    COALESCE(json_extract_string(industries, '$[0].label'), '') AS industry
                FROM company_profiles
                WHERE wikidata_id = ?
                LIMIT 1
                """,
                [query.upper()],
            ).fetchall()
            for wid, name, country, industry in id_rows:
                results.setdefault(
                    wid,
                    {
                        "wikidata_id": wid,
                        "name": name,
                        "country": country or None,
                        "industry": industry or None,
                    },
                )

    return list(results.values())[:25]


@router.get("/company/{wikidata_id}")
async def get_company_detail(
    wikidata_id: str,
    admin_ctx: AdminContext = Depends(require_admin),
) -> Dict[str, Any]:
    """Return detailed information for a single company."""

    if not _duckdb_available():
        return _bq_company_detail(wikidata_id)

    with _open_duckdb(read_only=True) as con:
        row = con.execute(
            """
            SELECT label, website, industries, countries, headquarters, revenue, employees, inception, last_refreshed
            FROM company_profiles
            WHERE wikidata_id = ?
            """,
            [wikidata_id],
        ).fetchone()

        if row is None:
            raise HTTPException(status_code=404, detail="Company not found")

        (
            label,
            website,
            industries_json,
            countries_json,
            headquarters_json,
            revenue_json,
            employees_json,
            inception_date,
            last_refreshed,
        ) = row

        industries = json.loads(industries_json) if industries_json else []
        countries = json.loads(countries_json) if countries_json else []
        headquarters = json.loads(headquarters_json) if headquarters_json else []
        revenue_data = json.loads(revenue_json) if revenue_json else None
        employees_data = json.loads(employees_json) if employees_json else None

        country_label = countries[0].get("label") if countries else None
        industry_label = industries[0].get("label") if industries else None
        hq_label = headquarters[0].get("label") if headquarters else None
        revenue_amount = (
            float(revenue_data.get("amount"))
            if revenue_data and revenue_data.get("amount") is not None
            else None
        )
        employees_count = (
            int(employees_data.get("count"))
            if employees_data and employees_data.get("count") is not None
            else None
        )
        founded_year = inception_date.year if inception_date else None
        last_updated = (
            last_refreshed.isoformat() if isinstance(last_refreshed, datetime) else None
        )

        ticker_row = con.execute(
            """
            SELECT id_value FROM company_external_ids
            WHERE wikidata_id = ? AND id_type = 'ticker'
            ORDER BY last_refreshed DESC
            LIMIT 1
            """,
            [wikidata_id],
        ).fetchone()
        stock_ticker = ticker_row[0] if ticker_row else None

        source_rows = con.execute(
            "SELECT DISTINCT source FROM company_external_ids WHERE wikidata_id = ?",
            [wikidata_id],
        ).fetchall()

    data_sources = [
        SOURCE_LABELS.get(
            (row[0] or "unknown"), (row[0] or "unknown").replace("_", " ").title()
        )
        for row in source_rows
    ]
    if not data_sources:
        data_sources = [SOURCE_LABELS.get("wikidata", "Wikidata")]

    return {
        "wikidata_id": wikidata_id,
        "name": label,
        "aliases": [],
        "description": "",
        "website": website,
        "headquarters": hq_label,
        "industry": industry_label,
        "country": country_label,
        "revenue": revenue_amount,
        "employees": employees_count,
        "stock_ticker": stock_ticker,
        "founded_year": founded_year,
        "data_sources": data_sources,
        "last_updated": last_updated,
    }


@router.post("/summary")
async def recompute_summary(
    admin_ctx: AdminContext = Depends(require_admin),
    db: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    """Re-run the coverage summary step and return the new metrics."""

    config = _load_config()
    coverage_path = _resolve_path(config["paths"]["coverage_json"])
    metrics: Dict[str, Any] = {}
    last_updated = None
    if coverage_path.exists():
        try:
            metrics = json.loads(coverage_path.read_text(encoding="utf-8"))
            last_updated = datetime.fromtimestamp(
                coverage_path.stat().st_mtime, tz=timezone.utc
            ).isoformat()
        except json.JSONDecodeError:
            metrics = {}

    state = _launch_summary_run()

    await log_admin_action(
        admin_ctx.user_id,
        admin_ctx.email,
        "wikidata_summary_refresh",
        db,
        details={"run_id": state.get("run_id"), "queued": True},
    )

    return {
        "status": "queued",
        "metrics": metrics,
        "duration_seconds": None,
        "coverage_path": str(coverage_path),
        "last_updated": last_updated,
        "run_state": state,
    }


@router.get("/history")
async def get_pipeline_history(
    limit: int = Query(10, ge=1, le=100),
    admin_ctx: AdminContext = Depends(require_admin),
) -> Dict[str, Any]:
    """Return the most recent pipeline runs recorded in the history log."""

    config = _load_config()
    history_path = _resolve_path(config["logging"]["history_file"])
    entries = _read_history(history_path, limit)
    return {
        "history": entries,
        "source_path": str(history_path),
    }


@router.get("/history/trends")
async def get_pipeline_trends(
    limit: int = Query(30, ge=1, le=200),
    admin_ctx: AdminContext = Depends(require_admin),
) -> Dict[str, Any]:
    """Return coverage trends derived from the history log."""

    config = _load_config()
    history_path = _resolve_path(config["logging"]["history_file"])
    entries = _read_history(history_path, limit)
    return {
        "runs": _compute_trends(list(reversed(entries))),
    }


@router.get("/queries")
async def list_custom_queries(
    admin_ctx: AdminContext = Depends(require_admin),
) -> Dict[str, Any]:
    with QUERIES_LOCK:
        return _load_queries()


@router.post("/queries")
async def create_custom_query(
    payload: Dict[str, Any] = Body(...),
    admin_ctx: AdminContext = Depends(require_admin),
    db: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    name = (payload.get("name") or "").strip()
    query = (payload.get("query") or "").strip()
    description = (payload.get("description") or "").strip() or None
    if not name or not query:
        raise HTTPException(status_code=400, detail="Both name and query are required")

    with QUERIES_LOCK:
        store = _load_queries()
        queries = store.setdefault("queries", [])
        query_id = str(uuid.uuid4())
        entry = {
            "id": query_id,
            "name": name,
            "description": description,
            "query": query,
            "created_at": datetime.now(timezone.utc).isoformat(),
            "updated_at": datetime.now(timezone.utc).isoformat(),
        }
        queries.append(entry)
        _save_queries(store)

    await log_admin_action(
        admin_ctx.user_id,
        admin_ctx.email,
        "wikidata_query_create",
        db,
        details={"id": query_id, "name": name},
    )

    return {"query": entry, "queries": store["queries"]}


@router.put("/queries/{query_id}")
async def update_custom_query(
    query_id: str,
    payload: Dict[str, Any] = Body(...),
    admin_ctx: AdminContext = Depends(require_admin),
    db: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    with QUERIES_LOCK:
        store = _load_queries()
        queries = store.setdefault("queries", [])
        for entry in queries:
            if entry.get("id") == query_id:
                if "name" in payload:
                    entry["name"] = payload["name"].strip()
                if "description" in payload:
                    entry["description"] = (
                        payload.get("description") or ""
                    ).strip() or None
                if "query" in payload:
                    new_query = (payload.get("query") or "").strip()
                    if not new_query:
                        raise HTTPException(
                            status_code=400, detail="Query text cannot be empty"
                        )
                    entry["query"] = new_query
                entry["updated_at"] = datetime.now(timezone.utc).isoformat()
                _save_queries(store)
                break
        else:
            raise HTTPException(status_code=404, detail="Query not found")

    await log_admin_action(
        admin_ctx.user_id,
        admin_ctx.email,
        "wikidata_query_update",
        db,
        details={"id": query_id},
    )

    return {"query": entry, "queries": store["queries"]}


@router.delete("/queries/{query_id}")
async def delete_custom_query(
    query_id: str,
    admin_ctx: AdminContext = Depends(require_admin),
    db: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    with QUERIES_LOCK:
        store = _load_queries()
        before = len(store.get("queries", []))
        store["queries"] = [
            entry for entry in store.get("queries", []) if entry.get("id") != query_id
        ]
        if len(store.get("queries", [])) == before:
            raise HTTPException(status_code=404, detail="Query not found")
        _save_queries(store)

    await log_admin_action(
        admin_ctx.user_id,
        admin_ctx.email,
        "wikidata_query_delete",
        db,
        details={"id": query_id},
    )

    return {"queries": store["queries"]}
