# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# ORCID: 0009-0006-4957-0540
# Email: Georgescottfoley@proton.me
# Licensed under the MIT License - see LICENSE file for details

"""
SpiderFoot OSINT Automation Skill

Orchestrate multi-source OSINT scans via a self-hosted SpiderFoot instance.
SpiderFoot coordinates DNS, WHOIS, SSL, Shodan, passive DNS, and dozens of
other modules into a single scan workflow.

ALL scan operations are OSINT-gated — ``start_spiderfoot_scan`` must pass
``confirm_osint_target()`` authorization before execution.

Setup:
    SPIDERFOOT_URL=http://localhost:5001    # SpiderFoot REST API base URL
    SPIDERFOOT_API_KEY=                    # Optional API key (if auth enabled)

Module presets:
    passive   — sfp_dns, sfp_whois, sfp_ssl, sfp_crt, sfp_pgp, sfp_passivedns
    standard  — passive + sfp_portscan_tcp, sfp_banner, sfp_http
    full      — all SpiderFoot modules (aggressive, slow)
"""

import logging
import os
import time
from typing import Optional

import httpx

logger = logging.getLogger(__name__)

REQUEST_TIMEOUT = 20

# Module preset shorthand → module list
_MODULE_PRESETS = {
    "passive": "sfp_dns,sfp_whois,sfp_ssl,sfp_crt,sfp_pgp,sfp_passivedns",
    "standard": (
        "sfp_dns,sfp_whois,sfp_ssl,sfp_crt,sfp_pgp,sfp_passivedns,"
        "sfp_portscan_tcp,sfp_banner,sfp_http"
    ),
    "full": "",  # empty string = all modules in SpiderFoot API
}


# ---------------------------------------------------------------------------
# Configuration helpers
# ---------------------------------------------------------------------------


def _get_config() -> dict:
    return {
        "url": os.environ.get("SPIDERFOOT_URL", "http://localhost:5001").rstrip("/"),
        "api_key": os.environ.get("SPIDERFOOT_API_KEY", ""),
    }


def _spiderfoot_available() -> bool:
    cfg = _get_config()
    return bool(cfg["url"])


def _api_get(path: str) -> dict:
    """GET from SpiderFoot API. Returns parsed JSON or raises."""
    cfg = _get_config()
    url = f"{cfg['url']}{path}"
    headers = {}
    if cfg["api_key"]:
        headers["X-SpiderFoot-API-Key"] = cfg["api_key"]
    with httpx.Client(timeout=REQUEST_TIMEOUT) as client:
        resp = client.get(url, headers=headers)
        resp.raise_for_status()
        return resp.json()


def _api_post(path: str, data: dict) -> dict:
    """POST to SpiderFoot API. Returns parsed JSON or raises."""
    cfg = _get_config()
    url = f"{cfg['url']}{path}"
    headers = {}
    if cfg["api_key"]:
        headers["X-SpiderFoot-API-Key"] = cfg["api_key"]
    with httpx.Client(timeout=REQUEST_TIMEOUT) as client:
        resp = client.post(url, data=data, headers=headers)
        resp.raise_for_status()
        return resp.json()


def _api_delete(path: str) -> dict:
    """DELETE via SpiderFoot API."""
    cfg = _get_config()
    url = f"{cfg['url']}{path}"
    headers = {}
    if cfg["api_key"]:
        headers["X-SpiderFoot-API-Key"] = cfg["api_key"]
    with httpx.Client(timeout=REQUEST_TIMEOUT) as client:
        resp = client.delete(url, headers=headers)
        resp.raise_for_status()
        return {"success": True}


# ---------------------------------------------------------------------------
# Risk level normalisation
# ---------------------------------------------------------------------------

_RISK_MAP = {
    "HIGH": "high",
    "MEDIUM": "medium",
    "LOW": "low",
    "INFO": "info",
    "SAFE": "info",
}


def _norm_risk(raw: str) -> str:
    return _RISK_MAP.get((raw or "").upper(), "info")


# ---------------------------------------------------------------------------
# Tools
# ---------------------------------------------------------------------------


def start_spiderfoot_scan(
    target: str,
    scan_name: str = "",
    modules: str = "passive",
    passive_only: bool = True,
) -> dict:
    """
    Start a new SpiderFoot OSINT scan against a target.

    This tool is OSINT-gated: ``confirm_osint_target(target=...)`` authorization
    must exist before calling.  The authorization is checked automatically by
    the tool executor — do not call this function directly without that gate.

    Args:
        target: Domain, IP address, email, or other indicator to scan.
        scan_name: Human-readable label for the scan. Defaults to "scan-<target>".
        modules: Module preset ("passive", "standard", "full") or a
            comma-separated SpiderFoot module list (e.g. "sfp_dns,sfp_whois").
            Defaults to "passive" (no direct contact with the target).
        passive_only: When True and modules is a preset, enforces the "passive"
            preset regardless of the modules argument.  Set False explicitly to
            allow active scanning.

    Returns:
        Dict with scan_id, scan_name, target, status, modules_used.
    """
    if not _spiderfoot_available():
        return {
            "format": "degraded",
            "message": "SpiderFoot not configured (SPIDERFOOT_URL not set)",
        }

    if passive_only and modules in _MODULE_PRESETS:
        modules = "passive"

    module_list = _MODULE_PRESETS.get(modules, modules)
    name = scan_name or f"scan-{target}"

    try:
        result = _api_post(
            "/api/v1/scan",
            {
                "scanname": name,
                "scantarget": target,
                "modulelist": module_list,
                "typelist": "",
            },
        )
        scan_id = result.get("id", result.get("scan_id", ""))
        return {
            "scan_id": scan_id,
            "scan_name": name,
            "target": target,
            "status": "STARTED",
            "modules_used": module_list or "(all modules)",
            "passive_only": passive_only,
        }
    except Exception as e:
        logger.warning("SpiderFoot start_scan failed for %s: %s", target, e)
        return {"format": "degraded", "message": f"Failed to start SpiderFoot scan: {e}"}


def get_scan_status(scan_id: str) -> dict:
    """
    Get the current status and progress of a SpiderFoot scan.

    Args:
        scan_id: Scan ID returned by start_spiderfoot_scan.

    Returns:
        Dict with scan_id, status (RUNNING/FINISHED/ABORTED/ERROR),
        start_time, elapsed_seconds, found_count.
    """
    if not _spiderfoot_available():
        return {
            "format": "degraded",
            "message": "SpiderFoot not configured (SPIDERFOOT_URL not set)",
        }

    try:
        data = _api_get(f"/api/v1/scanstatus/{scan_id}")
        # SpiderFoot returns a list of [status, target, scan_name, start_time, ...]
        if isinstance(data, list) and data:
            row = data[0]
            status = row[0] if len(row) > 0 else "UNKNOWN"
            start_time = row[3] if len(row) > 3 else ""
            elapsed = 0
            if start_time:
                try:
                    started = time.mktime(time.strptime(start_time, "%Y-%m-%d %H:%M:%S"))
                    elapsed = int(time.time() - started)
                except Exception:
                    pass
            found_count = row[4] if len(row) > 4 else 0
            return {
                "scan_id": scan_id,
                "status": status,
                "start_time": start_time,
                "elapsed_seconds": elapsed,
                "found_count": int(found_count) if found_count else 0,
            }
        # Some versions return a dict directly
        if isinstance(data, dict):
            return {
                "scan_id": scan_id,
                "status": data.get("status", "UNKNOWN"),
                "start_time": data.get("start_time", ""),
                "elapsed_seconds": data.get("elapsed", 0),
                "found_count": data.get("found_count", 0),
            }
        return {"scan_id": scan_id, "status": "UNKNOWN", "raw": str(data)}
    except Exception as e:
        logger.warning("SpiderFoot get_scan_status failed for %s: %s", scan_id, e)
        return {"format": "degraded", "message": f"Failed to get scan status: {e}"}


def get_scan_findings(
    scan_id: str,
    type_filter: Optional[str] = None,
    limit: int = 100,
) -> dict:
    """
    Retrieve findings from a completed or running SpiderFoot scan.

    Args:
        scan_id: Scan ID returned by start_spiderfoot_scan.
        type_filter: Filter by SpiderFoot event type. Common values:
            IP_ADDRESS, DOMAIN_NAME, EMAIL_ADDRESS, PHONE_NUMBER,
            SOCIAL_MEDIA, LEAKED_CREDENTIALS, VULNERABILITY,
            INTERNET_NAME, NETBLOCK_OWNER, RAW_FILE_META_DATA.
            Pass None to retrieve all finding types.
        limit: Maximum number of findings to return. Default 100.

    Returns:
        Dict with findings (list), total_count, type_breakdown (dict),
        and risk_breakdown (dict).
    """
    if not _spiderfoot_available():
        return {
            "format": "degraded",
            "message": "SpiderFoot not configured (SPIDERFOOT_URL not set)",
        }

    try:
        path = f"/api/v1/scaneventresults/{scan_id}"
        if type_filter:
            path += f"/{type_filter}"
        data = _api_get(path)

        # SpiderFoot returns a list of rows: [type, data, source_module, risk, ...]
        findings = []
        type_breakdown: dict = {}
        risk_breakdown: dict = {}

        rows = data if isinstance(data, list) else []
        for row in rows[:limit]:
            if not isinstance(row, list) or len(row) < 2:
                continue
            ftype = row[0] if len(row) > 0 else "UNKNOWN"
            fdata = row[1] if len(row) > 1 else ""
            source = row[2] if len(row) > 2 else ""
            risk = _norm_risk(row[3] if len(row) > 3 else "info")

            findings.append(
                {
                    "type": ftype,
                    "data": fdata,
                    "source_module": source,
                    "risk_level": risk,
                }
            )
            type_breakdown[ftype] = type_breakdown.get(ftype, 0) + 1
            risk_breakdown[risk] = risk_breakdown.get(risk, 0) + 1

        return {
            "scan_id": scan_id,
            "findings": findings,
            "total_count": len(findings),
            "type_filter": type_filter,
            "type_breakdown": type_breakdown,
            "risk_breakdown": risk_breakdown,
        }
    except Exception as e:
        logger.warning("SpiderFoot get_scan_findings failed for %s: %s", scan_id, e)
        return {"format": "degraded", "message": f"Failed to get scan findings: {e}"}


def list_spiderfoot_scans() -> dict:
    """
    List all SpiderFoot scans — running, finished, and aborted.

    Returns:
        Dict with scans (list), total (int), running (int).
        Each scan has id, name, target, status, found_count, started.
    """
    if not _spiderfoot_available():
        return {
            "scans": [],
            "total": 0,
            "running": 0,
            "format": "degraded",
            "message": "SpiderFoot not configured (SPIDERFOOT_URL not set)",
        }

    try:
        data = _api_get("/api/v1/scanlist")
        scans = []
        running = 0

        rows = data if isinstance(data, list) else []
        for row in rows:
            if not isinstance(row, list) or len(row) < 3:
                continue
            scan_id = row[0] if len(row) > 0 else ""
            name = row[1] if len(row) > 1 else ""
            target = row[2] if len(row) > 2 else ""
            status = row[3] if len(row) > 3 else "UNKNOWN"
            started = row[4] if len(row) > 4 else ""
            found = int(row[5]) if len(row) > 5 and row[5] else 0

            scans.append(
                {
                    "id": scan_id,
                    "name": name,
                    "target": target,
                    "status": status,
                    "found_count": found,
                    "started": started,
                }
            )
            if status == "RUNNING":
                running += 1

        return {
            "scans": scans,
            "total": len(scans),
            "running": running,
        }
    except Exception as e:
        logger.warning("SpiderFoot list_scans failed: %s", e)
        return {
            "scans": [],
            "total": 0,
            "running": 0,
            "format": "degraded",
            "message": f"Failed to list SpiderFoot scans: {e}",
        }


def export_scan_report(scan_id: str, format: str = "markdown") -> dict:
    """
    Aggregate scan findings into a structured report, deduplicating by type and risk.

    Args:
        scan_id: Scan ID to report on.
        format: Output format — "markdown" (human-readable) or "json" (structured).

    Returns:
        Dict with report (str or dict), scan_id, finding_count, sections.
    """
    if not _spiderfoot_available():
        return {
            "format": "degraded",
            "message": "SpiderFoot not configured (SPIDERFOOT_URL not set)",
        }

    # Fetch status + findings
    status_result = get_scan_status(scan_id)
    findings_result = get_scan_findings(scan_id, limit=500)

    if "format" in findings_result and findings_result["format"] == "degraded":
        return findings_result

    scan_status = status_result.get("status", "UNKNOWN")
    target = status_result.get("target", scan_id)
    findings = findings_result.get("findings", [])
    type_breakdown = findings_result.get("type_breakdown", {})
    risk_breakdown = findings_result.get("risk_breakdown", {})

    # Group findings by risk level for the report
    by_risk: dict = {"high": [], "medium": [], "low": [], "info": []}
    for f in findings:
        risk = f.get("risk_level", "info")
        bucket = by_risk.get(risk, by_risk["info"])
        bucket.append(f)

    # Deduplicate by (type, data)
    seen: set = set()
    deduped = []
    for f in findings:
        key = (f.get("type", ""), f.get("data", ""))
        if key not in seen:
            seen.add(key)
            deduped.append(f)

    sections = {
        "high_risk": [f for f in deduped if f.get("risk_level") == "high"],
        "medium_risk": [f for f in deduped if f.get("risk_level") == "medium"],
        "low_risk": [f for f in deduped if f.get("risk_level") == "low"],
        "informational": [f for f in deduped if f.get("risk_level") == "info"],
    }

    if format == "json":
        return {
            "scan_id": scan_id,
            "target": target,
            "status": scan_status,
            "finding_count": len(deduped),
            "type_breakdown": type_breakdown,
            "risk_breakdown": risk_breakdown,
            "sections": sections,
        }

    # Markdown report
    lines = [
        "# SpiderFoot Scan Report",
        "",
        f"**Target**: {target}  ",
        f"**Scan ID**: {scan_id}  ",
        f"**Status**: {scan_status}  ",
        f"**Unique Findings**: {len(deduped)}  ",
        "",
        "## Risk Summary",
        "",
    ]
    for risk_level in ("high", "medium", "low", "info"):
        count = risk_breakdown.get(risk_level, 0)
        if count:
            lines.append(f"- **{risk_level.upper()}**: {count}")
    lines.append("")

    for section_key, section_label in (
        ("high_risk", "High Risk Findings"),
        ("medium_risk", "Medium Risk Findings"),
        ("low_risk", "Low Risk Findings"),
        ("informational", "Informational"),
    ):
        items = sections[section_key]
        if not items:
            continue
        lines.append(f"## {section_label}")
        lines.append("")
        for item in items[:50]:  # cap each section at 50 in markdown view
            lines.append(
                f"- **{item.get('type', '')}**: `{item.get('data', '')}` _(via {item.get('source_module', '')})_"
            )
        if len(items) > 50:
            lines.append(f"  _(+{len(items) - 50} more)_")
        lines.append("")

    report_text = "\n".join(lines)
    return {
        "scan_id": scan_id,
        "target": target,
        "status": scan_status,
        "finding_count": len(deduped),
        "report": report_text,
        "sections": {k: len(v) for k, v in sections.items()},
    }


# ---------------------------------------------------------------------------
# TOOLS registry (loaded by the Familiar skill loader)
# ---------------------------------------------------------------------------

TOOLS = [
    {
        "name": "start_spiderfoot_scan",
        "description": (
            "Start a multi-source OSINT scan using SpiderFoot. "
            "SpiderFoot orchestrates DNS, WHOIS, SSL, Shodan, passive DNS, "
            "certificate transparency, and dozens of other modules. "
            "OSINT-gated: confirm_osint_target() authorization required first. "
            "Default module preset is 'passive' — no direct contact with the target. "
            "Set passive_only=False and modules='standard' for active scanning. "
            "Returns scan_id — poll with get_scan_status(), retrieve results with get_scan_findings()."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "target": {
                    "type": "string",
                    "description": "Domain, IP address, email, or other indicator to scan",
                },
                "scan_name": {
                    "type": "string",
                    "description": "Human-readable label for the scan (auto-generated if omitted)",
                },
                "modules": {
                    "type": "string",
                    "description": (
                        "Module preset: 'passive' (default, no direct contact), "
                        "'standard' (safe active), 'full' (all modules, aggressive). "
                        "Or a comma-separated SpiderFoot module list."
                    ),
                    "default": "passive",
                },
                "passive_only": {
                    "type": "boolean",
                    "description": (
                        "When True (default), enforces passive module preset. "
                        "Set False only for explicitly authorized active scanning."
                    ),
                    "default": True,
                },
            },
            "required": ["target"],
        },
        "handler": start_spiderfoot_scan,
        "category": "osint",
    },
    {
        "name": "get_scan_status",
        "description": (
            "Check the status of a running or completed SpiderFoot scan. "
            "Returns status (RUNNING/FINISHED/ABORTED/ERROR), elapsed time, and findings count. "
            "Poll this while status is RUNNING, then call get_scan_findings when FINISHED."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "scan_id": {
                    "type": "string",
                    "description": "Scan ID returned by start_spiderfoot_scan",
                },
            },
            "required": ["scan_id"],
        },
        "handler": get_scan_status,
        "category": "osint",
    },
    {
        "name": "get_scan_findings",
        "description": (
            "Retrieve findings from a SpiderFoot scan. "
            "Each finding has a type (IP_ADDRESS, DOMAIN_NAME, EMAIL_ADDRESS, "
            "PHONE_NUMBER, SOCIAL_MEDIA, LEAKED_CREDENTIALS, VULNERABILITY, etc.), "
            "data value, source module, and risk level. "
            "Use type_filter to narrow results to a specific indicator class. "
            "Returns type_breakdown and risk_breakdown for easy triage."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "scan_id": {
                    "type": "string",
                    "description": "Scan ID to retrieve findings for",
                },
                "type_filter": {
                    "type": "string",
                    "description": (
                        "Optional SpiderFoot event type filter: "
                        "IP_ADDRESS, DOMAIN_NAME, EMAIL_ADDRESS, PHONE_NUMBER, "
                        "SOCIAL_MEDIA, LEAKED_CREDENTIALS, VULNERABILITY, "
                        "INTERNET_NAME, NETBLOCK_OWNER, RAW_FILE_META_DATA"
                    ),
                },
                "limit": {
                    "type": "integer",
                    "description": "Maximum findings to return (default 100)",
                    "default": 100,
                },
            },
            "required": ["scan_id"],
        },
        "handler": get_scan_findings,
        "category": "osint",
    },
    {
        "name": "list_spiderfoot_scans",
        "description": (
            "List all SpiderFoot scans — running, finished, and aborted. "
            "Use to review investigation history or find a scan_id from a prior session."
        ),
        "input_schema": {
            "type": "object",
            "properties": {},
            "required": [],
        },
        "handler": list_spiderfoot_scans,
        "category": "osint",
    },
    {
        "name": "export_scan_report",
        "description": (
            "Generate a structured report from a SpiderFoot scan. "
            "Deduplicates findings, groups by risk level, and formats as markdown or JSON. "
            "Use after a scan finishes to create a shareable investigation report."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "scan_id": {
                    "type": "string",
                    "description": "Scan ID to report on",
                },
                "format": {
                    "type": "string",
                    "enum": ["markdown", "json"],
                    "description": "Output format: 'markdown' (default) or 'json'",
                    "default": "markdown",
                },
            },
            "required": ["scan_id"],
        },
        "handler": export_scan_report,
        "category": "osint",
    },
]
