"""
core/url_scan.py — VaultHound v1.0.0
Live URL scanner — fetches HTML, JS, and API responses, scans for secrets.
"""

from __future__ import annotations

import re
import ssl
import urllib.request
import urllib.error
from concurrent.futures import ThreadPoolExecutor, as_completed
from urllib.parse import urljoin, urlparse

from core.scanner import Scanner
from utils.logger import safe_print, log
from utils.models import SecretFinding

# SSL context — ignore cert errors for pentest targets
_SSL_CTX = ssl.create_default_context()
_SSL_CTX.check_hostname = False
_SSL_CTX.verify_mode    = ssl.CERT_NONE

_HEADERS = {
    "User-Agent": "Mozilla/5.0 (compatible; VaultHound/1.0; SecurityScanner)",
    "Accept":     "text/html,application/json,*/*",
}

# Regex to extract JS URLs from HTML
_JS_SRC_RE   = re.compile(r'<script[^>]+src=["\']([^"\']+)["\']', re.I)
_LINK_RE     = re.compile(r'href=["\']([^"\'#?]+)["\']', re.I)
_API_RESP_RE = re.compile(r'(?:fetch|axios|XMLHttpRequest)[^"\']*["\'](/api/[^"\']+)["\']', re.I)

# Common paths that often expose secrets
PROBE_PATHS = [
    "/.env",
    "/.env.local",
    "/.env.production",
    "/config.json",
    "/settings.json",
    "/app.config.js",
    "/config/database.yml",
    "/config/secrets.yml",
    "/api/config",
    "/api/settings",
    "/api/env",
    "/debug",
    "/debug/vars",
    "/actuator/env",          # Spring Boot
    "/actuator/configprops",  # Spring Boot
    "/__debug__/",            # Django debug
    "/rails/info/properties", # Rails
]


def _fetch(url: str, timeout: int = 10) -> tuple[str, str]:
    """
    Fetch a URL. Returns (content, content_type).
    Returns ("", "") on error.
    """
    try:
        req = urllib.request.Request(url, headers=_HEADERS)
        with urllib.request.urlopen(req, timeout=timeout, context=_SSL_CTX) as resp:
            content_type = resp.headers.get("Content-Type", "")
            raw = resp.read(500_000)  # 500KB max
            return raw.decode("utf-8", errors="replace"), content_type
    except urllib.error.HTTPError as e:
        if e.code in (401, 403):
            return f"HTTP {e.code}", ""
        return "", ""
    except Exception as e:
        log.debug(f"Fetch error {url}: {e}")
        return "", ""


def _extract_js_urls(html: str, base_url: str) -> list[str]:
    """Extract all script src URLs from HTML."""
    urls = []
    for match in _JS_SRC_RE.finditer(html):
        src = match.group(1)
        if src.startswith("//"):
            src = "https:" + src
        elif not src.startswith("http"):
            src = urljoin(base_url, src)
        urls.append(src)
    return urls


def _extract_links(html: str, base_url: str, max_depth: int) -> list[str]:
    """Extract internal links for crawling."""
    base_parsed = urlparse(base_url)
    links = []
    for match in _LINK_RE.finditer(html):
        href = match.group(1)
        if href.startswith("http"):
            parsed = urlparse(href)
            if parsed.netloc == base_parsed.netloc:
                links.append(href)
        elif href.startswith("/"):
            links.append(f"{base_parsed.scheme}://{base_parsed.netloc}{href}")
    return links[:50]  # cap crawl breadth


def scan_url(
    target_url:  str,
    scanner:     Scanner,
    scan_js:     bool = True,
    crawl_depth: int  = 2,
    threads:     int  = 10,
    timeout:     int  = 10,
) -> tuple[list[SecretFinding], list[str]]:
    """
    Scan a live URL for secrets.
    - Fetches target HTML
    - Scans HTML for inline secrets
    - Optionally fetches and scans all linked JS files
    - Probes common sensitive paths
    - Crawls internal links up to crawl_depth

    Returns:
        (findings, urls_scanned)
    """
    all_findings:  list[SecretFinding] = []
    urls_scanned:  list[str]           = []
    visited:       set[str]            = set()

    def process_url(url: str, label: str = "") -> list[SecretFinding]:
        if url in visited:
            return []
        visited.add(url)
        urls_scanned.append(url)

        content, ctype = _fetch(url, timeout)
        if not content:
            return []

        findings = scanner.scan_text(content, source=url)
        if findings:
            safe_print(f"  [danger]✘ {len(findings)} secret(s)[/] in [dim]{label or url}[/]")
        return findings

    safe_print(f"[info]  Scanning URL: {target_url}[/]")

    # ── Phase 1: Main page ─────────────────────────────────────────────────
    main_content, _ = _fetch(target_url, timeout)
    if main_content:
        visited.add(target_url)
        urls_scanned.append(target_url)
        findings = scanner.scan_text(main_content, source=target_url)
        all_findings.extend(findings)
        if findings:
            safe_print(f"  [danger]✘ {len(findings)} secret(s)[/] in [dim]{target_url}[/]")

    # ── Phase 2: JS files ──────────────────────────────────────────────────
    if scan_js and main_content:
        js_urls = _extract_js_urls(main_content, target_url)
        safe_print(f"[info]  Found {len(js_urls)} JS file(s) to scan...[/]")

        with ThreadPoolExecutor(max_workers=threads) as ex:
            futures = {
                ex.submit(process_url, url, f"JS:{url.split('/')[-1]}"): url
                for url in js_urls
            }
            for fut in as_completed(futures):
                try:
                    all_findings.extend(fut.result())
                except Exception as e:
                    log.debug(f"JS scan error: {e}")

    # ── Phase 3: Probe sensitive paths ────────────────────────────────────
    parsed   = urlparse(target_url)
    base_url = f"{parsed.scheme}://{parsed.netloc}"
    probe_urls = [base_url + path for path in PROBE_PATHS]

    safe_print(f"[info]  Probing {len(probe_urls)} sensitive paths...[/]")
    with ThreadPoolExecutor(max_workers=threads) as ex:
        futures = {
            ex.submit(process_url, url, url.replace(base_url, "")): url
            for url in probe_urls
        }
        for fut in as_completed(futures):
            try:
                all_findings.extend(fut.result())
            except Exception as e:
                log.debug(f"Probe error: {e}")

    # ── Phase 4: Crawl internal links ─────────────────────────────────────
    if crawl_depth > 1 and main_content:
        links = _extract_links(main_content, target_url, crawl_depth)
        safe_print(f"[info]  Crawling {len(links)} internal link(s) (depth={crawl_depth})...[/]")

        with ThreadPoolExecutor(max_workers=threads) as ex:
            futures = {ex.submit(process_url, url): url for url in links}
            for fut in as_completed(futures):
                try:
                    all_findings.extend(fut.result())
                except Exception as e:
                    log.debug(f"Crawl error: {e}")

    return all_findings, urls_scanned
