import hashlib
import inspect
import json
import pickle
import random
import re
import socket
import traceback
from concurrent.futures import ThreadPoolExecutor, as_completed

import urllib3
from lxml import html

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
from bson import ObjectId
from datetime import datetime
from pathlib import Path
from typing import Optional, Dict, Any, List, Tuple, Union
import requests
from pymongo.errors import DuplicateKeyError
from requests.adapters import HTTPAdapter
from requests.cookies import RequestsCookieJar
from requests.models import Response as RequestsResponse
from urllib3.util.retry import Retry
from pymongo import MongoClient
import sys, threading, time, os
def get_db():
    try:
        from config import db
        return db
    except ImportError:
        raise RuntimeError(
            "config.py with `db` must exist in the project root"
        )
db = get_db()
LOCAL_FILE_HEADER = "X-Local-File"
class CustomLogger:
    __slots__ = (
        "_fh", "_color", "_lock",
        "_write", "_last_sec", "_last_ts",
        "_origin"
    )

    # Colors mapped to:
    # 0=OK, 1=INFO, 2=WARN, 3=DEBUG, 4=ERROR
    COLORS = (
        "\033[32m",  # OK (green)
        "\033[37m",  # INFO (white)
        "\033[33m",  # WARN (yellow)
        "\033[36m",  # DEBUG (cyan)
        "\033[31m",  # ERROR (red)
    )
    RESET = "\033[0m"

    def __init__(self, name: str, logfile: str = None, use_color: bool = True):
        # Normalize module name
        name = name.replace("\\", "/")
        parent = os.path.basename(os.path.dirname(name))
        base = os.path.basename(name).rsplit(".", 1)[0]
        self._origin = f"{parent}_{base}" if parent else base

        # Logfile (if any)
        self._fh = open(logfile, "a", buffering=16384,
                        encoding="utf-8", errors="ignore") if logfile else None

        # Terminal color support
        if use_color and sys.stdout.isatty():
            self._color = self.COLORS
        else:
            self._color = ("", "", "", "", "")
        self._write = sys.stdout.write

        # Timestamp cache
        self._last_sec = -1
        self._last_ts = ""

        # Thread-safety
        self._lock = threading.Lock()

    # ---------------------------------------------------------
    # Cached timestamp (once per second)
    # ---------------------------------------------------------
    def _ts(self):
        now = time.time()
        sec = int(now)
        if sec != self._last_sec:
            self._last_ts = time.strftime("%H:%M:%S", time.localtime(sec))
            self._last_sec = sec
        return self._last_ts

    def _log(self, idx, level, msg):
        c = self._color[idx]
        r = self.RESET if c else ""

        # try:
        #     f = sys._getframe(2)
        #     p = f.f_code.co_filename
        #
        #     # find last slash or backslash
        #     s = p.rfind("/")
        #     if s < 0:
        #         s = p.rfind("\\")
        #
        #     # find parent directory slash
        #     s2 = p.rfind("/", 0, s)
        #     if s2 < 0:
        #         s2 = p.rfind("\\", 0, s)
        #
        #     # extract parent + file(no ext)
        #     short = p[s2 + 1:s] + "_" + p[s + 1:p.rfind(".")]
        #
        #     prefix = f"{short}:{f.f_lineno}"
        # except Exception:
        #     prefix = self._origin

        line = f"{c}[{level:^4}] {self._origin} - {msg}{r}\n"

        with self._lock:
            self._write(line)
            if self._fh:
                self._fh.write(line)

    # ---------------------------------------------------------
    # Public API
    # ---------------------------------------------------------
    def success(self, msg):
        self._log(0, "DONE", str(msg))

    def info(self, msg):
        self._log(1, "INFO", str(msg))

    def warning(self, msg):
        self._log(2, "WARN", str(msg))

    def debug(self, msg):
        trace = sys.gettrace()
        if trace and "pydevd" in trace.__module__:
            self._log(3, "DEBUG", str(msg))

    def error(self, msg):
        if isinstance(msg, Exception):
            # Get the last traceback frame (where the exception occurred)
            tb_frame = traceback.extract_tb(msg.__traceback__)[-1]
            filename = tb_frame.filename
            lineno = tb_frame.lineno
            funcname = tb_frame.name  # function or method name

            # Simplify parent dir + file base
            parent = os.path.basename(os.path.dirname(filename)).strip("_")
            file_base = os.path.splitext(os.path.basename(filename))[0].strip("_")
            short = f"{parent}_{file_base}" if parent else file_base

            # Compose clean log prefix
            prefix = f"{short}.{funcname}:{lineno}"

            # Include exception type for clarity
            formatted_msg = f"{prefix} {type(msg).__name__}: {msg}"
            self._log(4, "ERROR", formatted_msg)
        else:
            self._log(4, "ERROR", str(msg))

    # ---------------------------------------------------------
    # Exception formatter
    # ---------------------------------------------------------

    # ---------------------------------------------------------
    # Shutdown
    # ---------------------------------------------------------
    def close(self):
        if self._fh:
            self._fh.flush()
            self._fh.close()


class Record:
    """
    Lightweight Mongo document wrapper (high-performance version).
    - No dict copying
    - Only DB writes, no local mutation
    - Minimal allocations for speed
    - Subscriptable for read-only access like a dict
    """

    __slots__ = ("_col", "_id", "logger", "PK", "_doc")

    _ts_format = "%Y%m%d%H%M%S"

    def __init__(self, doc, collection, logger=None):
        if not logger:
            logger = CustomLogger(name=__file__)
        if isinstance(doc, tuple):
            doc = doc[0]
        if not isinstance(doc, dict):
            raise TypeError("Record() requires dict")

        self.logger = logger
        self._col = collection
        self._doc = doc  # store original dict for subscript access

        _id = doc.get("_id")
        if isinstance(_id, dict) and "$oid" in _id:
            _id = ObjectId(_id["$oid"])
        self._id = _id
        self.PK = doc.get("PK")
        if not self.PK:
            self.PK = doc.get("pk")

    # -----------------------------
    # Dict-like read access
    # -----------------------------
    def __getitem__(self, key):
        if key in self.__slots__:
            return getattr(self, key)
        return self._doc.get(key)

    def get(self, key, default=None):
        return self._doc.get(key, default)

    # -----------------------------
    # Helpers
    # -----------------------------
    @staticmethod
    def _timestamp():
        return datetime.now().strftime(Record._ts_format)

    @staticmethod
    def _parse_timestamp(ts):
        try:
            return datetime.strptime(ts, Record._ts_format)
        except Exception:
            return None

    def _encode_state(self, success: bool, ip: str):
        return f"{'T' if success else 'F'}|{self._timestamp()}|{ip or get_local_ip()}"

    # -----------------------------
    # Writes
    # -----------------------------
    def update(self, update_dict: dict, log_update: bool = True):
        if not update_dict:
            return False

        res = self._col.update_one({"_id": self._id}, {"$set": update_dict})
        return bool(res.modified_count)

    def mark_done(self, state: str, ip: str = None, **meta):
        ip = ip or get_local_ip()
        encoded = self._encode_state(True, ip)

        update_doc = {
            f"stats.{state}": encoded,
            state: "done",
            f"ip_{state}": ip
        }
        if meta:
            update_doc[f"meta.{state}"] = meta

        res = self._col.update_one({"_id": self._id}, {"$set": update_doc})
        self.logger.success(
            f"[{state}] for Record [{self.PK or self._id}] done"
            + (f" | {meta}" if meta else "")
        )
        return bool(res.modified_count)

    def mark_fail(self, state: str, ip: str = None, **meta):
        ip = ip or get_local_ip()
        encoded = self._encode_state(False, ip)

        update_doc = {
            f"stats.{state}": encoded,
            state: "failed",
            f"ip_{state}": ip
        }
        if meta:
            update_doc[f"meta.{state}"] = meta

        res = self._col.update_one({"_id": self._id}, {"$set": update_doc})
        self.logger.error(
            f"[{state}] for Record [{self.PK or self._id}] failed"
            + (f" | {meta}" if meta else "")
        )
        return bool(res.modified_count)

    def mark_reset(self, state: str):
        res = self._col.update_one(
            {"_id": self._id},
            {"$unset": {f"stats.{state}": "", f"meta.{state}": ""},
             "$set": {state: "pending"}}
        )
        self.logger.info(f"Record [{self.PK or self._id}]: [{state}] reset")
        return bool(res.modified_count)


logger = CustomLogger(name=__file__)


def fetch(
        url: str,
        method: str = "GET",
        params: Optional[Dict[str, Any]] = None,
        cookies: Optional[Dict[str, str]] = None,
        headers: Optional[Dict[str, str]] = None,
        data: Optional[Dict[str, Any]] = None,
        json: Optional[Dict[str, Any]] = None,
        save_dir: Optional[str] = None,
        filename: Optional[str] = None,
        proxies: Optional[Dict[str, str]] = None,
        refresh: bool = False,
        verify :bool = False,
        valid_rules: Optional[Dict[str, Any]] = None,
        session: Optional[requests.Session] = None,
        timeout: float = 10.0,
        backoff_factor: float = 0.1,
        max_retries: int = 0,

) -> Union[RequestsResponse, Exception]:
    try:
        session = session or requests.session()
        file_path = Path(save_dir) / filename if save_dir and filename else None
        is_pdf = filename and filename.lower().endswith(".pdf")

        # -------------------------------
        # CACHE LOAD
        # -------------------------------
        if file_path and file_path.exists() and not refresh:
            logger.info(f"{'Read':^5}: {filename} {url}")
            try:
                if is_pdf:
                    content = file_path.read_bytes()
                    response = RequestsResponse()
                    response.status_code = 200
                    response._content = content
                    response.url = url
                    response.reason = "OK"
                    response.headers["Content-Type"] = "application/pdf"
                    response.headers[LOCAL_FILE_HEADER] = f"file://{file_path.resolve().as_posix()}"
                    response.cookies = RequestsCookieJar()
                    return response

                text = file_path.read_text(encoding="utf-8")
                if valid_rules:
                    clean, errors = validate_text(text, valid_rules)
                    if not clean or errors:
                        if valid_rules.get('remove')==True:
                            errors.append('REMOVED AS REQUESTED')
                            logger.error(f'REMOVED AS REQUESTED : {file_path}')
                            os.remove(file_path)
                        return Exception("\n".join(errors))
                    text = clean

                response = RequestsResponse()
                response.status_code = 200
                response._content = text.encode("utf-8")
                response.url = url
                response.reason = "OK"
                response.headers["Content-Type"] = "text/html; charset=utf-8"
                response.headers[LOCAL_FILE_HEADER] = f"file://{file_path.resolve().as_posix()}"
                response.cookies = RequestsCookieJar()
                return response

            except Exception as e:
                return ValueError(f"Cache read error: {e}")

        # -------------------------------
        # RETRIES
        # -------------------------------
        if max_retries > 0:
            retries = Retry(
                total=max_retries,
                read=max_retries,
                connect=max_retries,
                backoff_factor=backoff_factor,
                status_forcelist=(500, 502, 503, 504, 403, 429),
                allowed_methods={"HEAD", "GET", "POST", "PUT", "DELETE", "OPTIONS", "PATCH"},
            )
            adapter = HTTPAdapter(max_retries=retries)
            session.mount("http://", adapter)
            session.mount("https://", adapter)

        # -------------------------------
        # NETWORK REQUEST
        # -------------------------------
        logger.info(f"{'Fresh':^5}: {filename} [{method}] {url}")
        MAX_ATTEMPTS = max_retries + 1

        for attempt in range(1, MAX_ATTEMPTS + 1):
            response = session.request(
                method=method.upper(),
                url=url,
                params=params,
                headers=headers,
                data=data,
                json=json,
                cookies=cookies,
                timeout=timeout,
                proxies=proxies,
                verify=verify
            )
            response.raise_for_status()
            if response.status_code == 200 and not response.content:
                logger.warning(f"{'Retrying':^10}: {filename} (attempt {attempt}/{MAX_ATTEMPTS}).")
                if attempt < MAX_ATTEMPTS:
                    time.sleep(backoff_factor * attempt)
                    continue
            break

        content_type = response.headers.get("Content-Type", "").lower()
        is_pdf_response = "application/pdf" in content_type or is_pdf

        # =========================================================================================
        # PDF LOGIC — integrates your "document_request" logic
        # =========================================================================================
        if is_pdf_response:
            file_size = len(response.content)

            if file_size < 1000:
                logger.warning(f"⚠️ PDF too small ({file_size} bytes), likely error page.")
                return response

            if "text/html" in content_type or "text/plain" in content_type:
                return response

            if file_path:
                try:
                    file_path.parent.mkdir(parents=True, exist_ok=True)

                    if file_path.exists():
                        logger.info(f"📄 Document cached: {file_path}")
                        return response

                    file_path.write_bytes(response.content)
                    response.headers[LOCAL_FILE_HEADER] = f"{file_path.resolve().as_uri()}"
                    logger.success(f"📄 PDF saved: {filename}")

                except Exception as e:
                    logger.error(f"Failed to save PDF: {e}")

            return response

        # =========================================================================================
        # TEXT / HTML RESPONSE HANDLING
        # =========================================================================================
        if valid_rules:
            clean, errors = validate_text(response.text, valid_rules)
            if errors:
                return ValueError("\n".join(errors))
            response._content = clean.encode("utf-8")

        if file_path:
            try:
                file_path.parent.mkdir(parents=True, exist_ok=True)
                file_path.write_text(response.text, encoding="utf-8")
                response.headers[LOCAL_FILE_HEADER] = f"{file_path.resolve().as_uri()}"
                logger.success(f"{'Pagesaved':^10}: {filename}")
            except IOError as e:
                logger.error(f"Failed to save: {e}")

        return response

    except Exception as outer_e:
        logger.error(outer_e)
        return outer_e



def save_pdf_response(response, filename, save_path="."):
    """Optimized PDF saving with minimal validation"""
    if not response or not hasattr(response, "status_code"):
        raise ValueError("Invalid response object")

    if not (200 <= response.status_code < 300):
        raise RuntimeError(f"HTTP error: {response.status_code}")

    if not filename or len(filename.strip()) < 3:
        raise ValueError("Invalid filename")

    if not filename.lower().endswith(".pdf"):
        filename += ".pdf"

    save_path = Path(save_path)
    save_path.mkdir(parents=True, exist_ok=True)

    file_path = save_path / filename

    with open(file_path, "wb") as f:
        for chunk in response.iter_content(chunk_size=8192):
            if chunk:
                f.write(chunk)

    if file_path.stat().st_size < 100:
        raise RuntimeError("File too small to be valid PDF")

    return filename
def validate_text(
        text: Union[str, bytes],
        rules: Optional[Dict[str, Any]] = None
) -> Tuple[Optional[str], Optional[List[str]]]:

    errors = []
    rules = rules or {}

    if isinstance(text, bytes):
        try:
            text = text.decode("utf-8", errors="replace")
        except Exception:
            return None, ["Unable to decode bytes input"]

    if not isinstance(text, str):
        return None, [f"Invalid type: {type(text).__name__}"]

    if not text:
        return None, ["Empty text"]

    if len(text) < 900:
        errors.append("Text shorter than minimum length: 900")

    # case_sensitive = rules.get("case_sensitive", False)
    # compare_text = text if case_sensitive else text.lower()
    compare_text = text.lower()

    def normalize(item):
        if isinstance(item, bytes):
            return item.decode("utf-8", errors="ignore")
        return str(item)

    # REQUIRED
    for item in rules.get("forbidden", []):
        item_str = normalize(item)
        item_cmp =  item_str.lower()
        if item_cmp in compare_text:
            errors.append(f"Forbidden found: '{item_str}'")


    for item in rules.get("required", []):
        item_str = normalize(item)
        item_cmp = item_str.lower()
        if item_cmp not in compare_text:
            errors.append(f"Required missing: '{item_str}'")


    # ✅ Final return
    if errors:
        return text, errors
    return text, None

def clean_dict(data: Dict[str, Any]) -> Dict[str, Any]:
    """
    Clean dictionary keys and values with comprehensive text cleaning.

    Key cleaning:
        - Convert to lowercase
        - Replace spaces with underscores
        - Keep only alphanumeric and underscore characters

    Value cleaning:
        - Decode HTML entities
        - Remove HTML tags, scripts, styles, comments
        - Remove non-printable unicode characters
        - Collapse multiple whitespace to single space
        - Normalize unicode characters
        - Recursively clean nested dicts and lists

    Args:
        data: Dictionary to clean

    Returns:
        Cleaned dictionary

    Raises:
        ValueError: If input is not a dictionary
    """
    import html
    import unicodedata

    CLEANR = re.compile(r'<.*?>|&([a-z0-9]+|#x[0-9]{1,6}|#x[0-9a-f]{1,6});|[\t\n\r]')

    def text_cleaner(text: str) -> str:
        """Deep clean text with HTML decoding and unicode normalization."""
        decodehtml = html.unescape(text.encode('utf-8', 'ignore').decode("utf-8"))
        htmltagremove = re.sub(CLEANR, '', decodehtml)
        escaperemove = re.sub(r'\s+', ' ', htmltagremove)
        return unicodedata.normalize('NFKD', escaperemove.strip()).encode('utf_8_sig', 'ignore').decode("utf_8_sig")

    def clean_string(text: str) -> str:
        """Clean string with HTML entity replacement and tag removal."""
        if not text:
            return ""

        if not isinstance(text, str):
            text = str(text)

        # Normalize common HTML entities
        replacements = {
            r"&gt;": ">",
            r"&lt;": "<",
            r"&amp;": "&",
            r"&nbsp;": " ",
            r"": "",
            r"[\r\n\t]+": " "
        }
        for pat, repl in replacements.items():
            text = re.sub(pat, repl, text, flags=re.IGNORECASE)

        # Remove scripts, styles, comments, tags
        text = re.sub(r"<script[^>]*>[\w\W]*?</script>", " ", text, flags=re.IGNORECASE)
        text = re.sub(r"\* style specs start[^>]*>[\w\W]*?style specs end *", " ", text, flags=re.IGNORECASE)
        text = re.sub(r"<style[^>]*>[\w\W]*?</style>", " ", text, flags=re.IGNORECASE)
        text = re.sub(r"<!--[\w\W]*?-->", " ", text, flags=re.IGNORECASE)
        text = re.sub(r"<[^>]+>", " ", text)
        text = re.sub(r"\s+", " ", text)
        return text_cleaner(text).strip()

    def clean_key(key):
        """Clean dictionary keys to lowercase alphanumeric with underscores."""
        if not isinstance(key, str):
            key = str(key)
        key = key.encode("utf-8", "ignore").decode("utf-8")
        key = key.lower().replace(" ", "_")
        return re.sub(r"[^a-z0-9_]", "", key)

    def clean_value(value):
        """Recursively clean values based on type."""
        if isinstance(value, str):
            return clean_string(value)
        elif isinstance(value, dict):
            return clean_dict(value)
        elif isinstance(value, list):
            return [clean_value(item) for item in value if not (isinstance(item, str) and not item.strip())]
        return value

    if not isinstance(data, dict):
        raise ValueError("Input must be dict")

    return {clean_key(k): clean_value(v) for k, v in data.items()}
def get_proxy():
    proxies = [
        "196.51.196.241:8800",
        "196.51.204.178:8800",
        "196.51.192.210:8800",
        "196.51.204.124:8800",
        "196.51.206.46:8800",
        "196.51.207.81:8800",
        "196.51.202.195:8800",
        "196.51.192.184:8800",
        "196.51.202.33:8800",
        "196.51.197.81:8800",
        "196.51.200.220:8800",
        "196.51.199.51:8800",
        "196.51.200.199:8800",
        "196.51.194.95:8800",
        "196.51.207.96:8800",
        "196.51.204.127:8800",
        "196.51.192.133:8800",
        "196.51.192.145:8800",
        "196.51.207.198:8800",
        "196.51.207.76:8800",
        "196.51.199.108:8800",
        "196.51.207.249:8800",
        "196.51.200.142:8800",
        "196.51.200.169:8800",
        "196.51.197.141:8800",
        "196.51.204.22:8800",
        "196.51.200.188:8800",
        "196.51.197.142:8800",
        "196.51.206.200:8800",
        "196.51.200.254:8800",
        "196.51.199.199:8800",
        "196.51.200.134:8800",
        "196.51.196.93:8800",
        "196.51.204.133:8800",
        "196.51.196.167:8800",
        "196.51.200.40:8800",
        "196.51.202.201:8800",
        "196.51.194.209:8800",
        "196.51.197.18:8800",
        "196.51.207.226:8800",
        "196.51.194.87:8800",
        "196.51.200.203:8800",
        "196.51.194.9:8800",
        "196.51.192.59:8800",
        "196.51.204.23:8800",
        "196.51.200.109:8800",
        "196.51.199.173:8800",
        "196.51.204.177:8800",
        "196.51.200.119:8800",
        "196.51.204.18:8800",
        "196.51.192.93:8800",
        "196.51.200.6:8800",
        "196.51.204.62:8800",
        "196.51.192.60:8800",
        "196.51.194.70:8800",
        "196.51.199.128:8800",
        "196.51.204.5:8800",
        "196.51.202.151:8800",
        "196.51.207.59:8800",
        "196.51.207.83:8800",
        "196.51.192.32:8800",
        "196.51.202.248:8800",
        "196.51.206.244:8800",
        "196.51.204.150:8800",
        "196.51.192.142:8800",
        "196.51.196.131:8800",
        "196.51.197.198:8800",
        "196.51.197.61:8800",
        "196.51.202.23:8800",
        "196.51.192.152:8800",
        "196.51.192.39:8800",
        "196.51.197.192:8800",
        "196.51.204.119:8800",
        "196.51.206.120:8800",
        "196.51.199.182:8800",
        "196.51.192.54:8800",
        "196.51.196.138:8800",
        "196.51.200.60:8800",
        "196.51.200.9:8800",
        "196.51.204.105:8800",
        "196.51.206.83:8800",
        "196.51.207.246:8800",
        "196.51.192.91:8800",
        "196.51.194.184:8800",
        "196.51.200.80:8800",
        "196.51.204.30:8800",
        "196.51.204.115:8800",
        "196.51.197.91:8800",
        "196.51.200.106:8800",
        "196.51.207.178:8800",
        "196.51.192.19:8800",
        "196.51.204.34:8800",
        "196.51.206.194:8800",
        "196.51.196.128:8800",
        "196.51.192.51:8800",
        "196.51.207.236:8800",
        "196.51.197.26:8800",
        "196.51.207.46:8800",
        "196.51.197.190:8800",
        "196.51.192.197:8800"
    ]
    username = "190170"
    password = "6dvfCe74k2Tp"
    ip_id = random.choice(proxies)
    proxy = {
        "http": f"http://{username}:{password}@{ip_id}",
        "https": f"http://{username}:{password}@{ip_id}"
    }
    # with open(r"\\192.168.2.163\d\proxifly.txt", "r") as f:
    #     proxies_list = [
    #         line.strip()
    #         for line in f
    #         if line.strip().lower().startswith("http")
    #     ]
    # rip_id = random.choice(proxies_list)
    # rproxy = {
    #     "http": f"{rip_id}",
    #     "https": f"{rip_id}"
    # }
    lum ={
"https": "brd-customer-xbyte-zone-zone_us:z3v95j5uqcba@zproxy.lum-superproxy.io:22225",
"http": "brd-customer-xbyte-zone-zone_us:z3v95j5uqcba@zproxy.lum-superproxy.io:22225"
}
    # return rproxy
    ip_id = '141.193.213.107'
    a = {
        "http": f"http://{ip_id}",
        "https": f"http://{ip_id}"
    }

    # # proxy = lum
    # print(proxy)
    return proxy
def start_feed(feed_name=None, interval=5):
    if feed_name is None:
        # Caller = previous stack frame
        caller_frame = inspect.stack()[1]
        feed_name = caller_frame.filename
    feed_name = f"{Path(feed_name).parent.name}_{Path(feed_name).stem}"
    ip = get_local_ip()
    collection = MongoClient("mongodb://admin:tP_kc8mn@192.168.2.163:27017/?authSource=admin")['health']['status']
    def heartbeat():
        while True:
            try:
                collection.update_one(
                    {"feed_name": feed_name, "worker_ip": ip},
                    {"$set": {"last_heartbeat": datetime.now().strftime("%Y-%m-%d %H:%M:%S")}},
                    upsert=True
                )
            except Exception as e:
                print(f"Heartbeat error for {ip}: {e}")
            time.sleep(interval)
    threading.Thread(target=heartbeat, daemon=True).start()
    collection.update_one(
        {"feed_name": feed_name, "worker_ip": ip},
        {"$set": {"last_heartbeat": datetime.now().strftime("%Y-%m-%d %H:%M:%S")}},
        upsert=True
    )
    return feed_name
def get_local_ip():
    hostname = socket.gethostname()
    # Get all addresses associated with the hostname
    addresses = socket.gethostbyname_ex(hostname)[2]
    for ip in addresses:
        # Check for private IPv4 ranges
        if ip.startswith("192."):
            if ip.startswith("192.") or ip.startswith("10.") or ip.startswith("172."):
                return ip




def get_hidden_fields(response):
    hidden_fields = {
        el.get("name"): el.get("value", "")
        for el in html.fromstring(response.text).xpath("//*[@type='hidden']")
    }

    if not hidden_fields:
        hidden_fields = {
            name: value
            for _, name, value in (
                m.groups() for m in re.finditer(
                r"\|(\d+)\|hiddenField\|([^|]+)\|([^|]*)",
                response.text
            )
            )
        }
    return hidden_fields

# **************************************************************************************
# **************************************************************************************
# **************************************************************************************




def is_valid_page_save(doc, PK):
    if not PK:
        return "missing PK"
    page_save = doc.get("page_save")
    if page_save.startswith("\\\\") or re.match(r"^[A-Za-z]:\\", page_save) or "\\" in page_save:
        return "invalid_path"
    if not isinstance(page_save, str) or not page_save.strip():
        return "empty"
    check_page_save = page_save.replace(".html", "").replace(".json", "")
    if str(PK) in check_page_save:
        return "valid"
    else:
        return "does not match PK"





def insert_to_db(data=None, retry=None, table_name=None):
    if not data:
        logger.error("insert_to_db: No data provided.")
        return False

    try:
        retry = str(retry)
    except Exception:
        pass
    PK = data.get('PK')
    if not PK :
        PK = data.get('pk')
    if not (retry and PK and table_name):
        logger.error("insert_to_db requires 'retry', 'PK', and 'table_name' parameters.")
        return False
    try:
        validation = is_valid_page_save(data, PK)
        if not isinstance(data, dict):
            logger.error("insert_to_db: Input data must be a dictionary.")
            return False
        # --- Validation Handling ---
        if validation == "empty":
            logger.warning("page_save missing or invalid key; skipping record.")
            return False
        elif validation == "does not match PK":
            logger.error(f"page_save mismatch or invalid extension (PK={PK}).")
            return False
        elif validation == "missing PK":
            logger.error("Record missing primary key (PK).")
            return False
        elif validation == "invalid_path":
            logger.error("page_save contains invalid path (UNC or drive-letter not allowed).")
            return False
        elif validation != "valid":
            logger.error(f"Unexpected validation result: {validation}.")
            return False
        # --- Clean and prepare record ---
        data = clean_dict(data)
        data['_id'] = hashlib.md5(json.dumps(data, sort_keys=True).encode()).hexdigest()
        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        data['_datetime'] = now
        data['_retry'] = retry
        # --- Insert ---

        db[table_name].insert_one(data)
        logger.info(f"Insert Success: <{table_name}>[{PK}]")
        return True
    except DuplicateKeyError:
        logger.info(f"Duplicate Skipped: <{table_name}>[{PK}]")
        return True
    except Exception as e:
        logger.exception(
            "Insert failed due to unexpected error",
            extra={"PK": PK, "table": table_name}
        )
        return False








def run_engine(col, query, worker, THREADS_START=False):
    hash = hashlib.md5(json.dumps(query, sort_keys=True).encode()).hexdigest()

    pickle_path = r"D:\Dharmik\threads.pkl"
    PICKLE_MAX_KEYS = 10_000   # hard cap to avoid unbounded growth

    # ---- load THREADS_START ----
    if not THREADS_START:
        try:
            if os.path.exists(pickle_path):
                with open(pickle_path, "rb") as f:
                    store = pickle.load(f)
                THREADS_START = store.get(hash, 30)
            else:
                THREADS_START = 30
        except Exception:
            THREADS_START = 30

    THREADS_MIN = 5
    THREADS_MAX = min(600, (os.cpu_count() or 4) * 70)
    THREAD_INC = 1.20
    THREAD_DEC = 0.90
    TASK_TIMEOUT = 120

    executor = ThreadPoolExecutor(max_workers=THREADS_MAX)

    try:
        docs = list(col.find(query)[:THREADS_START])
        if not docs:
            print("[ENGINE] No tasks found")
        else:
            success = fail = 0
            futures = {executor.submit(worker, d): d for d in docs}

            for fut in as_completed(futures):
                doc = futures[fut]
                try:
                    fut.result(timeout=TASK_TIMEOUT)
                    success += 1
                except Exception as e:
                    fail += 1
                    doc_pk = doc.get('PK') or doc.get('pk') or '<missing PK>'
                    print(f"[FAIL] {doc_pk} → {e}")
                    raise

            total = success + fail
            fail_rate = fail / total if total else 0

            if fail_rate >= 0.90:
                print("[FATAL] fail_rate ≥ 0.90 > exiting")
                sys.exit(5)

            prev = THREADS_START
            if fail > success:
                THREADS_START = max(THREADS_MIN, int(THREADS_START * THREAD_DEC))
                SCALER = f"{prev} < {THREADS_START}"
            elif success > fail:
                THREADS_START = min(THREADS_MAX, int(THREADS_START * THREAD_INC))
                SCALER = f"{prev} > {THREADS_START}"
            else:
                SCALER = ''

            print(f"[{SCALER}] [Success: {success} | Fail: {fail} | Rate: {fail_rate:.2f} | THREADS: {prev}]")

        # ---- save THREADS_START ----
        try:
            store = {}
            if os.path.exists(pickle_path):
                with open(pickle_path, "rb") as f:
                    store = pickle.load(f)

            store[hash] = THREADS_START

            # cap size (drop oldest entries)
            if len(store) > PICKLE_MAX_KEYS:
                for k in list(store.keys())[:len(store) - PICKLE_MAX_KEYS]:
                    store.pop(k, None)

            with open(pickle_path, "wb") as f:
                pickle.dump(store, f, protocol=pickle.HIGHEST_PROTOCOL)

        except Exception:
            pass

    except KeyboardInterrupt:
        print("[ENGINE] Interrupted by user")
    except Exception:
        import traceback
        traceback.print_exc()
        print("[ENGINE] Fatal crash")
        sys.exit(1)
    finally:
        executor.shutdown(wait=False)

