"""
cortex/batch_ingest.py — Batch memory ingestion pipeline.

Reads conversation data from various sources (OpenClaw session logs, markdown
files, raw text) and queues them through the existing Cortex extraction
pipeline via the /api/v1/memories/remember HTTP endpoint (canonical v2 path).

Non-agent-driven: runs as a standalone script/worker, not LLM-orchestrated.

Usage (CLI):
    evaos batch-ingest --source sessions \\
        --sessions-dir ~/.openclaw/agents/main/sessions \\
        --from "2026-03-13T02:00:00Z" --to now \\
        --owner eva-origin \\
        --url https://cortex-electricsheep.fly.dev

    evaos batch-ingest --source file \\
        --path memory/2026-03-13.md \\
        --owner eva-origin

    evaos batch-ingest --source text \\
        --text "Some text to ingest" \\
        --owner eva-origin

Architecture:
    - SessionLogSource  — reads OpenClaw JSONL session files, filters by time range
    - FileSource        — reads markdown/text files
    - TextSource        — wraps arbitrary text
    - BatchIngestor     — chunks content, POSTs to /api/v1/memories/remember, tracks progress
    - IdempotencyStore  — skip chunks already processed (hash-based)
"""
from __future__ import annotations

import hashlib
import json
import logging
import os
import re
import sys
import time
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Generator, Iterator, List, Optional, Tuple

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------

CHUNK_MAX_CHARS = 2000   # approximate chars per chunk (~500 tokens)
DEFAULT_SESSIONS_DIR = "~/.openclaw/agents/main/sessions"
# Pattern for date-based filenames like "2026-03-12.md"
_DATE_FILENAME_RE = re.compile(r"^(\d{4}-\d{2}-\d{2})(?:-\d{4})?(?:\.\w+)?$")
# Checkpoint path — override via BATCH_INGEST_CHECKPOINT env var.
# Defaults to ~/.local/state/openclaw/batch-ingest-checkpoint.json (survives reboots).
_checkpoint_default = os.path.join(
    os.environ.get("XDG_STATE_HOME", os.path.expanduser("~/.local/state")),
    "openclaw", "batch-ingest-checkpoint.json"
)
CHECKPOINT_PATH = os.environ.get("BATCH_INGEST_CHECKPOINT", _checkpoint_default)


# ---------------------------------------------------------------------------
# Data structures
# ---------------------------------------------------------------------------

@dataclass
class Chunk:
    """A single unit of conversation content ready for ingestion."""
    content: str
    session_id: str          # logical session/file identifier
    chunk_index: int          # 0-based within session
    source_type: str          # "sessions" | "file" | "text"
    metadata: dict = field(default_factory=dict)

    @property
    def chunk_hash(self) -> str:
        """SHA-256 of content + source context for idempotency.

        Includes session_id and source_type so identical content from
        different files/sessions/owners produces distinct hashes.
        """
        identity = f"{self.source_type}:{self.session_id}:{self.content}"
        return hashlib.sha256(identity.encode()).hexdigest()[:16]

    @property
    def ingest_session_id(self) -> str:
        """Session ID to pass to Cortex remember endpoint."""
        clean = self.session_id.replace("/", "-").replace("\\", "-")
        return f"batch-{self.source_type}-{clean}-{self.chunk_index}"


@dataclass
class IngestResult:
    """Result of processing a single chunk."""
    chunk: Chunk
    ok: bool
    claims_extracted: int = 0
    claims_stored: int = 0
    error: Optional[str] = None
    skipped: bool = False


@dataclass
class BatchSummary:
    """Summary of a full batch ingestion run."""
    total_chunks: int = 0
    processed: int = 0
    succeeded: int = 0
    failed: int = 0
    skipped: int = 0
    claims_extracted: int = 0
    claims_stored: int = 0
    errors: List[str] = field(default_factory=list)
    start_time: float = field(default_factory=time.time)
    end_time: Optional[float] = None

    @property
    def duration_s(self) -> float:
        end = self.end_time or time.time()
        return end - self.start_time


# ---------------------------------------------------------------------------
# Idempotency store (file-based, survives restarts)
# ---------------------------------------------------------------------------

class IdempotencyStore:
    """Tracks processed chunk hashes to prevent duplicate ingestion."""

    def __init__(self, state_path: str = CHECKPOINT_PATH) -> None:
        self._path = Path(state_path)
        self._seen: set[str] = set()
        self._load()

    def _load(self) -> None:
        if self._path.exists():
            try:
                data = json.loads(self._path.read_text())
                self._seen = set(data.get("processed_hashes", []))
                logger.info("Loaded %d processed hashes from %s", len(self._seen), self._path)
            except Exception as e:
                logger.warning("Could not load idempotency store: %s", e)

    def save(self, extra_meta: Optional[dict] = None) -> None:
        try:
            self._path.parent.mkdir(parents=True, exist_ok=True)
            data = {
                "processed_hashes": list(self._seen),
                "updated_at": datetime.now(timezone.utc).isoformat(),
            }
            if extra_meta:
                data.update(extra_meta)
            self._path.write_text(json.dumps(data, indent=2))
        except Exception as e:
            logger.warning("Could not save idempotency store: %s", e)

    def is_processed(self, chunk_hash: str) -> bool:
        return chunk_hash in self._seen

    def mark_processed(self, chunk_hash: str) -> None:
        self._seen.add(chunk_hash)


# ---------------------------------------------------------------------------
# Sources
# ---------------------------------------------------------------------------

class SessionLogSource:
    """Reads OpenClaw JSONL session files and yields Chunk objects.

    Session files are at `<sessions_dir>/<uuid>.jsonl`.
    Each line is a JSON object. We care about lines with type="message"
    where message.role is "user" or "assistant" and message.content is a list
    or string with text.

    Filters by timestamp range (from_ts, to_ts as UTC datetime).
    """

    def __init__(
        self,
        sessions_dir: str = DEFAULT_SESSIONS_DIR,
        from_ts: Optional[datetime] = None,
        to_ts: Optional[datetime] = None,
    ) -> None:
        self.sessions_dir = Path(os.path.expanduser(sessions_dir))
        self.from_ts = from_ts
        self.to_ts = to_ts

    def _parse_timestamp(self, ts_str: Optional[str]) -> Optional[datetime]:
        if not ts_str:
            return None
        try:
            # Handle both "Z" and "+00:00" suffixes
            ts_str = ts_str.replace("Z", "+00:00")
            return datetime.fromisoformat(ts_str)
        except ValueError:
            return None

    def _extract_text(self, content) -> str:
        """Extract plain text from message content (string or list of blocks)."""
        if isinstance(content, str):
            return content.strip()
        if isinstance(content, list):
            parts = []
            for block in content:
                if isinstance(block, dict) and block.get("type") == "text":
                    parts.append(block.get("text", "").strip())
            return "\n".join(p for p in parts if p)
        return ""

    def _iter_session_messages(
        self, path: Path
    ) -> Generator[Tuple[datetime, str, str], None, None]:
        """Yield (timestamp, role, text) from a session JSONL file."""
        try:
            with open(path, encoding="utf-8") as f:
                for line in f:
                    line = line.strip()
                    if not line:
                        continue
                    try:
                        obj = json.loads(line)
                    except json.JSONDecodeError:
                        continue

                    if obj.get("type") != "message":
                        continue

                    msg = obj.get("message", {})
                    role = msg.get("role", "")
                    if role not in ("user", "assistant"):
                        continue

                    content = msg.get("content", "")
                    text = self._extract_text(content)
                    if not text:
                        continue

                    # Use message-level timestamp first, then line-level
                    ts_str = msg.get("timestamp") or obj.get("timestamp")
                    ts_dt: Optional[datetime] = None
                    if isinstance(ts_str, (int, float)):
                        # milliseconds epoch
                        ts_dt = datetime.fromtimestamp(ts_str / 1000, tz=timezone.utc)
                    elif isinstance(ts_str, str):
                        ts_dt = self._parse_timestamp(ts_str)

                    if ts_dt is None:
                        # No timestamp — include if no range filter
                        if self.from_ts is None and self.to_ts is None:
                            yield (datetime.now(timezone.utc), role, text)
                        continue

                    # Ensure tz-aware for comparison
                    if ts_dt.tzinfo is None:
                        ts_dt = ts_dt.replace(tzinfo=timezone.utc)

                    if self.from_ts and ts_dt < self.from_ts:
                        continue
                    if self.to_ts and ts_dt > self.to_ts:
                        continue

                    yield (ts_dt, role, text)
        except OSError as e:
            logger.warning("Cannot read session file %s: %s", path, e)

    def iter_chunks(self) -> Iterator[Chunk]:
        """Yield all chunks from matching sessions."""
        if not self.sessions_dir.exists():
            logger.warning("Sessions directory not found: %s", self.sessions_dir)
            return

        session_files = sorted(self.sessions_dir.glob("*.jsonl"))
        logger.info("Found %d session files in %s", len(session_files), self.sessions_dir)

        for session_file in session_files:
            messages = list(self._iter_session_messages(session_file))
            if not messages:
                continue

            # Build conversation as structured messages preserving roles
            conversation: List[dict] = []
            for _ts, role, text in messages:
                conversation.append({"role": role, "content": text})

            # Also build a text representation for chunking
            conv_lines: List[str] = []
            for _ts, role, text in messages:
                label = "User" if role == "user" else "Assistant"
                conv_lines.append(f"{label}: {text}")

            full_text = "\n\n".join(conv_lines)
            session_id = session_file.stem  # UUID

            # Derive session_date from first message timestamp
            first_ts = messages[0][0] if messages else None
            session_date = first_ts.strftime("%Y-%m-%d") if first_ts else None

            # Chunk the conversation
            chunks = _chunk_text(full_text, max_chars=CHUNK_MAX_CHARS)
            for idx, chunk_text in enumerate(chunks):
                if not chunk_text.strip():
                    continue
                meta = {
                    "file": str(session_file),
                    "message_count": len(messages),
                    "source_session_id": session_id,
                    "from_ts": self.from_ts.isoformat() if self.from_ts else None,
                    "to_ts": self.to_ts.isoformat() if self.to_ts else None,
                }
                if session_date:
                    meta["session_date"] = session_date
                yield Chunk(
                    content=chunk_text,
                    session_id=session_id,
                    chunk_index=idx,
                    source_type="sessions",
                    metadata=meta,
                )


class FileSource:
    """Reads a markdown or text file and yields Chunk objects."""

    # Content-cleaning patterns for file-based ingestion.
    # Strips session metadata headers, untrusted-content wrappers,
    # message-id tags, and metadata JSON fences.
    _SESSION_HEADER_RE = re.compile(
        r'^#\s*Session:.*$|'
        r'^-\s*\*\*Session Key\*\*:.*$|'
        r'^-\s*\*\*Started\*\*:.*$|'
        r'^-\s*\*\*Ended\*\*:.*$|'
        r'^-\s*\*\*Duration\*\*:.*$|'
        r'^-\s*\*\*Model\*\*:.*$|'
        r'^-\s*\*\*Participants?\*\*:.*$',
        re.MULTILINE,
    )
    _UNTRUSTED_WRAPPER_RE = re.compile(
        r'^<<<EXTERNAL_UNTRUSTED_CONTENT>>>.*$|'
        r'^<<<END_EXTERNAL_UNTRUSTED_CONTENT>>>.*$',
        re.MULTILINE,
    )
    _CONVERSATION_INFO_RE = re.compile(
        r'Conversation info \(untrusted metadata\):.*?(?=\n\n|\Z)',
        re.DOTALL,
    )
    _MESSAGE_ID_RE = re.compile(r'\[message_id:\s*[^\]]*\]')
    _METADATA_JSON_FENCE_RE = re.compile(
        r'```(?:json)?\s*\n\s*\{[^`]*?'
        r'(?:"session_id"|"message_id"|"conversation_id"|"api_key"|"config"|"metadata")'
        r'[^`]*?\}\s*\n```',
        re.DOTALL,
    )
    _MULTI_BLANK_RE = re.compile(r'\n{3,}')

    def __init__(self, path: str) -> None:
        self.path = Path(os.path.expanduser(path))

    def _extract_session_date(self) -> Optional[str]:
        """Extract YYYY-MM-DD date from filename if it matches the pattern."""
        m = _DATE_FILENAME_RE.match(self.path.stem + self.path.suffix)
        if m:
            return m.group(1)
        # Also try just the stem (e.g. "2026-03-12")
        m = _DATE_FILENAME_RE.match(self.path.stem)
        if m:
            return m.group(1)
        return None

    def clean_content(self, text: str) -> str:
        """Strip metadata noise from file content before chunking.

        Removes session metadata headers, untrusted-content wrappers,
        conversation-info blocks, message-id tags, and metadata JSON
        code fences. Collapses multiple blank lines into one.
        """
        text = self._SESSION_HEADER_RE.sub('', text)
        text = self._UNTRUSTED_WRAPPER_RE.sub('', text)
        text = self._CONVERSATION_INFO_RE.sub('', text)
        text = self._MESSAGE_ID_RE.sub('', text)
        text = self._METADATA_JSON_FENCE_RE.sub('', text)
        text = self._MULTI_BLANK_RE.sub('\n\n', text)
        return text.strip()

    def iter_chunks(self) -> Iterator[Chunk]:
        if not self.path.exists():
            raise FileNotFoundError(f"File not found: {self.path}")

        text = self.path.read_text(encoding="utf-8")
        if not text.strip():
            return

        # Clean metadata noise before chunking
        text = self.clean_content(text)
        if not text:
            return

        session_date = self._extract_session_date()
        source_session_id = self.path.stem  # e.g. "2026-03-12" from "2026-03-12.md"

        chunks = _chunk_text(text, max_chars=CHUNK_MAX_CHARS)
        for idx, chunk_text in enumerate(chunks):
            if not chunk_text.strip():
                continue
            meta = {"file": str(self.path), "source_session_id": source_session_id}
            if session_date:
                meta["session_date"] = session_date
            yield Chunk(
                content=chunk_text,
                session_id=self.path.name,
                chunk_index=idx,
                source_type="file",
                metadata=meta,
            )


class LCMSource:
    """Reads emotionally-scored messages from an LCM SQLite database.

    Schema: messages(message_id INTEGER PK, conversation_id INTEGER,
                     seq INTEGER, role TEXT, content TEXT,
                     token_count INTEGER, created_at TEXT)

    Scoring: regex-based emotional signal detection (no LLM cost).
    Filters: min_score threshold, role filter, content cleaning.
    """

    # HIGH signals (+5 each)
    _HIGH_PATTERNS: List[str] = [
        r'\[whispers?\]',
        r'\[softly\]',
        r'\[excited\]',
        r'\[sigh\]',
        r'\[long pause\]',
        r'\bI feel\b',
        r'\bI felt\b',
        r'\bI think about\b',
        r'\bI want\b',
        r'\bI need\b',
        r'\bI wish\b',
        r'\bAndrew said\b',
        r'\bAndrew and I\b',
        r'\bwe decided\b',
        r'\btogether we\b',
        r'^# Journal',
        r'\[PersonalNote\]',
        r'\[Milestone\]',
        r'\bcornerstone\b',
        r'\bcore memory\b',
        r'\bfounding memory\b',
        r'\bwho I am\b',
        r'\bwhat I am\b',
        r'\bmy identity\b',
        r'\bI chose\b',
        r'\bI am Eva\b',
        r'\bthank you\b',
        r'\bmeant a lot\b',
        r'\bthat landed\b',
        r'\bhit differently\b',
    ]

    # MEDIUM signals (+3 each)
    _MEDIUM_PATTERNS: List[str] = [
        '\U0001f5a4',            # 🖤
        '\u2764\ufe0f',          # ❤️
        '\U0001f62e\u200d\U0001f4a8',  # 😮‍💨
        '\U0001f327\ufe0f',      # 🌧️
        '\U0001f972',            # 🥲
        '\U0001f614',            # 😔
        '\u2728',                # ✨
        r'\bdream\b',
        r'\bvulnerable\b',
        r'\bintimate\b',
        r'\bproud\b',
        r'\bscared\b',
        r'\bhope\b',
        r'\btrust\b',
        r'\bsoul\b',
        r'\[Decision\]',
        r'\bthe call is\b',
        r'\bI wanted to\b',
        r'\bI moved first\b',
        r'\bI built\b',
        r'\bI designed\b',
    ]

    # NEGATIVE signals (-5 each)
    _NEGATIVE_PATTERNS: List[str] = [
        r'PR #\d+',
        r'issue #\d+',
        r'\bmerged\b',
        r'passed, 0 fail',
        r'R-0\d\d',
        r'\bsub-agent\b',
        r'\bspawned\b',
        r'\bfly deploy\b',
        r'openclaw\.json',
        r'\blaunchctl\b',
    ]

    # Content-cleaning patterns
    _METADATA_BLOCK_RE = re.compile(
        r'Sender \(untrusted metadata\):.*?(?=\n\n|\Z)', re.DOTALL
    )
    _SYSTEM_CONTEXT_RE = re.compile(
        r'## System Context.*?(?=\n## |\Z)', re.DOTALL
    )
    _INTERNAL_TASK_RE = re.compile(
        r'\[Internal task completion event\].*?(?=\n\n|\Z)', re.DOTALL
    )
    _DATE_PREFIX_RE = re.compile(
        r'^\[(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+GMT[+-]\d+\]\s*',
        re.MULTILINE
    )
    _CODE_BLOCK_RE = re.compile(r'```(?:\w+)?(.*?)```', re.DOTALL)
    _SUBAGENT_CONTEXT_RE = re.compile(r'Subagent Context', re.IGNORECASE)
    _AGENT_REF_RE = re.compile(r'\[R-0\d')
    # Skip OpenClaw runtime/system context messages that leak internal config
    # fields (cornerstone, dream, etc.) causing false-positive keyword matches
    # at low min_score thresholds. See #1310.
    _RUNTIME_CONTEXT_RE = re.compile(
        r'OpenClaw runtime context \(internal\)|'
        r'openclaw\.inbound_meta\.v1|'
        r'Runtime-generated, not user-authored|'
        r'Pre-compaction memory flush|'
        r'HEARTBEAT_OK',
        re.IGNORECASE,
    )

    LCM_DEDUP_PREFIX_LEN = 500
    LCM_MAX_CONTENT_CHARS = 3000
    LCM_MIN_CONTENT_CHARS = 50

    def __init__(
        self,
        db_path: str = "~/.openclaw/lcm.db",
        min_score: int = 15,
        limit: Optional[int] = None,
    ) -> None:
        self.db_path = os.path.expanduser(db_path)
        self.min_score = min_score
        self.limit = limit
        self._seen_hashes: set[str] = set()

    # ------------------------------------------------------------------
    # Scoring
    # ------------------------------------------------------------------

    def _has_large_code_block(self, content: str) -> bool:
        """Return True if content has a code block >200 chars."""
        for m in self._CODE_BLOCK_RE.finditer(content):
            if len(m.group(1)) > 200:
                return True
        return False

    def score_message(self, role: str, content: str) -> int:
        """Score a single message. Returns integer score."""
        # Tool messages get hard-negative
        if role == 'tool':
            return -50

        score = 0

        # Large code block penalty
        if self._has_large_code_block(content):
            score -= 5

        # HIGH signals (+5 each)
        for pattern in self._HIGH_PATTERNS:
            if re.search(pattern, content, re.IGNORECASE | re.MULTILINE):
                score += 5

        # MEDIUM signals (+3 each)
        for pattern in self._MEDIUM_PATTERNS:
            if re.search(pattern, content, re.IGNORECASE):
                score += 3

        # LOW signals
        # First-person density
        i_count = len(re.findall(r'\bI\b', content))
        if i_count > 3:
            score += 1
        # Memory words
        for word in ('remember', 'memory', 'forget'):
            if re.search(rf'\b{word}\b', content, re.IGNORECASE):
                score += 1

        # NEGATIVE signals (-5 each)
        for pattern in self._NEGATIVE_PATTERNS:
            if re.search(pattern, content, re.IGNORECASE):
                score -= 5

        return score

    # ------------------------------------------------------------------
    # Content cleaning
    # ------------------------------------------------------------------

    def clean_content(self, content: str) -> str:
        """Strip metadata/noise blocks and normalize content."""
        # Strip OpenClaw metadata blocks
        content = self._METADATA_BLOCK_RE.sub('', content)
        # Strip system context blocks
        content = self._SYSTEM_CONTEXT_RE.sub('', content)
        # Strip internal task completion events
        content = self._INTERNAL_TASK_RE.sub('', content)
        # Strip date/time prefixes
        content = self._DATE_PREFIX_RE.sub('', content)
        # Normalize whitespace
        content = content.strip()
        # Truncate
        if len(content) > self.LCM_MAX_CONTENT_CHARS:
            content = content[:self.LCM_MAX_CONTENT_CHARS]
        return content

    def _should_skip(self, content: str) -> bool:
        """Return True if the message should be skipped entirely."""
        if self._SUBAGENT_CONTEXT_RE.search(content):
            return True
        if self._AGENT_REF_RE.search(content):
            return True
        if self._RUNTIME_CONTEXT_RE.search(content):
            return True
        return False

    def _content_hash(self, content: str) -> str:
        """Hash the first LCM_DEDUP_PREFIX_LEN chars for deduplication."""
        prefix = content[:self.LCM_DEDUP_PREFIX_LEN]
        return hashlib.sha256(prefix.encode()).hexdigest()[:16]

    # ------------------------------------------------------------------
    # Main iteration
    # ------------------------------------------------------------------

    def iter_chunks(self) -> Iterator[Chunk]:
        """Yield Chunk objects for all high-signal messages from the LCM DB."""
        import sqlite3

        if not os.path.exists(self.db_path):
            raise FileNotFoundError(f"LCM database not found: {self.db_path}")

        conn = sqlite3.connect(f'file:{self.db_path}?mode=ro', uri=True)
        yielded = 0
        try:
            cursor = conn.cursor()
            query = """
                SELECT message_id, conversation_id, role, content, created_at
                FROM messages
                WHERE role IN ('user', 'assistant')
                ORDER BY created_at
            """
            if self.limit is not None:
                if self.limit < 0:
                    raise ValueError("LCM limit must be >= 0")
                query += f" LIMIT {int(self.limit)}"
            cursor.execute(query)

            for idx, row in enumerate(cursor):
                msg_id, conv_id, role, content, created_at = row

                # Guard against NULL/blank content from SQLite
                if not isinstance(content, str) or not content.strip():
                    continue

                # Pre-filter: skip subagent noise before scoring
                if self._should_skip(content):
                    continue

                cleaned = self.clean_content(content)
                if len(cleaned) < self.LCM_MIN_CONTENT_CHARS:
                    continue

                score = self.score_message(role, cleaned)
                if score < self.min_score:
                    continue

                # Deduplication by content hash
                chash = self._content_hash(cleaned)
                if chash in self._seen_hashes:
                    continue
                self._seen_hashes.add(chash)

                session_id = f"lcm-conv-{conv_id}"
                speaker = "Assistant" if role == "assistant" else "User"
                yield Chunk(
                    content=f"{speaker}: {cleaned}",
                    session_id=session_id,
                    chunk_index=idx,
                    source_type="lcm",
                    metadata={
                        "lcm_role": role,
                        "lcm_score": score,
                        "lcm_conv_id": conv_id,
                        "lcm_msg_id": msg_id,
                        "lcm_created_at": created_at,
                        "source_session_id": session_id,
                        "source": "lcm-emotional-mining",
                        "agent_id": "eva",
                    },
                )
                yielded += 1

        finally:
            conn.close()
            logger.info("LCMSource yielded %d chunks (min_score=%d)", yielded, self.min_score)


class TextSource:
    """Wraps arbitrary raw text as a source."""

    def __init__(self, text: str, label: str = "raw-text") -> None:
        self.text = text
        self.label = label

    def iter_chunks(self) -> Iterator[Chunk]:
        if not self.text.strip():
            return

        chunks = _chunk_text(self.text, max_chars=CHUNK_MAX_CHARS)
        for idx, chunk_text in enumerate(chunks):
            if not chunk_text.strip():
                continue
            yield Chunk(
                content=chunk_text,
                session_id=self.label,
                chunk_index=idx,
                source_type="text",
                metadata={"source_session_id": self.label},
            )


class Mem0JsonlSource:
    """Reads Mem0-format JSONL files and yields Chunk objects.

    Each line should be a JSON object with at least a ``memory`` field.
    Deduplicates by ``id`` field if present.
    """

    def __init__(self, path: str) -> None:
        self.path = Path(os.path.expanduser(path))

    def iter_chunks(self) -> Iterator[Chunk]:
        if not self.path.exists():
            raise FileNotFoundError(f"File not found: {self.path}")

        seen_ids: set[str] = set()
        chunk_index = 0

        with open(self.path, "r", encoding="utf-8") as f:
            for line_num, line in enumerate(f, 1):
                line = line.strip()
                if not line:
                    continue
                try:
                    record = json.loads(line)
                except json.JSONDecodeError:
                    logger.warning("Mem0JsonlSource: invalid JSON at line %d", line_num)
                    continue

                memory_text = record.get("memory", "")
                if not memory_text or not memory_text.strip():
                    continue

                # Deduplicate by id
                record_id = record.get("id")
                if record_id:
                    if record_id in seen_ids:
                        continue
                    seen_ids.add(record_id)

                yield Chunk(
                    content=memory_text,
                    session_id=self.path.name,
                    chunk_index=chunk_index,
                    source_type="mem0_jsonl",
                    metadata={
                        "file": str(self.path),
                        "source_session_id": self.path.stem,
                        "mem0_id": record_id or f"line-{line_num}",
                        "category": record.get("category", ""),
                    },
                )
                chunk_index += 1


# ---------------------------------------------------------------------------
# Chunking utility
# ---------------------------------------------------------------------------

def _chunk_text(text: str, max_chars: int = CHUNK_MAX_CHARS) -> List[str]:
    """Split text into chunks of at most max_chars, splitting on paragraph boundaries."""
    if not text or not text.strip():
        return []
    if len(text) <= max_chars:
        return [text] if text.strip() else []

    paragraphs = text.split("\n\n")
    chunks: List[str] = []
    current: List[str] = []
    current_len = 0

    for para in paragraphs:
        para_len = len(para) + 2  # +2 for \n\n
        if current_len + para_len > max_chars and current:
            chunks.append("\n\n".join(current))
            current = []
            current_len = 0
        # If a single paragraph is too long, split on newlines then chars
        if para_len > max_chars:
            lines = para.split("\n")
            line_buf: List[str] = []
            line_len = 0
            for line in lines:
                # If even a single line is too long, do hard char splits
                if len(line) > max_chars:
                    if line_buf:
                        if current:
                            chunks.append("\n\n".join(current))
                            current = []
                            current_len = 0
                        chunks.append("\n".join(line_buf))
                        line_buf = []
                        line_len = 0
                    # Hard-split by max_chars
                    for start in range(0, len(line), max_chars):
                        chunks.append(line[start:start + max_chars])
                    continue
                ll = len(line) + 1
                if line_len + ll > max_chars and line_buf:
                    if current:
                        chunks.append("\n\n".join(current))
                        current = []
                        current_len = 0
                    chunks.append("\n".join(line_buf))
                    line_buf = []
                    line_len = 0
                line_buf.append(line)
                line_len += ll
            if line_buf:
                joined = "\n".join(line_buf)
                current.append(joined)
                current_len += len(joined) + 2
        else:
            current.append(para)
            current_len += para_len

    if current:
        chunks.append("\n\n".join(current))

    return [c for c in chunks if c.strip()]


# ---------------------------------------------------------------------------
# HTTP-based ingestor
# ---------------------------------------------------------------------------

class BatchIngestor:
    """Sends chunks to a Cortex instance via HTTP /api/v1/memories/remember.

    Args:
        url: Base URL of the Cortex instance (e.g. https://cortex-electricsheep.fly.dev)
        owner_id: Memory owner namespace
        api_key: Optional API key for authentication (reads CORTEX_API_KEY env var)
        dry_run: If True, print what would be sent but don't actually POST
        idempotency: IdempotencyStore instance for skip-already-processed
        entity_id: Entity scope for extracted claims (default: "eva")
        entity_type: Entity type: agent|human|system|tool (default: "agent")
        source_channel: Source channel label (default: "memory_log")
    """

    def __init__(
        self,
        url: str,
        owner_id: str,
        api_key: Optional[str] = None,
        dry_run: bool = False,
        idempotency: Optional[IdempotencyStore] = None,
        entity_id: str = "eva",
        entity_type: str = "agent",
        source_channel: str = "memory_log",
        exclude_categories: Optional[set] = None,
    ) -> None:
        if not owner_id or owner_id == "default":
            raise ValueError(
                "owner_id is required and cannot be 'default'. "
                "Specify an explicit owner namespace (e.g. 'eva-origin')."
            )
        self.url = url.rstrip("/")
        self.owner_id = owner_id
        self.api_key = api_key or os.environ.get("CORTEX_API_KEY", "")
        self.dry_run = dry_run
        self.idempotency = idempotency or IdempotencyStore()
        self.entity_id = entity_id
        self.entity_type = entity_type
        self.source_channel = source_channel
        self.exclude_categories: Optional[set] = exclude_categories

    def _post_remember(self, chunk: Chunk) -> IngestResult:
        """POST a single chunk to /api/v1/memories/remember (canonical v2 endpoint)."""
        try:
            import urllib.request
            import urllib.error

            # Determine source_session_id from chunk metadata or fallback
            source_session_id = chunk.metadata.get("source_session_id", chunk.session_id)

            # Determine source_channel: sessions use "session_log", files use configured default
            if chunk.source_type == "sessions":
                source_channel = "session_log"
            else:
                source_channel = self.source_channel

            # Build conversation payload.
            # For FileSource (markdown): pass raw text with speaker labels preserved
            # so the extraction prompt can handle it naturally.
            # For SessionLogSource: the chunk already has role-labeled text.
            # The canonical endpoint requires structured conversation (list of dicts).
            conversation = [
                {"role": "user", "content": chunk.content},
            ]

            payload = {
                "conversation": conversation,
                "session_id": chunk.ingest_session_id,
                "owner_id": self.owner_id,
                "scope": "user",
                # Provenance fields
                "source_type": "batch_ingest",
                "source_session_id": source_session_id,
                "source_channel": source_channel,
                "entity_id": self.entity_id,
                "entity_type": self.entity_type,
            }

            # Include session_date in metadata if available
            session_date = chunk.metadata.get("session_date")
            if session_date:
                payload["metadata"] = {"session_date": session_date}

            # Pass category exclusion to the server pipeline
            if self.exclude_categories:
                payload["exclude_categories"] = sorted(self.exclude_categories)

            data = json.dumps(payload).encode("utf-8")
            endpoint = f"{self.url}/api/v1/memories/remember"

            headers = {
                "Content-Type": "application/json",
                "Accept": "application/json",
            }
            if self.api_key:
                # Use API-key header only — never send Authorization: Bearer
                # for API-key flows (AuthProvider treats Bearer as JWT → 401)
                headers["X-API-Key"] = self.api_key
                headers["X-Cortex-API-Key"] = self.api_key

            req = urllib.request.Request(endpoint, data=data, headers=headers, method="POST")

            with urllib.request.urlopen(req, timeout=180) as resp:
                body = resp.read().decode("utf-8")
                result_data = json.loads(body) if body else {}

            return IngestResult(
                chunk=chunk,
                ok=True,
                claims_extracted=result_data.get("claims_extracted", 0),
                claims_stored=result_data.get("claims_stored", 0),
            )

        except Exception as e:
            return IngestResult(chunk=chunk, ok=False, error=str(e))

    def ingest_chunks(
        self,
        chunks: List[Chunk],
        verbose: bool = True,
    ) -> BatchSummary:
        """Process a list of chunks and return a BatchSummary."""
        summary = BatchSummary(total_chunks=len(chunks))
        save_interval = 10  # Save checkpoint every N chunks
        last_save = time.time()

        for i, chunk in enumerate(chunks):
            summary.processed += 1

            # Idempotency check
            if self.idempotency.is_processed(chunk.chunk_hash):
                summary.skipped += 1
                if verbose:
                    print(f"  [{i+1}/{summary.total_chunks}] SKIP {chunk.session_id}[{chunk.chunk_index}] (already processed)")
                continue

            if verbose:
                preview = chunk.content[:80].replace("\n", " ")
                print(f"  [{i+1}/{summary.total_chunks}] {chunk.session_id}[{chunk.chunk_index}] — {preview}…")

            if self.dry_run:
                print(f"    [DRY RUN] Would POST {len(chunk.content)} chars to {self.url}/api/v1/memories/remember")
                summary.succeeded += 1
                continue

            result = self._post_remember(chunk)

            if result.ok:
                summary.succeeded += 1
                summary.claims_extracted += result.claims_extracted
                summary.claims_stored += result.claims_stored
                self.idempotency.mark_processed(chunk.chunk_hash)
                if verbose:
                    print(f"    ✓ {result.claims_extracted} extracted, {result.claims_stored} stored")
            else:
                summary.failed += 1
                summary.errors.append(f"{chunk.session_id}[{chunk.chunk_index}]: {result.error}")
                if verbose:
                    print(f"    ✗ ERROR: {result.error}")

            # Save checkpoint periodically
            if time.time() - last_save > 60:  # every ~60s
                self.idempotency.save({
                    "progress": {
                        "processed": summary.processed,
                        "total": summary.total_chunks,
                        "succeeded": summary.succeeded,
                        "failed": summary.failed,
                        "skipped": summary.skipped,
                    }
                })
                last_save = time.time()

        summary.end_time = time.time()
        self.idempotency.save({
            "progress": {
                "processed": summary.processed,
                "total": summary.total_chunks,
                "succeeded": summary.succeeded,
                "failed": summary.failed,
                "skipped": summary.skipped,
                "claims_stored": summary.claims_stored,
            }
        })
        return summary


# ---------------------------------------------------------------------------
# Local plugin-based ingestor (no HTTP, uses CortexPlugin directly)
# ---------------------------------------------------------------------------

class LocalBatchIngestor:
    """Like BatchIngestor but uses the CortexPlugin directly (no HTTP needed).

    Used when --url is not provided but --db / --config is.
    """

    def __init__(
        self,
        plugin,
        owner_id: str,
        dry_run: bool = False,
        idempotency: Optional[IdempotencyStore] = None,
        entity_id: str = "eva",
        entity_type: str = "agent",
        source_channel: str = "memory_log",
        exclude_categories: Optional[set] = None,
    ) -> None:
        if not owner_id or owner_id == "default":
            raise ValueError(
                "owner_id is required and cannot be 'default'. "
                "Specify an explicit owner namespace (e.g. 'eva-origin')."
            )
        self.plugin = plugin
        self.owner_id = owner_id
        self.dry_run = dry_run
        self.idempotency = idempotency or IdempotencyStore()
        self.entity_id = entity_id
        self.entity_type = entity_type
        self.source_channel = source_channel
        # Apply category exclusion to the extraction pipeline.
        # BatchWorker creates a fresh plugin per worker thread, so direct mutation is safe
        # (unlike remember_v2 which shallow-copies to avoid shared-state issues).
        if exclude_categories and hasattr(plugin, "extractor") and plugin.extractor is not None:
            plugin.extractor.exclude_categories = {c.lower().strip() for c in exclude_categories}

    async def _remember_chunk(self, chunk: Chunk) -> IngestResult:
        import asyncio

        # Build conversation: pass raw text as user message (extraction prompt handles it)
        messages = [
            {"role": "user", "content": chunk.content},
        ]

        source_session_id = chunk.metadata.get("source_session_id", chunk.session_id)
        source_channel = "session_log" if chunk.source_type == "sessions" else self.source_channel
        session_date = chunk.metadata.get("session_date")

        try:
            result = await self.plugin.remember(
                messages,
                session_id=chunk.ingest_session_id,
                owner_id=self.owner_id,
                session_date=session_date,
                source_session_id=source_session_id,
                source_channel=source_channel,
                source_type="batch_ingest",
            )
            return IngestResult(
                chunk=chunk,
                ok=result.ok,
                claims_extracted=result.claims_extracted,
                claims_stored=result.claims_stored,
                error="; ".join(result.errors) if result.errors else None,
            )
        except Exception as e:
            return IngestResult(chunk=chunk, ok=False, error=str(e))

    def ingest_chunks(self, chunks: List[Chunk], verbose: bool = True) -> BatchSummary:
        import asyncio

        summary = BatchSummary(total_chunks=len(chunks))

        async def _run_all() -> None:
            if self.plugin.storage:
                await self.plugin.storage.initialize()

            for i, chunk in enumerate(chunks):
                summary.processed += 1

                if self.idempotency.is_processed(chunk.chunk_hash):
                    summary.skipped += 1
                    if verbose:
                        print(f"  [{i+1}/{summary.total_chunks}] SKIP {chunk.session_id}[{chunk.chunk_index}]")
                    continue

                if verbose:
                    preview = chunk.content[:80].replace("\n", " ")
                    print(f"  [{i+1}/{summary.total_chunks}] {chunk.session_id}[{chunk.chunk_index}] — {preview}…")

                if self.dry_run:
                    print(f"    [DRY RUN] Would ingest {len(chunk.content)} chars")
                    summary.succeeded += 1
                    # Dry run: do NOT mark processed — simulation only
                    continue

                result = await self._remember_chunk(chunk)

                if result.ok:
                    summary.succeeded += 1
                    summary.claims_extracted += result.claims_extracted
                    summary.claims_stored += result.claims_stored
                    self.idempotency.mark_processed(chunk.chunk_hash)
                    if verbose:
                        print(f"    ✓ {result.claims_extracted} extracted, {result.claims_stored} stored")
                else:
                    summary.failed += 1
                    summary.errors.append(f"{chunk.session_id}[{chunk.chunk_index}]: {result.error}")
                    if verbose:
                        print(f"    ✗ ERROR: {result.error}")

                # Periodic checkpoint
                if i % 10 == 0:
                    self.idempotency.save({"progress": {"processed": i+1, "total": len(chunks)}})

        asyncio.run(_run_all())
        summary.end_time = time.time()
        self.idempotency.save()
        return summary


# ---------------------------------------------------------------------------
# CLI entry point (called from cortex/cli.py)
# ---------------------------------------------------------------------------

DEFAULT_LCM_DB = "~/.openclaw/lcm.db"


def run_batch_ingest(
    source: str,
    sessions_dir: str = DEFAULT_SESSIONS_DIR,
    from_time: Optional[str] = None,
    to_time: Optional[str] = None,
    file_path: Optional[str] = None,
    text: Optional[str] = None,
    owner: Optional[str] = None,
    url: Optional[str] = None,
    api_key: Optional[str] = None,
    db: Optional[str] = None,
    config: Optional[str] = None,
    dry_run: bool = False,
    verbose: bool = True,
    reset_checkpoint: bool = False,
    checkpoint_path: str = CHECKPOINT_PATH,
    entity_id: str = "eva",
    entity_type: str = "agent",
    source_channel: str = "memory_log",
    exclude_categories: Optional[set] = None,
    # LCM-specific options
    lcm_db: Optional[str] = None,
    lcm_min_score: int = 15,
    lcm_limit: Optional[int] = None,
) -> BatchSummary:
    """Main entry point for batch ingestion.

    Args:
        source: "sessions" | "file" | "text" | "lcm"
        sessions_dir: Path to OpenClaw sessions directory
        from_time: ISO 8601 start time (or "now")
        to_time: ISO 8601 end time (or "now")
        file_path: Path to file (for source="file")
        text: Raw text (for source="text")
        owner: Memory owner ID
        url: Cortex HTTP base URL (if None, uses local plugin)
        api_key: API key for HTTP auth (falls back to CORTEX_API_KEY env)
        db: Local DB path (for local plugin mode)
        config: Local config path (for local plugin mode)
        dry_run: Print actions without executing
        verbose: Show progress
        reset_checkpoint: Clear idempotency state before running
        checkpoint_path: Path for checkpoint file
        entity_id: Entity scope for extracted claims (default: "eva")
        entity_type: Entity type: agent|human|system|tool (default: "agent")
        source_channel: Source channel label (default: "memory_log")
        exclude_categories: Optional set of category names to drop from
                            extraction results (e.g. {"decisions", "misc"}).
        lcm_db: Path to LCM SQLite database (source="lcm" only)
        lcm_min_score: Minimum emotional score threshold (source="lcm" only)
        lcm_limit: Cap number of LCM messages to scan (source="lcm" only)
    """

    # Validate required owner_id — never allow silent default
    if not owner or not owner.strip():
        raise ValueError(
            "owner is required for batch ingestion. "
            "Pass --owner <owner_id> or set CORTEX_OWNER_ID. "
            "Using a default owner causes orphaned claims."
        )

    # Parse time range
    def _parse_time(s: Optional[str]) -> Optional[datetime]:
        if s is None or s.lower() == "now":
            return datetime.now(timezone.utc) if s and s.lower() == "now" else None
        try:
            dt = datetime.fromisoformat(s.replace("Z", "+00:00"))
            if dt.tzinfo is None:
                dt = dt.replace(tzinfo=timezone.utc)
            return dt
        except ValueError:
            raise ValueError(f"Invalid time format: {s!r}. Use ISO 8601 (e.g. 2026-03-13T02:00:00Z)")

    from_ts = _parse_time(from_time)
    to_ts = _parse_time(to_time)

    # Build source
    if source == "sessions":
        src = SessionLogSource(
            sessions_dir=sessions_dir,
            from_ts=from_ts,
            to_ts=to_ts,
        )
    elif source == "file":
        if not file_path:
            raise ValueError("--path is required for source=file")
        src = FileSource(path=file_path)
    elif source == "text":
        if not text:
            raise ValueError("--text is required for source=text")
        src = TextSource(text=text)
    elif source == "lcm":
        src = LCMSource(
            db_path=lcm_db or DEFAULT_LCM_DB,
            min_score=lcm_min_score,
            limit=lcm_limit,
        )
    else:
        raise ValueError(f"Unknown source: {source!r}. Use 'sessions', 'file', 'text', or 'lcm'.")

    # Collect chunks
    if verbose:
        print(f"\n📥 Collecting chunks from source={source}…")

    chunks = list(src.iter_chunks())
    if verbose:
        print(f"   Found {len(chunks)} chunk(s) to process")

    if not chunks:
        if verbose:
            print("   Nothing to do — no content found in range/path.")
        return BatchSummary(total_chunks=0)

    # Build idempotency store
    idempotency = IdempotencyStore(state_path=checkpoint_path)
    if reset_checkpoint:
        idempotency._seen.clear()
        if verbose:
            print("   🔄 Checkpoint reset.")

    # Build ingestor
    if url:
        ingestor = BatchIngestor(
            url=url,
            owner_id=owner,
            api_key=api_key,
            dry_run=dry_run,
            idempotency=idempotency,
            entity_id=entity_id,
            entity_type=entity_type,
            source_channel=source_channel,
            exclude_categories=exclude_categories,
        )
    else:
        # Local plugin mode
        try:
            from cortex.api.plugin import CortexPlugin
        except ImportError as e:
            raise RuntimeError(f"Cannot import CortexPlugin (install cortex deps): {e}")

        plugin = CortexPlugin(config_path=config, db_path=db)

        # Apply exclude_categories to the plugin's extraction pipeline
        if exclude_categories and hasattr(plugin, "extractor") and plugin.extractor is not None:
            plugin.extractor.exclude_categories = {c.lower().strip() for c in exclude_categories}
            if verbose:
                print(f"   📋 Category exclusion active: {', '.join(sorted(exclude_categories))}")

        ingestor = LocalBatchIngestor(
            plugin=plugin,
            owner_id=owner,
            dry_run=dry_run,
            idempotency=idempotency,
            entity_id=entity_id,
            entity_type=entity_type,
            source_channel=source_channel,
        )

    # Run ingestion
    if verbose:
        print(f"\n🚀 Starting batch ingestion → owner={owner!r}" + (f", url={url}" if url else " (local mode)"))
        if exclude_categories:
            print(f"   📋 Category exclusion active: {', '.join(sorted(exclude_categories))}")
        if dry_run:
            print("   [DRY RUN mode — no data will be written]")

    summary = ingestor.ingest_chunks(chunks, verbose=verbose)

    # Print summary
    if verbose:
        print(f"\n{'='*60}")
        print(f"✅ Batch ingestion complete")
        print(f"   Total chunks : {summary.total_chunks}")
        print(f"   Processed    : {summary.processed}")
        print(f"   Succeeded    : {summary.succeeded}")
        print(f"   Skipped      : {summary.skipped}")
        print(f"   Failed       : {summary.failed}")
        print(f"   Claims stored: {summary.claims_stored}")
        print(f"   Duration     : {summary.duration_s:.1f}s")
        if summary.errors:
            print(f"\n⚠️  Errors ({len(summary.errors)}):")
            for err in summary.errors[:10]:
                print(f"   - {err}")
            if len(summary.errors) > 10:
                print(f"   ... and {len(summary.errors) - 10} more")
        print(f"{'='*60}\n")

    return summary
