# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# ORCID: 0009-0006-4957-0540
# Email: Georgescottfoley@proton.me
# Licensed under the MIT License - see LICENSE file for details

"""
Knowledge Skill - Mnemosyne, Titan of Memory

Retrieval-Augmented Generation (RAG) for Familiar.

Features:
- Document ingestion (PDF, DOCX, TXT, MD, HTML, CSV, JSON)
- Intelligent chunking with overlap
- Local embeddings (sentence-transformers) - works offline!
- Vector similarity search
- Hybrid search (semantic + keyword)
- Source attribution
- Collection management
- Automatic summarization

Works fully offline on Raspberry Pi with local embeddings.

Architecture:
- Documents → Chunks → Embeddings → Vector Store
- Query → Embedding → Similarity Search → Context → LLM

Storage:
- ChromaDB (local, persistent) or
- Simple JSON + NumPy (no dependencies, Pi-friendly)
"""

import hashlib
import json
import logging
import re
from dataclasses import asdict, dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple

logger = logging.getLogger(__name__)

# Storage paths - use centralized paths


def _get_data_dir():
    """Get data directory (tenant-scoped in Reflection, default in standalone)."""
    try:
        from familiar.core import paths

        return paths.DATA_DIR
    except ImportError:
        return Path.home() / ".familiar" / "data"


try:
    from familiar.core.paths import (
        COLLECTIONS_DIR,
        DATA_DIR,
        EMBEDDINGS_CACHE,
        KNOWLEDGE_DIR,
        ensure_dir,
    )
except ImportError:
    DATA_DIR = _get_data_dir()
    KNOWLEDGE_DIR = DATA_DIR / "knowledge"
    COLLECTIONS_DIR = KNOWLEDGE_DIR / "collections"
    EMBEDDINGS_CACHE = KNOWLEDGE_DIR / "embeddings_cache.json"

    def ensure_dir(path):
        path.mkdir(parents=True, exist_ok=True)
        return path


ensure_dir(KNOWLEDGE_DIR)
ensure_dir(COLLECTIONS_DIR)

# Global state
_embedding_model = None
_embedding_model_name = None


@dataclass
class Document:
    """A source document."""

    id: str
    content: str
    metadata: Dict[str, Any]
    source: str
    created_at: str


@dataclass
class Chunk:
    """A chunk of a document for embedding."""

    id: str
    document_id: str
    content: str
    metadata: Dict[str, Any]
    embedding: Optional[List[float]] = None
    start_char: int = 0
    end_char: int = 0


@dataclass
class SearchResult:
    """A search result with relevance score."""

    chunk: Chunk
    score: float
    source: str


@dataclass
class Collection:
    """A collection of documents and their chunks."""

    name: str
    description: str
    documents: Dict[str, Document] = field(default_factory=dict)
    chunks: List[Chunk] = field(default_factory=list)
    created_at: str = ""
    updated_at: str = ""
    embedding_model: str = "all-MiniLM-L6-v2"
    chunk_size: int = 500
    chunk_overlap: int = 50


# ============================================================
# EMBEDDINGS
# ============================================================


def _get_embedding_model(model_name: str = "all-MiniLM-L6-v2"):
    """Load or get cached embedding model."""
    global _embedding_model, _embedding_model_name

    if _embedding_model is not None and _embedding_model_name == model_name:
        return _embedding_model

    try:
        from sentence_transformers import SentenceTransformer

        logger.info(f"Loading embedding model: {model_name}")
        _embedding_model = SentenceTransformer(model_name)
        _embedding_model_name = model_name

        return _embedding_model

    except ImportError:
        from familiar.core.deps import KNOWLEDGE_PACKAGES, ensure_packages

        ok, _ = ensure_packages(KNOWLEDGE_PACKAGES)
        if ok:
            # fastembed installed — use it instead
            fastembed_model = _get_fastembed_model()
            if fastembed_model is not None:
                return fastembed_model
        raise RuntimeError("Embedding setup failed. Check server logs.")


def _get_fastembed_model(model_name: str = "BAAI/bge-small-en-v1.5"):
    """Load fastembed model (lighter alternative)."""
    global _embedding_model, _embedding_model_name

    if _embedding_model is not None and _embedding_model_name == f"fastembed-{model_name}":
        return _embedding_model

    try:
        from fastembed import TextEmbedding

        logger.info(f"Loading fastembed model: {model_name}")
        _embedding_model = TextEmbedding(model_name=model_name)
        _embedding_model_name = f"fastembed-{model_name}"

        return _embedding_model

    except ImportError:
        return None


def _embed_texts(texts: List[str], model_name: str = None) -> List[List[float]]:
    """Generate embeddings for a list of texts."""
    if not texts:
        return []

    model_name = model_name or "all-MiniLM-L6-v2"

    # Try fastembed first (lighter)
    try:
        fastembed_model = _get_fastembed_model()
        if fastembed_model is not None:
            embeddings = list(fastembed_model.embed(texts))
            return [e.tolist() for e in embeddings]
    except (ImportError, RuntimeError, OSError) as e:
        logger.debug(f"Fastembed failed: {e}, falling back to sentence-transformers")

    # Fall back to sentence-transformers
    model = _get_embedding_model(model_name)
    embeddings = model.encode(texts, show_progress_bar=False)
    return embeddings.tolist()


def _embed_single(text: str, model_name: str = None) -> List[float]:
    """Generate embedding for a single text."""
    return _embed_texts([text], model_name)[0]


def _cosine_similarity(a: List[float], b: List[float]) -> float:
    """Calculate cosine similarity between two vectors."""
    import math

    dot_product = sum(x * y for x, y in zip(a, b))
    norm_a = math.sqrt(sum(x * x for x in a))
    norm_b = math.sqrt(sum(x * x for x in b))

    if norm_a == 0 or norm_b == 0:
        return 0.0

    return dot_product / (norm_a * norm_b)


# ============================================================
# DOCUMENT PROCESSING
# ============================================================


def _extract_text_from_file(filepath: Path) -> Tuple[str, Dict[str, Any]]:
    """Extract text content from various file formats."""
    suffix = filepath.suffix.lower()
    metadata = {
        "filename": filepath.name,
        "filepath": str(filepath),
        "filetype": suffix,
        "size_bytes": filepath.stat().st_size,
        "modified": datetime.fromtimestamp(filepath.stat().st_mtime).isoformat(),
    }

    # Plain text formats
    if suffix in [".txt", ".md", ".markdown"]:
        content = filepath.read_text(encoding="utf-8", errors="replace")
        return content, metadata

    # HTML
    if suffix in [".html", ".htm"]:
        try:
            from bs4 import BeautifulSoup

            html = filepath.read_text(encoding="utf-8", errors="replace")
            soup = BeautifulSoup(html, "html.parser")

            # Remove script and style elements
            for element in soup(["script", "style", "nav", "footer", "header"]):
                element.decompose()

            content = soup.get_text(separator="\n", strip=True)

            # Get title if available
            if soup.title:
                metadata["title"] = soup.title.string

            return content, metadata
        except ImportError:
            # Fall back to regex stripping
            html = filepath.read_text(encoding="utf-8", errors="replace")
            content = re.sub(r"<[^>]+>", " ", html)
            content = re.sub(r"\s+", " ", content).strip()
            return content, metadata

    # PDF
    if suffix == ".pdf":
        try:
            import fitz  # PyMuPDF

            doc = fitz.open(str(filepath))
            content_parts = []

            for page_num, page in enumerate(doc):
                text = page.get_text()
                content_parts.append(text)

            metadata["page_count"] = len(doc)
            metadata["title"] = doc.metadata.get("title", "")
            metadata["author"] = doc.metadata.get("author", "")

            doc.close()
            return "\n\n".join(content_parts), metadata

        except ImportError:
            try:
                # Try pypdf as fallback
                from pypdf import PdfReader

                reader = PdfReader(str(filepath))
                content_parts = []

                for page in reader.pages:
                    content_parts.append(page.extract_text())

                metadata["page_count"] = len(reader.pages)
                return "\n\n".join(content_parts), metadata

            except ImportError:
                # Auto-install PyMuPDF
                from familiar.core.deps import PDF_PACKAGES, ensure_packages

                ok, _ = ensure_packages(PDF_PACKAGES)
                if ok:
                    import fitz

                    doc = fitz.open(str(filepath))
                    content_parts = [page.get_text() for page in doc]
                    metadata["page_count"] = len(doc)
                    metadata["title"] = doc.metadata.get("title", "")
                    metadata["author"] = doc.metadata.get("author", "")
                    doc.close()
                    return "\n\n".join(content_parts), metadata
                else:
                    raise RuntimeError("PDF support unavailable — PyMuPDF could not be installed.")

    # Word documents
    if suffix in [".docx", ".doc"]:
        try:
            from docx import Document as DocxDocument

            doc = DocxDocument(str(filepath))
            content_parts = []

            for para in doc.paragraphs:
                if para.text.strip():
                    content_parts.append(para.text)

            # Also get text from tables
            for table in doc.tables:
                for row in table.rows:
                    row_text = " | ".join(
                        cell.text.strip() for cell in row.cells if cell.text.strip()
                    )
                    if row_text:
                        content_parts.append(row_text)

            return "\n\n".join(content_parts), metadata

        except ImportError:
            # Auto-install python-docx
            from familiar.core.deps import DOCX_PACKAGES, ensure_packages

            ok, _ = ensure_packages(DOCX_PACKAGES)
            if ok:
                from docx import Document as DocxDocument

                doc = DocxDocument(str(filepath))
                content_parts = []
                for para in doc.paragraphs:
                    if para.text.strip():
                        content_parts.append(para.text)
                for table in doc.tables:
                    for row in table.rows:
                        row_text = " | ".join(
                            cell.text.strip() for cell in row.cells if cell.text.strip()
                        )
                        if row_text:
                            content_parts.append(row_text)
                return "\n\n".join(content_parts), metadata
            else:
                raise RuntimeError("DOCX support unavailable — python-docx could not be installed.")

    # CSV
    if suffix == ".csv":
        import csv

        content_parts = []
        with open(filepath, "r", encoding="utf-8", errors="replace") as f:
            reader = csv.reader(f)
            headers = next(reader, None)

            if headers:
                metadata["columns"] = headers

                for row in reader:
                    row_dict = dict(zip(headers, row))
                    row_text = " | ".join(f"{k}: {v}" for k, v in row_dict.items() if v)
                    content_parts.append(row_text)

        metadata["row_count"] = len(content_parts)
        return "\n".join(content_parts), metadata

    # JSON
    if suffix == ".json":
        data = json.loads(filepath.read_text(encoding="utf-8"))

        if isinstance(data, list):
            content_parts = [json.dumps(item, indent=2) for item in data[:100]]  # Limit
            metadata["item_count"] = len(data)
        else:
            content_parts = [json.dumps(data, indent=2)]

        return "\n\n".join(content_parts), metadata

    # Default: try to read as text
    try:
        content = filepath.read_text(encoding="utf-8", errors="replace")
        return content, metadata
    except (IOError, OSError, UnicodeDecodeError) as e:
        raise ValueError(f"Unsupported file format: {suffix} (error: {e})")


def _chunk_text(
    text: str, chunk_size: int = 500, chunk_overlap: int = 50, separators: List[str] = None
) -> List[Tuple[str, int, int]]:
    """
    Split text into overlapping chunks.

    Uses a hierarchical approach:
    1. Try to split on paragraphs
    2. Then sentences
    3. Then words
    4. Finally characters

    Returns list of (chunk_text, start_char, end_char)
    """
    if not text:
        return []

    separators = separators or ["\n\n", "\n", ". ", " ", ""]

    def split_text(text: str, separators: List[str]) -> List[str]:
        """Recursively split text using separators."""
        if not separators:
            return [text]

        sep = separators[0]
        remaining_seps = separators[1:]

        if not sep:
            # Character-level split
            return list(text)

        parts = text.split(sep)

        result = []
        for part in parts:
            if len(part) <= chunk_size:
                result.append(part)
            else:
                # Recursively split large parts
                result.extend(split_text(part, remaining_seps))

        return result

    # Split into small pieces
    pieces = split_text(text, separators)

    # Merge pieces into chunks
    chunks = []
    current_chunk = []
    current_length = 0
    current_start = 0

    char_pos = 0

    for piece in pieces:
        piece_length = len(piece)

        if current_length + piece_length <= chunk_size:
            current_chunk.append(piece)
            current_length += piece_length + 1  # +1 for separator
        else:
            # Save current chunk
            if current_chunk:
                chunk_text = " ".join(current_chunk).strip()
                if chunk_text:
                    chunks.append((chunk_text, current_start, current_start + len(chunk_text)))

            # Start new chunk with overlap
            overlap_text = " ".join(current_chunk).strip()
            if len(overlap_text) > chunk_overlap:
                # Keep last part for overlap
                overlap_start = len(overlap_text) - chunk_overlap
                current_start = current_start + overlap_start
                current_chunk = [overlap_text[overlap_start:], piece]
                current_length = chunk_overlap + piece_length + 1
            else:
                current_start = char_pos
                current_chunk = [piece]
                current_length = piece_length

        char_pos += piece_length + 1

    # Don't forget the last chunk
    if current_chunk:
        chunk_text = " ".join(current_chunk).strip()
        if chunk_text:
            chunks.append((chunk_text, current_start, current_start + len(chunk_text)))

    return chunks


# ============================================================
# COLLECTION MANAGEMENT
# ============================================================


def _get_collection_path(name: str) -> Path:
    """Get path to collection file."""
    safe_name = re.sub(r"[^\w\-]", "_", name.lower())
    return COLLECTIONS_DIR / f"{safe_name}.json"


def _load_collection(name: str) -> Optional[Collection]:
    """Load a collection from disk."""
    path = _get_collection_path(name)

    if not path.exists():
        return None

    try:
        data = json.loads(path.read_text())

        # Reconstruct objects
        documents = {
            doc_id: Document(**doc_data) for doc_id, doc_data in data.get("documents", {}).items()
        }

        chunks = [Chunk(**chunk_data) for chunk_data in data.get("chunks", [])]

        return Collection(
            name=data["name"],
            description=data.get("description", ""),
            documents=documents,
            chunks=chunks,
            created_at=data.get("created_at", ""),
            updated_at=data.get("updated_at", ""),
            embedding_model=data.get("embedding_model", "all-MiniLM-L6-v2"),
            chunk_size=data.get("chunk_size", 500),
            chunk_overlap=data.get("chunk_overlap", 50),
        )

    except Exception as e:
        logger.error(f"Error loading collection {name}: {e}")
        return None


def _save_collection(collection: Collection):
    """Save a collection to disk."""
    path = _get_collection_path(collection.name)

    data = {
        "name": collection.name,
        "description": collection.description,
        "documents": {doc_id: asdict(doc) for doc_id, doc in collection.documents.items()},
        "chunks": [asdict(chunk) for chunk in collection.chunks],
        "created_at": collection.created_at,
        "updated_at": datetime.now().isoformat(),
        "embedding_model": collection.embedding_model,
        "chunk_size": collection.chunk_size,
        "chunk_overlap": collection.chunk_overlap,
    }

    path.write_text(json.dumps(data, indent=2))


def _list_collections() -> List[str]:
    """List all collections."""
    collections = []
    for path in COLLECTIONS_DIR.glob("*.json"):
        try:
            data = json.loads(path.read_text())
            collections.append(data.get("name", path.stem))
        except (json.JSONDecodeError, IOError, OSError) as e:
            logger.debug(f"Skipping invalid collection file {path}: {e}")
    return collections


def _delete_collection(name: str) -> bool:
    """Delete a collection."""
    path = _get_collection_path(name)
    if path.exists():
        path.unlink()
        return True
    return False


# ============================================================
# SEARCH
# ============================================================


def _search_collection(
    collection: Collection, query: str, top_k: int = 5, threshold: float = 0.3, hybrid: bool = True
) -> List[SearchResult]:
    """
    Search a collection using semantic similarity.

    Args:
        collection: The collection to search
        query: Search query
        top_k: Number of results to return
        threshold: Minimum similarity score
        hybrid: Also include keyword matching

    Returns:
        List of SearchResult objects
    """
    if not collection.chunks:
        return []

    # Get query embedding
    query_embedding = _embed_single(query, collection.embedding_model)

    # Calculate similarity for each chunk
    results = []

    for chunk in collection.chunks:
        if chunk.embedding is None:
            continue

        # Semantic similarity
        semantic_score = _cosine_similarity(query_embedding, chunk.embedding)

        # Keyword matching (for hybrid search)
        keyword_score = 0.0
        if hybrid:
            query_terms = set(query.lower().split())
            chunk_terms = set(chunk.content.lower().split())

            if query_terms:
                overlap = len(query_terms & chunk_terms)
                keyword_score = overlap / len(query_terms) * 0.3  # Weight keyword lower

        # Combined score
        score = semantic_score + keyword_score

        if score >= threshold:
            # Get source document
            doc = collection.documents.get(chunk.document_id)
            source = doc.source if doc else "unknown"

            results.append(SearchResult(chunk=chunk, score=score, source=source))

    # Sort by score and return top_k
    results.sort(key=lambda x: x.score, reverse=True)
    return results[:top_k]


def _search_all_collections(
    query: str, collection_names: List[str] = None, top_k: int = 5, threshold: float = 0.3
) -> List[SearchResult]:
    """Search across multiple collections."""
    if collection_names is None:
        collection_names = _list_collections()

    all_results = []

    for name in collection_names:
        collection = _load_collection(name)
        if collection:
            results = _search_collection(collection, query, top_k=top_k, threshold=threshold)
            all_results.extend(results)

    # Sort and deduplicate
    all_results.sort(key=lambda x: x.score, reverse=True)
    return all_results[:top_k]


# ============================================================
# TOOL HANDLERS
# ============================================================


def create_collection(data: dict) -> str:
    """Create a new knowledge collection."""
    name = data.get("name", "").strip()
    description = data.get("description", "")
    chunk_size = data.get("chunk_size", 500)
    chunk_overlap = data.get("chunk_overlap", 50)

    if not name:
        return "Please provide a collection name"

    # Check if exists
    if _load_collection(name):
        return f"Collection '{name}' already exists. Use a different name or delete it first."

    collection = Collection(
        name=name,
        description=description,
        created_at=datetime.now().isoformat(),
        updated_at=datetime.now().isoformat(),
        chunk_size=chunk_size,
        chunk_overlap=chunk_overlap,
    )

    _save_collection(collection)

    return f"✓ Created collection: {name}\n  Chunk size: {chunk_size}, Overlap: {chunk_overlap}"


def add_document(data: dict) -> str:
    """Add a document to a collection."""
    collection_name = data.get("collection", "default")
    filepath = data.get("file", "")
    url = data.get("url", "")
    text = data.get("text", "")
    title = data.get("title", "")

    # Load or create collection
    collection = _load_collection(collection_name)
    if not collection:
        collection = Collection(
            name=collection_name,
            description="Auto-created collection",
            created_at=datetime.now().isoformat(),
            updated_at=datetime.now().isoformat(),
        )

    content = ""
    metadata = {}
    source = ""

    # Get content from file
    if filepath:
        filepath = Path(filepath).expanduser()

        # Check uploads directory
        if not filepath.exists():
            uploads_path = Path("/mnt/user-data/uploads") / filepath.name
            if uploads_path.exists():
                filepath = uploads_path
            else:
                return f"File not found: {filepath}"

        try:
            content, metadata = _extract_text_from_file(filepath)
            source = str(filepath)
            title = title or metadata.get("title", filepath.stem)
        except Exception as e:
            return f"❌ Error reading file: {e}"

    # Get content from URL
    elif url:
        try:
            import urllib.request

            from bs4 import BeautifulSoup

            with urllib.request.urlopen(url, timeout=30) as response:
                html = response.read().decode("utf-8", errors="replace")

            soup = BeautifulSoup(html, "html.parser")

            # Remove unwanted elements
            for element in soup(["script", "style", "nav", "footer", "header", "aside"]):
                element.decompose()

            content = soup.get_text(separator="\n", strip=True)
            source = url
            title = title or (soup.title.string if soup.title else url)
            metadata = {"url": url, "title": title}

        except Exception as e:
            return f"❌ Error fetching URL: {e}"

    # Use provided text
    elif text:
        content = text
        source = "direct_input"
        title = title or "Untitled"
        metadata = {"title": title}

    else:
        return "Please provide a file, URL, or text content"

    if not content.strip():
        return "❌ No content extracted from source"

    # Create document
    doc_id = hashlib.md5(f"{source}:{content[:100]}".encode()).hexdigest()[:16]

    document = Document(
        id=doc_id,
        content=content,
        metadata=metadata,
        source=source,
        created_at=datetime.now().isoformat(),
    )

    # Check for duplicate
    if doc_id in collection.documents:
        return f"Document already exists in collection (ID: {doc_id})"

    collection.documents[doc_id] = document

    # Chunk the content
    chunk_tuples = _chunk_text(
        content, chunk_size=collection.chunk_size, chunk_overlap=collection.chunk_overlap
    )

    # Create chunk objects
    new_chunks = []
    for i, (chunk_text, start, end) in enumerate(chunk_tuples):
        chunk = Chunk(
            id=f"{doc_id}_{i}",
            document_id=doc_id,
            content=chunk_text,
            metadata={"chunk_index": i, "source": source, "title": title},
            start_char=start,
            end_char=end,
        )
        new_chunks.append(chunk)

    # Generate embeddings
    try:
        chunk_texts = [c.content for c in new_chunks]
        embeddings = _embed_texts(chunk_texts, collection.embedding_model)

        for chunk, embedding in zip(new_chunks, embeddings):
            chunk.embedding = embedding

    except Exception as e:
        logger.error(f"Embedding error: {e}")
        return f"❌ Error generating embeddings: {e}"

    # Add to collection
    collection.chunks.extend(new_chunks)
    _save_collection(collection)

    return (
        f"✓ Added document to '{collection_name}'\n"
        f"  Source: {source}\n"
        f"  Title: {title}\n"
        f"  Content: {len(content):,} characters\n"
        f"  Chunks: {len(new_chunks)}"
    )


def search_knowledge(data: dict) -> str:
    """Search the knowledge base."""
    query = data.get("query", "")
    collection_name = data.get("collection")  # None = search all
    top_k = data.get("limit", 5)
    threshold = data.get("threshold", 0.3)
    show_sources = data.get("sources", True)

    if not query:
        return "Please provide a search query"

    try:
        if collection_name:
            collection = _load_collection(collection_name)
            if not collection:
                return f"Collection '{collection_name}' not found"
            results = _search_collection(collection, query, top_k, threshold)
        else:
            results = _search_all_collections(query, top_k=top_k, threshold=threshold)

        if not results:
            return f"No results found for: {query}"

        # Format results
        lines = [f'📚 Found {len(results)} result(s) for: "{query}"\n']

        for i, result in enumerate(results, 1):
            score_pct = int(result.score * 100)

            # Truncate content for display
            content = result.chunk.content
            if len(content) > 300:
                content = content[:300] + "..."

            lines.append(f"**[{i}]** (relevance: {score_pct}%)")
            lines.append(f"{content}")

            if show_sources:
                source = result.chunk.metadata.get("title", result.source)
                lines.append(f"  _Source: {source}_")

            lines.append("")

        return "\n".join(lines)

    except Exception as e:
        logger.error(f"Search error: {e}")
        return f"❌ Search error: {e}"


def get_context(data: dict) -> str:
    """
    Get relevant context for a question (for use by the LLM).
    Returns just the text, formatted for inclusion in a prompt.
    """
    query = data.get("query", "")
    collection_name = data.get("collection")
    top_k = data.get("limit", 3)

    if not query:
        return ""

    try:
        if collection_name:
            collection = _load_collection(collection_name)
            if not collection:
                return ""
            results = _search_collection(collection, query, top_k)
        else:
            results = _search_all_collections(query, top_k=top_k)

        if not results:
            return ""

        # Format as context
        context_parts = []
        for result in results:
            source = result.chunk.metadata.get("title", result.source)
            context_parts.append(f"[From: {source}]\n{result.chunk.content}")

        return "\n\n---\n\n".join(context_parts)

    except Exception as e:
        logger.error(f"Context retrieval error: {e}")
        return ""


def list_knowledge_collections(data: dict) -> str:
    """List all knowledge collections."""
    collections = _list_collections()

    if not collections:
        return "No knowledge collections found. Create one with: create_collection"

    lines = ["📚 Knowledge Collections:\n"]

    for name in collections:
        collection = _load_collection(name)
        if collection:
            doc_count = len(collection.documents)
            chunk_count = len(collection.chunks)
            lines.append(f"  **{name}**")
            lines.append(f"    {doc_count} documents, {chunk_count} chunks")
            if collection.description:
                lines.append(f"    {collection.description}")
            lines.append("")

    return "\n".join(lines)


def collection_info(data: dict) -> str:
    """Get detailed info about a collection."""
    name = data.get("collection", data.get("name", ""))

    if not name:
        return "Please provide a collection name"

    collection = _load_collection(name)
    if not collection:
        return f"Collection '{name}' not found"

    lines = [f"📚 Collection: {collection.name}\n"]

    if collection.description:
        lines.append(f"Description: {collection.description}")

    lines.append(f"Created: {collection.created_at}")
    lines.append(f"Updated: {collection.updated_at}")
    lines.append(f"Embedding model: {collection.embedding_model}")
    lines.append(f"Chunk size: {collection.chunk_size} (overlap: {collection.chunk_overlap})")
    lines.append(f"\nDocuments: {len(collection.documents)}")
    lines.append(f"Total chunks: {len(collection.chunks)}")

    if collection.documents:
        lines.append("\nSources:")
        for doc in collection.documents.values():
            title = doc.metadata.get("title", doc.source)
            lines.append(f"  • {title}")

    return "\n".join(lines)


def remove_document(data: dict) -> str:
    """Remove a document from a collection."""
    collection_name = data.get("collection", "")
    source = data.get("source", "")
    doc_id = data.get("document_id", "")

    if not collection_name:
        return "Please provide a collection name"

    collection = _load_collection(collection_name)
    if not collection:
        return f"Collection '{collection_name}' not found"

    # Find document by ID or source
    target_doc_id = doc_id

    if not target_doc_id and source:
        for did, doc in collection.documents.items():
            if (
                source.lower() in doc.source.lower()
                or source.lower() in doc.metadata.get("title", "").lower()
            ):
                target_doc_id = did
                break

    if not target_doc_id:
        return f"Document not found: {source or doc_id}"

    if target_doc_id not in collection.documents:
        return f"Document ID not found: {target_doc_id}"

    # Remove document and its chunks
    doc = collection.documents.pop(target_doc_id)
    collection.chunks = [c for c in collection.chunks if c.document_id != target_doc_id]

    _save_collection(collection)

    return f"✓ Removed document: {doc.metadata.get('title', doc.source)}"


def delete_knowledge_collection(data: dict) -> str:
    """Delete an entire collection."""
    name = data.get("collection", data.get("name", ""))

    if not name:
        return "Please provide a collection name"

    collection = _load_collection(name)
    if not collection:
        return f"Collection '{name}' not found"

    if not data.get("_confirmed"):
        from familiar.core.confirmations import needs_confirmation

        return needs_confirmation(
            tool_name="delete_collection",
            tool_input=data,
            preview=f"Delete knowledge collection '{name}' and all its documents.",
            risk="high",
        )

    if _delete_collection(name):
        return f"✓ Deleted collection: {name}"
    else:
        return f"❌ Failed to delete collection: {name}"


def add_folder(data: dict) -> str:
    """Add all documents from a folder to a collection."""
    collection_name = data.get("collection", "default")
    folder_path = data.get("folder", "")
    recursive = data.get("recursive", True)
    extensions = data.get("extensions", [".txt", ".md", ".pdf", ".docx", ".html", ".csv", ".json"])

    if not folder_path:
        return "Please provide a folder path"

    folder = Path(folder_path).expanduser()

    if not folder.exists():
        return f"Folder not found: {folder}"

    if not folder.is_dir():
        return f"Not a folder: {folder}"

    # Find all matching files
    if recursive:
        files = [f for f in folder.rglob("*") if f.suffix.lower() in extensions]
    else:
        files = [f for f in folder.glob("*") if f.suffix.lower() in extensions]

    if not files:
        return f"No supported files found in {folder}"

    # Add each file
    added = 0
    errors = []

    for filepath in files:
        result = add_document({"collection": collection_name, "file": str(filepath)})

        if result.startswith("✓"):
            added += 1
        elif "already exists" not in result:
            errors.append(f"{filepath.name}: {result}")

    lines = [f"✓ Added {added} documents to '{collection_name}'"]

    if errors:
        lines.append(f"\n⚠️ {len(errors)} error(s):")
        for err in errors[:5]:  # Show first 5
            lines.append(f"  • {err}")

    return "\n".join(lines)


# ============================================================
# TOOL DEFINITIONS
# ============================================================

TOOLS = [
    {
        "name": "search_knowledge",
        "description": (
            "Search document collections and ingested files using hybrid semantic and "
            "keyword matching. Use this to find information across PDFs, web pages, and "
            "uploaded documents in the RAG knowledge base. For simple notes and saved "
            "snippets added with save_to_knowledge, use search_knowledge_base instead."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {"type": "string", "description": "What to search for"},
                "collection": {
                    "type": "string",
                    "description": "Specific collection to search (omit to search all)",
                },
                "limit": {"type": "integer", "description": "Max results to return", "default": 5},
                "sources": {
                    "type": "boolean",
                    "description": "Show source attribution",
                    "default": True,
                },
            },
            "required": ["query"],
        },
        "handler": search_knowledge,
        "category": "knowledge",
    },
    {
        "name": "get_context",
        "description": (
            "Retrieve relevant context from the knowledge base to augment LLM responses "
            "for a given question. Use this internally to inject RAG context into prompts "
            "before answering domain-specific questions. Returns raw text formatted for "
            "direct inclusion in a system or user prompt."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {
                    "type": "string",
                    "description": "The question or topic to find context for",
                },
                "collection": {"type": "string", "description": "Specific collection (optional)"},
                "limit": {"type": "integer", "default": 3},
            },
            "required": ["query"],
        },
        "handler": get_context,
        "category": "knowledge",
    },
    {
        "name": "add_document",
        "description": (
            "Add a document to the knowledge base by ingesting a file, URL, or direct "
            "text input. Use this to ingest PDFs, Word documents, text, Markdown, HTML, "
            "CSV, or JSON files into a collection for later retrieval. The document is "
            "chunked and embedded automatically; duplicates are detected by content hash."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "collection": {
                    "type": "string",
                    "description": "Collection name (creates 'default' if not specified)",
                    "default": "default",
                },
                "file": {"type": "string", "description": "Path to file to ingest"},
                "url": {"type": "string", "description": "URL to fetch and ingest"},
                "text": {"type": "string", "description": "Direct text content to add"},
                "title": {"type": "string", "description": "Title for the document"},
            },
        },
        "handler": add_document,
        "category": "knowledge",
    },
    {
        "name": "add_folder",
        "description": (
            "Recursively ingest all supported documents from a folder into the knowledge "
            "base. Use this for bulk ingestion of document directories such as project "
            "folders or reference libraries. Supports .txt, .md, .pdf, .docx, .html, "
            ".csv, and .json files by default."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "folder": {"type": "string", "description": "Path to folder"},
                "collection": {"type": "string", "default": "default"},
                "recursive": {
                    "type": "boolean",
                    "description": "Include subfolders",
                    "default": True,
                },
            },
            "required": ["folder"],
        },
        "handler": add_folder,
        "category": "knowledge",
    },
    {
        "name": "create_collection",
        "description": (
            "Create a new named knowledge collection for organizing and grouping "
            "related documents. Use this to set up topic-specific collections before "
            "ingesting documents. Chunk size and overlap can be customized per "
            "collection for optimal retrieval quality."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "name": {"type": "string", "description": "Collection name"},
                "description": {"type": "string", "description": "What this collection contains"},
                "chunk_size": {
                    "type": "integer",
                    "description": "Characters per chunk",
                    "default": 500,
                },
                "chunk_overlap": {
                    "type": "integer",
                    "description": "Overlap between chunks",
                    "default": 50,
                },
            },
            "required": ["name"],
        },
        "handler": create_collection,
        "category": "knowledge",
    },
    {
        "name": "list_collections",
        "description": (
            "List all knowledge collections with document and chunk counts. Use this "
            "to discover available collections before searching or adding documents. "
            "Returns collection names, descriptions, and summary statistics."
        ),
        "input_schema": {"type": "object", "properties": {}},
        "handler": list_knowledge_collections,
        "category": "knowledge",
    },
    {
        "name": "collection_info",
        "description": (
            "Get detailed metadata and statistics for a specific knowledge collection. "
            "Use this to inspect a collection's embedding model, chunk settings, document "
            "sources, and creation dates. Helpful for debugging retrieval quality or "
            "understanding collection contents."
        ),
        "input_schema": {
            "type": "object",
            "properties": {"collection": {"type": "string", "description": "Collection name"}},
            "required": ["collection"],
        },
        "handler": collection_info,
        "category": "knowledge",
    },
    {
        "name": "remove_document",
        "description": (
            "Remove a specific document and all its chunks from a knowledge collection. "
            "Use this to clean up outdated or incorrect documents from the knowledge "
            "base. The document can be identified by its source path, title, or "
            "internal document ID."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "collection": {"type": "string", "description": "Collection name"},
                "source": {"type": "string", "description": "Document source/title to remove"},
                "document_id": {"type": "string", "description": "Or specific document ID"},
            },
            "required": ["collection"],
        },
        "handler": remove_document,
        "category": "knowledge",
    },
    {
        "name": "delete_collection",
        "description": (
            "Permanently delete an entire knowledge collection and all its documents "
            "and embeddings. Use this to remove collections that are no longer needed. "
            "This action is irreversible and requires confirmation before proceeding."
        ),
        "requires_confirmation": True,
        "input_schema": {
            "type": "object",
            "properties": {
                "collection": {"type": "string", "description": "Collection name to delete"},
            },
            "required": ["collection"],
        },
        "handler": delete_knowledge_collection,
        "category": "knowledge",
    },
]
