#!/usr/bin/env python3
"""
Novyx Core - Batch & Streaming Operations (Phase 13)
=====================================================
Low-memory streaming and batch processing for large artifact collections.

Features:
- Streaming artifact loading (ijson) - O(1) memory per artifact
- Batch create/update operations (up to 100 per request)
- Generator-based iteration for scalability
- Multi-tenant aware streaming

Usage:
    from tools.batch import stream_artifacts, batch_create

    # Stream artifacts with low memory
    for batch in stream_artifacts(Path("memory"), batch_size=100):
        process_batch(batch)

    # Batch create artifacts
    results = batch_create(artifacts_list, tenant_id="acme-corp")
"""

import json
import os
from pathlib import Path
from typing import Generator, List, Dict, Any, Optional, Tuple
from datetime import datetime

# Optional ijson for streaming JSON parsing
try:
    import ijson
    IJSON_AVAILABLE = True
except ImportError:
    IJSON_AVAILABLE = False

# Configuration
MAX_BATCH_SIZE = 100
DEFAULT_BATCH_SIZE = 50
STREAM_BUFFER_SIZE = 8192  # 8KB read buffer


def stream_artifacts(
    directory: Path,
    *,
    batch_size: int = DEFAULT_BATCH_SIZE,
    tenant_id: Optional[str] = None,
    file_pattern: str = "*.jsonld"
) -> Generator[List[Dict[str, Any]], None, None]:
    """
    Stream artifacts from directory in memory-efficient batches.

    Uses generator pattern to yield batches without loading all files into memory.
    Each batch contains fully parsed artifact dicts ready for processing.

    Args:
        directory: Root directory to scan for artifacts
        batch_size: Number of artifacts per batch (default: 50, max: 100)
        tenant_id: Optional tenant ID to filter by subdirectory
        file_pattern: Glob pattern for artifact files (default: *.jsonld)

    Yields:
        List of artifact dicts, each batch up to batch_size items

    Example:
        >>> for batch in stream_artifacts(Path("memory"), batch_size=100):
        ...     print(f"Processing {len(batch)} artifacts")
        ...     for artifact in batch:
        ...         process(artifact)
    """
    batch_size = min(batch_size, MAX_BATCH_SIZE)

    # Determine scan directory
    scan_dir = directory
    if tenant_id:
        scan_dir = directory / tenant_id

    if not scan_dir.exists():
        return

    batch = []

    for jsonld_path in scan_dir.rglob(file_pattern):
        try:
            artifact = _load_artifact_streaming(jsonld_path)
            if artifact:
                artifact['_path'] = str(jsonld_path)
                artifact['_tenant'] = tenant_id
                batch.append(artifact)

                if len(batch) >= batch_size:
                    yield batch
                    batch = []

        except (json.JSONDecodeError, IOError, OSError) as e:
            # Log but continue - don't fail entire stream for one bad file
            continue

    # Yield remaining items
    if batch:
        yield batch


def _load_artifact_streaming(path: Path) -> Optional[Dict[str, Any]]:
    """
    Load a single artifact using streaming parser if available.

    Falls back to standard json.load() if ijson not installed.

    Args:
        path: Path to .jsonld file

    Returns:
        Parsed artifact dict, or None on error
    """
    if IJSON_AVAILABLE:
        # Use ijson for streaming parse (lower peak memory)
        try:
            with open(path, 'rb') as f:
                # ijson.items parses incrementally
                # For small files, this is similar to json.load
                # For large files, it prevents loading entire file into memory
                parser = ijson.parse(f)
                result = {}
                current_key = None
                current_value = None
                array_stack = []

                for prefix, event, value in parser:
                    if event == 'map_key':
                        current_key = value
                    elif event in ('string', 'number', 'boolean', 'null'):
                        if current_key and not array_stack:
                            result[current_key] = value
                        elif array_stack:
                            array_stack[-1].append(value)
                    elif event == 'start_array':
                        new_array = []
                        if current_key and not array_stack:
                            result[current_key] = new_array
                        array_stack.append(new_array)
                    elif event == 'end_array':
                        if len(array_stack) > 1:
                            completed = array_stack.pop()
                            array_stack[-1].append(completed)
                        elif array_stack:
                            array_stack.pop()
                    elif event == 'start_map':
                        if array_stack:
                            new_map = {}
                            array_stack[-1].append(new_map)
                    elif event == 'end_map':
                        pass

                return result if result else None

        except Exception:
            # Fall back to standard loading on any ijson error
            pass

    # Standard json.load fallback
    try:
        with open(path, 'r', encoding='utf-8') as f:
            return json.load(f)
    except Exception:
        return None


def stream_artifact_ids(
    directory: Path,
    *,
    tenant_id: Optional[str] = None
) -> Generator[Tuple[str, Path], None, None]:
    """
    Stream artifact IDs without loading full content (ultra-low memory).

    Useful for indexing, counting, or selective loading.

    Args:
        directory: Root directory to scan
        tenant_id: Optional tenant filter

    Yields:
        Tuple of (artifact_id, file_path)

    Example:
        >>> ids = list(stream_artifact_ids(Path("memory")))
        >>> print(f"Found {len(ids)} artifacts")
    """
    scan_dir = directory / tenant_id if tenant_id else directory

    if not scan_dir.exists():
        return

    for jsonld_path in scan_dir.rglob("*.jsonld"):
        try:
            # Quick extraction of ID without full parse
            artifact_id = _extract_id_quick(jsonld_path)
            if artifact_id:
                yield (artifact_id, jsonld_path)
        except Exception:
            continue


def _extract_id_quick(path: Path) -> Optional[str]:
    """
    Extract artifact ID from file without full parse.

    Reads first ~1KB to find uuid/Persistent_ID field.
    """
    try:
        with open(path, 'r', encoding='utf-8') as f:
            # Read first chunk
            chunk = f.read(1024)

            # Quick string search for ID patterns
            import re

            # Try uuid first (v1.1.0 schema)
            match = re.search(r'"uuid"\s*:\s*"([^"]+)"', chunk)
            if match:
                return match.group(1)

            # Try Persistent_ID (v1.0 schema)
            match = re.search(r'"Persistent_ID"\s*:\s*"([^"]+)"', chunk)
            if match:
                return match.group(1)

            # Try @id
            match = re.search(r'"@id"\s*:\s*"([^"]+)"', chunk)
            if match:
                return match.group(1)

    except Exception:
        pass

    return None


def batch_create(
    artifacts: List[Dict[str, Any]],
    *,
    tenant_id: Optional[str] = None,
    directory: Optional[Path] = None,
    skip_versioning: bool = False
) -> Dict[str, Any]:
    """
    Create multiple artifacts in a single batch operation.

    Validates and persists up to MAX_BATCH_SIZE artifacts atomically.
    Each artifact is created with proper hashing and indexing.

    Args:
        artifacts: List of artifact payloads (max 100)
        tenant_id: Optional tenant ID for multi-tenant storage
        directory: Base directory (default: memory/)
        skip_versioning: Skip version tracking for bulk imports

    Returns:
        Dict with created count, failed count, and details

    Raises:
        ValueError: If batch exceeds MAX_BATCH_SIZE

    Example:
        >>> results = batch_create([
        ...     {"@type": "novyx:Decision", "title": "Decision 1", ...},
        ...     {"@type": "novyx:Decision", "title": "Decision 2", ...},
        ... ], tenant_id="acme-corp")
        >>> print(f"Created: {results['created']}")
    """
    if len(artifacts) > MAX_BATCH_SIZE:
        raise ValueError(f"Batch size {len(artifacts)} exceeds maximum {MAX_BATCH_SIZE}")

    if not artifacts:
        return {
            "created": 0,
            "failed": 0,
            "results": [],
            "errors": []
        }

    from tools.factory import create_artifact, persist_artifact
    from tools.artifacts import get_artifact_config

    base_dir = directory or Path("memory")
    results = []
    errors = []

    for i, artifact_payload in enumerate(artifacts):
        try:
            # Detect artifact type
            artifact_type = _detect_artifact_type(artifact_payload)

            if not artifact_type:
                errors.append({
                    "index": i,
                    "error": "Could not determine artifact type from @type field"
                })
                continue

            # Get artifact config
            config = get_artifact_config(artifact_type)
            if not config:
                errors.append({
                    "index": i,
                    "error": f"Unknown artifact type: {artifact_type}"
                })
                continue

            # Create artifact (tenant_id is set on the artifact)
            artifact = create_artifact(artifact_type, artifact_payload, tenant_id=tenant_id)

            # Determine output directory (extract subdir from output_dir path)
            # persist_artifact handles tenant subdirectory internally
            subdir = Path(config['output_dir']).name
            output_dir = base_dir / subdir

            # Persist (reads tenant_id from artifact)
            artifact_path = persist_artifact(
                artifact,
                output_dir,
                skip_versioning=skip_versioning
            )

            results.append({
                "index": i,
                "artifact_id": artifact.get("uuid"),
                "path": str(artifact_path),
                "type": artifact_type
            })

        except Exception as e:
            errors.append({
                "index": i,
                "error": str(e)
            })

    return {
        "created": len(results),
        "failed": len(errors),
        "results": results,
        "errors": errors
    }


def _detect_artifact_type(payload: Dict[str, Any]) -> Optional[str]:
    """
    Detect artifact type from @type field.

    Handles both array and string @type values, with novyx: prefix.
    """
    type_value = payload.get("@type")

    if not type_value:
        return None

    # Handle array of types
    if isinstance(type_value, list):
        for t in type_value:
            detected = _normalize_type(t)
            if detected:
                return detected
        return None

    # Handle single string type
    return _normalize_type(type_value)


def _normalize_type(type_str: str) -> Optional[str]:
    """Normalize type string to lowercase artifact type name."""
    if not type_str:
        return None

    # Remove namespace prefix
    if ":" in type_str:
        type_str = type_str.split(":")[-1]

    # Map to known types
    type_lower = type_str.lower()

    known_types = {
        "decision": "decision",
        "apiendpoint": "apiendpoint",
        "graphqloperation": "graphqloperation",
        "tenant": "tenant",
        "version": "version",
        "digitaldocument": "decision",  # Default fallback
    }

    return known_types.get(type_lower)


def batch_validate(
    directory: Path,
    *,
    tenant_id: Optional[str] = None,
    batch_size: int = DEFAULT_BATCH_SIZE
) -> Generator[Dict[str, Any], None, None]:
    """
    Validate artifacts in streaming batches.

    Yields validation results as they complete, enabling progress tracking.
    Uses Sentinel's validate_artifact for accurate file-based hash validation.

    Args:
        directory: Directory to validate
        tenant_id: Optional tenant filter
        batch_size: Artifacts per validation batch

    Yields:
        Validation result dict for each batch

    Example:
        >>> for result in batch_validate(Path("memory")):
        ...     print(f"Batch: {result['passed']}/{result['total']} passed")
    """
    from tools.sentinel import NovyxSentinel
    from tools.util import calculate_hash

    # Create sentinel instance for validation
    sentinel = NovyxSentinel(root_dir=directory, verbose=False, enforce_graph=False)

    for batch in stream_artifacts(directory, batch_size=batch_size, tenant_id=tenant_id):
        passed = 0
        failed = 0
        errors = []

        for artifact in batch:
            try:
                # Get the file path from the artifact metadata
                artifact_path_str = artifact.get('_path')
                if not artifact_path_str:
                    failed += 1
                    errors.append({
                        "path": None,
                        "error": "Missing _path metadata"
                    })
                    continue

                artifact_path = Path(artifact_path_str)

                # Get stored hash from the artifact
                stored_hash = (
                    artifact.get('novyx:integrityHash') or
                    artifact.get('Integrity_Hash')
                )

                if not stored_hash or stored_hash in ('pending_calculation', 'TBD', ''):
                    # No valid hash to verify - count as passed (no hash constraint)
                    passed += 1
                    continue

                # Calculate hash from the FILE (not the dict) for accurate comparison
                # This matches how the hash was originally computed
                computed_hash = calculate_hash(artifact_path)

                # Compare full hashes for integrity verification
                if computed_hash == stored_hash:
                    passed += 1
                else:
                    failed += 1
                    errors.append({
                        "path": artifact_path_str,
                        "error": f"Hash mismatch: expected {stored_hash[:16]}..., got {computed_hash[:16]}..."
                    })

            except Exception as e:
                failed += 1
                errors.append({
                    "path": artifact.get('_path'),
                    "error": str(e)
                })

        yield {
            "batch_size": len(batch),
            "passed": passed,
            "failed": failed,
            "errors": errors
        }


def count_artifacts(
    directory: Path,
    *,
    tenant_id: Optional[str] = None
) -> int:
    """
    Count artifacts without loading content (fast).

    Uses file counting only - no parsing required.

    Args:
        directory: Directory to count
        tenant_id: Optional tenant filter

    Returns:
        Total artifact count
    """
    scan_dir = directory / tenant_id if tenant_id else directory

    if not scan_dir.exists():
        return 0

    return sum(1 for _ in scan_dir.rglob("*.jsonld"))


# Module-level convenience for checking ijson availability
def is_streaming_available() -> bool:
    """Check if streaming JSON parsing is available."""
    return IJSON_AVAILABLE


if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser(description="Novyx Batch Operations")
    parser.add_argument("--directory", default="memory", help="Directory to process")
    parser.add_argument("--count", action="store_true", help="Count artifacts")
    parser.add_argument("--stream-test", action="store_true", help="Test streaming")
    parser.add_argument("--batch-size", type=int, default=50, help="Batch size for streaming")
    parser.add_argument("--tenant", help="Filter by tenant ID")

    args = parser.parse_args()
    directory = Path(args.directory)

    if args.count:
        count = count_artifacts(directory, tenant_id=args.tenant)
        print(f"Total artifacts: {count}")

    elif args.stream_test:
        print(f"Streaming test (batch_size={args.batch_size}):")
        print(f"ijson available: {IJSON_AVAILABLE}")

        total = 0
        batches = 0
        for batch in stream_artifacts(directory, batch_size=args.batch_size, tenant_id=args.tenant):
            batches += 1
            total += len(batch)
            print(f"  Batch {batches}: {len(batch)} artifacts")

        print(f"Total: {total} artifacts in {batches} batches")

    else:
        parser.print_help()
