"""End-to-end data sync pipeline: GraphQL API → Polars → Database."""

# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/10_utils_sync.ipynb.

# %% ../nbs/10_utils_sync.ipynb 2
from __future__ import annotations
import polars as pl
from .utils_graphql import GraphQLClient
from .utils_polars_mapper import map_and_upsert, apply_schema
from typing import Dict, Optional, List
import logging
from nbdev.showdoc import show_doc

logger = logging.getLogger(__name__)

# %% auto 0
__all__ = ['logger', 'sync_external_data', 'sync_incremental']

# %% ../nbs/10_utils_sync.ipynb 5
async def sync_external_data(
    client: GraphQLClient, # Initialized GraphQL client
    query_template: str, # GraphQL query with $cursor variable
    variables: dict, # Initial variables (e.g., {'cursor': None})
    items_path: list[str], # Path to data list in response
    cursor_path: list[str], # Path to next cursor
    table_name: str, # Target database table
    key_col: str, # Primary key for upsert
    db_uri: str, # Database connection string
    column_map: dict = None, # Optional column renaming
    type_map: dict = None, # Optional type conversions
    has_next_path: list[str] = None, # Optional hasNextPage path
    batch_size: int = 5000 # Max rows per batch (pagination page size)
) -> Dict[str, int]:
    """Sync external data from GraphQL API to database (streaming, memory-efficient)."""
    total_records = 0
    batch_count = 0
    
    # Stream paginated data
    async for batch in client.fetch_pages_generator(
        query_template=query_template,
        variables=variables,
        items_path=items_path,
        cursor_path=cursor_path,
        has_next_path=has_next_path
    ):
        batch_count += 1
        
        # Convert to Polars DataFrame
        df = pl.DataFrame(batch)
        logger.info(f"Batch {batch_count}: Converted {len(df)} records to DataFrame")
        
        # Apply type conversions if specified
        if type_map:
            df = apply_schema(df, type_map)
            logger.info(f"Batch {batch_count}: Applied type conversions")
        
        # Upsert to database
        map_and_upsert(
            df=df,
            table_name=table_name,
            key_col=key_col,
            db_uri=db_uri,
            column_map=column_map
        )
        
        total_records += len(df)
        logger.info(f"Batch {batch_count}: Upserted {len(df)} records (total: {total_records})")
    
    return {
        'total_records': total_records,
        'batches': batch_count
    }

# %% ../nbs/10_utils_sync.ipynb 8
async def sync_incremental(
    client: GraphQLClient, # Initialized GraphQL client
    query_template: str, # GraphQL query with $cursor and $last_sync variables
    last_sync_time: str, # ISO timestamp (e.g., '2024-01-15T10:00:00Z')
    items_path: list[str], # Path to data list
    cursor_path: list[str], # Path to cursor
    table_name: str, # Target table
    key_col: str, # Primary key
    db_uri: str, # Database connection
    column_map: dict = None, # Optional column map
    type_map: dict = None, # Optional type map
    has_next_path: list[str] = None # Optional hasNextPage path
) -> Dict[str, any]:
    """Incremental sync: fetch only records updated after last sync timestamp."""
    from datetime import datetime
    
    # Add last_sync to variables
    variables = {
        'cursor': None,
        'last_sync': last_sync_time
    }
    
    # Execute sync
    stats = await sync_external_data(
        client=client,
        query_template=query_template,
        variables=variables,
        items_path=items_path,
        cursor_path=cursor_path,
        table_name=table_name,
        key_col=key_col,
        db_uri=db_uri,
        column_map=column_map,
        type_map=type_map,
        has_next_path=has_next_path
    )
    
    # Add current timestamp
    stats['last_sync_time'] = datetime.utcnow().isoformat() + 'Z'
    
    return stats
