# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/05a_pipeline_batch.ipynb (unless otherwise specified).

__all__ = ['download_missing_eumetsat_files', 'DataFrame', 'download_missing_data_pipeline']

# Cell
import pandas as pd
import xarray as xr
import os
import glob
import dotenv
import warnings
from dagster import execute_pipeline, pipeline, solid, Field, OutputDefinition, DagsterType, Output

from IPython.display import JSON

from satip import eumetsat, reproj, io, gcp_helpers
from .mario import (df_metadata_to_dt_to_fp_map,
                         reproject_datasets,
                         save_metadata,
                         compress_and_save_datasets,
                         compress_export_then_delete_raw)

# Cell
#exports
warnings.filterwarnings('ignore', message='divide by zero encountered in true_divide')
warnings.filterwarnings('ignore', message='invalid value encountered in sin')
warnings.filterwarnings('ignore', message='invalid value encountered in cos')
warnings.filterwarnings('ignore', message='invalid value encountered in subtract')
warnings.filterwarnings('ignore', message='You will likely lose important projection information when converting to a PROJ string from another format. See: https://proj.org/faq.html#what-is-the-best-format-for-describing-coordinate-reference-systems')

# Cell

# create pandas DataFrame type definition for Dagster
DataFrame = DagsterType(
    name="DataFrame",
    type_check_fn=lambda _, x: isinstance(x, pd.DataFrame),
)

@solid(output_defs=[OutputDefinition(name='df_new_metadata', dagster_type=DataFrame, is_required=False)])
def download_missing_eumetsat_files(context, env_vars_fp: str, data_dir: str, metadata_db_fp: str, debug_fp: str, table_id: str, project_id: str, start_date: str='', end_date: str=''):
    _ = dotenv.load_dotenv(env_vars_fp)
    dm = eumetsat.DownloadManager(os.environ.get('USER_KEY'), os.environ.get('USER_SECRET'), data_dir, metadata_db_fp, debug_fp, slack_webhook_url=os.environ.get('SLACK_WEBHOOK_URL'), slack_id=os.environ.get('SLACK_ID'))

    missing_datasets = io.identifying_missing_datasets(start_date, end_date)
    context.log.info(f"Missing data: {len(missing_datasets)}")

    df_new_metadata = dm.download_datasets(missing_datasets)

    # if df_new_metadata is None, pipeline will skip subsequent solids
    if df_new_metadata is None:
        context.log.info("*******************")
        context.log.info("Files already in zarr. Exiting.")
        context.log.info("*******************")
        return

    yield Output(df_new_metadata, 'df_new_metadata')

# Cell
@pipeline
def download_missing_data_pipeline():
    # Retrieving data, reprojecting, compressing, and saving to GCP
    df_new_metadata = download_missing_eumetsat_files()
    datetime_to_filepath = df_metadata_to_dt_to_fp_map(df_new_metadata)
    ds_combined_reproj = reproject_datasets(datetime_to_filepath)
    ds_combined_compressed = compress_and_save_datasets(ds_combined_reproj)

    ready_to_delete = save_metadata(ds_combined_compressed, df_new_metadata)
    compress_export_then_delete_raw(ready_to_delete)