# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs-dev/1_02_ml.ipynb.

# %% ../nbs-dev/1_02_ml.ipynb 4
from __future__ import annotations
from IPython.display import Image, display
from pathlib import Path
from pprint import pprint
from typing import Any, List
from zipfile import ZipFile
from .core import get_config_value
from .ipython import run_cli

import configparser
import datetime as dt
import json
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import shutil
import subprocess

# %% auto 0
__all__ = ['are_features_consistent', 'kaggle_setup_colab', 'kaggle_list_files', 'kaggle_download_competition_files',
           'fastbook_on_colab']

# %% ../nbs-dev/1_02_ml.ipynb 6
def are_features_consistent(
    df1:pd.DataFrame,                 # First set, typically the training set
    df2:pd.DataFrame,                 # Second set, typically the test set or inference set 
    dependent_variables:list[str] = None, # List of column name(s) for dependent variables
    raise_error:bool = False,             # True to raise an error if not consistent
)-> bool :                                # True if features in train and test datasets are consistent, False otherwise
    """Verify that features/columns in training and test sets are consistent"""
    if dependent_variables is None:
        features_df1 = df1.columns
    else:
        features_df1 = df1.drop(dependent_variables, axis=1).columns
    features_df2 = df2.columns
    features_diff = set(features_df1).symmetric_difference(features_df2)
    if features_diff == set():
        return True
    else:
        if raise_error:
            raise ValueError(f"Discrepancy between training and test feature set: {features_diff}")
        else: return False

# %% ../nbs-dev/1_02_ml.ipynb 18
def kaggle_setup_colab(path_to_config_file:Path|str = None      # path to the configuration file (e.g. config.cfg)
                      ):
    """Update kaggle API and create security key json file from config file on Google Drive"""
    # Create API security key file
    path_to_kaggle = Path('/root/.kaggle')
    os.makedirs(path_to_kaggle, exist_ok=True)
    
    # Validate path_to_config
    if path_to_config_file is None:
        path_to_config_file = Path('/content/gdrive/MyDrive/private-across-accounts/config-api-keys.cfg')
    if isinstance(path_to_config_file, str): 
        path_to_config_file = Path(path_to_config_file)
    if not path_to_config_file.is_file():
        raise ValueError(f"No file at {path_to_config_file.absolute()}. Check the path")
    
    # retrieve configuration, create token and save it
    username = get_config_value('kaggle', 'kaggle_username', path_to_config_file=path_to_config_file)
    key = get_config_value('kaggle', 'kaggle_key', path_to_config_file=path_to_config_file)

    api_token = {"username": username, "key": key}
    with open(path_to_kaggle / 'kaggle.json', 'w') as file:
        json.dump(api_token, file)
        os.fchmod(file.fileno(), 600)

    # Update kaggle API software
    run_cli('pip install -Uqq kaggle --upgrade')

# %% ../nbs-dev/1_02_ml.ipynb 20
def kaggle_list_files(code:str = None,          # code for the kaggle competition or dataset
                      mode:str ='competitions'  # mode: `competitions` or `datasets`
                     ):
    """List all files available in the competition or dataset for the passed code"""
    if code is None:
        print(f"code is None, please provide the code of the kaggle competition or dataset")
        return 'Failed'
    elif mode not in ['competitions', 'datasets']:
        print(f"mode must be either 'competitions' or 'datasets', not {mode}")
        return 'Failed'
    else:
        print(f"Listing the files available for {mode}: <{code}>")
        run_cli(f"kaggle {mode} files {code}")

    if mode == 'competitions':
        print(f"{'=' * 140}")
        print(f"Make sure to set the parameters for <{code}> in next cell:")
        print(f" - kaggle_project_folder_name: string with name of the project folder")
        print(f" - train_files: list of files to place into the <train> folder")
        print(f" - test_files: list of files to place into the <test> folder")
        print(f" - submit_files: list of files to place into the <submit> folder")
        print(f"{'=' * 140}")

# %% ../nbs-dev/1_02_ml.ipynb 21
def kaggle_download_competition_files(
    competition_code:str = None, 
    train_files:list() = [], 
    test_files:list = [], 
    submit_files:list = [], 
    project_folder:str = 'ds'
    ):
    """download all files for passed competition, unzip them if required, move them to train, test and submit folders

    competition_code: str       code of the kaggle competition
    train_files: list of str    names of files to be moved into train folder
    test_files: list of str     names of files to be moved into test folder
    submit_files: list of str   names of files to be moved into submit folder
    """
    if competition_code is None:
        print(f"competition_code is None, please provide the code of the kaggle competition")
        return 'Failed'
    else:
        list_of_datasets = {'train': train_files,
                            'test': test_files,
                            'submit': submit_files}

        # creating a project directory and set paths
        if not os.path.exists(project_folder):
            os.makedirs(project_folder)

        path2datasets = Path(f"/content/{project_folder}")
        path2datasets_str = str(path2datasets.absolute())

        # download all files from kaggle
        run_cli(f"kaggle competitions download -c {competition_code} -p {path2datasets}")

        print(f"{'=' * 140}")
        print('Downloaded files:')
        for f in [item for item in path2datasets.iterdir() if item.is_file()]:
            print(f" - {f}")
        print(f"{'=' * 140}")

        # Unzip all zipped files
        for f in path2datasets.glob('*.zip'):
            print(f"Unzipping {f.name}")
            zip_f = ZipFile(f)
            zip_f.extractall(path=path2datasets)
            os.remove(f)
        print(f"{'=' * 140}")

        # Move all data files to the correct data folder
        for dataset_folder, files in list_of_datasets.items():
            if not os.path.exists(f'{project_folder}/{dataset_folder}'):
                os.makedirs(f'{project_folder}/{dataset_folder}')

            for f in files:
                print(f"Moving {f} to {dataset_folder}")
                p2f = path2datasets / f
                if p2f.suffix == '.csv':
                    shutil.move(path2datasets / f, path2datasets / dataset_folder / f)
                else:
                    msg = f"Does not support {p2f.name}'s extension {p2f.suffix}"
                    raise RuntimeError(msg)

        print(f"{'=' * 140}")
        print('Done loading Kaggle files and moving them to corresponding folders')

# %% ../nbs-dev/1_02_ml.ipynb 23
def fastbook_on_colab():
    """Set up environment to run fastbook notebooks for colab"""
    instructions = ['pip install -Uqq fastbook',
                    'wget -O utils.py https://raw.githubusercontent.com/vtecftwy/fastbook/walk-thru/utils.py',
                    'wget -O fastbook_utils.py https://raw.githubusercontent.com/vtecftwy/fastbook/walk-thru/fastbook_utils.py'
                    ]
