# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs-dev/1_02_ml.ipynb.

# %% ../nbs-dev/1_02_ml.ipynb 4
from __future__ import annotations
from IPython.display import Image, display
from pathlib import Path
from pprint import pprint
from typing import Any, List
from zipfile import ZipFile

import configparser
import datetime as dt
import json
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import shutil
import subprocess

# %% auto 0
__all__ = ['are_features_consistent', 'kaggle_setup_colab', 'kaggle_list_files', 'kaggle_download_competition_files',
           'fastbook_on_colab']

# %% ../nbs-dev/1_02_ml.ipynb 6
def are_features_consistent(
    df1:pd.DataFrame,                 # First set, typically the training set
    df2:pd.DataFrame,                 # Second set, typically the test set or inference set 
    dependent_variables:list[str] = None, # List of column name(s) for dependent variables
    raise_error:bool = False,             # True to raise an error if not consistent
)-> bool :                                # True if features in train and test datasets are consistent, False otherwise
    """Verify that features/columns in training and test sets are consistent"""
    if dependent_variables is None:
        features_df1 = df1.columns
    else:
        features_df1 = df1.drop(dependent_variables, axis=1).columns
    features_df2 = df2.columns
    features_diff = set(features_df1).symmetric_difference(features_df2)
    if features_diff == set():
        return True
    else:
        if raise_error:
            raise ValueError(f"Discrepancy between training and test feature set: {features_diff}")
        else: return False

# %% ../nbs-dev/1_02_ml.ipynb 18
def kaggle_setup_colab(path_to_config_file:Path|str = None      # path to the configuration file (e.g. config.cfg)
                      ):
    """Update kaggle API and create security key json file from config file on Google Drive"""
    # Create API security key file
    path_to_kaggle = Path('/root/.kaggle')
    os.makedirs(path_to_kaggle, exist_ok=True)
    
    # Validate path_to_config
    if path_to_config_file is None:
        path_to_config_file = Path('/content/gdrive/MyDrive/private-across-accounts/config-api-keys.cfg')
    if isinstance(path_to_config_file, str): 
        path_to_config_file = Path(path_to_config_file)
    if not path_to_config_file.is_file():
        raise ValueError(f"No file at {path_to_config_file.absolute()}. Check the path")
    
    # retrieve configuration, create token and save it
    username = get_config_value('kaggle', 'kaggle_username', path_to_config_file=path_to_config_file)
    key = get_config_value('kaggle', 'kaggle_key', path_to_config_file=path_to_config_file)

    api_token = {"username": username, "key": key}
    with open(path_to_kaggle / 'kaggle.json', 'w') as file:
        json.dump(api_token, file)
        os.fchmod(file.fileno(), 600)

    # Update kaggle API software
    run_cli('pip install -Uqq kaggle --upgrade')

# %% ../nbs-dev/1_02_ml.ipynb 20
def kaggle_list_files(code:str = None,          # code for the kaggle competition or dataset
                      mode:str ='competitions'  # mode: `competitions` or `datasets`
                     ):
    """List all files available in the competition or dataset for the passed code"""
    if code is None:
        print(f"code is None, please provide the code of the kaggle competition or dataset")
        return 'Failed'
    elif mode not in ['competitions', 'datasets']:
        print(f"mode must be either 'competitions' or 'datasets', not {mode}")
        return 'Failed'
    else:
        print(f"Listing the files available for {mode}: <{code}>")
        run_cli(f"kaggle {mode} files {code}")

    if mode == 'competitions':
        print(f"{'=' * 140}")
        print(f"Make sure to set the parameters for <{code}> in next cell:")
        print(f" - kaggle_project_folder_name: string with name of the project folder")
        print(f" - train_files: list of files to place into the <train> folder")
        print(f" - test_files: list of files to place into the <test> folder")
        print(f" - submit_files: list of files to place into the <submit> folder")
        print(f"{'=' * 140}")

# %% ../nbs-dev/1_02_ml.ipynb 21
def kaggle_download_competition_files(
    competition_code:str = None, 
    train_files:list() = [], 
    test_files:list = [], 
    submit_files:list = [], 
    project_folder:str = 'ds'
    ):
    """download all files for passed competition, unzip them if required, move them to train, test and submit folders

    competition_code: str       code of the kaggle competition
    train_files: list of str    names of files to be moved into train folder
    test_files: list of str     names of files to be moved into test folder
    submit_files: list of str   names of files to be moved into submit folder
    """
    if competition_code is None:
        print(f"competition_code is None, please provide the code of the kaggle competition")
        return 'Failed'
    else:
        list_of_datasets = {'train': train_files,
                            'test': test_files,
                            'submit': submit_files}

        # creating a project directory and set paths
        if not os.path.exists(project_folder):
            os.makedirs(project_folder)

        path2datasets = Path(f"/content/{project_folder}")
        path2datasets_str = str(path2datasets.absolute())

        # download all files from kaggle
        run_cli(f"kaggle competitions download -c {competition_code} -p {path2datasets}")

        print(f"{'=' * 140}")
        print('Downloaded files:')
        for f in [item for item in path2datasets.iterdir() if item.is_file()]:
            print(f" - {f}")
        print(f"{'=' * 140}")

        # Unzip all zipped files
        for f in path2datasets.glob('*.zip'):
            print(f"Unzipping {f.name}")
            zip_f = ZipFile(f)
            zip_f.extractall(path=path2datasets)
            os.remove(f)
        print(f"{'=' * 140}")

        # Move all data files to the correct data folder
        for dataset_folder, files in list_of_datasets.items():
            if not os.path.exists(f'{project_folder}/{dataset_folder}'):
                os.makedirs(f'{project_folder}/{dataset_folder}')

            for f in files:
                print(f"Moving {f} to {dataset_folder}")
                p2f = path2datasets / f
                if p2f.suffix == '.csv':
                    shutil.move(path2datasets / f, path2datasets / dataset_folder / f)
                else:
                    msg = f"Does not support {p2f.name}'s extension {p2f.suffix}"
                    raise RuntimeError(msg)

        print(f"{'=' * 140}")
        print('Done loading Kaggle files and moving them to corresponding folders')

# %% ../nbs-dev/1_02_ml.ipynb 23
def fastbook_on_colab():
    """Set up environment to run fastbook notebooks for colab"""
    instructions = ['pip install -Uqq fastbook',
                    'wget -O utils.py https://raw.githubusercontent.com/vtecftwy/fastbook/walk-thru/utils.py',
                    'wget -O fastbook_utils.py https://raw.githubusercontent.com/vtecftwy/fastbook/walk-thru/fastbook_utils.py'
                    ]
