# camels #
# https://confluence.ecmwf.int/display/COPSRV/GloFAS+Climate+Data+Store
# https://www.sciencedirect.com/journal/data-in-brief
# https://data.mendeley.com/datasets/5vzp6svhwh/4
# https://zenodo.org/record/4218413#.YA6w7BZS-Uk
# https://www.sciencedirect.com/search?qs=time%20series&pub=Data%20in%20Brief&cid=311593
# https://doi.pangaea.de/10.1594/PANGAEA.898217
# https://doi.pangaea.de/10.1594/PANGAEA.811992  # protected
# https://doi.pangaea.de/10.1594/PANGAEA.905446
# https://doi.pangaea.de/10.1594/PANGAEA.900958
# https://doi.pangaea.de/10.1594/PANGAEA.831193
# https://data.world/datagov-uk/223a8f60-e3ac-4a25-987d-587cc3a92fa1
# https://www.bafg.de/GRDC/EN/04_spcldtbss/41_ARDB/arcticHycos.html?nn=201574 flow dataset
# https://doi.pangaea.de/10.1594/PANGAEA.924561 Air temp Tiangin china


# https://zenodo.org/records/14450214

# https://zenodo.org/record/3712407#.YExYDtyRWUk
# https://zenodo.org/record/3844201#.YExYi9yRWUk
# https://zenodo.org/record/1471322#.YExYftyRWUk
# https://zenodo.org/record/3961605#.YExYcdyRWUk
# https://zenodo.org/record/1452383#.YExZRdyRWUk
# https://zenodo.org/record/4428151#.YExZPdyRWUk
# https://zenodo.org/record/3903238#.YExZItyRWUk
# https://zenodo.org/record/3670864#.YExZFdyRWUk
# https://zenodo.org/record/3834623#.YExY7tyRWUk
# https://zenodo.org/record/4029572#.YExY5NyRWUk
# https://zenodo.org/record/4552842#.YExaR9yRWUk
# https://zenodo.org/record/3466097#.YExaQ9yRWUk
# https://zenodo.org/record/4327078#.YExaCdyRWUk
# https://zenodo.org/record/3712397#.YExbsdyRWUk
# https://zenodo.org/record/3560706#.YExeztyRWUk
# https://zenodo.org/record/3698998#.YExekdyRWUk
# https://zenodo.org/record/3564237#.YExlh9yRWUl
# https://zenodo.org/record/581145#.YExeV9yRWUk
# https://zenodo.org/record/3978225#.YExeEtyRWUk
# https://zenodo.org/record/3763766#.YExdntyRWUk
# https://zenodo.org/record/3744217#.YExdi9yRWUk
# https://zenodo.org/record/3948568#.YExdeNyRWUk
# https://zenodo.org/record/3538207#.YExdbtyRWUk
# https://zenodo.org/record/1486058#.YExc-dyRWUk
# https://zenodo.org/record/3561032#.YExc7tyRWUk
# https://zenodo.org/record/1466038#.YExc3dyRWUk
# https://zenodo.org/record/581186#.YExcz9yRWUk
# https://zenodo.org/record/4572636#.YExcwNyRWUk
# https://zenodo.org/record/1267837#.YExcZNyRWUk
# https://zenodo.org/record/3808223#.YExcX9yRWUk
# https://zenodo.org/record/4447435#.YExcWtyRWUk
# https://zenodo.org/record/1300354#.YExcVdyRWUk
# https://zenodo.org/record/4308036#.YExcJdyRWUk
# https://zenodo.org/record/3459610#.YExhNNyRWUk
# https://zenodo.org/record/3763342#.YExhCdyRWUk
# https://zenodo.org/record/4559571#.YExhBNyRWUk
# https://zenodo.org/record/3663630#.YExg89yRWUk
# https://zenodo.org/record/4382937#.YExg7dyRWUk
# https://zenodo.org/record/3876148#.YExgUdyRWUk
# https://zenodo.org/record/3982962#.YExgTdyRWUk
# https://zenodo.org/record/2559480#.YExvWtyRWUk
# https://zenodo.org/record/4094684#.YExvS9yRWUk
# https://zenodo.org/record/2596929#.YExvP9yRWUk
# https://zenodo.org/record/977773#.YExvEtyRWUk
# https://zenodo.org/record/3520146#.YExu_tyRWUk
# https://zenodo.org/record/3836648#.YExu09yRWUk
# https://zenodo.org/record/4290294#.YExo5tyRWUk
# https://zenodo.org/record/2728636#.YEx4EdyRWUk
# https://zenodo.org/record/3581187#.YEx5CNyRWUk
# https://zenodo.org/record/3946242#.YEx5FtyRWUk
# https://zenodo.org/record/883100#.YEx5L9yRWUk
# https://zenodo.org/record/3239401#.YEx5gtyRWUk
# https://zenodo.org/record/4183611#.YEx5vNyRWUk
# https://zenodo.org/record/4559696#.YEx5xdyRWUk
# https://zenodo.org/record/3776011#.YEx6YdyRWUk
# https://zenodo.org/record/4315647#.YEx6v9yRWUk
# https://zenodo.org/record/1185084#.YEx77NyRWUk
# https://zenodo.org/record/4271209#.YEx7z9yRWUk
# https://zenodo.org/record/4570780#.YEx7y9yRWUk
# https://zenodo.org/record/3593395#.YEx7x9yRWUk
# https://zenodo.org/record/3632501#.YEx7qtyRWUk
# https://zenodo.org/record/1122635#.YEx7ndyRWUk
# https://zenodo.org/record/3893897#.YEx7gNyRWUk
# https://zenodo.org/record/4395737#.YEx7a9yRWUk
# https://zenodo.org/record/3779473#.YEx7aNyRWUk
# https://zenodo.org/record/1226394#.YEx7O9yRWUk
# https://zenodo.org/record/4391461#.YEx7MtyRWUk
# https://zenodo.org/record/4247833#.YEx7HtyRWUk
# https://zenodo.org/record/1486058#.YEx7G9yRWUk
# https://zenodo.org/record/3928587#.YEx7E9yRWUk
# https://zenodo.org/record/4341521#.YEx7DdyRWUk
# https://zenodo.org/record/3974871#.YEx7CdyRWUk
# https://zenodo.org/record/1298526#.YEx7B9yRWUk
# https://zenodo.org/record/57293#.YEx6_dyRWUk
# https://zenodo.org/record/4268711#.YEx6-9yRWUk
# https://zenodo.org/record/322827#.YEx69tyRWUk
# https://zenodo.org/record/1050301#.YEx6y9yRWUk
# https://zenodo.org/record/4734372#.YKc9QKGRWUk
# https://www.nature.com/articles/s41597-019-0288-y#Abs1
# https://catalogue-imos.aodn.org.au/geonetwork/srv/api/records/9e2ba32a-5da3-4ea5-b750-e6279680dd71
#https://daac.ornl.gov/cgi-bin/dsviewer.pl?ds_id=1566
# https://doi.pangaea.de/10.1594/PANGAEA.885860
# https://doi.pangaea.de/10.1594/PANGAEA.913939
# https://www.nature.com/articles/s41597-019-0346-5#Sec8


# ETP
# https://zenodo.org/record/4038399#.YEx6INyRWUk
# https://zenodo.org/record/4601596#.YEx6M9yRWUk
# https://zenodo.org/record/3981919#.YEx6ONyRWUk
# https://zenodo.org/record/4271331#.YEx6PdyRWUk
# https://zenodo.org/record/3726856#.YEx6RdyRWUk
# https://zenodo.org/record/4580292#.YEx6TtyRWUk
# https://zenodo.org/record/1044306#.YEx6UNyRWUk
# https://zenodo.org/record/3891936#.YEx7S9yRWUk
# https://zenodo.org/record/4060319#.YEx7QtyRWUk


# rr
# https://zenodo.org/record/3341592#.YEx5RtyRWUk
# https://zenodo.org/record/3931582#.YEx5W9yRWUk
# https://zenodo.org/record/3528098#.YEx64NyRWUk
# https://hess.copernicus.org/articles/25/3105/2021/
# https://www.nature.com/articles/s41597-019-0282-4#Sec12
# https://www.nature.com/articles/sdata201880#Tab3
# https://edg.epa.gov/metadata/catalog/search/resource/details.page?uuid=https://doi.org/10.23719/1378947

# air
# https://zenodo.org/record/4311854#.YExpwNyRWUk
# https://zenodo.org/record/4281271#.YExpYNyRWUk


# ocean
# https://zenodo.org/record/4600696#.YExpSdyRWUk


# Water Quality
# https://zenodo.org/record/1495558#.YExqFtyRWUk

# Flow
# https://zenodo.org/record/3941890#.YExp5NyRWUk
# https://zenodo.org/record/1206188#.YExn-dyRWUk
# https://zenodo.org/record/4394503#.YEx6ndyRWUk
# https://zenodo.org/record/3240954#.YEx6s9yRWUk


# Groundwater
# https://zenodo.org/record/3887120#.YExq1tyRWUk
# https://zenodo.org/record/3928587#.YExnztyRWUk
# https://zenodo.org/record/1158631#.YEx7ZdyRWUk
# https://zenodo.org/record/4139912#.YEx7XdyRWUk


# Weather
# https://zenodo.org/record/3678799#.YExsP9yRWUk
# https://zenodo.org/record/3679247#.YExsOdyRWUk
# https://zenodo.org/record/3678789#.YExsN9yRWUk
# https://zenodo.org/record/4567325#.YExqjtyRWUk
# https://zenodo.org/record/3549899#.YExqNdyRWUk
# https://zenodo.org/record/4319773#.YExoq9yRWUk
# https://zenodo.org/record/4319770#.YExooNyRWUk
# https://zenodo.org/record/4319756#.YExnl9yRWUk
# https://zenodo.org/record/854619#.YExnityRWUk
# https://essd.copernicus.org/articles/13/1289/2021/
# https://essd.copernicus.org/articles/13/1307/2021/
# https://www.tr32db.uni-koeln.de/search/view.php?dataID=1786
# https://doi.org/10.3334/ORNLDAAC/1840

#DWD
# https://opendata.dwd.de/climate_environment/CDC/observations_germany/

# geologic
# https://zenodo.org/record/4536561#.YExpQNyRWUk
# https://zenodo.org/record/2549499#.YExo09yRWUk


# 2D time series datasets
# https://zenodo.org/record/1135230#.YExYotyRWUk
# https://zenodo.org/record/2630456#.YExb4tyRWUk
# https://zenodo.org/record/4559368#.YExd1NyRWUk
# https://zenodo.org/record/4542076#.YExuxtyRWUk
# https://zenodo.org/record/4489056#.YExoBtyRWUk
# https://zenodo.org/record/1157344#.YExnqNyRWUk
# https://www.nature.com/articles/s41597-020-0450-6
# https://www.nature.com/articles/sdata201542#Abs1
# https://www.nature.com/articles/s41597-019-0228-x
# https://zenodo.org/record/4058167
# https://www.nature.com/articles/sdata2018224#Sec10


# soil
# https://www.tr32db.uni-koeln.de/search/view.php?dataID=1839
# https://www.tr32db.uni-koeln.de/search/view.php?dataID=1838
# https://www.tr32db.uni-koeln.de/search/view.php?dataID=1837
# https://www.tr32db.uni-koeln.de/search/view.php?dataID=1760
# https://www.tr32db.uni-koeln.de/search/view.php?dataID=1761
import os

import numpy as np

from .download_pangaea import PanDataSet
from .utils import maybe_download, download_and_unzip


SEP = os.sep
# TODO, add visualization

# TODO all available datasets should be available using a single interface instead of importing each separately


class Datasets(object):
    """
    This is the base class for datasets

    Note:
        We don't host datasets. Each dataset is downloaded fromt he target remote
        server and saved into local disk.
    """
    def __init__(
            self,
            name=None,
            units=None,
            path:str = None,
            verbosity:int = 1,
            processes:int = None,
            remove_zip:bool = False,
            overwrite:bool = False,
            float_precision:str = np.float32,
            integer_precision:str = np.int32
    ):
        """
        Arguments:
            name : str (default=None)
                name of dataset
            units : str, (default=None)
                the unit system being used
            path : str (default=None)
                path where the data is available (manually downloaded).
                If None, it will be downloaded
            processes : int
                number of processes to use for parallel processing
            verbosity : int
                determines the amount of information to be printed
            remove_zip : bool
                whether to remove the zip files after unz
        """
        if name is None:
            name = self.__class__.__name__

        if units is not None:
            assert units in ['si', 'imperial', 'metric']

        self.units = units
        self.name = name
        self.processes = processes
        self.verbosity = verbosity
        self.path = path
        self.remove_zip = remove_zip
        self.overwrite = overwrite
        self.fp = float_precision
        self.ip = integer_precision

    @property
    def url(self):
        raise NotImplementedError(f"url must be defined.")

    @property
    def base_ds_dir(self):
        """Base datasets directory"""
        return os.path.join(os.path.dirname(__file__), 'data')

    @property
    def path(self):
        return self._path

    @path.setter
    def path(self, x):
        if x is None:
            # path is not given dataset is not downloaded yet
            if self.__class__.__name__.startswith('CAMELS'):
                x = os.path.join(self.camels_dir, self.__class__.__name__)
            else:
                x = os.path.join(self.base_ds_dir, self.__class__.__name__)

            if not os.path.exists(x):
                os.makedirs(x)
        else:
            #assert os.path.exists(x), f"The path {x} does not exist"
            if not os.path.exists(x) and self.verbosity>0:
                pass  # todo : why this check?
                # print(f"The path {x} does not exist. Creating the directory.")
            x = os.path.join(x, self.__class__.__name__)
        # sanity_check(self.name, x)
        self._path = x

    def _download(self, overwrite=False, **kwargs):
        """
        Downloads the dataset. If already downloaded, then

        Parameters
        -----------
        overwrite : bool
        **kwargs :
            any keyword arguments for maybe_download function
        """
        maybe_download(
            self.path, 
            overwrite=overwrite,
            url=self.url, 
            name=self.name,
            verbosity=self.verbosity,
            **kwargs
            )
        return

    def _download_and_unzip(self):
        # todo : this function should not exist, we should always call _download
        download_and_unzip(self.path, self.url, verbosity=self.verbosity)
        return

    def download_from_pangaea(self, overwrite=False):

        if os.path.exists(self.path):
            if overwrite:
                if self.verbosity: print("removing previously downloaded data and downloading again")
            else:
                if self.verbosity: print(f"The path {self.path} already exists.")
                self.data_files = [f for f in os.listdir(self.path) if f.endswith('.txt')]
                self.metadata_files = [f for f in os.listdir(self.path) if f.endswith('.json')]
                if len(self.data_files) == 0:
                    if self.verbosity: print(f"The path {self.path} is empty so downloading the files again")
                    self._download_from_pangaea()
        else:
            os.makedirs(self.path)
            self._download_from_pangaea()
        return

    def _download_from_pangaea(self):
        self.data_files = []
        self.metadata_files = []
        ds = PanDataSet(self.url)
        kids = ds.children()
        if len(kids) > 1:
            for kid in kids:
                kid_ds = PanDataSet(kid)
                fname = kid_ds.download(self.path)
                self.metadata_files.append(fname + '._metadata.json')
                self.data_files.append(fname + '.txt')
        else:
            fname = ds.download(self.path)
            self.metadata_files.append(fname + '._metadata.json')
            self.data_files.append(fname + '.txt')
        return
    
    def remove_zip_files(self):
        for f in os.listdir(self.path):
            if f.endswith('.zip') or f.endswith(".gz"):
                os.remove(os.path.join(self.path, f))
        return

    def maybe_remove_zip_files(self):
        if self.remove_zip:
            self.remove_zip_files()
        return
