# camels #
# https://confluence.ecmwf.int/display/COPSRV/GloFAS+Climate+Data+Store
# https://www.sciencedirect.com/journal/data-in-brief
# https://data.mendeley.com/datasets/5vzp6svhwh/4
# https://zenodo.org/record/4218413#.YA6w7BZS-Uk
# https://www.sciencedirect.com/search?qs=time%20series&pub=Data%20in%20Brief&cid=311593
# https://doi.pangaea.de/10.1594/PANGAEA.898217
# https://doi.pangaea.de/10.1594/PANGAEA.811992  # protected
# https://doi.pangaea.de/10.1594/PANGAEA.905446
# https://doi.pangaea.de/10.1594/PANGAEA.900958
# https://doi.pangaea.de/10.1594/PANGAEA.831193
# https://data.world/datagov-uk/223a8f60-e3ac-4a25-987d-587cc3a92fa1
# https://www.bafg.de/GRDC/EN/04_spcldtbss/41_ARDB/arcticHycos.html?nn=201574 flow dataset
# https://doi.pangaea.de/10.1594/PANGAEA.924561 Air temp Tiangin china


# https://zenodo.org/record/3712407#.YExYDtyRWUk
# https://zenodo.org/record/3844201#.YExYi9yRWUk
# https://zenodo.org/record/1471322#.YExYftyRWUk
# https://zenodo.org/record/3961605#.YExYcdyRWUk
# https://zenodo.org/record/1452383#.YExZRdyRWUk
# https://zenodo.org/record/4428151#.YExZPdyRWUk
# https://zenodo.org/record/3903238#.YExZItyRWUk
# https://zenodo.org/record/3670864#.YExZFdyRWUk
# https://zenodo.org/record/3834623#.YExY7tyRWUk
# https://zenodo.org/record/4029572#.YExY5NyRWUk
# https://zenodo.org/record/4552842#.YExaR9yRWUk
# https://zenodo.org/record/3466097#.YExaQ9yRWUk
# https://zenodo.org/record/4327078#.YExaCdyRWUk
# https://zenodo.org/record/3712397#.YExbsdyRWUk
# https://zenodo.org/record/3560706#.YExeztyRWUk
# https://zenodo.org/record/3698998#.YExekdyRWUk
# https://zenodo.org/record/3564237#.YExlh9yRWUl
# https://zenodo.org/record/581145#.YExeV9yRWUk
# https://zenodo.org/record/3978225#.YExeEtyRWUk
# https://zenodo.org/record/3763766#.YExdntyRWUk
# https://zenodo.org/record/3744217#.YExdi9yRWUk
# https://zenodo.org/record/3948568#.YExdeNyRWUk
# https://zenodo.org/record/3538207#.YExdbtyRWUk
# https://zenodo.org/record/1486058#.YExc-dyRWUk
# https://zenodo.org/record/3561032#.YExc7tyRWUk
# https://zenodo.org/record/1466038#.YExc3dyRWUk
# https://zenodo.org/record/581186#.YExcz9yRWUk
# https://zenodo.org/record/4572636#.YExcwNyRWUk
# https://zenodo.org/record/1267837#.YExcZNyRWUk
# https://zenodo.org/record/3808223#.YExcX9yRWUk
# https://zenodo.org/record/4447435#.YExcWtyRWUk
# https://zenodo.org/record/1300354#.YExcVdyRWUk
# https://zenodo.org/record/4308036#.YExcJdyRWUk
# https://zenodo.org/record/3459610#.YExhNNyRWUk
# https://zenodo.org/record/3763342#.YExhCdyRWUk
# https://zenodo.org/record/4559571#.YExhBNyRWUk
# https://zenodo.org/record/3663630#.YExg89yRWUk
# https://zenodo.org/record/4382937#.YExg7dyRWUk
# https://zenodo.org/record/3876148#.YExgUdyRWUk
# https://zenodo.org/record/3982962#.YExgTdyRWUk
# https://zenodo.org/record/2559480#.YExvWtyRWUk
# https://zenodo.org/record/4094684#.YExvS9yRWUk
# https://zenodo.org/record/2596929#.YExvP9yRWUk
# https://zenodo.org/record/977773#.YExvEtyRWUk
# https://zenodo.org/record/3520146#.YExu_tyRWUk
# https://zenodo.org/record/3836648#.YExu09yRWUk
# https://zenodo.org/record/4290294#.YExo5tyRWUk
# https://zenodo.org/record/2728636#.YEx4EdyRWUk
# https://zenodo.org/record/3581187#.YEx5CNyRWUk
# https://zenodo.org/record/3946242#.YEx5FtyRWUk
# https://zenodo.org/record/883100#.YEx5L9yRWUk
# https://zenodo.org/record/3239401#.YEx5gtyRWUk
# https://zenodo.org/record/4183611#.YEx5vNyRWUk
# https://zenodo.org/record/4559696#.YEx5xdyRWUk
# https://zenodo.org/record/3776011#.YEx6YdyRWUk
# https://zenodo.org/record/4315647#.YEx6v9yRWUk
# https://zenodo.org/record/1185084#.YEx77NyRWUk
# https://zenodo.org/record/4271209#.YEx7z9yRWUk
# https://zenodo.org/record/4570780#.YEx7y9yRWUk
# https://zenodo.org/record/3593395#.YEx7x9yRWUk
# https://zenodo.org/record/3632501#.YEx7qtyRWUk
# https://zenodo.org/record/1122635#.YEx7ndyRWUk
# https://zenodo.org/record/3893897#.YEx7gNyRWUk
# https://zenodo.org/record/4395737#.YEx7a9yRWUk
# https://zenodo.org/record/3779473#.YEx7aNyRWUk
# https://zenodo.org/record/1226394#.YEx7O9yRWUk
# https://zenodo.org/record/4391461#.YEx7MtyRWUk
# https://zenodo.org/record/4247833#.YEx7HtyRWUk
# https://zenodo.org/record/1486058#.YEx7G9yRWUk
# https://zenodo.org/record/3928587#.YEx7E9yRWUk
# https://zenodo.org/record/4341521#.YEx7DdyRWUk
# https://zenodo.org/record/3974871#.YEx7CdyRWUk
# https://zenodo.org/record/1298526#.YEx7B9yRWUk
# https://zenodo.org/record/57293#.YEx6_dyRWUk
# https://zenodo.org/record/4268711#.YEx6-9yRWUk
# https://zenodo.org/record/322827#.YEx69tyRWUk
# https://zenodo.org/record/1050301#.YEx6y9yRWUk
# https://zenodo.org/record/4734372#.YKc9QKGRWUk
# https://www.nature.com/articles/s41597-019-0288-y#Abs1
# https://catalogue-imos.aodn.org.au/geonetwork/srv/api/records/9e2ba32a-5da3-4ea5-b750-e6279680dd71
#https://daac.ornl.gov/cgi-bin/dsviewer.pl?ds_id=1566
# https://doi.pangaea.de/10.1594/PANGAEA.885860
# https://doi.pangaea.de/10.1594/PANGAEA.913939
# https://www.nature.com/articles/s41597-019-0346-5#Sec8


# ETP
# https://zenodo.org/record/4038399#.YEx6INyRWUk
# https://zenodo.org/record/4601596#.YEx6M9yRWUk
# https://zenodo.org/record/3981919#.YEx6ONyRWUk
# https://zenodo.org/record/4271331#.YEx6PdyRWUk
# https://zenodo.org/record/3726856#.YEx6RdyRWUk
# https://zenodo.org/record/4580292#.YEx6TtyRWUk
# https://zenodo.org/record/1044306#.YEx6UNyRWUk
# https://zenodo.org/record/3891936#.YEx7S9yRWUk
# https://zenodo.org/record/4060319#.YEx7QtyRWUk


# rr
# https://zenodo.org/record/3341592#.YEx5RtyRWUk
# https://zenodo.org/record/3931582#.YEx5W9yRWUk
# https://zenodo.org/record/3528098#.YEx64NyRWUk
# https://hess.copernicus.org/articles/25/3105/2021/
# https://www.nature.com/articles/s41597-019-0282-4#Sec12
# https://www.nature.com/articles/sdata201880#Tab3
# https://edg.epa.gov/metadata/catalog/search/resource/details.page?uuid=https://doi.org/10.23719/1378947

# air
# https://zenodo.org/record/4311854#.YExpwNyRWUk
# https://zenodo.org/record/4281271#.YExpYNyRWUk


# ocean
# https://zenodo.org/record/4600696#.YExpSdyRWUk


# Water Quality
# https://zenodo.org/record/1495558#.YExqFtyRWUk
#https://www.nature.com/articles/sdata201798#Sec18
# https://www.nature.com/articles/s41597-020-0455-1#Sec11

# Flow
# https://zenodo.org/record/3941890#.YExp5NyRWUk
# https://zenodo.org/record/1206188#.YExn-dyRWUk
# https://zenodo.org/record/4394503#.YEx6ndyRWUk
# https://zenodo.org/record/3240954#.YEx6s9yRWUk


# Groundwater
# https://zenodo.org/record/3887120#.YExq1tyRWUk
# https://zenodo.org/record/3928587#.YExnztyRWUk
# https://zenodo.org/record/1158631#.YEx7ZdyRWUk
# https://zenodo.org/record/4139912#.YEx7XdyRWUk


# Weather
# https://zenodo.org/record/3678799#.YExsP9yRWUk
# https://zenodo.org/record/3679247#.YExsOdyRWUk
# https://zenodo.org/record/3678789#.YExsN9yRWUk
# https://zenodo.org/record/4567325#.YExqjtyRWUk
# https://zenodo.org/record/3549899#.YExqNdyRWUk
# https://zenodo.org/record/4319773#.YExoq9yRWUk
# https://zenodo.org/record/4319770#.YExooNyRWUk
# https://zenodo.org/record/4319756#.YExnl9yRWUk
# https://zenodo.org/record/854619#.YExnityRWUk
# https://essd.copernicus.org/articles/13/1289/2021/
# https://essd.copernicus.org/articles/13/1307/2021/
# https://www.tr32db.uni-koeln.de/search/view.php?dataID=1786
# https://doi.org/10.3334/ORNLDAAC/1840

#DWD
# https://opendata.dwd.de/climate_environment/CDC/observations_germany/

# geologic
# https://zenodo.org/record/4536561#.YExpQNyRWUk
# https://zenodo.org/record/2549499#.YExo09yRWUk


# 2D time series datasets
# https://zenodo.org/record/1135230#.YExYotyRWUk
# https://zenodo.org/record/2630456#.YExb4tyRWUk
# https://zenodo.org/record/4559368#.YExd1NyRWUk
# https://zenodo.org/record/4542076#.YExuxtyRWUk
# https://zenodo.org/record/4489056#.YExoBtyRWUk
# https://zenodo.org/record/1157344#.YExnqNyRWUk
# https://www.nature.com/articles/s41597-020-0450-6
# https://www.nature.com/articles/sdata201542#Abs1
# https://www.nature.com/articles/s41597-019-0228-x
# https://zenodo.org/record/4058167
# https://www.nature.com/articles/sdata2018224#Sec10


# soil
# https://www.tr32db.uni-koeln.de/search/view.php?dataID=1839
# https://www.tr32db.uni-koeln.de/search/view.php?dataID=1838
# https://www.tr32db.uni-koeln.de/search/view.php?dataID=1837
# https://www.tr32db.uni-koeln.de/search/view.php?dataID=1760
# https://www.tr32db.uni-koeln.de/search/view.php?dataID=1761


import os
import glob
import random
import zipfile
import warnings
import shutil
from typing import Union, Tuple

try:
    import netCDF4
except ModuleNotFoundError:
    netCDF4 = None

try:
    import shapefile
except ModuleNotFoundError:
    shapefile = None

try:
    from shapely.geometry import shape, mapping
    from shapely.ops import unary_union
except (ModuleNotFoundError, OSError):
    shape, mapping, unary_union = None, None, None

try:
    import fiona
except ModuleNotFoundError:
    fiona = None

try:
    import xarray as xr
except ModuleNotFoundError:
    xr = None

import numpy as np
import pandas as pd

from ai4water.preprocessing.spatial_utils import find_records
from .download_pangaea import PanDataSet
from .download_zenodo import download_from_zenodo
from .utils import download, download_all_http_directory
from .utils import check_attributes, sanity_check
from ai4water.preprocessing.resample import Resampler

SEP = os.sep
# TODO, add visualization

# TODO all available datasets should be available using a single interface instead of importing each separately
DATASETS = [
    'ISWDC',
    'SEBAL_ET_CHINA',
    'GeoChemMatane',
    'PrecipBerlin',
    'HydroChemJava',
    'WaterChemVictoriaLakes',
    'WaterChemEcuador',
    'HydrocarbonsGabes',
    'SedimentAmersee',
    'FlowTetRiver',
    'HoloceneTemp',
    'RiverTempEroo',
    'StreamTempSpain',
    'FlowSedDenmark',
    'FlowSamoylov',
    'EtpPcpSamoylov',
    'RiverIsotope',
    'WQCantareira',
    'RiverTempSpain',
    'HydrometricParana',
    'FlowBenin',
    'YamaguchiClimateJp',
    'WQJordan2',
    'WQJordan',
    'Weisssee'
            ]


class Datasets(object):
    """
    Base class for datasets

    Note:
        We don't host datasets. Each dataset is downloaded fromt he target remote
        server and saved into local disk.
    """
    def __init__(self, name=None, units=None):
        """
        Arguments:
            name :
            units :
        """
        if name is None:
            name = self.__class__.__name__

        if units is not None:
            assert units in ['si', 'imperial', 'metric']

        self.units = units
        self.name = name

    @property
    def url(self):
        raise NotImplementedError

    @property
    def base_ds_dir(self):
        """Base datasets directory"""
        return os.path.join(os.path.dirname(__file__), 'data')

    @property
    def ds_dir(self):
        _dir = os.path.join(self.base_ds_dir, self.__class__.__name__)
        if not os.path.exists(_dir):
            os.makedirs(_dir)
        return _dir

    def _download(self, overwrite=False):
        """Downloads the dataset. If already downloaded, then"""
        if os.path.exists(self.ds_dir) and len(os.listdir(self.ds_dir)) > 0:
            if overwrite:
                print(f"removing previous data directory {self.ds_dir} and downloading new")
                shutil.rmtree(self.ds_dir)
                self._download_and_unzip()
            else:
                sanity_check(self.name, self.ds_dir)
                print(f"""
Not downloading the data since the directory 
{self.ds_dir} already exists.
Use overwrite=True to remove previously saved files and download again""")
        else:
            self._download_and_unzip()
        return

    def _download_and_unzip(self):
        if not os.path.exists(self.ds_dir):
            os.makedirs(self.ds_dir)
        if isinstance(self.url, str):
            if 'zenodo' in self.url:
                download_from_zenodo(self.ds_dir, self.url)
            else:
                download(self.url, self.ds_dir)
            self._unzip()
        elif isinstance(self.url, list):
            for url in self.url:
                if 'zenodo' in url:
                    download_from_zenodo(self.ds_dir, url)
                else:
                    download(url, self.ds_dir)
            self._unzip()
        elif isinstance(self.url, dict):
            for fname, url in self.url.items():
                if 'zenodo' in url:
                    download_from_zenodo(self.ds_dir, url)
                else:
                    download(url, os.path.join(self.ds_dir, fname))
            self._unzip()
        return

    def _unzip(self, dirname=None):
        """unzip all the zipped files in a directory"""
        if dirname is None:
            dirname = self.ds_dir

        all_files = glob.glob(f"{dirname}/*.zip")
        for f in all_files:
            src = os.path.join(dirname, f)
            trgt = os.path.join(dirname, f.split('.zip')[0])
            if not os.path.exists(trgt):
                print(f"unziping {src} to {trgt}")
                with zipfile.ZipFile(os.path.join(dirname, f), 'r') as zip_ref:
                    try:
                        zip_ref.extractall(os.path.join(dirname, f.split('.zip')[0]))
                    except OSError:
                        filelist = zip_ref.filelist
                        for _file in filelist:
                            if '.txt' in _file.filename or '.csv' in _file.filename or '.xlsx' in _file.filename:
                                zip_ref.extract(_file)

        # extracting tar.gz files todo, check if zip files can also be unpacked by the following oneliner
        gz_files = glob.glob(f"{self.ds_dir}/*.gz")
        for f in gz_files:
            shutil.unpack_archive(f, self.ds_dir)

        return

    def download_from_pangaea(self, overwrite=False):

        if os.path.exists(self.ds_dir):
            if overwrite:
                print("removing previously downloaded data and downloading again")
            else:
                print(f"The path {self.ds_dir} already exists.")
                self.data_files = [f for f in os.listdir(self.ds_dir) if f.endswith('.txt')]
                self.metadata_files = [f for f in os.listdir(self.ds_dir) if f.endswith('.json')]
                if len(self.data_files) == 0:
                    print(f"The path {self.ds_dir} is empty so downloading the files again")
                    self._download_from_pangaea()
        else:
            self._download_from_pangaea()
        return

    def _download_from_pangaea(self):
        self.data_files = []
        self.metadata_files = []
        ds = PanDataSet(self.url)
        kids = ds.children()
        if len(kids) > 1:
            for kid in kids:
                kid_ds = PanDataSet(kid)
                fname = kid_ds.download(self.ds_dir)
                self.metadata_files.append(fname + '._metadata.json')
                self.data_files.append(fname + '.txt')
        else:
            fname = ds.download(self.ds_dir)
            self.metadata_files.append(fname + '._metadata.json')
            self.data_files.append(fname + '.txt')
        return


class Weisssee(Datasets):

    dynamic_attributes = ['Precipitation_measurements',
                          'long_wave_upward_radiation',
                          'snow_density_at_30cm',
                          'long_wave_downward_radiation'
                          ]

    url = '10.1594/PANGAEA.898217'

    def fetch(self, **kwargs):
        self.download_from_pangaea()
        data = {}
        for f in self.data_files:
            fpath = os.path.join(self.ds_dir, f)
            df = pd.read_csv(fpath, **kwargs)

            if 'index_col' in kwargs:
                df.index = pd.to_datetime(df.index)

            data[f.split('.txt')[0]] = df

        return data


class ETP_CHN_SEBAL(Datasets):

    url = "https://zenodo.org/record/4218413#.YBNhThZS-Ul"


class ISWDC(Datasets):

    url = "https://zenodo.org/record/2616035#.YBNl5hZS-Uk"


class WQJordan(Weisssee):
    """Jordan River water quality data of 9 variables for two variables."""
    url = 'https://doi.pangaea.de/10.1594/PANGAEA.919103'


class WQJordan2(Weisssee):
    """Stage and Turbidity data of Jordan River"""
    url = '10.1594/PANGAEA.919104'


class YamaguchiClimateJp(Weisssee):
    """Daily climate and flow data of Japan from 2006 2018"""
    url = "https://doi.pangaea.de/10.1594/PANGAEA.909880"


class FlowBenin(Weisssee):
    """Flow data"""
    url = "10.1594/PANGAEA.831196"


class HydrometricParana(Weisssee):
    """Daily and monthly water level and flow data of Parana river Argentina
    from 1875 to 2017."""
    url = "https://doi.pangaea.de/10.1594/PANGAEA.882613"


class RiverTempSpain(Weisssee):
    """Daily mean stream temperatures in Central Spain for different periods."""
    url = "https://doi.pangaea.de/10.1594/PANGAEA.879494"


class WQCantareira(Weisssee):
    """Water quality and quantity primary data from field campaigns in the Cantareira Water Supply System,
     period Oct. 2013 - May 2014"""
    url = "https://doi.pangaea.de/10.1594/PANGAEA.892384"


class RiverIsotope(Weisssee):
    """399 δ18O and δD values in river surface waters of Indian River"""
    url = "https://doi.pangaea.de/10.1594/PANGAEA.912582"


class EtpPcpSamoylov(Weisssee):
    """Evpotranspiration and Precipitation at station TOWER on Samoylov Island Russia
     from 20110524 to 20110819 with 30 minute frequency"""
    url = "10.1594/PANGAEA.811076"


class FlowSamoylov(Weisssee):
    """Net lateral flow at station INT2 on Samoylov Island Russia
    from 20110612 to 20110819 with 30 minute frequency"""
    url = "10.1594/PANGAEA.811072"


class FlowSedDenmark(Weisssee):
    """Flow and suspended sediment concentration fields over tidal bedforms, ADCP profile"""
    url = "10.1594/PANGAEA.841977"


class StreamTempSpain(Weisssee):
    """Daily Mean Stream Temperature at station Tormes3, Central Spain from 199711 to 199906."""
    url = "https://doi.pangaea.de/10.1594/PANGAEA.879507"


class RiverTempEroo(Weisssee):
    """Water temperature records in the Eroo River and some tributaries (Selenga River basin, Mongolia, 2011-2012)"""
    url = "10.1594/PANGAEA.890070"


class HoloceneTemp(Weisssee):
    """Holocene temperature reconstructions for northeastern North America and the northwestern Atlantic,
     core Big_Round_Lake."""
    url = "10.1594/PANGAEA.905446"


class FlowTetRiver(Weisssee):
    """Daily mean river discharge at meteorological station Perpignan upstream, Têt basin France from 1980
    to 2000."""
    url = "10.1594/PANGAEA.226925"


class SedimentAmersee(Weisssee):
    """Occurence of flood laminae in sediments of Ammersee"""
    url = "10.1594/PANGAEA.746240"


class HydrocarbonsGabes(Weisssee):
    """Concentration and geological parameters of n-alkanes and n-alkenes in surface sediments from the Gulf of Gabes,
     Tunisia"""
    url = "10.1594/PANGAEA.774595"


class WaterChemEcuador(Weisssee):
    """weekly and biweekly Water chemistry of cloud forest streams at baseflow conditions,
     Rio San Francisco, Ecuador"""
    url = "10.1594/PANGAEA.778629"


class WaterChemVictoriaLakes(Weisssee):
    """Surface water chemistry of northern Victoria Land lakes"""
    url = "10.1594/PANGAEA.807883"


class HydroChemJava(Weisssee):
    """Hydrochemical data from subsurface rivers, coastal and submarine springsin a karstic region
     in southern Java."""
    url = "10.1594/PANGAEA.882178"


class PrecipBerlin(Weisssee):
    """Sub-hourly Berlin Dahlem precipitation time-series 2001-2013"""
    url = "10.1594/PANGAEA.883587"


class GeoChemMatane(Weisssee):
    """Geochemical data collected in shallow groundwater and river water in a subpolar environment
     (Matane river, QC, Canada)."""
    url = "10.1594/PANGAEA.908290"


class HydroMeteorAndes(Datasets):
    """Hydrometeriological dataset of tropical Andes region"""
    url = ["https://springernature.figshare.com/ndownloader/files/10514506",
           "https://springernature.figshare.com/ndownloader/files/10514509"]


class WeatherJena(Datasets):
    """
    10 minute weather dataset of Jena, Germany hosted at https://www.bgc-jena.mpg.de/wetter/index.html
    from 2002 onwards.
    """
    url = "https://www.bgc-jena.mpg.de/wetter/weather_data.html"

    def __init__(self, obs_loc='roof'):
        """
        The ETP data is collected at three different locations i.e. roof, soil and saale(hall).
        Arguments:
            obs_loc str: location of observation.
        """

        if obs_loc not in ['roof', 'soil', 'saale']:
            raise ValueError
        self.obs_loc = obs_loc

        super().__init__()

        sub_dir = os.path.join(self.ds_dir, self.obs_loc)

        if not os.path.exists(sub_dir):
            os.makedirs(sub_dir)

        download_all_http_directory(self.url, sub_dir, match_name=self.obs_loc)
        unzip_all_in_dir(sub_dir, 'zip')

    def fetch(self,
              st: str = None,
              en: str = None
              ) -> pd.DataFrame:
        """
        Fetches the time series data between given period as pandas dataframe.
        Arguments:
            st : start of data to be fetched. If None, the data from start will
                be retuned.
            en : end of data to be fetched. If None, the data from till end
                be retuned.
        Returns:
            a pandas dataframe.
        """

        sub_dir = os.path.join(self.ds_dir, self.obs_loc)
        all_files = glob.glob(f"{sub_dir}/*.csv")

        df = pd.DataFrame()
        for fpath in all_files:
            f_df = pd.read_csv(fpath, index_col='Date Time',
                               encoding='unicode_escape', na_values=-9999)
            f_df.index = pd.DatetimeIndex(f_df.index)
            df = pd.concat([df, f_df])  # todo, such concatenation is slow.

        df = df.sort_index()

        if st is None:
            st = df.index[0]
        if en is None:
            en = df.index[-1]

        return df[st:en]


class SWECanada(Datasets):
    """
    Daily Canadian historical Snow Water Equivalent dataset from 1928 to 2020
    https://doi.org/10.1080/07055900.2019.1598843
    """
    url = "https://doi.org/10.5194/essd-2021-160"
    feaures = ['snw', 'snd', 'den']
    q_flags = ['data_flag_snw', 'data_flag_snd', 'qc_flag_snw', 'qc_flag_snd']

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        self._download()

    def stations(self) -> list:
        nc = netCDF4.Dataset(os.path.join(self.ds_dir, 'CanSWE-CanEEN_1928-2020_v1.nc'))
        s = nc['station_id'][:]
        return s.tolist()

    @property
    def start(self):
        return '19280101'

    @property
    def end(self):
        return '20200731'

    def fetch(self,
              station_id: Union[None, str, float, int, list] = None,
              features: Union[None, str, list] = None,
              q_flags: Union[None, str, list] = None,
              st=None,
              en=None,
              ) -> dict:
        """
        Fetches time series data from selected stations.
        Arguments:
            station_id : station/stations to be retrieved. In None, then data
                          from all stations will be returned.
            features : Names of features to be retrieved. Following features
                are allowed:
                    - 'snw' snow water equivalent kg/m3
                    - 'snd' snow depth m
                    - 'den' snowpack bulk density kg/m3
                If None, then all three features will be retrieved.
            q_flags : If None, then no qflags will be returned. Following q_flag
                values are available.
                    - 'data_flag_snw'
                    - 'data_flag_snd'
                    - 'qc_flag_snw'
                    - 'qc_flag_snd'
            st : start of data to be retrieved
            en : end of data to be retrived.
        returns:
            a dictionary of dataframes of shape (st:en, features + q_flags) whose
            length is equal to length of stations being considered.
        """

        if station_id is None:
            station_id = self.stations()
        elif isinstance(station_id, str):
            station_id = [station_id]
        elif isinstance(station_id, list):
            pass
        elif isinstance(station_id, int):
            station_id = random.sample(self.stations(), station_id)
        elif isinstance(station_id, float):
            num_stations = int(len(self.stations()) * station_id)
            station_id = random.sample(self.stations(), num_stations)

        stns = self.stations()
        stn_id_dict = {k: v for k, v in zip(stns, np.arange(len(stns)))}
        stn_id_dict_inv = {v: k for k, v in stn_id_dict.items()}
        stn_ids = [stn_id_dict[i] for i in station_id]

        features = check_attributes(features, self.feaures)
        qflags = []
        if q_flags is not None:
            qflags = check_attributes(q_flags, self.q_flags)

        features_to_fetch = features + qflags

        all_stn_data = {}
        for stn in stn_ids:

            stn_df = self.fetch_station_attributes(stn, features_to_fetch, st=st, en=en)
            all_stn_data[stn_id_dict_inv[stn]] = stn_df

        return all_stn_data

    def fetch_station_attributes(self,
                                 stn,
                                 features_to_fetch,
                                 st=None,
                                 en=None,
                                 ) -> pd.DataFrame:
        """fetches attributes of one station"""

        # st, en = self._check_length(st, en)

        nc = netCDF4.Dataset(os.path.join(self.ds_dir, 'CanSWE-CanEEN_1928-2020_v1.nc'))

        stn_df = pd.DataFrame(columns=features_to_fetch)

        for var in nc.variables:
            if var in features_to_fetch:
                ma = np.array(nc[var][:])
                ma[ma == nc[var]._FillValue] = np.nan
                ta = ma[stn, :]  # target array of on station
                s = pd.Series(ta, index=pd.date_range(self.start, self.end, freq='D'), name=var)
                stn_df[var] = s[st:en]

        nc.close()

        return stn_df


class RRLuleaSweden(Datasets):
    """
    Rainfall runoff data for an urban catchment from 2016-2019
    https://doi.org/10.5194/hess-24-869-2020
    """
    url = "https://zenodo.org/record/3931582"


class RRAlpileCatchments(Datasets):
    """
    Modelled runoff in contrasting Alpine catchments in Austria from 1981 to 2100
    using 14 models.
    past 1981 - 2010
    future
    https://hess.copernicus.org/preprints/hess-2021-92/
    """
    url = "https://zenodo.org/record/4539986"


class ETPAgroForestGermany(Datasets):
    """
    Evapotranspiration over agroforestry sites in Germany
    https://doi.org/10.5194/bg-17-5183-2020
    SiteName_Landuse_Content_Figures_Tables.csv
    """
    url = "https://zenodo.org/record/4038399"


class ETPTelesinaItaly(Datasets):
    """
    Daily rain and reference evapotranspiration for three years 2002-2004
    """
    url = "https://zenodo.org/record/3726856"


class MtropicsLaos(Datasets):
    """
    Downloads and prepares hydrological, climate and land use data for Laos from
    Mtropics_ website and ird_ data servers.

    See the notebook_ for exploratory data analysis of this data

    Methods
    -------
        - fetch_lu
        - fetch_ecoli
        - fetch_rain_gauges
        - fetch_weather_station_data
        - fetch_pcp
        - fetch_hydro
        - make_regression

    .. _Mtropics:
        https://mtropics.obs-mip.fr/catalogue-m-tropics/

    .. _notebook:
        https://nbviewer.jupyter.org/github/AtrCheema/AI4Water/blob/master/examples/datasets/MtropicsLaos.ipynb

    .. _ird:
        https://dataverse.ird.fr/dataset.xhtml?persistentId=doi:10.23708/EWOYNK
    """
    target = ['Ecoli_mpn100']

    url = {
        'lu.zip':
            "https://services.sedoo.fr/mtropics/data/v1_0/download?collectionId=0f1aea48-2a51-9b42-7688-a774a8f75e7a",
        'pcp.zip':
            "https://services.sedoo.fr/mtropics/data/v1_0/download?collectionId=3c870a03-324b-140d-7d98-d3585a63e6ec",
        'hydro.zip':
            "https://services.sedoo.fr/mtropics/data/v1_0/download?collectionId=389bbea0-7279-12c1-63d0-cfc4a77ded87",
        'rain_guage.zip':
            "https://services.sedoo.fr/mtropics/data/v1_0/download?collectionId=7bc45591-5b9f-a13d-90dc-f2a75b0a15cc",
        'weather_station.zip':
            "https://services.sedoo.fr/mtropics/data/v1_0/download?collectionId=353d7f00-8d6a-2a34-c0a2-5903c64e800b",
        'ecoli_data.csv':
            "https://dataverse.ird.fr/api/access/datafile/5435",
        "ecoli_dict.csv":
            "https://dataverse.ird.fr/api/access/datafile/5436",
        "soilmap.zip":
            "https://dataverse.ird.fr/api/access/datafile/5430",
        "subs1.zip":
            "https://dataverse.ird.fr/api/access/datafile/5432"
    }

    physio_chem_features = {
        "T_deg": "T",
        "EC_s/cm": "EC",
        "DO_percent": "DOpercent",
        "DO_mgl": "DO",
        "pH": "pH",
        "ORP_mV": "ORP",  # stream water oxidation-reduction potential
        "Turbidity_NTU": "Turbidity",
        "TSS_gL": "TSS",

                            }

    weather_station_data = ['air_temp', 'rel_hum', 'wind_speed', 'sol_rad']
    inputs = weather_station_data + ['water_level', 'pcp', 'susp_pm']

    def __init__(self, **kwargs):

        if xr is None:
            raise ModuleNotFoundError("xarray must be installed to use datasets sub-module")

        super().__init__(**kwargs)
        self._download()

        # we need to pre-process the land use shapefiles
        in_dir = os.path.join(self.ds_dir, 'lu')
        out_dir = os.path.join(self.ds_dir, 'lu1')
        if not os.path.exists(out_dir):
            os.makedirs(out_dir)

        files = glob.glob(f'{in_dir}/*.shp')
        for fpath in files:
            f = os.path.basename(fpath)
            shp_file = os.path.join(in_dir, f)
            op = os.path.join(out_dir, f)

            _process_laos_shpfiles(shp_file, op)

    def fetch_lu(self, processed=False):
        """returns landuse_ data as list of shapefiles.

        .. _landuse:
            https://doi.org/10.1038/s41598-017-04385-2"""
        lu_dir = os.path.join(self.ds_dir, f"{'lu1' if processed else 'lu'}")
        files = glob.glob(f'{lu_dir}/*.shp')
        return files

    def fetch_physiochem(
            self,
            features: Union[list, str] = 'all',
            st: Union[str, pd.Timestamp] = '20110525 10:00:00',
            en: Union[str, pd.Timestamp] = '20210406 15:05:00',
    ) -> pd.DataFrame:
        """
        Fetches physio-chemical features of Huoy Pano catchment Laos.

        Arguments:
            st : start of data.
            en :end of data.
            features :
                The physio-chemical features to fetch. Following features
                are available

                - 'T',
                - 'EC',
                - 'DOpercent',
                - 'DO',
                - 'pH',
                - 'ORP',
                - 'Turbidity',
                'TSS'

        Returns:
            a pandas dataframe
        """

        if isinstance(features, list):
            _features = []
            for f in features:
                _features.append(self.physio_chem_features[f])
        else:
            assert isinstance(features, str)
            if features == 'all':
                _features = features
            else:
                _features = self.physio_chem_features[features]

        features = check_attributes(_features, list(self.physio_chem_features.values()))

        fname = os.path.join(self.ds_dir, 'ecoli_data.csv')
        df = pd.read_csv(fname, sep='\t')
        df.index = pd.to_datetime(df['Date_Time'])

        df = df[features]

        col_names = {v: k for k, v in self.physio_chem_features.items() if v in features}

        df = df.rename(columns=col_names)

        return df.loc[st:en]

    def fetch_ecoli(
            self,
            features: Union[list, str] = 'Ecoli_mpn100',
            st: Union[str, pd.Timestamp] = '20110525 10:00:00',
            en: Union[str, pd.Timestamp] = '20210406 15:05:00',
            remove_duplicates: bool = True,
    ) -> pd.DataFrame:
        """
        Fetches E. coli data collected at the outlet. See Ribolzi_ et al., 2021
        and Boithias_ et al., 2021 for reference.
        NaNs represent missing values. The data is randomly sampled between 2011
        to 2021 during rainfall events. Total 368 E. coli observation points are available now.

        Arguments:
            st :
                start of data. By default the data is fetched from the point it
                is available.
            en :
                end of data. By default the data is fetched til the point it is
                available.
            features :
                E. coli concentration data. Following data are available

                - Ecoli_LL_mpn100: Lower limit of the confidence interval
                - Ecoli_mpn100: Stream water Escherichia coli concentration
                - Ecoli_UL_mpn100: Upper limit of the confidence interval
            remove_duplicates : whether to remove duplicates or not. This is because
                some values were recorded within a minute,
        Returns:
            a pandas dataframe consisting of features as columns.

        .. _Ribolzi:
            https://dataverse.ird.fr/dataset.xhtml?persistentId=doi:10.23708/EWOYNK

        .. _Boithias:
            https://doi.org/10.1002/hyp.14126

        """
        fname = os.path.join(self.ds_dir, 'ecoli_data.csv')
        df = pd.read_csv(fname, sep='\t')
        df.index = pd.to_datetime(df['Date_Time'])

        available_features = {
            "Ecoli_LL_mpn100": "E-coli_4dilutions_95%-CI-LL",  # Lower limit of the confidence interval
            "Ecoli_mpn100": "E-coli_4dilutions",  # Stream water Escherichia coli concentration
            "Ecoli_UL_mpn100": "E-coli_4dilutions_95%-CI-UL"  # Upper limit of the confidence interval
        }
        if isinstance(features, list):
            _features = []
            for f in features:
                _features.append(available_features[f])
        else:
            assert isinstance(features, str)
            if features == 'all':
                _features = features
            else:
                _features = available_features[features]

        features = check_attributes(_features, list(available_features.values()))

        if remove_duplicates:
            df = df[~df.index.duplicated(keep='first')]

        df = df.sort_index()

        df = df[features]

        col_names = {v: k for k, v in available_features.items() if v in features}

        df = df.rename(columns=col_names)

        return df.loc[st:en]

    def fetch_rain_gauges(
            self,
            st: Union[str, pd.Timestamp] = "20010101",
            en: Union[str, pd.Timestamp] = "20191231",
    ) -> pd.DataFrame:
        """
        fetches data from 7 rain gauges_ which is collected at daily time step
        from 2001 to 2019.

        Arguments:
            st : start of data. By default the data is fetched from the point it
                is available.
            en : end of data. By default the data is fetched til the point it is
                available.
        Returns:
            a dataframe of 7 columns, where each column represnets a rain guage
            observations. The length of dataframe depends upon range defined by
            `st` and `en` arguments.

        .. _gauges:
            https://doi.org/10.1038/s41598-017-04385-2
        """
        # todo, does nan means 0 rainfall?
        fname = os.path.join(self.ds_dir, 'rain_guage', 'rain_guage.nc')
        if not os.path.exists(fname):
            files = glob.glob(f"{os.path.join(self.ds_dir, 'rain_guage')}/*.xlsx")
            df = pd.DataFrame()
            for f in files:
                _df = pd.read_excel(f, sheet_name='Daily', usecols=['R1', 'R2', 'R3', 'R4', 'R5', 'R6', 'R7'])
                df = pd.concat([df, _df])

            df = df.reset_index(drop=True)  # index is of type Int64Index
            df.to_xarray().to_netcdf(fname)

        else:  # feather file already exists so load from it
            df = xr.load_dataset(fname).to_dataframe()

        df.index = pd.date_range('20010101', periods=len(df), freq='D')

        return df[st:en]

    def fetch_weather_station_data(
            self,
            st: Union[str, pd.Timestamp] = "20010101 01:00:00",
            en: Union[str, pd.Timestamp] = "20200101 00:00:00",
            freq: str = 'H'
    ) -> pd.DataFrame:
        """
        fetches hourly weather_ station data which consits of air temperature,
        humidity, wind speed and solar radiation.
        Arguments:
            st : start of data to be feteched.
            en : end of data to be fetched.
            freq : frequency at which the data is to be fetched.
        Returns:
            a pandas dataframe consisting of 4 columns

        .. _weather:
            https://doi.org/10.1038/s41598-017-04385-2
        """

        fname = os.path.join(self.ds_dir, 'weather_station', 'weather_stations.f')
        if not os.path.exists(fname):
            files = glob.glob(f"{os.path.join(self.ds_dir, 'weather_station')}/*.xlsx")
            df = pd.DataFrame()
            for f in files:
                _df = pd.read_excel(f, sheet_name='Hourly', usecols=['T', 'H', 'W', 'Gr'])
                df = pd.concat([df, _df])

            df = df.reset_index(drop=True)  # index is of type Int64Index
            df.to_xarray().to_netcdf(fname)
        else:  # feather file already exists so load from it
            df = xr.load_dataset(fname).to_dataframe()

        df.columns = self.weather_station_data

        df.index = pd.date_range('20010101 01:00:00', periods=len(df), freq='H')

        return df[st:en]

    def fetch_pcp(self,
                  st: Union[str, pd.Timestamp] = '20010101 00:06:00',
                  en: Union[str, pd.Timestamp] = '20200101 00:06:00',
                  freq: str = '6min'
                  ) -> pd.DataFrame:
        """
        Fetches the precipitation_ data which is collected at 6 minutes time-step
        from 2001 to 2020.
        Arguments:
            st : starting point of data to be fetched.
            en : end point of data to be fetched.
            freq : frequency at which the data is to be returned.
        Returns:
            pandas dataframe of precipitation data

        .. _precipitation:
            https://doi.org/10.1038/s41598-017-04385-2
        """
        # todo allow change in frequency

        fname = os.path.join(self.ds_dir, 'pcp', 'pcp.nc')
        # feather file does not exist
        if not os.path.exists(fname):
            files = glob.glob(f"{os.path.join(self.ds_dir, 'pcp')}/*.xlsx")
            df = pd.DataFrame()
            for f in files:
                _df = pd.read_excel(f, sheet_name='6mn', usecols=['Rfa'])
                df = pd.concat([df, _df])

            df = df.reset_index(drop=True)
            df.to_xarray().to_netcdf(fname)
        else:  # feather file already exists so load from it
            df = xr.load_dataset(fname).to_dataframe()

        df.index = pd.date_range('20010101 00:06:00', periods=len(df), freq='6min')
        df.columns = ['pcp']

        return df[st:en]

    def fetch_hydro(self,
                    st: Union[str, pd.Timestamp] = '20010101 00:06:00',
                    en: Union[str, pd.Timestamp] = '20200101 00:06:00',
                    ) -> Tuple[pd.DataFrame, pd.DataFrame]:
        """
        fetches water level and suspended particulate matter. Both data are from
        2001 to 2019 but are randomly sampled.
        Arguments:
            st : starting point of data to be fetched.
            en : end point of data to be fetched.
        Returns:
            a tuple of pandas dataframes of water level and suspended particulate
            matter.
        """
        wl_fname = os.path.join(self.ds_dir, 'hydro', 'wl.nc')
        spm_fname = os.path.join(self.ds_dir, 'hydro', 'spm.nc')
        if not os.path.exists(wl_fname):
            print("reading data from xlsx files and saving them in netcdf format.")
            print("This will happen only once but will save io time.")
            files = glob.glob(f"{os.path.join(self.ds_dir, 'hydro')}/*.xlsx")
            wl = pd.DataFrame()
            spm = pd.DataFrame()
            for f in files:
                _df = pd.read_excel(f, sheet_name='Aperiodic')
                _wl = _df[['Date', 'Time', 'RWL04']]
                _wl.index = pd.to_datetime(_wl['Date'].astype(str) + ' ' + _wl['Time'].astype(str))
                _spm = _df[['Date.1', 'Time.1', 'SPM04']]
                _spm = _spm.iloc[_spm.first_valid_index():_spm.last_valid_index()]
                if os.path.basename(f) == 'OMPrawdataLaos2016.xlsx':
                    _spm.iloc[166] = ['2016-07-01', '20:43:47', 1.69388]
                    _spm.iloc[247] = ['2016-07-23', '12:57:47', 8.15714]
                    _spm.iloc[248] = ['2016-07-23', '17:56:47', 0.5]
                    _spm.iloc[352] = ['2016-08-16', '03:08:17', 1.12711864406]
                if os.path.basename(f) == 'OMPrawdataLaos2017.xlsx':
                    _spm.index = pd.to_datetime(_spm['Date.1'].astype(str))
                else:
                    _spm.index = pd.to_datetime(_spm['Date.1'].astype(str) + ' ' + _spm['Time.1'].astype(str))
                wl = pd.concat([wl, _wl['RWL04']])
                spm = pd.concat([spm, _spm['SPM04']])

            wl.columns = ['water_level']
            wl = wl.reset_index()
            wl.to_xarray().to_netcdf(wl_fname)
            spm.columns = ['susp_pm']
            spm = spm.reset_index()
            spm.to_xarray().to_netcdf(spm_fname)
        else:
            wl = xr.load_dataset(wl_fname).to_dataframe(['index'])
            spm = xr.load_dataset(spm_fname).to_dataframe(['index'])

        # wl.index = pd.to_datetime(wl.pop('index'))
        # spm.index = pd.to_datetime(spm.pop('index'))
        wl = wl[~wl.index.duplicated(keep='first')]
        spm = spm[~spm.index.duplicated(keep='first')]

        # FutureWarning: Value based partial slicing on non-monotonic DatetimeIndexes
        return wl.loc[st:en], spm.loc[st:en]

    def make_classification(self,
                            input_features: Union[None, list] = None,
                            output_features: Union[str, list] = None,
                            st: Union[None, str] = "20110525 14:00:00",
                            en: Union[None, str] = "20181027 00:00:00",
                            freq: str = "6min",
                            threshold: Union[int, dict] = 400,
                            lookback_steps: int = None,
                            ) -> pd.DataFrame:
        """
        Makes a classification problem.

        Arguments:
            input_features :
                names of inputs to use.
            output_features :
                feature/features to consdier as target/output/label
            st :
                starting date of data
            en :
                end date of data
            freq :
                frequency of data
            threshold :
                threshold to use to determine classes. Values greater than
                equal to threshold are set to 1 while values smaller than threshold
                are set to 0. The value of 400 is chosen for E. coli to make the
                the number 0s and 1s balanced. It should be noted that US-EPA recommends
                threshold value of 400 cfu/ml.
            lookback_steps:
                the number of previous steps to use. If this argument is used,
                the resultant dataframe will have (ecoli_observations * lookback_steps)
                rows. The resulting index will not be continuous.

        returns :
            a dataframe of shape `(inputs+target, st:en)`

        Example:
            >>> from ai4water.datasets import MtropicsLaos
            >>> laos = MtropicsLaos()
            >>> df = laos.make_classification()
        """
        thresholds = {
            'Ecoli_mpn100': 400
        }

        target: list = check_attributes(output_features, self.target)

        data = self._make_ml_problem(input_features, target, st, en, freq)

        if len(target) == 1:
            threshold = threshold or thresholds[target[0]]
        else:
            raise ValueError

        s = data[target[0]]
        s[s < threshold] = 0
        s[s >= threshold] = 1

        data[target[0]] = s

        if lookback_steps:
            return consider_lookback(data, lookback_steps, target)
        return data

    def make_regression(self,
                        input_features: Union[None, list] = None,
                        output_features: Union[str, list] = "Ecoli_mpn100",
                        st: Union[None, str] = "20110525 14:00:00",
                        en: Union[None, str] = "20181027 00:00:00",
                        freq: str = "6min",
                        lookback_steps: int = None
                        ) -> pd.DataFrame:
        """
        Makes a regression problem using hydrological, environmental,
        and water quality data of Huoay pano.

        Parameters
        ----------
            input_features :
                names of inputs to use. By default following features
                are used as input

                - ``air_temp`
                - ``rel_hum``
                - ``wind_speed`
                - ``sol_rad`
                - ``water_level`
                - ``pcp``
                - ``susp_pm`'

            output_features : feature/features to consdier as target/output/label
            st :
                starting date of data
            en :
                end date of data
            freq : frequency of data
            lookback_steps:
                the number of previous steps to use. If this argument is used,
                the resultant dataframe will have (ecoli_observations * lookback_steps)
                rows. The resulting index will not be continuous.

        Returns
        -------
            a dataframe of shape (inputs+target, st - en)

        Example
        -------
            >>> from ai4water.datasets import MtropicsLaos
            >>> laos = MtropicsLaos()
            >>> ins = ['pcp', 'air_temp']
            >>> out = ['Ecoli_mpn100']
            >>> reg_data = laos.make_regression(ins, out, '20110101', '20181231')

        todo add HRU definition
        """
        data = self._make_ml_problem(input_features, output_features, st, en, freq)

        if lookback_steps:
            return consider_lookback(data, lookback_steps, output_features)
        return data

    def _make_ml_problem(self, input_features, output_features, st, en, freq):
        inputs = check_attributes(input_features, self.inputs)
        target = check_attributes(output_features, self.target)
        features_to_fetch = inputs + target

        pcp = self.fetch_pcp(st=st, en=en)
        pcp = pcp.interpolate('linear', limit=5)
        pcp = pcp.fillna(0.0)

        w = self.fetch_weather_station_data(st=st, en=en)
        w = w.interpolate()
        w = w.bfill()
        assert int(w.isna().sum().sum()) == 0

        w.columns = ['air_temp', 'rel_hum', 'wind_speed', 'sol_rad']
        w_6min = Resampler(w,
                           freq=freq,
                           how={'air_temp': 'linear',
                                'rel_hum': 'linear',
                                'wind_speed': 'linear',
                                'sol_rad': 'linear'
                                }
                           )()

        ecoli = self.fetch_ecoli(st=st, en=en)
        ecoli = ecoli.dropna()
        ecoli_6min = ecoli.resample(freq).mean()

        wl, spm = self.fetch_hydro(st=st, en=en)
        wl_6min = wl.resample(freq).first().interpolate(method="linear")
        spm_6min = spm.resample(freq).first().interpolate(method='linear')

        data = pd.concat([w_6min.loc[st:en],
                          pcp.loc[st:en],
                          wl_6min.loc[st:en],
                          spm_6min.loc[st:en],
                          ecoli_6min.loc[st:en],
                          ], axis=1)

        if data['water_level'].isna().sum() < 15:
            data['water_level'] = data['water_level'].bfill()  # only 11 nan present at start
            data['water_level'] = data['water_level'].ffill()  # only 1 nan is present at ned

        if data['susp_pm'].isna().sum() < 40:
            data['susp_pm'] = data['susp_pm'].bfill()  # only 26 nan is present at ned
            data['susp_pm'] = data['susp_pm'].ffill()  # only 9 nan is present

        return data.loc[st:en, features_to_fetch]


class MtropcsThailand(Datasets):
    url = {
        "pcp.zip":
            "https://services.sedoo.fr/mtropics/data/v1_0/download?collectionId=27c65b5f-59cb-87c1-4fdf-628e6143d8c4",
        # "hydro.zip":
        #    "https://services.sedoo.fr/mtropics/data/v1_0/download?collectionId=9e6f7144-8984-23bd-741a-06378fabd72",
        "rain_gauge.zip":
            "https://services.sedoo.fr/mtropics/data/v1_0/download?collectionId=0a12ffcf-42bc-0289-1c55-a769ef19bb16",
        "weather_station.zip":
            "https://services.sedoo.fr/mtropics/data/v1_0/download?collectionId=fa0bca5f-caee-5c68-fed7-544fe121dcf5 "
    }

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self._download()


class MtropicsVietnam(Datasets):
    url = {
        "pcp.zip":
            "https://services.sedoo.fr/mtropics/data/v1_0/download?collectionId=d74ab1b0-379b-71cc-443b-662a73b7f596",
        "hydro.zip":
            "https://services.sedoo.fr/mtropics/data/v1_0/download?collectionId=85fb6717-4095-a2a2-34b5-4f1b70cfd304",
        # "lu.zip":
        #    "https://services.sedoo.fr/mtropics/data/v1_0/download?collectionId=c3724992-a043-4bbf-8ac1-bc6f9a608c1c",
        "rain_guage.zip":
            "https://services.sedoo.fr/mtropics/data/v1_0/download?collectionId=3d3382d5-08c1-2595-190b-8568a1d2d6af",
        "weather_station.zip":
            "https://services.sedoo.fr/mtropics/data/v1_0/download?collectionId=8df40086-4232-d8d0-a1ed-56c860818989"
    }

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self._download()


def unzip_all_in_dir(dir_name, ext=".gz"):
    gz_files = glob.glob(f"{dir_name}/*{ext}")
    for f in gz_files:
        shutil.unpack_archive(f, dir_name)


def _process_laos_shpfiles(shape_file, out_path):

    if fiona is None:
        warnings.warn("preprocessing of shapefiles can not be done because no fiona installation is found.")
        return

    shp_reader = shapefile.Reader(shape_file)

    container = {
        'Forest': [],
        'Culture': [],
        'Fallow': [],
        'Teak': [],
        # 'others': []
    }

    for i in range(shp_reader.numRecords):
        lu = find_records(shape_file, 'LU3', i)
        shp = shp_reader.shape(i)
        if shp.shapeType == 0:
            continue
        geom = shape(shp.__geo_interface__)
        if lu.startswith('Forest'):
            container['Forest'].append(geom)
        elif lu.startswith('Culture'):
            container['Culture'].append(geom)
        elif lu.startswith('Fallow'):
            container['Fallow'].append(geom)
        elif lu.startswith('Teak'):
            container['Teak'].append(geom)
        else:  # just consider all others as 'culture' for siplicity
            container['Culture'].append(geom)
            # container['others'].append(geom)

    # Define a polygon feature geometry with one attribute
    schema = {
        'geometry': 'Polygon' if os.path.basename(shape_file) in ['LU2000.shp', 'LU2001.shp'] else 'MultiPolygon',
        'properties': {'id': 'int',
                       'NAME': 'str',
                       'area': 'float'},
    }

    # Write a new Shapefile
    with fiona.open(out_path, 'w', 'ESRI Shapefile', schema) as c:
        for idx, lu in enumerate(list(container.keys())):
            geoms = container[lu]
            poly = unary_union([shape(s.__geo_interface__) for s in geoms])

            assert poly.is_valid

            c.write({
                'geometry': mapping(poly),
                'properties': {'id': idx,
                               'NAME': lu,
                               'area': poly.area},
            })


def consider_lookback(df:pd.DataFrame, lookback:int, col_name:str)->pd.DataFrame:
    """selects rows from dataframe considering lookback based upon nan
     values in col_name"""

    if isinstance(col_name, list):
        assert len(col_name) == 1
        col_name = col_name[0]

    if not isinstance(col_name, str):
        raise NotImplementedError

    start = False
    steps = 0

    masks = np.full(len(df), False)

    for idx, ecoli in enumerate(df[col_name].values[::-1]):
        if not ecoli != ecoli:
            start = True
            steps = 0

        if start and steps < lookback:
            masks[idx] = True
            steps += 1

        # if we have started counting but the limit has reached
        if start and steps > lookback:
            start = False

    return df.iloc[masks[::-1]]