# AUTOGENERATED! DO NOT EDIT! File to edit: 00_data.ipynb (unless otherwise specified).

__all__ = ['get_chart_list', 'get_chart', 'get_charts', 'get_data', 'get_alarm_log', 'get_allmetrics',
           'get_allmetrics_async']

# Cell
# export
from typing import Union
import re
import time
import asks
from asks import BasicAuth
import trio
import pandas as pd
import requests
from requests.auth import HTTPBasicAuth
from .wrangle import drop_low_uniqueness_cols, drop_low_std_cols

# Cell


def get_chart_list(host: str = '127.0.0.1:19999', starts_with: str = None, ends_with: str = None, protocol: str = 'http',
    verify: Union[str, bool] = True) -> list:
    """Get list of all available charts on a `host`.

    ##### Parameters:
    - **host** `str` The host we want to get a list of available charts from.
    - **starts_with** `str` A string to filter the list of charts returns to just those that start with `starts_with`.
    - **ends_with** `str` A string to filter the list of charts returns to just those that end with `ends_with`.
    - **protocol** `str` 'http' or 'https'.
    - **verify** `Union[str, bool]` `verify` parameter to be set to `requests` for SSL cert verification.

    ##### Returns:
    - **chart_list** `list` A list of availalbe charts.

    """
    url = f"{protocol}://{host}/api/v1/charts"
    r = requests.get(url, verify=verify)
    charts = r.json().get('charts')
    chart_list = [chart for chart in charts]
    if starts_with:
        chart_list = [chart for chart in chart_list if chart.startswith(starts_with)]
    if ends_with:
        chart_list = [chart for chart in chart_list if chart.endswith(ends_with)]
    return chart_list



# Cell


async def get_chart(api_call: str, data: list, col_sep: str ='|', numeric_only: bool = True, float_size: str = 'float64',
                    host_prefix: bool = False, host_sep: str = ':'):
    """Get data for an individual chart.

    ##### Parameters:
    - **api_call** `tuple` A tuple of (`url`,`chart`) for the url to pull data from and chart it represents.
    - **data** `list` A list for dataframes for each chart to be appended to.
    - **col_sep** `str` A character for separating chart and dimension in column names of dataframe.
    - **numeric_only** `bool` Set to true if you want to filter out any non numeric data.
    - **float_size** `str` float size to use if would like to save some memory, eg can use 'float32' or 'float16'.
    - **host_prefix** `bool` True to prefix each colname with the corresponding host.
    - **host_sep** `str` A character for separating host and chart and dimensions in column names of dataframe.

    """
    url, chart, host, user, pwd = api_call
    if user and pwd:
        user_pwd = (user, pwd)
        r = await asks.get(url, auth=BasicAuth(user_pwd))
    else:
        r = await asks.get(url)
    r_json = r.json()
    df = pd.DataFrame(r_json['data'], columns=['time_idx'] + r_json['labels'][1:])
    if host_prefix:
        df = df.set_index(['time_idx']).add_prefix(f'{host}{host_sep}{chart}{col_sep}')
    else:
        df['host'] = host
        df = df.set_index(['host','time_idx']).add_prefix(f'{chart}{col_sep}')
    if numeric_only:
        df = df._get_numeric_data().astype(float_size)
    data.append(df)



# Cell

async def get_charts(api_calls: list, col_sep: str ='|', timeout: int = 60, numeric_only: bool = True, float_size: str = 'float64',
                     host_prefix: bool = False, host_sep: str = ':') -> pd.DataFrame:
    """Create a nursey to make seperate async calls to get each chart.

    ##### Parameters:
    - **api_calls** `list` A list of tuple's of [(`url`,`chart`),...] of api calls that need to be made.
    - **col_sep** `str` A character for separating chart and dimension in column names of dataframe.
    - **timeout** `int` The number of seconds for trio to [move_on_after](https://trio.readthedocs.io/en/stable/reference-core.html#trio.move_on_after).
    - **numeric_only** `bool` Set to true if you want to filter out any non numeric data.
    - **float_size** `str` float size to use if would like to save some memory, eg can use 'float32' or 'float16'.
    - **host_prefix** `bool` True to prefix each colname with the corresponding host.
    - **host_sep** `str` A character for separating host and chart and dimensions in column names of dataframe.

    ##### Returns:
    - **df** `pd.DataFrame` A pandas dataframe with all chart data outer joined based on time index.

    """
    n_hosts = len(set([x[2] for x in api_calls]))
    data = []
    with trio.move_on_after(timeout):
        async with trio.open_nursery() as nursery:
            for api_call in api_calls:
                nursery.start_soon(get_chart, api_call, data, col_sep, numeric_only, float_size, host_prefix, host_sep)
    if n_hosts == 1 or host_prefix:
        df = pd.concat(data, join='outer', axis=1, sort=True)
    else:
        df = pd.concat(data, join='outer', axis=0, sort=True)
    return df



# Cell


def get_data(hosts: list = ['london.my-netdata.io'], charts: list = ['system.cpu'], after: int = -60,
             before: int = 0, points: int = 0, col_sep: str = '|', numeric_only: bool = True,
             ffill: bool = True, diff: bool = False, timeout: int = 60, nunique_thold = None,
             std_thold: float = None, index_as_datetime: bool = False, freq: str = 'infer',
             group: str = 'average', sort_cols: bool = True, user: str = None, pwd: str = None,
             protocol: str = 'http', sort_rows: bool = True, float_size: str = 'float64',
             host_charts_dict: dict = None, host_prefix: bool = False, host_sep: str = ':',
             charts_regex: str = None, verify: Union[str, bool] = True, dimensions: str = '*',
             options: str = '') -> pd.DataFrame:
    """Define api calls to make and any post processing to be done.

    ##### Parameters:
    - **hosts** `list` A list of hosts to pull data from.
    - **charts** `list` A list of charts to pull data for.
    - **after** `int` The timestamp or relative integer from which to pull data after.
    - **before** `int` The timestamp or relative integer from which to pull data before.
    - **points** `int` The `points` parameter to pass to the api call if need to aggregate data in some way.
    - **col_sep** `str` A character for separating chart and dimension in column names of dataframe.
    - **numeric_only** `bool` Set to true if you want to filter out any non numeric data.
    - **ffill** `bool` Set to true if you want to forward fill any null or missing values.
    - **diff** `bool` Set to true if you want to get the difference of metrics as opposed to their raw value.
    - **timeout** `int` The number of seconds for trio to [move_on_after](https://trio.readthedocs.io/en/stable/reference-core.html#trio.move_on_after).
    - **nunique_thold** [`float`,`int`] If defined calls function to filter cols with low number of unique values.
    - **std_thold** `float` If defined calls function to filter cols with low standard deviation.
    - **index_as_datetime** `bool` If true, set the index to be a pandas datetime.
    - **freq** `str` Freq to be passed to pandas datetime index.
    - **group** `str` The grouping function to use in the netdata api call.
    - **sort_cols** `bool` True to sort columns by name.
    - **user** `str` A username to use if netdata is password protected.
    - **pwd** `str` A password to use if netdata is password protected.
    - **protocol** `str` 'http' or 'https'.
    - **sort_rows** `bool` True to sort rows by index.
    - **float_size** `str` float size to use if would like to save some memory, eg can use 'float32' or 'float16'.
    - **host_charts_dict** `dict` dictionary of hosts to pull for where each value is list of relevant charts to pull from that host.
    - **host_prefix** `bool` True to prefix each colname with the corresponding host.
    - **host_sep** `str` A character for separating host and chart and dimensions in column names of dataframe.
    - **charts_regex** `str` A regex expression for charts you want data for.
    - **verify** `Union[str, bool]` `verify` parameter to be set to `requests` for SSL cert verification.
    - **dimensions** `str` The `dimensions` parameter to pass to the api call, defaults to '*' for all dimensions.
    - **options** `str` The `options` parameter to pass to the api call, defaults to '' to just accept defaults.

    ##### Returns:
    - **df** `pd.DataFrame` A pandas dataframe with all chart data outer joined based on time index and any post processing done.

    """
    # if hosts is a string make it a list of one
    if isinstance(hosts, str):
        hosts = [hosts]

    # get list of host chart tuples we need to get data for
    if host_charts_dict:
        host_charts = [(k, v) for k in host_charts_dict for v in host_charts_dict[k]]
        hosts = list(set(host_charts_dict.keys()))
    elif charts_regex:
        charts_regex = re.compile(charts_regex)
        host_charts = [(host, chart) for host in hosts for chart in list(filter(charts_regex.match, get_chart_list(host, verify=verify)))]
    elif charts == ['all']:
        host_charts = [(host, chart) for host in hosts for chart in get_chart_list(host, verify=verify)]
    else:
        host_charts = [(host, chart) for host in hosts for chart in charts]

    # define list of all api calls to be made
    api_calls = [
        (
            f'{protocol}://{host_chart[0]}/api/v1/data?chart={host_chart[1]}&after={after}&before={before}&points={points}&format=json&group={group}&dimensions={dimensions}&options={options}',
            host_chart[1],
            host_chart[0],
            user,
            pwd
        )
        for host_chart in host_charts
    ]
    # get the data
    df = trio.run(get_charts, api_calls, col_sep, timeout, numeric_only, float_size, host_prefix, host_sep)
    # post process the data
    if host_prefix:
        df = df.groupby(by=['time_idx']).max()
    else:
        df = df.groupby(by=['host','time_idx']).max()
    if len(hosts) == 1:
        df = df.reset_index(level=0, drop=True)
    if sort_rows:
        df = df.sort_index()
    if ffill:
        df = df.ffill()
    if diff:
        df = df.diff().dropna(how='all')
    if nunique_thold:
        df = drop_low_uniqueness_cols(df, nunique_thold)
    if std_thold:
        df = drop_low_std_cols(df, std_thold)
    if index_as_datetime:
        df = df.set_index(pd.DatetimeIndex(pd.to_datetime(df.index, unit='s'), freq=freq))
    if sort_cols:
        df = df.reindex(sorted(df.columns), axis=1)
    return df



# Cell


def get_alarm_log(host: str = '127.0.0.1:19999', datetimes: bool = True, user: str = None,
                  pwd: str = None, protocol: str = 'http', include_children: bool = False) -> pd.DataFrame:
    """Get alarm log from `host`.

    ##### Parameters:
    - **host** `str` The host we want to get the alarm log from.
    - **user** `str` A username to use if netdata is password protected.
    - **pwd** `str` A password to use if netdata is password protected.
    - **protocol** `str` 'http' or 'https'.
    - **include_children** `bool` 'True' to include alarm log for all children streamed to host.

    ##### Returns:
    - **df** `pd.DataFrame` A df of the alarm_log.

    """

    def get_alarm_log_df(protocol, host, user, pwd, child=None):
        if child:
            url = f"{protocol}://{host}/host/{child}/api/v1/alarm_log"
        else:
            url = f"{protocol}://{host}/api/v1/alarm_log"
        if user and pwd:
            r = requests.get(url, auth=HTTPBasicAuth(user, pwd))
        else:
            r = requests.get(url)
        alarm_log = r.json()
        df = pd.DataFrame(alarm_log)
        return df

    def get_children(protocol, host):
        r = requests.get(f"{protocol}://{host}/api/v1/info")
        children = r.json()['mirrored_hosts']
        children.pop(0)
        return children

    df = get_alarm_log_df(protocol, host, user, pwd)
    if include_children:
        children = get_children(protocol, host)
        for child in children:
            df = df.append(get_alarm_log_df(protocol, host, user, pwd, child))
    if datetimes:
        for col in ['when', 'delay_up_to_timestamp']:
            df[col] = pd.to_datetime(df[col], unit='s')
    return df



# Cell


def get_allmetrics(host='london.my-netdata.io', charts: list = None, wide: bool = False, col_sep: str = '|', sort_cols: bool = True,
                   user: str = None, pwd: str = None, protocol: str = 'http', numeric_only: bool = True,
                   float_size: str = 'float64', host_charts_dict: dict = None, host_prefix: bool = False,
                   host_sep: str = ':') -> pd.DataFrame:
    """Get allmetrics into a df.

    ##### Parameters:
    - **host** `str` The host we want to get the alarm log from.
    - **charts** `list` A list of charts to pull data for.
    - **wide** `bool` True if you want to return the data in wide format as opposed to long.
    - **user** `str` A username to use if netdata is password protected.
    - **pwd** `str` A password to use if netdata is password protected.
    - **protocol** `str` 'http' or 'https'.
    - **numeric_only** `bool` Set to true if you want to filter out any non numeric data.
    - **float_size** `str` float size to use if would like to save some memory, eg can use 'float32' or 'float16'.

    ##### Returns:
    - **df** `pd.DataFrame` A df of the latest data from allmetrics.

    """

    if not host_charts_dict:
        host_charts_dict = {host: charts}

    data = []
    for host in host_charts_dict:
        charts = host_charts_dict[host]
        url = f'{protocol}://{host}/api/v1/allmetrics?format=json'
        if user and pwd:
            raw_data = requests.get(url, auth=HTTPBasicAuth(user, pwd)).json()
        else:
            raw_data = requests.get(url).json()
        if charts is None:
            charts = list(raw_data.keys())
        for k in raw_data:
            if k in charts:
                time = raw_data[k]['last_updated']
                dimensions = raw_data[k]['dimensions']
                for dimension in dimensions:
                    # [time, chart, name, value]
                    if host_prefix:
                        data.append(
                            [time, f"{host}{host_sep}{k}", f"{host}{host_sep}{k}{col_sep}{dimensions[dimension]['name']}", dimensions[dimension]['value']]
                        )
                    else:
                        data.append(
                            [time, k, "{}{}{}".format(k, col_sep, dimensions[dimension]['name']), dimensions[dimension]['value']]
                        )

    df = pd.DataFrame(data, columns=['time','chart','dimension','value'])
    if wide:
        df = df[['dimension', 'value']].groupby('dimension').mean().reset_index().pivot_table(columns=['dimension'])
        if sort_cols:
            df = df.reindex(sorted(df.columns), axis=1)
        if numeric_only:
            df = df._get_numeric_data().astype(float_size)
    return df



# Cell


async def _get_allmetrics_async_single(api_call: str, data: list, col_sep: str ='|', numeric_only: bool = True, float_size: str = 'float64',
                    host_prefix: bool = False, host_sep: str = ':', wide: bool = False, sort_cols: bool = True):
    """Get all metrics for individual host.

    ##### Parameters:
    - **api_call** `tuple` A tuple of (`url`,`chart`) for the url to pull data from and chart it represents.
    - **data** `list` A list for dataframes for each chart to be appended to.
    - **col_sep** `str` A character for separating chart and dimension in column names of dataframe.
    - **numeric_only** `bool` Set to true if you want to filter out any non numeric data.
    - **float_size** `str` float size to use if would like to save some memory, eg can use 'float32' or 'float16'.
    - **host_prefix** `bool` True to prefix each colname with the corresponding host.
    - **host_sep** `str` A character for separating host and chart and dimensions in column names of dataframe.

    """
    url, host, charts, user, pwd = api_call
    if user and pwd:
        user_pwd = (user, pwd)
        r = await asks.get(url, auth=BasicAuth(user_pwd))
    else:
        r = await asks.get(url)
    raw_data = r.json()
    if charts is None:
        charts = list(raw_data.keys())
    tmp_data = []
    for k in raw_data:
        if k in charts:
            time = raw_data[k]['last_updated']
            dimensions = raw_data[k]['dimensions']
            for dimension in dimensions:
                # [time, chart, name, value]
                if host_prefix:
                    tmp_data.append(
                        [time, f"{host}{host_sep}{k}", f"{host}{host_sep}{k}{col_sep}{dimensions[dimension]['name']}", dimensions[dimension]['value']]
                    )
                else:
                    tmp_data.append(
                        [time, k, "{}{}{}".format(k, col_sep, dimensions[dimension]['name']), dimensions[dimension]['value']]
                    )
    df = pd.DataFrame(tmp_data, columns=['time','chart','dimension','value'])
    df['host'] = host
    if wide:
        df = df[['dimension', 'value']].groupby('dimension').mean().reset_index().pivot_table(columns=['dimension'])
        if sort_cols:
            df = df.reindex(sorted(df.columns), axis=1)
        if numeric_only:
            df = df._get_numeric_data().astype(float_size)

    data.append(df)



# Cell


async def _get_allmetrics_async_runner(api_calls: list, col_sep: str ='|', timeout: int = 60, numeric_only: bool = True, float_size: str = 'float64',
                     host_prefix: bool = False, host_sep: str = ':', wide: bool = False, sort_cols: bool = True) -> pd.DataFrame:
    """Create a nursey to make seperate async calls to get each chart.

    ##### Parameters:
    - **api_calls** `list` A list of tuple's of [(`url`,`chart`),...] of api calls that need to be made.
    - **col_sep** `str` A character for separating chart and dimension in column names of dataframe.
    - **timeout** `int` The number of seconds for trio to [move_on_after](https://trio.readthedocs.io/en/stable/reference-core.html#trio.move_on_after).
    - **numeric_only** `bool` Set to true if you want to filter out any non numeric data.
    - **float_size** `str` float size to use if would like to save some memory, eg can use 'float32' or 'float16'.
    - **host_prefix** `bool` True to prefix each colname with the corresponding host.
    - **host_sep** `str` A character for separating host and chart and dimensions in column names of dataframe.

    ##### Returns:
    - **df** `pd.DataFrame` A pandas dataframe with all chart data outer joined based on time index.

    """
    n_hosts = len(set([x[1] for x in api_calls]))
    data = []
    with trio.move_on_after(timeout):
        async with trio.open_nursery() as nursery:
            for api_call in api_calls:
                nursery.start_soon(_get_allmetrics_async_single, api_call, data, col_sep, numeric_only, float_size, host_prefix, host_sep, wide, sort_cols)
    if n_hosts == 1:
        df = pd.concat(data, join='outer', axis=1, sort=True)
    else:
        df = pd.concat(data, join='outer', axis=0, sort=True)
    return df



# Cell


def get_allmetrics_async(host_charts_dict: dict = None, col_sep: str = '|', numeric_only: bool = True,
                   timeout: int = 60, index_as_datetime: bool = False, freq: str = 'infer', sort_cols: bool = True, user: str = None,
                   pwd: str = None, protocol: str = 'http', float_size: str = 'float64',
                   host_prefix: bool = False, host_sep: str = ':', wide: bool = False) -> pd.DataFrame:
    """Define api calls to make and any post processing to be done.

    ##### Parameters:
    - **col_sep** `str` A character for separating chart and dimension in column names of dataframe.
    - **numeric_only** `bool` Set to true if you want to filter out any non numeric data.
    - **timeout** `int` The number of seconds for trio to [move_on_after](https://trio.readthedocs.io/en/stable/reference-core.html#trio.move_on_after).
    - **index_as_datetime** `bool` If true, set the index to be a pandas datetime.
    - **freq** `str` Freq to be passed to pandas datetime index.
    - **sort_cols** `bool` True to sort columns by name.
    - **user** `str` A username to use if netdata is password protected.
    - **pwd** `str` A password to use if netdata is password protected.
    - **protocol** `str` 'http' or 'https'.
    - **float_size** `str` float size to use if would like to save some memory, eg can use 'float32' or 'float16'.
    - **host_charts_dict** `dict` dictionary of hosts to pull for where each value is list of relevant charts to pull from that host.
    - **host_prefix** `bool` True to prefix each colname with the corresponding host.
    - **host_sep** `str` A character for separating host and chart and dimensions in column names of dataframe.

    ##### Returns:
    - **df** `pd.DataFrame` A pandas dataframe with all chart data outer joined based on time index and any post processing done.

    """

    # define list of all api calls to be made
    api_calls = [
        (f'{protocol}://{host}/api/v1/allmetrics?format=json', host, host_charts_dict[host], user, pwd)
        for host in host_charts_dict
    ]
    # get the data
    df = trio.run(_get_allmetrics_async_runner, api_calls, col_sep, timeout, numeric_only, float_size, host_prefix, host_sep, wide, sort_cols)
    #df = df.max().to_frame()
    df = df.groupby(by=df.index).max()
    if index_as_datetime:
        df['time_idx'] = int(time.time())
        df = df.set_index('time_idx')
    if sort_cols:
        df = df.reindex(sorted(df.columns), axis=1)
    return df

