import logging
import os
import shutil
import subprocess
import datetime
from itertools import combinations, chain
from time import sleep
from pathlib import Path
import inspect
from appdirs import user_data_dir

import plotly.express as px
import pandas as pd
import numpy as np
from plotly.offline import plot
from wiliot_deployment_tools.common.debug import debug_print, is_databricks
from os.path import exists
import tabulate
try:
    from wiliot_deployment_tools.internal.utils.network_packet_list import NetworkPacketList, network_data_preparation
    print('NetworkPacketList imported (running internal version)')
except ImportError:
    pass
from wiliot_deployment_tools.common.utils import convert_timestamp_to_datetime, current_timestamp, mstimestamp_to_timezone


if is_databricks():
    logging.getLogger("py4j.java_gateway").setLevel(logging.ERROR)

class WiliotTableError(Exception):
    pass

class WiliotDatabricksUtils:

    def __init__(self, spark):
        self.spark = spark
        self.is_databricks = is_databricks()

    def get_seen_tags(self, table, start_time, end_time, tags_list=None, bridges_list=None, gateways_list=None):
        """
        does an SQL query of the packet data table between specified timestamps
        filters the data for specified tags/bridges/gateways (if specified)
        returns relevant values from the table
        :type table: str
        :param table: name of data table
        :type start_time: float
        :param start_time: time filter start timestamp (UTC milliseconds)
        :type end_time: float
        :param end_time: time filter end timestamp (UTC milliseconds)
        :type tags_list: list
        :param tags_list: list of tags to filter from the data
        :type bridges_list: list
        :param bridges_list: list of bridges to filter from the data
        :type gateways_list: list
        :param gateways_list: list of gateways to filter from the data
        :rtype: pandas DataFrame
        :return: dataframe of data from table
        """
        # TODO - debug the SQL Query with different types of lists
        if bridges_list is not None:
            if len(bridges_list) == 1:
                bridges_list = list(bridges_list) + [""]
            bridges_list = tuple(bridges_list)
        if gateways_list is not None:
            if len(gateways_list) == 1:
                gateways_list = list(gateways_list) + [""]
            gateways_list = tuple(gateways_list)
        if tags_list is not None:
            if len(tags_list) == 1:
                tags_list = list(tags_list) + [""]
            tags_list = tuple(tags_list)
        # adding search by date to improve search time (most tables in data bricks partition is by date)
        query_start_datetime = convert_timestamp_to_datetime(str(start_time))
        query_end_datetime = convert_timestamp_to_datetime(str(end_time))
        start_datetime = query_start_datetime - datetime.timedelta(hours=24)
        end_datetime = query_end_datetime + datetime.timedelta(hours=24)
        start_date = datetime.datetime.strftime(start_datetime, '%Y%m%d')
        end_date = datetime.datetime.strftime(end_datetime, '%Y%m%d')
        sql_method = f"""
        select *
        from {table}
        where date between {start_date} and {end_date} 
        and timestamp between {start_time} and {end_time} 
        """
        if tags_list is not None and tags_list != ():
            sql_method = sql_method + f"""and externalId in {tags_list}
        """
        if bridges_list is not None and bridges_list != ():
            sql_method = sql_method + f"""and bridgeId in {bridges_list}
        """
        if gateways_list is not None and gateways_list != ():
            sql_method = sql_method + f"""and gatewayId in {gateways_list}
        """
        if self.spark is not None:
            debug_print('Running SQL query...', center=True)
            debug_print(sql_method)
            query_data = self.spark.sql(sql_method)
            query_data = query_data.toPandas()
            return query_data
        else:
            raise EnvironmentError("SQL query can only run in databricks")

    def get_seen_events(self, table, start_time, end_time, bridges_list=None, gateways_list=None, platform=False):
        """
        does an SQL query of the packet data table between specified timestamps
        filters the data for specified tags/bridges/gateways (if specified)
        returns relevant values from the table
        :type table: str
        :param table: name of data table
        :type start_time: float
        :param start_time: time filter start timestamp (UTC milliseconds)
        :type end_time: float
        :param end_time: time filter end timestamp (UTC milliseconds)
        :type bridges_list: list
        :param bridges_list: list of bridges to filter the data by
        :type gateways_list: list
        :param gateways_list: list of gateways to filter the data by
        :type platform: bool
        :param platform: platform/management (true/false)
        :rtype: pandas DataFrame
        :return: dataframe of data from table
        """
        # TODO - debug the SQL Query with different types of lists
        devices_list = ()
        if bridges_list is not None:
            if len(bridges_list) == 1:
                devices_list = list(bridges_list) + [""]
            devices_list = tuple(devices_list)
        if gateways_list is not None:
            if len(gateways_list) == 1 and devices_list == ():
                devices_list = list(gateways_list) + [""]
            elif devices_list is not None:
                devices_list = devices_list + tuple(gateways_list)
            devices_list = tuple(devices_list)

        # adding search by date to improve search time (most tables in data bricks partition is by date)
        query_start_datetime = convert_timestamp_to_datetime(str(start_time))
        query_end_datetime = convert_timestamp_to_datetime(str(end_time))
        start_datetime = query_start_datetime - datetime.timedelta(hours=36)
        end_datetime = query_end_datetime + datetime.timedelta(hours=36)
        start_date = datetime.datetime.strftime(start_datetime, '%Y%m%d')
        end_date = datetime.datetime.strftime(end_datetime, '%Y%m%d')
        sql_method = f"""
        select *
        from {table}
        where date between {start_date} and {end_date} 
        and start between {start_time} and {end_time} 
        """
        if not platform:
            connectivity_event = "name = 'NTWK'"
            id_filter = "id"
        else:
            connectivity_event = "eventName = 'connectivity'"
            id_filter = "assetId"

        if devices_list is not None:
            sql_method = sql_method + f"""and ({connectivity_event} or {id_filter} in {devices_list})
        """
        else:
            sql_method = sql_method + f"""and {connectivity_event}"""
        if self.spark is not None:
            debug_print('Running SQL query...', center=True)
            debug_print(sql_method)
            query_data = self.spark.sql(sql_method)
            query_data = query_data.toPandas()
            return query_data
        else:
            raise EnvironmentError("SQL query can only run in databricks")

    def get_sequence_id_data(self, table, start_time, end_time, gateways_list=None):
        """
        does an SQL query of the packet data table between specified timestamps
        returns only sequence id
        :type table: str
        :param table: name of data table
        :type start_time: float
        :param start_time: time filter start timestamp (UTC milliseconds)
        :type end_time: float
        :param end_time: time filter end timestamp (UTC milliseconds)
        :type gateways_list: list
        :param gateways_list: list of gateways to filter the data by
        :rtype: pandas DataFrame
        :return: dataframe of data from table
        """

        if gateways_list is not None:
            if len(gateways_list) == 1:
                gateways_list = list(gateways_list) + [""]
            gateways_list = tuple(gateways_list)
        # adding search by date to improve search time (most tables in data bricks partition is by date)
        query_start_datetime = convert_timestamp_to_datetime(str(start_time))
        query_end_datetime = convert_timestamp_to_datetime(str(end_time))
        start_datetime = query_start_datetime - datetime.timedelta(hours=36)
        end_datetime = query_end_datetime + datetime.timedelta(hours=36)
        start_date = datetime.datetime.strftime(start_datetime, '%Y%m%d')
        end_date = datetime.datetime.strftime(end_datetime, '%Y%m%d')
        sql_method = f"""
        select sequenceId, timestamp, gatewayId, gatewayName
        from {table}
        where date between {start_date} and {end_date} 
        and timestamp between {start_time} and {end_time} 
        """
        if gateways_list is not None:
            sql_method = sql_method + f"""and gatewayId in {gateways_list}
        """
        if self.spark is not None:
            debug_print('Running SQL query...', center=True)
            debug_print(sql_method)
            query_data = self.spark.sql(sql_method)
            query_data = query_data.toPandas()
            return query_data
        else:
            raise EnvironmentError("SQL query can only run in databricks")

    def get_statistics_data(self, table, start_time, end_time, gateways_list=None):
        """
        does an SQL query of the statistics data table between specified timestamps
        :type table: str
        :param table: name of data table
        :type start_time: float
        :param start_time: time filter start timestamp (UTC milliseconds)
        :type end_time: float
        :param end_time: time filter end timestamp (UTC milliseconds)
        :type gateways_list: list
        :param gateways_list: list of gateways to filter the data by
        :rtype: pandas DataFrame
        :return: dataframe of data from table
        """

        if gateways_list is not None:
            if len(gateways_list) == 1:
                gateways_list = list(gateways_list) + [""]
            gateways_list = tuple(gateways_list)
        # adding search by date to improve search time (most tables in data bricks partition is by date)
        query_start_datetime = convert_timestamp_to_datetime(str(start_time))
        query_end_datetime = convert_timestamp_to_datetime(str(end_time))
        start_datetime = query_start_datetime - datetime.timedelta(hours=36)
        end_datetime = query_end_datetime + datetime.timedelta(hours=36)
        start_date = datetime.datetime.strftime(start_datetime, '%Y%m%d')
        end_date = datetime.datetime.strftime(end_datetime, '%Y%m%d')
        sql_method = f"""
        select *
        from {table}
        where date between {start_date} and {end_date} 
        and timestamp between {start_time} and {end_time} 
        """
        if gateways_list is not None:
            sql_method = sql_method + f"""and gatewayId in {gateways_list}
        """
        if self.spark is not None:
            debug_print('Running SQL query...', center=True)
            debug_print(sql_method)
            query_data = self.spark.sql(sql_method)
            query_data = query_data.toPandas()
            return query_data
        else:
            raise EnvironmentError("SQL query can only run in databricks")

    def get_heartbeat_data(self, table, start_time, end_time, gateways_list=None, bridges_list=None):
        """
        does an SQL query of the statistics data table between specified timestamps
        :type table: str
        :param table: name of data table
        :type start_time: float
        :param start_time: time filter start timestamp (UTC milliseconds)
        :type end_time: float
        :param end_time: time filter end timestamp (UTC milliseconds)
        :type gateways_list: list
        :param gateways_list: list of gateways to filter the data by
        :rtype: pandas DataFrame
        :return: dataframe of data from table
        """

        if gateways_list is not None:
            if len(gateways_list) == 1:
                gateways_list = list(gateways_list) + [""]
            gateways_list = tuple(gateways_list)
        
        if bridges_list is not None:
            if len(bridges_list) == 1:
                bridges_list = list(bridges_list) + [""]
            bridges_list = tuple(bridges_list)
        
        # adding search by date to improve search time (most tables in data bricks partition is by date)
        query_start_datetime = convert_timestamp_to_datetime(str(start_time))
        query_end_datetime = convert_timestamp_to_datetime(str(end_time))
        start_datetime = query_start_datetime - datetime.timedelta(hours=36)
        end_datetime = query_end_datetime + datetime.timedelta(hours=36)
        start_date = datetime.datetime.strftime(start_datetime, '%Y%m%d')
        end_date = datetime.datetime.strftime(end_datetime, '%Y%m%d')
        sql_method = f"""
        select *
        from {table}
        where date between {start_date} and {end_date} 
        and timestamp between {start_time} and {end_time} 
        """
        if gateways_list is not None:
            sql_method = sql_method + f"and gatewayId in {gateways_list}\n"
        if bridges_list is not None:
            sql_method = sql_method + f"and bridgeId in {bridges_list}\n"
        if self.spark is not None:
            debug_print('Running SQL query...', center=True)
            debug_print(sql_method)
            query_data = self.spark.sql(sql_method)
            query_data = query_data.toPandas()
            return query_data
        else:
            raise EnvironmentError("SQL query can only run in databricks")

    
    def get_num_seen_bridges(self, table, start_time, end_time, gateways_list=None):
        """
        gets number of unique bridges seen by each gateways (bridge has to send data packets)
        :type table: str
        :param table: name of data table
        :type start_time: float
        :param start_time: time filter start timestamp (UTC milliseconds)
        :type end_time: float
        :param end_time: time filter end timestamp (UTC milliseconds)
        :type gateways_list: list
        :param gateways_list: list of gateways to filter the data by
        :return: dictionary of number of seen bridges per gateway
        """
        if gateways_list is not None:
            if len(gateways_list) == 1:
                gateways_list = list(gateways_list) + [""]
            gateways_list = tuple(gateways_list)
        # adding search by date to improve search time (most tables in data bricks partition is by date)
        query_start_datetime = convert_timestamp_to_datetime(str(start_time))
        query_end_datetime = convert_timestamp_to_datetime(str(end_time))
        start_datetime = query_start_datetime - datetime.timedelta(hours=36)
        end_datetime = query_end_datetime + datetime.timedelta(hours=36)
        start_date = datetime.datetime.strftime(start_datetime, '%Y%m%d')
        end_date = datetime.datetime.strftime(end_datetime, '%Y%m%d')
        sql_method = f"""
        select gatewayId, count(distinct bridgeId) as countBridge
        from {table}
        where date between {start_date} and {end_date} 
        and timestamp between {start_time} and {end_time} 
        """
        if gateways_list is not None:
            sql_method = sql_method + f"""and gatewayId in {gateways_list}
        """
        sql_method = sql_method + "group by gatewayId"
        if self.spark is not None:
            debug_print('Running SQL query...', center=True)
            debug_print(sql_method)
            query_data = self.spark.sql(sql_method)
            query_data = query_data.toPandas()
            query_data = dict(zip(query_data['gatewayId'], query_data['countBridge']))
            return query_data
        else:
            raise EnvironmentError("SQL query can only run in databricks")


    def get_sequence_id_loss(self, packets_table, statistics_table, start_time, end_time, gateways_list=None, sync_tables=True):
        """
        calculates sequence ID loss (takes reboot into consideration) per GW in table
        :type packets_table: str | DataFrame
        :param packets_table: name of data table | data table DataFrame
        :type statistics_table: str | DataFrame
        :param statistics_table: name of statistics table | statistics table DataFrame
        :type start_time: float
        :param start_time: time filter start timestamp (UTC milliseconds)
        :type end_time: float
        :param end_time: time filter end timestamp (UTC milliseconds)
        :type gateways_list: list
        :param gateways_list: list of gateways to filter the data by
        :type sync_tables: bool
        :param sync_tables: sync table start/end by timestamp (relevant for running directly after DB table updates)
        :return: dictionary of statistics
        """

        RESET_THRESHOLD = 1000
        
        def sync_data_and_statistics_tables(data_table, statistics_table, gw_list):
            """
            sync data and statistics table by timestamps
            """
            def get_first_stat_timestamp(statistics_table):
                first_stat_timestamp = 0
                for gw in statistics_table['gatewayId'].unique():
                    gw_first_stat_timestamp = np.sort(statistics_table[statistics_table['gatewayId'] == gw]['timestamp'])[0]
                    if gw_first_stat_timestamp > first_stat_timestamp:
                        first_stat_timestamp = gw_first_stat_timestamp
                return first_stat_timestamp
            
            # 'sync' statistics table and data table latest timestamps:
            max_data_timestamp = data_table['timestamp'].max()
            max_stats_timestamp = statistics_table['timestamp'].max()
            last_synced = min(max_data_timestamp, max_stats_timestamp)
            # truncate end of data tables to timeslot
            data_table = data_table[data_table['timestamp']<= last_synced]
            statistics_table = statistics_table[statistics_table['timestamp'] <= last_synced]
            
            first_stat_timestamp = get_first_stat_timestamp(statistics_table)
            stat_start_timestamp = first_stat_timestamp - (datetime.timedelta(seconds = 60).total_seconds() * 1000)
            data_start_timestamp = data_table['timestamp'].min()
            while data_start_timestamp > stat_start_timestamp:
                statistics_table = statistics_table[statistics_table['timestamp']>first_stat_timestamp]
                first_stat_timestamp = get_first_stat_timestamp(statistics_table)
                stat_start_timestamp = first_stat_timestamp - (datetime.timedelta(seconds = 60).total_seconds() * 1000)

            return data_table, statistics_table
                

        
        def get_stats_for_continuous_sequence_ids(sequence):
            """
            generate statistics for contiuous sequence IDs
            :type sequence: list
            :param sequence: array of sequence ID
            :rtype: dict
            :return: dictionary of statistics
            """
            stats = {}

            # remove duplicates
            sequence = np.unique(sequence)
            # sort by descending order
            sequence = -np.sort(-sequence)

            s_max = np.max(sequence)
            s_min = np.min(sequence)
            s_rec = len(np.unique(sequence)) # TODO - Compare with len(sequence) / num of coupled packets from table
            s_expected = (s_max - s_min)+1
            stats['maxSequenceId'] = s_max
            stats['minSequenceId'] = s_min
            stats['receivedSequenceIds'] = s_rec
            stats['expectedSequenceIds'] = s_expected
            return stats

        def process_sequences(df_array, num_mgmt_packets=0):
            """
            generate statistics for (normalized by duration) for array of sequence IDs (compensated for GW Reboots)
            :type df_array: list of DataFrames
            :param df_array: list of DataFrames with continuous sequence IDs
            :type num_mgmt_packets: int
            :param num_mgmt_packets: number of management packets
            :rtype: dict
            :return: dictionary of statistics
            """
            total_stats = {}
            total_received_packets = num_mgmt_packets
            total_expected_packets = 0
            for df in df_array:
                stats = get_stats_for_continuous_sequence_ids(df['sequenceId'])
                total_received_packets += stats['receivedSequenceIds']
                total_expected_packets += stats['expectedSequenceIds']
            if total_expected_packets == 0:
                breakpoint()
            loss_percentage = (1 - (total_received_packets / total_expected_packets) )* 100
            total_stats['totalManagementPackets'] = num_mgmt_packets
            total_stats['totalDataPackets'] = total_received_packets - num_mgmt_packets
            total_stats['totalReceivedPackets'] = total_received_packets
            total_stats['totalExpectedPackets'] = total_expected_packets
            total_stats['lossPercentage'] = round(loss_percentage, 3)
            total_stats['numResets'] = len(df_array)-1
            return total_stats

        results = {}
        if isinstance(packets_table, pd.DataFrame):
            data = packets_table
        else:
            data = self.get_sequence_id_data(packets_table, start_time, end_time, gateways_list).sort_values(by='timestamp', ascending=False)
        gw_list = data['gatewayId'].unique()
        if gw_list is None:
            return None
        if isinstance(statistics_table, pd.DataFrame):
            statistics_data = statistics_table
        else:
            statistics_data = self.get_statistics_data(statistics_table, start_time, end_time, gw_list)
        if sync_tables:
            data, statistics_data = sync_data_and_statistics_tables(data, statistics_data, gw_list)

        for gw in gw_list:
            gw_name = data[data['gatewayId'] == gw]['gatewayName'].iloc[0]
            num_mgmt_packets = statistics_data[statistics_data['gatewayId'] == gw]['managementPktCount'].sum()
            gw_results = []
            gw_df = data[data['gatewayId'] == gw].reset_index()
            gw_df['diff'] = gw_df['sequenceId'].diff()
            gw_df['reset'] = np.where(gw_df['diff']>RESET_THRESHOLD, True, False)
            gw_resets = gw_df[gw_df['reset'] == True]
            sequences = np.array_split(gw_df, gw_resets.index)
            gw_results = process_sequences(sequences, num_mgmt_packets)
            gw_results.update({'gwName': gw_name})
            results[gw] = gw_results
        return results



    def get_amount_of_unique_tags_per_data_path(self, table, start_time, end_time, gateways_list=None,
                                                tags_to_ignore=None, bridges_list=None):
        """
        does an SQL query of the packet data table between specified timestamps
        returns amount of unique externalIds per data path (bridge->gw) in the given timeframe
        :type table: str
        :param table: name of data table
        :type start_time: float
        :param start_time: time filter start timestamp (UTC milliseconds)
        :type end_time: float
        :param end_time: time filter end timestamp (UTC milliseconds)
        :type tags_to_ignore: list
        :param tags_to_ignore: list of tags to ignore in the query (will not be counted)
        :type gateways_list: list
        :param gateways_list: list of gateways to filter the data by
        :type bridges_list: list
        :param bridges_list: list of bridges to filter the data by
        :rtype: pandas DataFrame
        :return: dataframe of data from table
        """
        if bridges_list is not None:
            if len(bridges_list) == 1:
                bridges_list = list(bridges_list) + [""]
            bridges_list = tuple(bridges_list)
        if gateways_list is not None:
            if len(gateways_list) == 1:
                gateways_list = list(gateways_list) + [""]
            gateways_list = tuple(gateways_list)
        if tags_to_ignore is not None:
            if len(tags_to_ignore) == 1:
                tags_to_ignore = list(tags_to_ignore) + [""]
            tags_to_ignore = tuple(tags_to_ignore)
        # adding search by date to improve search time (most tables in data bricks partition is by date)
        query_start_datetime = convert_timestamp_to_datetime(str(start_time))
        query_end_datetime = convert_timestamp_to_datetime(str(end_time))
        start_datetime = query_start_datetime - datetime.timedelta(hours=36)
        end_datetime = query_end_datetime + datetime.timedelta(hours=36)
        start_date = datetime.datetime.strftime(start_datetime, '%Y%m%d')
        end_date = datetime.datetime.strftime(end_datetime, '%Y%m%d')
        sql_method = f"""
        select gatewayId, bridgeId, count(distinct externalId)
        from {table}
        where date between {start_date} and {end_date} 
        and timestamp between {start_time} and {end_time} 
        """
        if tags_to_ignore is not None and tags_to_ignore != ():
            sql_method = sql_method + f"""and externalId not in {tags_to_ignore}
        """
        if bridges_list is not None and bridges_list != ():
            sql_method = sql_method + f"""and bridgeId in {bridges_list}
        """
        if gateways_list is not None and gateways_list != ():
            sql_method = sql_method + f"""and gatewayId in {gateways_list}
        """
        sql_method = sql_method + f"""group by gatewayId, bridgeId   order by gatewayId, bridgeId"""
        if self.spark is not None:
            debug_print('Running SQL query...', center=True)
            debug_print(sql_method)
            query_data = self.spark.sql(sql_method)
            query_data = query_data.toPandas()
            return query_data
        else:
            raise EnvironmentError("SQL query can only run in databricks")

    def get_amount_of_unique_tags_per_brg(self, table, start_time, end_time, tags_to_ignore=None, bridges_list=None):
        """
        does an SQL query of the packet data table between specified timestamps
        returns amount of unique externalIds per data path (bridge->gw) in the given timeframe
        :type table: str
        :param table: name of data table
        :type start_time: float
        :param start_time: time filter start timestamp (UTC milliseconds)
        :type end_time: float
        :param end_time: time filter end timestamp (UTC milliseconds)
        :type tags_to_ignore: list
        :param tags_to_ignore: list of tags to ignore in the query (will not be counted)
        :type gateways_list: list
        :param gateways_list: list of gateways to filter the data by
        :type bridges_list: list
        :param bridges_list: list of bridges to filter the data by
        :rtype: pandas DataFrame
        :return: dataframe of data from table
        """
        if bridges_list is not None:
            if len(bridges_list) == 1:
                bridges_list = list(bridges_list) + [""]
            bridges_list = tuple(bridges_list)
        if tags_to_ignore is not None:
            if len(tags_to_ignore) == 1:
                tags_to_ignore = list(tags_to_ignore) + [""]
            tags_to_ignore = tuple(tags_to_ignore)
        # adding search by date to improve search time (most tables in data bricks partition is by date)
        query_start_datetime = convert_timestamp_to_datetime(str(start_time))
        query_end_datetime = convert_timestamp_to_datetime(str(end_time))
        start_datetime = query_start_datetime - datetime.timedelta(hours=36)
        end_datetime = query_end_datetime + datetime.timedelta(hours=36)
        start_date = datetime.datetime.strftime(start_datetime, '%Y%m%d')
        end_date = datetime.datetime.strftime(end_datetime, '%Y%m%d')
        sql_method = f"""
        select gatewayId, bridgeId, count(distinct externalId)
        from {table}
        where date between {start_date} and {end_date} 
        and timestamp between {start_time} and {end_time} 
        """
        if tags_to_ignore is not None and tags_to_ignore != ():
            sql_method = sql_method + f"""and externalId not in {tags_to_ignore}
        """
        if bridges_list is not None and bridges_list != ():
            sql_method = sql_method + f"""and bridgeId in {bridges_list}
        """
        sql_method = sql_method + f"""group by bridgeId   order by bridgeId"""
        if self.spark is not None:
            debug_print('Running SQL query...', center=True)
            debug_print(sql_method)
            query_data = self.spark.sql(sql_method)
            query_data = query_data.toPandas()
            return query_data
        else:
            raise EnvironmentError("SQL query can only run in databricks")

    def get_amount_of_unique_tags_per_gw(self, table, start_time, end_time, gateways_list=None, tags_to_ignore=None):
        """
        does an SQL query of the packet data table between specified timestamps
        returns amount of unique externalIds per data path (bridge->gw) in the given timeframe
        :type table: str
        :param table: name of data table
        :type start_time: float
        :param start_time: time filter start timestamp (UTC milliseconds)
        :type end_time: float
        :param end_time: time filter end timestamp (UTC milliseconds)
        :type tags_to_ignore: list
        :param tags_to_ignore: list of tags to ignore in the query (will not be counted)
        :type gateways_list: list
        :param gateways_list: list of gateways to filter the data by
        :rtype: pandas DataFrame
        :return: dataframe of data from table
        """

        if gateways_list is not None:
            if len(gateways_list) == 1:
                gateways_list = list(gateways_list) + [""]
            gateways_list = tuple(gateways_list)
        if tags_to_ignore is not None:
            if len(tags_to_ignore) == 1:
                tags_to_ignore = list(tags_to_ignore) + [""]
            tags_to_ignore = tuple(tags_to_ignore)
        # adding search by date to improve search time (most tables in data bricks partition is by date)
        query_start_datetime = convert_timestamp_to_datetime(str(start_time))
        query_end_datetime = convert_timestamp_to_datetime(str(end_time))
        start_datetime = query_start_datetime - datetime.timedelta(hours=36)
        end_datetime = query_end_datetime + datetime.timedelta(hours=36)
        start_date = datetime.datetime.strftime(start_datetime, '%Y%m%d')
        end_date = datetime.datetime.strftime(end_datetime, '%Y%m%d')
        sql_method = f"""
        select gatewayId, count(distinct externalId)
        from {table}
        where date between {start_date} and {end_date} 
        and timestamp between {start_time} and {end_time} 
        """
        if tags_to_ignore is not None and tags_to_ignore != ():
            sql_method = sql_method + f"""and externalId not in {tags_to_ignore}
        """
        if gateways_list is not None and gateways_list != ():
            sql_method = sql_method + f"""and gatewayId in {gateways_list}
        """
        sql_method = sql_method + f"""group by gatewayId   order by gatewayId"""
        if self.spark is not None:
            debug_print('Running SQL query...', center=True)
            debug_print(sql_method)
            query_data = self.spark.sql(sql_method)
            query_data = query_data.toPandas()
            return query_data
        else:
            raise EnvironmentError("SQL query can only run in databricks")

    def get_last_data(self, table, mins=60, tags_list=None, bridges_list=None, gateways_list=None):
        """
        function querys back specified minutes from data table (counts back from last entry in table)
        :type table: str
        :param table: databricks data table name
        :type mins: int
        :param mins: minutes to query back, defaults to 1 hour
        :param tags_list: tags list
        :type tags_list: list
        :param tags_list: list of tags to filter from the data
        :type bridges_list: list
        :param bridges_list: list of bridger to filter from the data
        :type gateways_list: list
        :param gateways_list: list of gateways to filter from the data
        :rtype: pandas DataFrame
        :return: dataframe of data from table
        """
        end_timestamp = self.get_last_entry_timestamp(table)
        end_datetime = mstimestamp_to_timezone(end_timestamp)
        start_timestamp = end_timestamp - (mins * 60 * 1000)
        start_datetime = mstimestamp_to_timezone(start_timestamp)
        debug_print(f'Getting last {mins} mins from {table}')
        debug_print(f'Last entry at {end_timestamp}')
        debug_print(f'querying {start_datetime} -> {end_datetime}')
        return self.get_seen_tags(table, start_timestamp, end_timestamp, tags_list, bridges_list, gateways_list)

    def get_last_entry_timestamp(self, table):
        """
        function gets name of data table and returns the timestamp of the last entry (in milliseconds)
        :type table: str
        :param table: name of table
        """
        today = datetime.datetime.now()
        yesterday = today-datetime.timedelta(days=1)
        today_date = datetime.datetime.strftime(today, '%Y%m%d')
        yesterday_date = datetime.datetime.strftime(yesterday, '%Y%m%d')

        # Try to get last timestamp from last day
        sql_method = f"""
        select MAX (timestamp)
        from {table}
        where date between {yesterday_date} and {today_date} 
        """
        if self.spark is not None:
            query_data = self.spark.sql(sql_method)
            query_data = query_data.toPandas()
            last_entry_ts = query_data.iloc[0][0]
            if not np.isnan(last_entry_ts):
                return query_data.iloc[0][0]
            else:
                # Query all packets in table
                sql_method = ''.join(sql_method.split('\n')[:-2])
                query_data = self.spark.sql(sql_method)
                query_data = query_data.toPandas()
                last_entry_ts = query_data.iloc[0][0]
                if not np.isnan(last_entry_ts):
                    return query_data.iloc[0][0]
                else:
                    raise WiliotTableError(f'Cannot get last entry, no entries in data table {table}')
        else:
            raise EnvironmentError("Unable to detect dbutils function")

    def wait_for_data(self, table, requested_timestamp, timeout_mins=70):
    # TODO - implement date search
        """
        function waits for data timed at requested_timestamp (or later) to appear in the data table.
        :type table: str
        :param table: data table name
        :type requested_timestamp: int
        :param requested_timestamp: timestamp to wait to appear in the table
        """
        last_entry_timestamp = self.get_last_entry_timestamp(table)
        table_ready = last_entry_timestamp >= requested_timestamp
        start = current_timestamp()
        while not table_ready:
            if (current_timestamp() - start) >= 1000 * 60 * timeout_mins:
                debug_print(f"""{timeout_mins} minute timeout reached! Table still does not have the requested data
                Run test again or get raw data with wait_for_data=False""")
                exit_notebook(f'Wait for table timeout reached after {timeout_mins} minutes')
            debug_print(
                f'Waiting for table to get data. \n'
                f'Requested timestamp {requested_timestamp} | ({mstimestamp_to_timezone(requested_timestamp)}) \n'
                f'Table latest timestamp {last_entry_timestamp} | ({mstimestamp_to_timezone(last_entry_timestamp)})')
            mins_behind = (requested_timestamp - last_entry_timestamp) / 60000
            debug_print(f'Data table {mins_behind} minutes behind , trying again in 30 seconds', center=True)
            sleep(30)
            last_entry_timestamp = self.get_last_entry_timestamp(table)
            table_ready = last_entry_timestamp >= requested_timestamp
        debug_print(
            f'Table is updated. \n'
            f'Requested timestamp {requested_timestamp}, table latest timestamp {last_entry_timestamp} \n'
            f'Table latest timestamp {last_entry_timestamp} | ({mstimestamp_to_timezone(last_entry_timestamp)})')
        mins_ahead = (last_entry_timestamp - requested_timestamp) / 60000
        debug_print(f'Data table {mins_ahead} minutes ahead', center=True)

    def get_seen_edge_devices_from_packets(self, table, start_time, end_time):
        """
        does an SQL query of the packet data table between specified timestamps
        returns dictionary of seen gateways and bridges from slice of packet table
        :type table: str
        :param table: name of data table
        :type start_time: float
        :param start_time: time filter start timestamp (UTC milliseconds)
        :type end_time: float
        :param end_time: time filter end timestamp (UTC milliseconds)
        :rtype: pandas DataFrame
        :return: dataframe of data from table
        """
        # adding search by date to improve search time (most tables in data bricks partition is by date)
        query_start_datetime = convert_timestamp_to_datetime(str(start_time))
        query_end_datetime = convert_timestamp_to_datetime(str(end_time))
        start_datetime = query_start_datetime - datetime.timedelta(hours=24)
        end_datetime = query_end_datetime + datetime.timedelta(hours=24)
        start_date = datetime.datetime.strftime(start_datetime, '%Y%m%d')
        end_date = datetime.datetime.strftime(end_datetime, '%Y%m%d')
        sql_method = f"""
        select gatewayId, bridgeId, gatewayName, max(timestamp) as timestamp
        from {table}
        where date between {start_date} and {end_date} 
        and timestamp between {start_time} and {end_time}
        group by gatewayId, gatewayName, bridgeId 
        """
        if self.spark is not None:
            debug_print('Running SQL query...', center=True)
            debug_print(sql_method)
            query_data = self.spark.sql(sql_method).toPandas()
            query_data['gatewayId'] = query_data['gatewayId'].str.upper()
            query_data['bridgeId'] = query_data['bridgeId'].str.upper()
            return query_data
        else:
            raise EnvironmentError("SQL query can only run in databricks")


def process_tagstats(params, tag_stats_df, working_directory, test_no):
    """
        function gets tag stats dataframe and creates downloadable cdf graph for specified parameters
        :type params: list
        :param params: tag stats metrics to create graphs for
        :type tag_stats_df: pandas DataFrame
        :param tag_stats_df: tag stats DataFrame (can be created using get_tagstats_from_test)
        :type working_directory: string
        :param working_directory: directory to save the graphs HTML files
        :type test_no: string
        :param test_no: test number
    """
    if params is None:
        debug_print('No parameters specified. Creating graphs for all possible parameters.')
        params = ['num_packets', 'num_cycles', 'sprinkler_counter_mean', 'sprinkler_counter_std',
                  'sprinkler_counter_min', 'sprinkler_counter_max', 'tbp_mean', 'tbp_std', 'tbp_min', 'tbp_max',
                  'tbp_num_vals', 'per_mean', 'per_std', 'rssi_mean', 'rssi_std', 'rssi_min', 'rssi_max', 'ttfp',
                  'ttfp_seconds', 'end_time', 'duration', 'rx_rate_normalized', 'rx_rate', 'charge_time_min',
                  'charge_time_max', 'packet_counter_min', 'packet_counter_max', 'packet_counter_first',
                  'estimated_packet_counter_resets', 'estimated_total_packet_rate', 'estimated_total_per',
                  'externalId']
    tests_list = tag_stats_df['testId'].unique()
    for param in params:
        debug_print(f"""*****************************{param}*****************************""")
        graph = pd.DataFrame()
        for test in tests_list:
            df = tag_stats_df[tag_stats_df.testId == test]
            test_curveset = cdf_dataframe(df, param, test)
            graph = pd.concat([test_curveset, graph])
            debug_print(f"""test {test} added to graph""")
        fig = px.scatter(graph, x=param, y='TagCount', color=['testName'])
        fig.update_layout(title=param)
        fig.update_yaxes()
        if param == 'ttfp_seconds':
            fig.update_xaxes(title_text='Time to first packet [seconds]')
        if param == 'estimated_total_packet_rate':
            fig.update_xaxes(title_text='Estimated total packet rate [counters/second]')
        filepath = f"""{working_directory}{test_no}_{param}_graph.html"""
        open(filepath, 'a').close()
        fig.write_html(filepath)
        # debug_print(filepath)
        # p = plot(fig, output_type='div')
        # display_html(p)
        create_download_link(filepath, file_name=f"""{param} graph""")

def file_exists(path):
    """
    function checks if file exists
    :type path: str
    :param path: path to file
    :rtype: bool
    :return: file exists
    """
    path = path.replace('/dbfs', 'dbfs:')
    try:
        db_utils().fs.head(path)
        return True
    except Exception:
        try:
            return exists(path)
        except Exception:
            return False

def get_tagstats_from_test(tests_config_df=None, tests_config_csv_path=None, rawdata_df=None,
                           rawdata_csv_path=None, tagdeployment_df=None, tagdeployment_csv_path=None):
    """
        function gets test config df/csv and rawdata (from datatables) df/csv as input
        the function processes the data and created a new dataframe containing tag statistics for each test.
        :type tests_config_df: pandas DataFrame
        :param tests_config_df: dataframe of test config
        :type tests_config_csv_path: string
        :param tests_config_csv_path: path to test config csv
        :type rawdata_df: pandas DataFrame
        :param rawdata_df: dataframe of raw data
        :type rawdata_csv_path: string
        :param rawdata_csv_path: path to raw data csv
        :type tagdeployment_df: pandas DataFrame
        :param tagdeployment_df:dataframe of tag deployment data
        :type tagdeployment_csv_path: string
        :param tagdeployment_csv_path: path to tag deployment data csv
    """
    if tests_config_csv_path is not None and not tests_config_df:
        tests_config_df = pd.read_csv(tests_config_csv_path)
    if rawdata_csv_path is not None and not rawdata_df:
        rawdata_df = pd.read_csv(rawdata_csv_path)
    if tagdeployment_csv_path is not None:
        tagdeployment_df = pd.read_csv(tagdeployment_csv_path)
    testid_list = tests_config_df['testId'].unique()
    master_tag_stats = pd.DataFrame()
    for testid in testid_list:
        if  testid not in rawdata_df.testId.unique():
            debug_print(f"""0 data for testId {testid} in raw data! please check tables/test config""")
            continue
        currenttest_df = rawdata_df[rawdata_df.testId == testid].copy()
        start_time = tests_config_df[tests_config_df.testId == testid].get('startTimestamp').item()
        debug_print(f"""***** Processing {testid} data. Test Timestart = {start_time}""")
        currenttest_packets = analyze_packets_data(packets_df=currenttest_df, test_time_start=start_time)
        tag_stats_dict = currenttest_packets.get_group_statistics(group_by_col='externalId')
        tag_stats_df = pd.DataFrame.from_dict(tag_stats_dict, orient='index')
        tag_stats_df['testId'] = testid
        master_tag_stats = pd.concat([master_tag_stats, tag_stats_df])
    if tagdeployment_df is not None:
        debug_print('***********Tag deployment information merged with tag stats from test***********')
        master_tag_stats = pd.merge(master_tag_stats, tagdeployment_df, how='outer', on='externalId'). \
            sort_values('testId').set_index('externalId')
    master_tag_stats.insert(18, 'ttfp_seconds', master_tag_stats.ttfp / 1000)
    return master_tag_stats

def massagedata(dataset, param):
    curveset = pd.DataFrame(columns=[param, 'TagCount'])
    count = 0
    reftme_stmp = None
    for index, row in dataset.iterrows():
        if pd.isna(row[param]):
            continue
        if reftme_stmp is None:
            reftme_stmp = row[param]
        if reftme_stmp == row[param]:
            count += 1
        else:
            curveset = curveset.append({param: reftme_stmp, 'TagCount': count}, ignore_index=True)
            reftme_stmp = row[param]
            count += 1
    curveset = curveset.append({param: reftme_stmp, 'TagCount': count}, ignore_index=True)
    return curveset

def display_html(html):
    """
    Use databricks displayHTML from an external package
    :type html: string
    :param html: html document to display
    """
    for frame in inspect.getouterframes(inspect.currentframe()):
        global_names = set(frame.frame.f_globals)
        # Use multiple functions to reduce risk of mismatch
        if all(v in global_names for v in ["displayHTML", "display", "spark"]):
            return frame.frame.f_globals["displayHTML"](html)
    raise EnvironmentError("Unable to detect displayHTML function")

def display_print(todisplay, console_only=False, **kwargs):
    """
    Use databricks display func from an external package.
    uses tabulate library to display dataframe in case running locally
    :type todisplay: pandas DataFrame
    :param todisplay: variable to display
    :type console_only: bool
    :param console_only: if true, prints the data table to console (even if running in DB notebook) using tabulate
    """
    if not console_only:
        for frame in inspect.getouterframes(inspect.currentframe()):
            # call dbutils display
            global_names = set(frame.frame.f_globals)
            if all(v in global_names for v in ["display"]):
                try:
                    return frame.frame.f_globals["display"](todisplay)
                except ValueError:
                    debug_print('ValueError when reading DataFrame! Trying to print in console', center=True)
                except KeyError:
                    debug_print('KeyError when reading DataFrame! Trying to print in console', center=True)
                except TypeError:
                    debug_print('TypeError when reading DataFrame! Trying to print in console', center=True)
    if isinstance(todisplay, pd.DataFrame) or console_only:
        debug_print('\n' + tabulate.tabulate(todisplay, **kwargs))
        return None
    raise EnvironmentError("Unable to detect Display function")

def db_utils_rm(dbfs_path):
    """
    helper function to remove files/folders from dbfs
    """
    dbfs_path = dbfs_path.replace('/dbfs', 'dbfs:')
    return db_utils().fs.rm(dbfs_path)

def db_utils():
    """
    Use databricks dbutils from an external package
    for example: to use dbutils.fs.head() - use db_utils().fs.head()
    """
    for frame in inspect.getouterframes(inspect.currentframe()):
        global_names = set(frame.frame.f_globals)
        # Use multiple functions to reduce risk of mismatch
        if all(v in global_names for v in ["dbutils"]):
            return frame.frame.f_globals["dbutils"]()
    raise EnvironmentError("Unable to detect dbutils function")

def get_secret(scope, key):
    """
    get databrickst secret, return None if not running in databricks
    :param scope: secret scope
    :param key: secret key
    """
    try:
        secret = db_utils().secrets.get(scope=scope, key=key)
        return secret
    except EnvironmentError:
        raise(EnvironmentError('Cannot get secret when not running in databricks!'))


def create_download_link(dbfs_path, file_name='file'):
    """
    accepts path to dbfs file, and creates a download link to the file in the notebook
    the function only works with files saved in /dbfs/FileStore
    :type dbfs_path: string
    :param dbfs_path: path to dbfs path (accepts either spark API or file API format)
    :type file_name: string
    :param file_name: name of file (this will be the name of the link created)
    """
    if is_databricks():
        if not ('/dbfs/FileStore' in dbfs_path or 'dbfs:/FileStore' in dbfs_path):
            raise ValueError('the path must start with /dbfs/FileStore or dbfs:/FileStore!')
        dbfs_path = dbfs_path.replace('/dbfs/FileStore', '/files')
        dbfs_path = dbfs_path.replace('dbfs:/FileStore', '/files')
        display_html(f"""<a href="{dbfs_path}" download>Download {file_name} </a>""")
    debug_print(f"""File available at {dbfs_path}""")

def cdf_dataframe(dataset, param, dataset_name, groupby=None, comp=False):
    """
    function accepts dataframe as input the function returns dataframe of CDF
    complementary cumulative distribution function (of tag count) according to param,
    and labels it by dataset_name
    :type dataset: pandas DataFrame
    :param dataset: dataset to create CDF
    :type param: string
    :param param: parameter to create the CDF
    :type dataset_name: string
    :param dataset_name: name to add to the result
    :type groupby: str
    :param groupby: added parameter (in dataset) to group results by
    :type comp: bool
    :param comp: whether to create CDF (comp = False) or CCDF (comp = True)
    """
    curveset = pd.DataFrame(columns=[param, 'TagCount', groupby])
    dataset = dataset.sort_values(param, ascending=True)
    if groupby is None:
        iterations = [None]
    else:
        iterations = dataset[groupby].unique()
    for group in iterations:
        group_curveset = pd.DataFrame(columns=[param, 'TagCount', groupby])
        if group is not None:
            group_dataset = dataset[dataset[groupby] == group]
        else:
            group_dataset = dataset
        values = group_dataset[param].unique()
        group_curveset[param] = values
        if group is not None:
            group_curveset[groupby] = group
        for index, value in group_curveset[param].items():
            biggerthan = group_dataset[group_dataset[param] > value][param].count()
            smallerorequals = group_dataset[group_dataset[param] <= value][param].count()
            if comp:
                group_curveset.at[index, 'TagCount'] = biggerthan
            else:
                group_curveset.at[index, 'TagCount'] = smallerorequals
        if comp:
            group_curveset['testName'] = dataset_name + '_CCDF'
        else:
            group_curveset['testName'] = dataset_name + '_CDF'
        curveset = pd.concat([curveset, group_curveset])

    return curveset

def powerset(iterable):
    s = list(iterable)
    return chain.from_iterable(combinations(s, r) for r in range(len(s) + 1))

def analyze_packets_data(packets_df=None, csv_path=None, test_time_start=None, test_time_conf_received=None,
                         no_duplications=True):

    """
    gets dataframe of packets / csv of packets, returns NetworkPacketList of packets
    :type csv_path: string
    :param csv_path: path to the packets csv file
    :type packets_df: dataframe
    :param packets_df: packets dataframe
    :type test_time_start: timestamp (int)
    :param test_time_start: UTC millisecond timestamp of the beginning of the test
    :type test_time_conf_received: timestamp (int)
    :param test_time_conf_received: UTC millisecond timestamp of the time the test configuration was received
    :type no_duplications: bool
    :param no_duplications: if False, will keep duplications of same rawPacket
    :return: NetworkPacketList created from .csv located in path
    """
    if test_time_start is None:
        debug_print('Test time start was not passed, some metrics will not be calculated!')
    else:
        debug_print(f'Test time start {test_time_start} | ')
    if csv_path is None and packets_df is None:
        debug_print("no data was given!()")
        return None
    if csv_path is not None:
        packets_df = pd.read_csv(csv_path)

    debug_print('Preprocessing raw data...')

    # remove duplicate rows
    dup_rows = packets_df.shape[0]
    packets_df.sort_values(by='timestamp', ascending=True, ignore_index=True, inplace=True)
    if no_duplications:
        packets_df.drop_duplicates('rawPacket', ignore_index=True, inplace=True)
        dup_rows = dup_rows - packets_df.shape[0]
        if dup_rows > 0:
            debug_print(f'{dup_rows} duplicate rows removed')

    # flag bugged tags packets
    flag_bugged_tags(packets_df)

    debug_print('Creating NetworkPacketList from DataFrame...')
    packets_list = NetworkPacketList(list_custom_data={'test_time_start': test_time_start})
    if test_time_conf_received is not None:
        packets_list.list_custom_data['test_time_conf_received'] = test_time_conf_received
    reconstructed_packet_df = network_data_preparation(packets_df=packets_df)

    packets_list = packets_list.import_packet_df(import_all=True, packet_df=reconstructed_packet_df)
    return packets_list

def create_zip(dir_path):
    """
    gets path to FileStore directory and creates a download link to a zip file of the files in given directory
    zip file will be saved inside the dir_path directory
    :type dir_path: string
    :param dir_path: path to FileStore directory to zip
    """
    if dir_path[-1] == '/':
        dir_path = dir_path[:-1]
    dir_name = dir_path.rsplit('/', 1)[-1]
    if file_exists(dir_path + '/' + dir_name + '.zip'):
        db_utils().fs.rm(dir_path.replace('/dbfs', 'dbfs:') + '/' + dir_name + '.zip')
    zip_tmp_path = shutil.make_archive('/tmp/zips/tmpzip', 'zip', dir_path)
    zip_download_path = dir_path.replace('/dbfs', 'dbfs:') + '/' + dir_name + '.zip'
    db_utils().fs.mv(f'file:{zip_tmp_path}', zip_download_path)
    create_download_link(zip_download_path, f'{dir_name}.zip')

def exit_notebook(message):
    """
    closes notebook and prints out error message
    :type message: str
    :param message: error message
    """
    debug_print(message, center=True)
    try:
        db_utils().notebook.exit(message)
    except EnvironmentError as e:
        raise Exception(message)

def save_df(df, path, name=None, withindex=False, silent=False):
    """
    saves DataFrame to path, displays the DataFrame and creates a download link
    :type df: pandas Dataframe
    :param df: dataframe to save
    :type path: str
    :param path: path to save can be entered either as filename to save or directory to save in
    :type name: str
    :param name: name of dataframe (this displays in the download link and as a header before the link
    :type withindex: bool
    :param withindex: flag to choose if to export the dataframe with/without index
    :type silent: bool
    :param silent: if true does not generate HTML link
    """
    # save as dataframe.csv if no filename given
    if path[-1] == '/':
        path = path + 'dataframe'
    if path[-4:] != '.csv':
        path = path + '.csv'
    if name is None:
        name = path.split()[-1]
    if is_databricks():
        path = path.replace(' ', '_')
    debug_print(f'Saving DataFrame at {path}')
    if not is_databricks():
        df.to_csv(path, index=withindex)
        return True
    try:
        df.to_csv(path, index=withindex)
    except OSError:
        mkdirs_path = path.replace('/dbfs', 'dbfs:').rsplit('/', 1)[0]
        db_utils().fs.mkdirs(mkdirs_path)
        df.to_csv(path, index=withindex)
    if not silent:
        if is_databricks():
            display_html(f'<h1>{name}</h1>')
            create_download_link(dbfs_path=path, file_name=name)
            display_print(pd.read_csv(path), headers="keys")
        else:
            debug_print(f'{name} available at {path}')

def initialize_logger(working_directory=None):
    """
    initializes the logger to print to log and to logfile, which by default is named by the current timestamp (ms)
    when calling the function.
    :param working_directory: working directory to save logfile (in case running locally)
    :type working_directory: str
    :return: logger fileHandler filename
    """
    logger = logging.getLogger()
    for handler in logger.handlers:
        try:
            # extract current filename from log
            filename = handler.baseFilename.split('\\')[-1].split('.')[0]
            debug_print(f'Logger already initialized! passing logfile {filename}')
            return filename
        except Exception:
            pass
    logger_filename = int(current_timestamp())
    if is_databricks():
        db_utils().fs.put(f'file:/databricks/driver/{logger_filename}.log', '', overwrite=True)
        logging.getLogger("py4j.java_gateway").setLevel(logging.ERROR)
        logging.basicConfig(level=logging.DEBUG,
                            format='%(asctime)s | %(levelname)s | %(message)s',
                            handlers=[
                                logging.FileHandler(f'{logger_filename}.log', 'a'),
                                # logging.handlers.RotatingFileHandler
                                logging.StreamHandler()
                            ])
        logging.getLogger().handlers[0].setLevel(logging.DEBUG)
        logging.getLogger().handlers[1].setLevel(logging.INFO)
        debug_print(f'logger initialized at {logger_filename}', center=True)
        debug_print(f'logfile located at file:/databricks/driver/{logger_filename}.log')
    else:
        if working_directory is None:
            working_directory = os.path.join(user_data_dir(), 'wiliot', 'deployment_tools')
        if not os.path.exists(working_directory):
            os.makedirs(working_directory)
        logging.basicConfig(level=logging.DEBUG,
                            format='%(asctime)s | %(levelname)s | %(message)s',
                            handlers=[
                                logging.FileHandler(f'{working_directory}/{logger_filename}.log', 'a'),
                                # logging.handlers.RotatingFileHandler
                                logging.StreamHandler()
                            ])
        # filter stream to show info and up
        logging.getLogger().handlers[0].setLevel(logging.DEBUG)
        logging.getLogger().handlers[1].setLevel(logging.INFO)
        debug_print(f'logger initialized at {logger_filename}', center=True)
        debug_print(f'logfile located at {working_directory}/{logger_filename}.log')
    logging.getLogger().setLevel(logging.DEBUG)
    return logger_filename

def create_logfile(logger_filename, working_directory, copy_filename):
    """
        function copies the logfile for recent run to the test working directory and creates link for download
        :type logger_filename: str
        :param logger_filename: filename of log file
        :type working_directory: str
        :param working_directory: directory to copy logfile to
        :type copy_filename: str
        :param copy_filename: name of logfile when copied
    """
    if not is_databricks():
        debug_print('Cannot create logfile, logfile available already!')
    debug_print('Creating Logfile...', center=True)
    debug_print(f'cp {logger_filename}.log {copy_filename}.log')
    subprocess.run(f'cp {logger_filename}.log {copy_filename}.log', shell=True)
    copy_directory = working_directory.replace('/dbfs', 'dbfs:')
    debug_print(f'copy directory {copy_directory}')
    db_utils().fs.mkdirs(f'{copy_directory}')
    debug_print(f'mkdirs {copy_directory}')
    db_utils().fs.cp(f'file:/databricks/driver/{copy_filename}.log', copy_directory)
    debug_print(f'cp file:/databricks/driver/{copy_filename}.log, copy_directory')
    create_download_link(working_directory + f'{copy_filename}.log', f'{copy_filename}.log')
    debug_print(f"create download link - {working_directory} + f'{copy_filename}.log', f'{copy_filename}.log'")

def get_packet_table_name(owner, env, platform=False):
    """
    function gets ownerId and environment and return the name of the packet table (in databricks)
    :type owner: str
    :param owner: ownerId
    :type env: str
    :param env: wiliot environment (prod/test/dev)
    :type platform: bool
    :param platform: wiliot platform
    :rtype: str
    :return: data table name
    """
    env = 'prod' if env is None else env
    if not platform:
        data_table = owner + '' + f'_packet_data_{env}'
    else:
        data_table = owner + f'_enriched_packets_data_{env}'
    if owner.isnumeric():
        data_table = '_' + data_table
    return data_table

def get_event_table_name(owner, env=None, platform=False):
    """
    function gets ownerId and environment and return the name of the event table (in databricks)
    :type owner: str
    :param owner: ownerId
    :type env: str
    :param env: wiliot environment (prod/test/dev)
    :type platform: bool
    :param platform: wiliot platform
    :rtype: str
    :return: data table name
    """
    env = 'prod' if env is None else env
    if not platform:
        event_table = owner + '' + f'_event_data_{env}'
    else:
        event_table = owner + '' + f'_assets_metrics_data_{env}'
    if owner.isnumeric():
        event_table = '_' + event_table
    return event_table

def get_heartbeat_table_name(env=None):
    """
    function gets environment and returns the name of the heartbeat table (in databricks)
    :type owner: str
    :param owner: ownerId
    :type env: str
    :param env: wiliot environment (prod/test/dev)
    :rtype: str
    :return: hearbeat table name
    """
    env = 'prod' if env is None else env
    hearbeat_table = f'_network_heartbeat_statistics_{env}'
    return hearbeat_table

def get_statistics_table_name(env=None):
    """
    function gets environment and returns the name of the heartbeat table (in databricks)
    :type owner: str
    :param owner: ownerId
    :type env: str
    :param env: wiliot environment (prod/test/dev)
    :rtype: str
    :return: hearbeat table name
    """
    env = 'prod' if env is None else env
    hearbeat_table = f'_network_data_statistics_{env}'
    return hearbeat_table

def get_configuration_table_name(owner, env=None):
    """
    function gets ownerId and environment and return the name of the configuration table (in databricks)
    :type owner: str
    :param owner: ownerId
    :type env: str
    :param env: wiliot environment (prod/test/dev)
    :rtype: str
    :return: configuration table name
    """
    env = 'prod' if env is None else env
    configuration_table = owner + '' + f'_network_configuration_{env}'
    if owner.isnumeric():
        configuration_table = '_' + configuration_table
    return configuration_table

def process_graph(fig, graph_name, display_graph=True, save_graph=False, directory=None, pass_as_is=False):
    """
    function processes graph - displays and saves the graph (according to entered flags)
    :type fig: plotly Figure
    :param fig: graph to display
    :type graph_name: str
    :param graph_name: name of graph, and the filename by which it will be saved
    :type display_graph: bool
    :param display_graph: flag to choose if to display the graph in DataBricks Notebook
    :type save_graph: bool
    :param save_graph: flag to choose if to save the graph
    :type directory: str
    :param directory: directory to save graph
    :type pass_as_is: bool
    :param pass_as_is: if true the fig entered is already a plot
    """
    debug_print(f'Processing Graph - {graph_name}', center=True)
    if save_graph:
        if directory is None:
            exit_notebook('Need to supply directory to save graph!')
        if directory[-1] != '/':
            directory = directory + '/'
        filepath = f"""{directory}{graph_name}.html"""
        Path(directory).mkdir(parents=True, exist_ok=True)
        open(filepath, 'a').close()
        fig.write_html(filepath)
        create_download_link(filepath, file_name=graph_name)
    if display_graph:
        if is_databricks():
            if not pass_as_is:
                fig = plot(fig, output_type='div')
            display_html(fig)
        else:
            fig.show(renderer="browser")

def get_spark():
    try:
        from pyspark.sql import SparkSession
        return SparkSession.builder.getOrCreate()
    except NameError:
        raise NameError('Spark is not installed!')


def analyze_packets_data_per_brg(df, brg_list, time_start, test_time_conf_received=None):
    df = df.drop_duplicates(subset=["rawPacket", "bridgeId"])
    tag_stats_df = pd.DataFrame()
    for brg in brg_list:
        brg_df = df[df['bridgeId'] == brg]
        debug_print(f'Processing {brg_df.shape[0]} packets from {brg}')
        if brg_df.empty:
            continue
        df_tagstats = analyze_packets_data(packets_df=brg_df, test_time_start=time_start,
                                           test_time_conf_received=test_time_conf_received)
        tag_stats_dict = df_tagstats.get_group_statistics(group_by_col='externalId')
        tag_stats_df_tmp = pd.DataFrame.from_dict(tag_stats_dict, orient='index')
        tag_stats_df_tmp['bridgeId'] = brg
        tag_stats_df = pd.concat([tag_stats_df, tag_stats_df_tmp])
    if 'externalId' not in tag_stats_df.columns:
        tag_stats_df['externalId'] = tag_stats_df.index
    tag_stats_df = tag_stats_df.sort_values('rx_rate_normalized', ascending=False).drop_duplicates(
        'externalId').sort_index()
    return tag_stats_df


def flag_bugged_tags(rawdata_df):
    """
    function gets raw data dataframe, adds 'isBugged' column
    'isBugged' is true for each packet of a bugged tag, false otherwise
    :type rawdata_df: pandas DataFrame
    :param rawdata_df: raw packet data
    """
    rawdata_df['isBugged'] = None
    unique_tags = rawdata_df['externalId'].unique()
    for tag in unique_tags:
        tmp_df = rawdata_df.loc[rawdata_df['externalId'] == tag]
        tmp_df = tmp_df[['packet_counter', 'timestamp']]
        tmp_df = tmp_df.sort_values(by=['timestamp'])
        prev_packet_counter = 0
        cycles = 0
        orig_cycles = tmp_df['packet_counter'].unique()
        for timestamp in tmp_df['timestamp'].unique():
            # TODO - see when this function throws error
            try:
                packet_counter = tmp_df.loc[tmp_df['timestamp'] == timestamp, 'packet_counter'].unique().item()
            except Exception:
                continue
            if packet_counter + 256 * cycles < prev_packet_counter:
                cycles = cycles + 1
            prev_packet_counter = packet_counter + 256 * cycles
            tmp_df.loc[tmp_df['timestamp'] == timestamp, 'packet_counter'] = prev_packet_counter
        tmp_df2 = tmp_df.diff(axis=0)
        tmp_df2['rate'] = (1000 * tmp_df2['packet_counter'] / tmp_df2['timestamp'])
        max_rate = tmp_df2['rate'].max()
        is_bugged = False
        if max_rate > 6:
            is_bugged = True
            debug_print(f'{tag} is bugged! Flagging packets')
        rawdata_df.loc[rawdata_df['externalId'] == tag, 'isBugged'] = is_bugged


def parse_commastring(string):
    """
    parse string with comma or comma+space seperated values to list
    :type string: str
    :param string: input
    :rtype: list
    :return: list of values
    """
    if type(string) == float:
        try:
            if np.isnan(string):
                return None
        except Exception as e:
            pass
    if type(string) == list:
        return string
    if string is None:
        return list()
    cmd_list = ''.join(string.split()).split(',')
    return cmd_list


def parse_commastring_array(array):
    """
    parses Pandas array (DataFrame column) to list of all unique values
    :type array: Pandas array
    :param array: array
    :rtype: list
    :return: list of unique values
    """
    if len(array) == 1:
        return parse_commastring(array[0])
    result = list()
    for item in array:
        item_values = parse_commastring(item)
        if len(item_values) == 1:
            item_values
            if item_values[0] not in result:
                result.extend(item_values)
        else:
            for value in item_values:
                if value not in result:
                    result.extend([value])
    return result