# -*- coding: utf-8 -*-
"""
Created on Wed May  5 08:29:00 2021

@author: Therese
"""

import glob
import avn.dataloading as dataloading
import avn.plotting as plotting
#import dataloading
#import plotting
import numpy as np
import pandas as pd
import librosa
import librosa.display
import sklearn
import os
import matplotlib.pyplot as plt
import random

class SegData:
    """
    Syllable segmentation data for many files from a single bird. 
    
    Attributes
    ----------
    Bird_ID: str
        String containing a unique identifier for subject bird.
        
    seg_table: pandas DataFrame
        Dataframe with columns `onsets`, `offsets` and `files`, which contains 
        the onset and offset times in seconds of every syllable, and the 
        syllable's corresponding file. This is generated by an 
        avn.segmentation.Segmenter class object with the function 
        `make_segmentation_table`. 
    
    true_seg_table: pandas DataFrame
        Dataframe with columns `files`, `labels`, `onsets` and `offsets`, 
        which contains ground truth segmentation information for calculation 
        of automatic segmentation metrics. `labels` contains only {'n', 's'}, 
        to indicate whether a row reflects a true song syllable ('s') or cage
        noise ('n'). 
        This can be imported from evsonganaly with the 
        `avn.dataloading.add_ev_song_truth_table()` function. 
        
    seg_metrics: pandas DataFrame
        Dataframe with columns `F1`, `precision` and `recall` conatining a 
        single row with each metric calculated by comparing segmentations in 
        `SegData.seg_table` to `SegData.true_seg_table`. 
        This is generated with the function `segmentation.Metrics.calc_F1()`. 
    
    Methods
    -------
    save_as_csv(out_folder_path)
        Saves the contents of SegData.seg_table and SegData.seg_metrics as csv 
        files in the `out_folder_path` directory. 
    """
    
    def __init__(self, Bird_ID, seg_table):
        """
        Parameters
        ----------
        Bird_ID : str
            String containing a unique identifier for subject bird.
        seg_table : pandas DataFrame
            Dataframe with columns `onsets`, `offsets` and `files`, which 
            contains the onset and offset times in seconds of every syllable, 
            and the syllable's corresponding file.

        """
        self.Bird_ID = Bird_ID
        self.seg_table = seg_table
        
    def save_as_csv(self, out_folder_path):
        """
        Saves SegData.seg_table and SegData.seg_metrics as csv files in the 
        `out_folder_path` directory. 

        Parameters
        ----------
        out_folder_path : str
            Path to local directory in which to save csv files.

        Returns
        -------
        None.
        
        Notes
        ------
        The SegData.seg_table file will be called "[Bird_ID]_seg_table.csv", 
        and the SegData.seg_metrics file will be called 
        "[Bird_ID]_seg_metrics.csv". 
        
        If either the .seg_table or .seg_metrics attributes do not exist, the 
        corresponding file will not be created. 

        """
        #check if SegData has a seg_table attribute. If so, save it. 
        if hasattr(self, 'seg_table'):
            self.seg_table.to_csv(out_folder_path + self.Bird_ID + "_seg_table.csv")
        #check if SegData has seg_metrics attribute. If so, save it. 
        if hasattr(self, 'seg_metrics'):
            self.seg_metrics.to_csv(out_folder_path + self.Bird_ID + "_seg_metrics.csv")

class Segmenter:
    """
    Parent class for automated syllable segmentation
    
    Attributes
    ----------
    None
    
    Methods
    -------
    make_segmentation_table(Bird_ID, song_folder_path, upper_threshold, lower_threshold)
        Generates a SegData object with syllable segmentation information for 
        all .wav files in `song_folder_path`
    
    get_seg_criteria(song)
        For threshold based segmentation, this calculates the song feature on 
        which to apply threshold segmentation (eg. RMSE, MFCC, MFCC Derivative)
    
    rescale(seg_criteria)
        Applies 0 to 1 min-max rescaling to a vector. 
    
    get_threshold(seg_criteria, thresh):
        Generates a flat threshold vetor for comparison to seg_criteria for 
        threshold based segmentation. 
        
    get_syll_onsets_offsets(seg_criteria, upper_thresh, lower_thresh, total_file_duration)
        Returns onsets and offset timestamps of all syllables in a file based 
        on seg_criteria threshold crossings. 
    
    Notes
    -----
    An instance of this parent class cannot be used to generate syllable
    segmentations, as the `get_seg_criteria()` function is not implemented. 
    Instead, please use one of the child classes which each use a different 
    segmentation criteria (e.g RMSE, RMSEDerivative, MFCC, MFCCDerivative) 

    """
    
    def __init__(self):
        pass
    
    def make_segmentation_table(self, Bird_ID, song_folder_path, upper_threshold, lower_threshold, 
                                max_syll_duration = 0.33, hop_length = 512, n_fft = 2048):
        """
        

        Parameters
        ----------
        Bird_ID : str
            String containing a unique identifier for subject bird.
        song_folder_path : str
            Path to a local directory containing all .wav files to be segmented.
        upper_threshold : float > lower_threshold
            Value of the upper segmentation criteria threshold for detecting 
            syllable onsets. 
        lower_threshold : float < upper_threshold
            Value of the lower segmentation criteria threshold used for detecting
            syllable offsets when onset to onset segmentation results in a 
            syllable longer than `max_syll_duration`.
            DESCRIPTION.
        max_syll_duration : float > 0, optional
            Maximum allowable syllable duration in seconds. The default is 0.33.
            If the gap between consecutive syllabel onsets is longer than this 
            value, the offset will be determined by lower threshold crossing. 
            If the lower threshold crossing still results in a syllable longer
            than this value, the syllable offset will be set to the onset
            timestamp + `max_syll_duration`.       
        hop_length : int > 0, optional
            The number of samples between successive frames used in the short
            term fourier transform to generate segmentation criteria values. 
            The default is 512.
        n_fft : int > 0, optional
            The length of the FFT window used to calculate the segmentation 
            criteria values. The default is 2048.

        Returns
        -------
        segmentation_data : avn.segmentation.SegData object
            SegData object with attributes `.Bird_ID` and `.seg_table`, where
            `.seg_table` is a pandas Dataframe with columns `onsets`, `offsets`
            and `files`, which contains the onset and offset times in seconds 
            of every syllable, and the syllable's corresponding file.

        """
        #get list of all .wav files in `song_folder_path`
        all_song_files = glob.glob(song_folder_path + "*.wav")
        
        all_data = None
        
        for song_file in all_song_files:
            
            #Load single song file
            song = dataloading.SongFile(song_file)
        
            #Calculate value of segmentation criteria (e.g. MFCC, RMSE)
            seg_criteria = self.get_seg_criteria(song, hop_length = hop_length, n_fft = n_fft)
        
            #Create thresholds 
            upper_thresh = self.__get_threshold(seg_criteria, thresh = upper_threshold)
            lower_thresh = self.__get_threshold(seg_criteria, thresh = lower_threshold)
        
            #Get syllable onset and offset indices
            syll_onset_indices, syll_offset_indices = self.__get_syll_onsets_offsets(seg_criteria, 
                                                                              upper_thresh, 
                                                                              lower_thresh, 
                                                                              song.duration,
                                                                              max_syll_duration = max_syll_duration)
            #Convert indices to times -- should this be a function?
            all_times = librosa.frames_to_time(np.arange(len(seg_criteria)), 
                                           sr = song.sample_rate, hop_length = hop_length, 
                                           n_fft = n_fft)
            syll_onsets = all_times[syll_onset_indices]
            syll_offsets = all_times[syll_offset_indices]
        
            #Package data into Dataframe
            curr_file_data = pd.DataFrame({"onsets": syll_onsets, 
                                           "offsets": syll_offsets})
            curr_file_data['files'] = song.file_name
            
            #add curr_file data to growing all_data DataFrame, which contains 
                #segmentation data for all files
            if all_data is not None:
              all_data = pd.concat([all_data, curr_file_data], ignore_index=True)
            else:
              all_data = curr_file_data
        
        #Create SegData object
        segmentation_data = SegData(Bird_ID, all_data)
        segmentation_data.song_folder_path = song_folder_path
        
        #Save n_fft and hop_length to SegData object for future plotting
        segmentation_data.hop_length = hop_length
        segmentation_data.n_fft = n_fft
        
        return segmentation_data
      
    def get_seg_criteria(self, song):
        raise NotImplementedError
    
    def rescale(self, seg_criteria):
        """
        Applies 0 to 1 min-max rescaling to a vector.

        Parameters
        ----------
        seg_criteria : numpy ndarray, 1D
            1 dimensional numpy array to be rescaled

        Returns
        -------
        seg_criteria : numpy ndarray, 1D
            1 dimensional numpy array rescaled between 0 and 1.
        """
        #initialize scaler
        scaler = sklearn.preprocessing.MinMaxScaler()
        #apply scaler to data
        seg_criteria = scaler.fit_transform(seg_criteria[:, np.newaxis])[:, 0]
        
        return seg_criteria
        
    def __get_threshold(self, seg_criteria, thresh):
        """
        Generates a flat threshold vetor for comparison to seg_criteria for 
        threshold based segmentation. 

        Parameters
        ----------
        seg_criteria : numpy ndarray, 1D
            1 dimensional numpy array containing the segmentation criteria 
            values for a wave file for treshold segmentation. 
        thresh : float
            Value of threshold to be created.

        Returns
        -------
        threshold_vector : numpy ndarray, 1D
            1 dimensional numpy array with the same dimensions as `seg_criteria`, 
            where every entry has the value `thresh`. 

        """
        threshold_vector = np.ones_like(seg_criteria) * thresh
    
        return threshold_vector
  
    def __get_syll_onsets_offsets(self, seg_criteria, upper_thresh, lower_thresh, total_file_duration, 
                            max_syll_duration = 0.3):
        """
        Returns onsets and offset timestamps of all syllables in a file based 
        on seg_criteria threshold crossings.

        Parameters
        ----------
        seg_criteria : numpy ndarray, 1D
            1 dimensional numpy array containing the segmentation criteria
            values for a wave file for threshold segmentation
        upper_thresh : numpy ndarray, 1D
            1 dimensional numpy array with the same dimensions as `seg_criteria`. 
            Contains the value of the upper threshold for detecting syllable 
            onsets. 
        lower_thresh : numpy ndarray, 1D
            1 dimensional numpy array with the same dimensions as `seg_criteria`. 
            Contains the value of the lower threshold for detecting syllable 
            offsets. 
        total_file_duration : float > 0
            Total duration of the song file in seconds. 
        max_syll_duration : float > 0, optional
            Maximum allowable syllable duration in seconds. The default is 0.33.
            If the gap between consecutive syllabel onsets is longer than this 
            value, the offset will be determined by lower threshold crossing. 
            If the lower threshold crossing still results in a syllable longer
            than this value, the syllable offset will be set to the onset
            timestamp + `max_syll_duration`.

        Returns
        -------
        syll_onsets: numpy ndarray, 1D
            1 dimensional numpy array containing the indices of all syllable 
            onsets in `seg_criteria`. 
        syll_offsets: numpy ndarray, 1D
            1 dimensional numpy array containing the indices of all syllable
            offsets in `seg_criteria`. 
            
        Notes
        -----
        This segmentation works by first finding every point where the 
        segmentation criteria (e.g. RMSE, MFCC Derivative etc.) crosses the 
        upper threshold. Those are all considered syllable onsets. 
        The onset of the following syllable will be used as the offset of the 
        previous syllable, so long as that doesn't result in a syllable with 
        a duration longer than `max_syll_duration`. If the syllable is longer, 
        then the offset is determined by finding the first point after the 
        onset where the segmentation criteria crosses the lower threshold. If 
        that still results in a syllable which is longer than `max_syll_duration`
        then the offset defaults to `max_syll_duration` seconds after the 
        syllable onset. 

        """

    
        #get array of boolean above or below upper thresh
        upper_thresh_position = seg_criteria >= upper_thresh
        #get array of positive or negative threshold crossings
        upper_thresh_crossing = np.diff(upper_thresh_position, prepend = 0)
        #get array with True at every upward crossing of upper thresh
        upper_thresh_passing = upper_thresh_crossing == 1
        
        #get array of boolean above or below lower thresh
        lower_thresh_position = seg_criteria < lower_thresh
        #get array of positive or negative threshold crossings
        lower_thresh_crossing = np.diff(lower_thresh_position, prepend = 0)
        #get array with True at every downward crossing of the lower thresh
        lower_thresh_passing = lower_thresh_crossing == 1 
        
        #Get indices of upward crossings of upper thresh
        upper_thresh_indices = np.nonzero(upper_thresh_passing)[0]
        
        #Add end of file to act as a final syllable onset. 
        #This way, all syllables will have an offset at or before the end of the file. 
        upper_thresh_indices = np.append(upper_thresh_indices, len(upper_thresh_passing)-1)
        
        #initialize vector to contain syllable offsets
        syll_offset = np.zeros_like(upper_thresh_indices)
      
        #determine how many frames corresponds to a syllable with duration `max_syll_duration`. 
        max_syll_duration_frames = seg_criteria.shape[0] / total_file_duration * max_syll_duration
        
        #loop through positive upper threshold crossings
        for i in np.arange(len(upper_thresh_indices)-1):
      
        #check whether gap between consecutive offsets is more than max_syll_duration seconds long
          if (upper_thresh_indices[i+1] - upper_thresh_indices[i]) >  max_syll_duration_frames:
            
            #if so, find the first negative lower threshold crossing after positive upper threshold crossing
            #and set that to the syllable offset. 
            for a in np.arange(upper_thresh_indices[i], np.floor(upper_thresh_indices[i] + max_syll_duration_frames)).astype(int):
          
              if lower_thresh_passing[a] == 1:
                syll_offset[i] = a 
                break
            
              #If the lower threshold isn't crossed before `max_syll_duration`s after the onset, 
              #then just set the offset to `max_syll_duration` seconds after onset.
              syll_offset[i] = np.floor(upper_thresh_indices[i] + max_syll_duration_frames)
      
        #Otherwise, the syllable offset is the onset of the following syllable. 
          else: 
            syll_offset[i] = upper_thresh_indices[i + 1]
          
        #Drop the last one because it is an artifact
        syll_offset = syll_offset[:-1]
        syll_onset = upper_thresh_indices[:-1]
      
        return(syll_onset, syll_offset)
      
      
      
class RMSE(Segmenter):
    """
    Child class of avn.segmentation.Segmenter(), which segments syllables based
    on root mean square energy (RMSE) threshold crossing.
    """
    
    def __init__(self):
        super().__init__()
        
    def get_seg_criteria(self, song, hop_length = 512, n_fft = 2048, 
                         bandpass = True, lower_cutoff = 200, upper_cutoff = 9000, 
                         rescale = True):
        """
        Calculates the RMSE at every frame of a song file for later use in 
        threshold crossing-based segmentation. 

        Parameters
        ----------
        song : avn.dataloading.SongFile class instance
            Contains audio data for a single song file
        hop_length : int > 0, optional
            The number of samples between successive frames used in the short
            term fourier transform to generate RMSE values. The default is 512.
        n_fft : int > 0, optional
            The length of the FFT window used to calculate the RMSE values. 
            The default is 2048.
        bandpass : bool, optional
            If True, the song will be bandpass filtered before calculating the 
            RMSE. If False, the RMSE will be calculated on the song as-is. 
            The default is True.
        lower_cutoff : float >=0, optional
            The lower frequency limit in Hz used to bandpass filter the input 
            wave before calculating the RMSE. The default is 200.
        upper_cutoff : float > lower_cutoff, optional
            The upper frequency limit in Hz used to bandpass filter the input
            wave before calculating the RMSE. The default is 9000.
        rescale : bool, optional
            If True, the RMSE will be min-max rescaled so that all values fall 
            between 0 and 1. This is meant to ensure consistency across 
            recordings. If False, the raw RMSE values will be returned. 
            The default is True.

        Returns
        -------
        rmse : numpy ndarray, 1D
            1 dimensional numpy array containing the RMSE values for the input
            wave.

        """

        #apply bandpass filter to wave if `bandpass` = True
        if bandpass:
            song.bandpass_filter(lower_cutoff, upper_cutoff)
        
        #calculate RMSE
        rmse = librosa.feature.rms(y = song.data, 
                                  hop_length = hop_length, 
                                  frame_length = n_fft)[0, :]
                                  
        #rescale all rmse values to a range of 0 to 1 if `rescale` = True
        if rescale:
            rmse = self.rescale(rmse)
        
        return rmse
    
class RMSEDerivative(Segmenter):
    """
    Child class of avn.segmentation.Segmenter(), which segments syllables based
    on threshold crossing of the first derivative of the root mean square
    energy (RMSE).
    """
    def __init__(self):
        super().__init__()
        
    def get_seg_criteria(self, song, hop_length = 512, n_fft = 2048, 
                         bandpass = True, lower_cutoff = 200, upper_cutoff = 9000, 
                         rescale = True, deriv_width = 3):
        """
        Calculates the first derivative of the RMSE at every frame of a song 
        file for later use in threshold crossing-based segmentation. 

        Parameters
        ----------
        song : avn.dataloading.SongFile class instance
            Contains audio data for a single song file
        hop_length : int > 0, optional
            The number of samples between successive frames used in the short
            term fourier transform to generate RMSE values. The default is 512.
        n_fft : int > 0, optional
            The length of the FFT window used to calculate the RMSE values. 
            The default is 2048.
        bandpass : bool, optional
            If True, the song will be bandpass filtered before calculating the 
            RMSE. If False, the RMSE will be calculated on the song as-is. 
            The default is True.
        lower_cutoff : float >=0, optional
            The lower frequency limit in Hz used to bandpass filter the input 
            wave before calculating the RMSE. The default is 200.
        upper_cutoff : float > lower_cutoff, optional
            The upper frequency limit in Hz used to bandpass filter the input
            wave before calculating the RMSE. The default is 9000.
        rescale : bool, optional
            If True, the RMSE will be min-max rescaled so that all values fall 
            between 0 and 1 before the first derivative is taken. This is meant 
            to ensure consistency across recordings. If False, the first 
            derivative is calculated on the raw RMSE. 
            The default is True.
        deriv_width: int >=3, odd, optional.
            Number of frames over which to compute the change in RMSE to 
            estimate the first derivative.

        Returns
        -------
        rmse_derivative : numpy ndarray, 1D
            1 dimensional numpy array containing an estimate of the first 
            derivative of the RMSE of the input wave.

        """
        #apply bandpass filter to song if `bandpass` = True
        if bandpass:
            song.bandpass_filter(lower_cutoff, upper_cutoff)
        
        #calculate RMSE
        rmse = librosa.feature.rms(y = song.data,
                                   hop_length = hop_length, 
                                   frame_length = n_fft)[0, :]
        
        #rescale  all rmse values to a range of 0 to 1 if `rescale` = True
        if rescale:
            rmse = self.rescale(rmse)
            
        #calculate derivative of rmse
        rmse_derivative = librosa.feature.delta(rmse, width = deriv_width)
        
        return rmse_derivative
    
    
class MFCC(Segmenter):
    """
    Child class of avn.segmentation.Segmenter(), which segments syllables based
    on threshold crossing of the first mel frequency cepstral coefficient 
    (MFCC).
    """
    def __init__(self):
        super().__init__()
            
    def get_seg_criteria(self, song, hop_length = 512, n_fft = 2048, 
                         bandpass = True, lower_cutoff = 200, upper_cutoff = 9000, 
                         rescale = True):
        """
        Calculates the first MFCC component at every frame of a song file for 
        later use in threshold crossing-based segmentation. 

        Parameters
        ----------
        song : avn.dataloading.SongFile class instance
            Contains audio data for a single song file
        hop_length : int > 0, optional
            The number of samples between successive frames used in the short
            term fourier transform to generate MFCC values. The default is 512.
        n_fft : int > 0, optional
            The length of the FFT window used to calculate the MFCC values. 
            The default is 2048.
        bandpass : bool, optional
            If True, the song will be bandpass filtered before calculating the 
            MFCC. If False, the MFCC will be calculated on the song as-is. 
            The default is True.
        lower_cutoff : float >=0, optional
            The lower frequency limit in Hz used to bandpass filter the input 
            wave before calculating the MFCC. The default is 200.
        upper_cutoff : float > lower_cutoff, optional
            The upper frequency limit in Hz used to bandpass filter the input
            wave before calculating the MFCC. The default is 9000.
        rescale : bool, optional
            If True, the MFCC will be min-max rescaled so that all values fall 
            between 0 and 1. This is meant to ensure consistency across 
            recordings. If False, the raw MFCC values will be returned. 
            The default is True.

        Returns
        -------
        mfcc : numpy ndarray, 1D
            1 dimensional numpy array containing the MFCC values for the input
            wave.

        """
        #apply bandpass filter to wave if `bandpass` = True
        if bandpass:
            song.bandpass_filter(lower_cutoff, upper_cutoff)
        
        #calculate first MFCC
        mfcc = librosa.feature.mfcc(song.data, song.sample_rate, n_mfcc = 1,
                                    hop_length = hop_length, 
                                    win_length = n_fft)[0, :]
                                  
        #rescale all mfcc values to a range of 0 to 1 if `rescale` = True
        if rescale:
            mfcc = self.rescale(mfcc)
        
        return mfcc
    
class MFCCDerivative(Segmenter):
    """
    Child class of avn.segmentation.Segmenter(), which segments syllables based
    on threshold crossing of an estimate of the first derivative of the first 
    mel frequency cepstral coefficient (MFCC).
    """
    def __init__(self):
        super().__init__()
        
    def get_seg_criteria(self, song, hop_length = 512, n_fft = 2048, 
                         bandpass = True, lower_cutoff = 200, upper_cutoff = 9000, 
                         rescale = True, deriv_width = 3):
        """
        Calculates the first derivative of the MFCC at every frame of a song 
        file for later use in threshold crossing-based segmentation. 

        Parameters
        ----------
        song : avn.dataloading.SongFile class instance
            Contains audio data for a single song file
        hop_length : int > 0, optional
            The number of samples between successive frames used in the short
            term fourier transform to generate MFCC values. The default is 512.
        n_fft : int > 0, optional
            The length of the FFT window used to calculate the MFCC values. 
            The default is 2048.
        bandpass : bool, optional
            If True, the song will be bandpass filtered before calculating the 
            MFCC. If False, the MFCC will be calculated on the song as-is. 
            The default is True.
        lower_cutoff : float >=0, optional
            The lower frequency limit in Hz used to bandpass filter the input 
            wave before calculating the MFCC. The default is 200.
        upper_cutoff : float > lower_cutoff, optional
            The upper frequency limit in Hz used to bandpass filter the input
            wave before calculating the MFCC. The default is 9000.
        rescale : bool, optional
            If True, the MFCC will be min-max rescaled so that all values fall 
            between 0 and 1 before the first derivative is taken. This is meant 
            to ensure consistency across recordings. If False, the first 
            derivative is calculated on the raw MFCC. 
            The default is True.
        deriv_width: int >=3, odd, optional.
            Number of frames over which to compute the change in MFCC to 
            estimate the first derivative.

        Returns
        -------
        mfcc_derivative : numpy ndarray, 1D
            1 dimensional numpy array containing an estimate of the first 
            derivative of the MFCC of the input wave.

        """

        #apply bandpass filter to wav if `bandpass` == True
        if bandpass:
            song.bandpass_filter(lower_cutoff, upper_cutoff)
        
        #calculate first MFCC
        mfcc = librosa.feature.mfcc(song.data, song.sample_rate, n_mfcc = 1, 
                                    hop_length = hop_length, 
                                    win_length = n_fft)[0, :]
        
        #rescale MFCC vector is `rescale` == True
        if rescale:
            mfcc = self.rescale(mfcc)
            
        #calculate derivative of rescaled MFCC
        mfcc_derivative = librosa.feature.delta(mfcc, width = deriv_width)
        
        return mfcc_derivative
    

class Metrics():
    """
    Contains functions for calculating segmentation accuracy metrics by comparing
    automatically generates syllable segmentations to 'ground truth' segmentations.
    """
    def __init__(self):
        pass
    
    def calc_F1 (seg_data, max_gap = 0.05):
        """
        Calculates the F1 score, precision and recall of syllable onsets in 
        seg_data.seg_table relative to seg_data.true_seg_table.

        Parameters
        ----------
        seg_data : avn.segmentation.SegData instance
            Instance of a SegData object which must have valid `.seg_table` 
            and `.true_seg_table` attributes.
        max_gap : float, optional
            The maximum allowable gap in seconds between a syllable onset in 
            `seg_data.seg_table` and in `seg_data.true_seg_table` that will 
            be considered a match. The default is 0.05.

        Returns
        -------
        seg_data : avn.segmentation.SegData instance
            This will be the same seg_data object as is passed as an argument, 
            with an added `.seg_metrics` attribute which contains a DataFrame
            with columns `F1`, `precision` and `recall` and a single row with 
            the value of each metric.

        """
        #make a copy of seg_data.true_seg_table
        truth_seg_table = seg_data.true_seg_table
        
        #initialize counters 
        all_true_positives = 0
        all_false_positives = 0
        all_false_negatives = 0
        
        #loop through each individual file in the segmenter.seg_table
        for current_file in np.unique(seg_data.seg_table['files']):
            #filter the seg_tables so that they contain only syllables from the 
            #current file. 
            seg_current_file = seg_data.seg_table[seg_data.seg_table['files'] == current_file]
            truth_current_file = truth_seg_table[truth_seg_table['files'] == current_file]
            
            #get best matches of truth to segmenter onsets
            seg_matches = Metrics.__get_best_matches(seg_current_file['onsets'], 
                                                truth_current_file['onsets'], max_gap)
            #get best matches of segmenter to truth onsets
            truth_matches = Metrics.__get_best_matches(truth_current_file['onsets'], 
                                             seg_current_file['onsets'], max_gap)
            
            #calculate F1 components for the given file
            true_positives, false_positives, false_negatives = Metrics.__calc_F1_components_per_file(seg_current_file, 
                                                                                           truth_current_file, 
                                                                                           seg_matches, 
                                                                                           truth_matches, 
                                                                                           max_gap)
            #add F1 components to cumulative sum
            all_true_positives += true_positives
            all_false_positives += false_positives
            all_false_negatives += false_negatives
            
        #calculate F1 score, precision and recall
        F1 = all_true_positives / (all_true_positives + 0.5 * (all_false_positives + all_false_negatives))
        precision = all_true_positives / (all_true_positives + all_false_positives)
        recall = all_true_positives / (all_true_positives + all_false_negatives)
        
        #format metrics info as a data frame
        segmentation_metrics = pd.DataFrame({"F1": [F1], 
                                   "precision" : [precision],  
                                   "recall" : [recall]})
        #add segmentation_metrics dataframe as an attribute of seg_data
        seg_data.seg_metrics = segmentation_metrics
        
        return seg_data
        
        
            
        
    def __get_best_matches(first_onsets, second_onsets, max_gap):
        """
        Finds the best unique matches between timestamps in two sets. These 
        timestamps can reflect syllable onsets or offsets generates differently.

        Parameters
        ----------
        first_onsets : pandas Series
            Contains the timestamps in seconds of syllable onsets calculated
            with a particular method.
        second_onsets : pandas Series
            Contains the timestamps in seconds of syllable onsets calculated 
            with a different method.
        max_gap : float > 0
            The maximum allowable gap in seconds between onsets in the
            `first_onsets` and `second_onsets` series where they will still be
            considered a match.

        Returns
        -------
        best_matches : numpy ndarray, 1D
            For every syllable onset in `first_onsets` this 1 dimensional 
            numpy array contains the index of the best unique match in 
            `second_onsets`. If there is no unique match within the allowable
            `max_gap`, the value at that index is `NaN`.

        """
        #Create a grid versions of first_onsets and second_onsets and use them
            #to calcualte the time delta between every possible pair of onsets
            #from first_onsets and second_onsets.
        first_grid, second_grid = np.meshgrid(first_onsets, second_onsets)
        delta_t_grid = abs(first_grid - second_grid)
        
        #set max gap threshold
            #wherever the time delta between onsets is larger than max_gap, 
            #set the value of that time delta to np.inf
        delta_t_grid = np.where(delta_t_grid > max_gap, np.inf, delta_t_grid)
        
        #if there are no syllables in either one of the onsets, return an 
            #empty array as there are no matches possible. 
        if delta_t_grid.shape[0] == 0:
            best_matches = np.array([])
            return best_matches 
        
        #find best matches (smallest delta t to a syllable in second_onsets for 
            #every syllable onset in first_onsets)
        best_matches = np.argmin(delta_t_grid, axis = 0).astype(float)
        
        #remove matches were delta t is > max_gap by setting the value of 
            #best_matches at that index to np.NaN
        for i, match_index in enumerate(best_matches):
            if np.isinf(delta_t_grid[int(match_index), i]):
              best_matches[i] = np.NaN 
        
        best_matches_previous = best_matches.copy()   
        
        #Deal with duplicate values by setting the second best matches to their second best pairs
        best_matches, delta_t_grid = Metrics.__correct_duplicates(best_matches, delta_t_grid)
        
        #check if there were changes made by checking for duplicates. If so, repeat duplicate check. 
        ##should this not be a while loop? as in you would want to keep checking if new duplicates arose as a result of checking for duplicates until there are none?
        if not np.allclose(best_matches, best_matches_previous):
            best_matches, delta_t_grid = Metrics.__correct_duplicates(best_matches, delta_t_grid)
        
        #make sure duplicate corrections didn't result in out of order matches. 
            #if they do, set the later match to np.NaN
        for i, curr_match in enumerate(best_matches):
             if i+1 < len(best_matches):
               if curr_match > best_matches[i+1]:
                 best_matches[i+1] = np.nan
        
        return best_matches
    
    
    def __correct_duplicates(best_matches, delta_t_grid):   
        """
        Removes duplicate matches from a set of best timestamp matches by 
        setting all but the single best match to an index to the second best match

        Parameters
        ----------
        best_matches : numpy ndarray, 1D
            Contains the index of the best match in one set of timestamps to 
            each timestamp in another set of timestamps. Generated by 
            `avn.segmentation.Metrics.__get_best_matches()`.
        delta_t_grid : numpy ndarray, 2D
            Array containing the absolute value of the time difference between 
            all possible pairs of timestamps in two sets being compared. 

        Returns
        -------
        best_matches : numpy ndarray, 1D
            Same as input `best_matches`, but with duplicate matches corrected.
        delta_t_grid : numpy nd array, 2D
            Same as input `delta_t_grid` but with values at duplicated positions
            set to np.inf so that second best match can be identified. 

        """
       #loop through all the best matches to check if any two indices match 
           #to the same timestamp. 
        for i, match_index in enumerate(best_matches):
           if np.isnan(match_index):
             continue
           #check if match index is duplicated
           if len(np.argwhere(best_matches == match_index)) > 1: 
             #create list of indices of duplicates
             duplicates = np.nonzero(best_matches == match_index)[0]
             #find which duplicate has the smallest delta t
             delta_ts = []
             for n in duplicates:
               delta_ts.append(delta_t_grid[int(best_matches[n]), n])
       
             #get all but the best matches of the duplicates
             bad_matches = np.delete(duplicates, np.argmin(delta_ts))
       
             #find second best matches for all but the best duplicate matches
             for bad_match in bad_matches:
               delta_t_grid[int(best_matches[bad_match]), bad_match] = np.inf
               best_matches[bad_match] = np.argmin(delta_t_grid[:, bad_match])
               if np.isinf(np.min(delta_t_grid[:, bad_match])):
                 best_matches[bad_match] = np.NaN
        return (best_matches, delta_t_grid)
        
    
    def __calc_F1_components_per_file(seg_current_file, 
                                    truth_current_file, 
                                    seg_matches, 
                                    truth_matches, max_gap):
        """
        Calculates the number of true positive, false positive and false 
        negative syllable onsets in `seg_current_file` within `max_gap` off a
        syllable onset in `truth_current_file`. 
        

        Parameters
        ----------
        seg_current_file : pandas DataFrame
            Dataframe with the columns `onsets` `offsets` and `files`, 
            containing the onset and offset times in seconds of every syllable
            in a single .wav file, generated through automated syllable 
            segmentation
        truth_current_file : pandas DataFrame
            Dataframe with the columns `onsets`, `offsets`, `labels` and `files`
            containing the ground truth onset and offset times in seconds of 
            every syllable in the same .wav file that was segmented in 
            `seg_current_file`. `labels` can be either 's' for syllable or 'n'
            for noise, where segmentation matches to noise are ignored when 
            calculating the F1 components. 
        seg_matches : numpy ndarray, 1D
            For every syllable onset in `seg_current_file`, this array contains
            the index of the best unique onset match in `truth_current_file`. 
            Generated with `avn.segmentation.Metrics.__get_best_matches()`.
        truth_matches : numpy ndarray, 1D
            For every syllable onset in `truth_current_file`, this array 
            contains the index of the best unique onset match in 
            `seg_current_file`. Generated with
            `avn.segmentation.Metrics.get_best_mtches()`.
        max_gap : float > 0
            The maximum allowable gap in seconds between onsets where they will
            still be considered a match

        Returns
        -------
        true_positives: int 
            The total number of true positive syllable onsets. ie the number of
            syllable onsets in `seg_current_file` which uniquely fall within 
            `max_gap` seconds of a true syllable onset in `truth_current_file`. 
        false_positives: int
            The total number of false positive syllable onsets. ie the number 
            of syllable onsets in `seg_current_file` which do not uniquely 
            fall within `max_gap` seconds of a true syllable onset in 
            `truth_current_file`. 
        false_negatives: int 
            the total number of false negative syllable onsets. ie the number 
            of true syllable onsets in `truth_current_file` which do not 
            uniquely fall within `max_gap` seconds of a syllable onset in 
            `seg_current_file`. 
            
        Notes
        ------
        No syllable onsets in `seg_current_file` which occur either before 
        the first true syllable or after the last true syllable in `true_seg_table`
        will be considered when calculating the F1 components. Additionally, 
        if `true_seg_table` contains noise in the middle of the song file 
        (ie a syllable labeled 'n'), no syllable onsets in `seg_current_file`
        which fall after the last stllable onset before noise and before the 
        first syllable onset after noise in `true_seg_table` will be considered. 
        This is meant to reduce the impact of matches or mismatches of cage 
        noise on the ultimate F1 score. 

        """
        truth_current_file_OG = truth_current_file.copy()
        
        #create dataframe with matches and timestamps for ease of filtering
        matches_and_times = pd.DataFrame({'onsets' : seg_current_file['onsets'], 
                                          'matches' : seg_matches})
        
        #check whether file contains any syllables. If not, skip it.
        if (truth_current_file['labels'] == 's').sum() == 0:
            true_positives = 0
            false_negatives = 0
            false_positives = 0
            return(true_positives, false_positives, false_negatives)
            
        #remove matches before first true syllable, as these likely reflect noise
        first_syllable_time = truth_current_file['onsets'].iloc[np.min(np.nonzero([truth_current_file['labels'] == 's'])[1])]
        matches_and_times = matches_and_times[matches_and_times['onsets'] >= first_syllable_time]
        
        #remove matches after last true syllable, as these likely reflect noise
        last_syllable_time = truth_current_file['offsets'].iloc[np.max(np.nonzero([truth_current_file['labels'] == 's'])[1])]
        matches_and_times = matches_and_times[matches_and_times['onsets'] < last_syllable_time]
        
        #find noise in the middle of the file
        truth_current_file = truth_current_file[(truth_current_file['onsets'] >= first_syllable_time) & 
                                                (truth_current_file['onsets'] < last_syllable_time)]
        middle_noises = np.nonzero([truth_current_file['labels'] == 'n'])[1]
        
        #remove matches that are between the last syllable before noise and the first syllable after noise
        for noise in middle_noises:
            all_sylls_before = truth_current_file[:noise]
            #special case where there is only one syllable before the first noise
            if len(all_sylls_before) == 1:
                last_time_before = all_sylls_before['offsets'].values[0]
            
            #all other cases
            else:
                last_syll_before = np.max(np.nonzero([all_sylls_before['labels'] == 's'])[1])
                last_time_before = all_sylls_before['offsets'].iloc[last_syll_before]
                
            all_sylls_after = truth_current_file[noise:]
            #special case where there is only one syllable after the last noise
            if len(all_sylls_after) == 1:
                first_time_after = all_sylls_after['onsets'].values[0]
            #all other cases
            else:
                first_syll_after = np.min(np.nonzero([all_sylls_after['labels'] == 's'])[1])
                first_time_after = all_sylls_after['onsets'].iloc[first_syll_after]
                
            matches_and_times = matches_and_times[(matches_and_times['onsets'] < last_time_before + max_gap) |
                    (matches_and_times['onsets'] > first_time_after - max_gap)]
            
        true_positives = np.isfinite(matches_and_times['matches']).sum()
        false_positives = np.isnan(matches_and_times['matches']).sum()
        
        #find all true syllables
        syllable_indices = np.nonzero([truth_current_file_OG['labels'] == 's'])[1]
        #select only matches to true syllables (not noise)
        true_matches = truth_matches[syllable_indices]
        #calculate total number of false negatives (true onsets without a match)
        false_negatives = np.isnan(true_matches).sum()
        
        return(true_positives, false_positives, false_negatives)
        
        
class Plot():
    """
    Contains functions for plotting automatically generated syllable segmentations, 
    segmentation criteria and/or ground truth syllable segmentations over 
    spectrograms. 
    """
    def __init__(self):
        pass
    
    def plot_segmentations(seg_data, seg_label, plot_ground_truth = False, 
                           true_label = 'Ground Truth', 
                           file_idx = 0, figsize = (20, 5), 
                           seg_attribute = 'onsets', plot_title = ""):
        """
        Plots the spectrogram of a given wave file with automatically generated 
        syllable onsets or offsets plotted over top. Ground truth segmentations
        can be plotted in addition to automatically generate segmentations 
        when available. 

        Parameters
        ----------
        seg_data : avn.segmentation.SegData object
            SegData object containing automatic syllable segmentations in it's 
            `.seg_table` attribute, and optionally also ground truth 
            segmentations in its `.true_seg_table` attribute. 
        seg_label : str
            Label for automatic syllable segmentations to be displayed in legend.
        plot_ground_truth : bool, optional
            If True, both the automatically generated and ground truth syllable
            segmentations will be plotted. If False, only automatically 
            generated syllable segmentations will be plotted. The default is 
            False.
        true_label : str, optional
            Label for ground truth syllable segmentations to be displayed in 
            legend. Only used if `plot_ground_truth` == True. 
            The default is 'Ground Truth'.
        file_idx : int >=0, <total number of files segmented in `seg_data`, optional
            The index of the single file within `seg_data` to be plotted. 
            The default is 0.
        figsize : tuple of floats, optional
            Specifies the dimensions of the output plot. The default is (20, 5).
        seg_attribute : {'onsets', 'offsets'}, optional
            Specifies whether syllable onset times or offset times should be 
            displayed. The default is 'onsets'.
        plot_title : str, optional
            Title of the generated plot. The default is "".

        Returns
        -------
        None.

        """
        #retrieve subset of seg_table which corresponds to individual file. 
        file_name = np.unique(seg_data.seg_table['files'])[file_idx]
        
        seg_table_current_file = seg_data.seg_table[seg_data.seg_table['files'] == file_name]
        
        #load individual song wave file
        song = dataloading.SongFile(seg_data.song_folder_path + "/" + file_name)
        
        #make spectrogram
        spectrogram = plotting.make_spectrogram(song)
        
        #plot spectrogram 
        plotting.plot_spectrogram(spectrogram, song.sample_rate, figsize = figsize)
        
        #if plot_ground_truth == True, retrieve subset of true_seg_table 
            #which corresponds to individual file and plot them
        if plot_ground_truth:
            
            true_table_current_file = seg_data.true_seg_table[seg_data.true_seg_table['files'] == file_name]
            #select only true syllables, to prevent true noise onsets from being plotted
            true_table_current_file = true_table_current_file[true_table_current_file['labels'] == 's']
            
            #plot seg onsets 
            plt.eventplot(seg_table_current_file[seg_attribute],
                      lineoffsets = 15000, linelengths = 10000, 
                      color = 'white', label = seg_label)
        
            #plot true onsets
            plt.eventplot(true_table_current_file[seg_attribute], 
                      lineoffsets = 5000, linelength = 10000, 
                      color = 'red', label = true_label)
        else:
            #plot seg onsets 
            plt.eventplot(seg_table_current_file[seg_attribute],
                      lineoffsets = 10000, linelengths = 20000, 
                      color = 'white', label = seg_label)
        
        plt.legend()
        plt.title(plot_title)
        plt.show()
        
    
    def plot_seg_criteria(seg_data, segmenter, label, file_idx = 0, figsize = (20, 5)):
        """
        Plots a given segmentation criteria (ie MFCC, RMSE, RMSE Derivative) 
        over the spectrogram of a given song file.

        Parameters
        ----------
        seg_data : avn.segmentation.SegData object
            SegData object with valid `.seg_table` and `.song_folder_path` 
            attributes.
        segmenter : avn.segmentation.Segmenter daughter class object
            The type of the segmenter (ie MFCC, RMSE, RMSEDerivative) determines
            which segmentation criteria will be plotted
        label : str
            Label of segmentation criteria to be displayed in plot legend.
        file_idx : int >=0, <total number of files segmented in `seg_data`, optional
            The index of the single file within `seg_data` to be plotted. 
            The default is 0.
        figsize : tuple of floats, optional
            Specifies the dimensions of the output plot. The default is (20, 5).

        Returns
        -------
        None.

        """
        #load a single file
        file_name = np.unique(seg_data.seg_table['files'])[file_idx]
        song = dataloading.SongFile(seg_data.song_folder_path + file_name)
        
        #make spectrogram
        spectrogram = plotting.make_spectrogram(song)
        
        #calculate segmentation criteria
        seg_criteria = segmenter.get_seg_criteria(song)
        
        #create figure and axes
        fig, ax = plt.subplots(figsize = figsize)
        #plot spectrogram
        plotting.plot_spectrogram(spectrogram, song.sample_rate, ax)
        
        #create second axes for plotting seg_criteria
        ax2 = ax.twinx()
        
        #plot seg_criteria
        x_axis = librosa.frames_to_time(np.arange(len(seg_criteria)), sr = song.sample_rate, 
                                        hop_length = 512, n_fft = seg_data.n_fft)
        ax2.plot(x_axis, seg_criteria, color = 'white', label = label)
        ax2.set_ylabel("Segmentation Criteria")
        ax2.legend();

        
        
        

class Utils:
    """
    Contains syllable segmentation utilities
    """

    def __init__():
        pass
    
    def threshold_optimization(segmenter, Bird_ID, song_folder_path, truth_table_path, 
                               threshold_range, threshold_step, lower_threshold):
        """
        Tests a range of upper threshold values for threshold-based segmentation
        to find the threshold which results in the best F1 score for one bird. 

        Parameters
        ----------
        segmenter : avn.segmentation.Segmenter daughter class object
            Determines the segmentation criteria that will be used for threshold
            segmentation (ie RMSE, MFCC, RMSEDerivative)
        Bird_ID : str
            String containing a unique identifier for subject bird.
        song_folder_path : str
            Path to the folder containing all .wav files to be segmented for 
            the subject bird.
        truth_table_path : str
            Path to the .csv file generated with evsonganaly which contains the 
            ground truth syllable segmentations. 
        threshold_range : tuple of floats
            Specifies the range thresholds to test. 
        threshold_step : float
            The size of the step between consequtive thresholds to be tested.
        lower_threshold : float
            Lower segmentation thrshold for determining syllable offsets (fixed).

        Returns
        -------
        optimal_threshold : float
            Value of the threshold which results in the best F1 score.
        peak_F1 : float
            F1 score of segmentation using optimal threshold.
        segmentation_scores : pandas DataFrame
            DataFrame with columns `F1`, `precision`, `recall`, `upper_threshold`, 
            and `lower_threshold` which contain the metrics of segmentation at 
            every segmentation threshold tested. This can be useful for plotting 
            the relationship between the threshold value and metrics.
            
        Notes
        -----
        Selecting a very wide `threshold_range` and small `threshold_step` can 
        make this quite slow to run. It is recommended to test a wide 
        `threshold_range` with large `threshold_step` and plot F1 across 
        threshold values using `segmentation_scores` initially, then once 
        you have a better idea of the ballpark of the peak threshold try using 
        a smaller `threshold_range` with a finer `threshold_step`.

        """
        
        #create array of thresholds to test
        thresholds = np.arange(threshold_range[0], threshold_range[1], threshold_step)
        segmentation_scores = pd.DataFrame()
        
        #loop over every threshold and calculate the metrics resulting from 
            #that segmentation.
        for threshold in thresholds:
            #Generate automatic segmentations with given threshold
            seg_data = segmenter.make_segmentation_table(Bird_ID, song_folder_path, 
                                              upper_threshold = threshold, lower_threshold = lower_threshold)
            #load ground truth segmentation from memory
            seg_data = dataloading.Utils.add_ev_song_truth_table(seg_data, truth_table_path)
            
            #calculate segmentation metrics
            seg_data = Metrics.calc_F1(seg_data)
            
            #package output into segmentation_score DataFrame
            segmentation_score = seg_data.seg_metrics
            segmentation_score['upper_threshold'] = threshold
            segmentation_score['lower_threshold'] = lower_threshold
            
            #append segmentation data from this threshold to growing DataFrame
            segmentation_scores = segmentation_scores.append(segmentation_score)
            
        #Find threshold which results in peak F1 score, and value of peak F1 score.    
        optimal_threshold = segmentation_scores['upper_threshold'].iloc[segmentation_scores['F1'].argmax()]
        peak_F1 = segmentation_scores['F1'].max()
        
        return optimal_threshold, peak_F1, segmentation_scores
    
    def calc_F1_many_birds(segmenter, Bird_IDs, 
                           folder_path, upper_threshold, lower_threshold, 
                           truth_table_suffix = "_syll_table.csv"):
        """
        Calculate the segmentation metrics for all birds in `Bird_IDs` with 
        a given method and threshold. 

        Parameters
        ----------
        segmenter : avn.segmentation.Segmenter daughter class object.
            Determines the segmentation method. 
        Bird_IDs : List of strings
            List of unique bird identifiers. These should correspond to the 
            names of subfolders within the `folder_path` directory.
        folder_path : str
            Path to a local directory containing subdirectories named with the 
            Bird IDs in `Bird_IDs`, which in turn contain the .wav files to be
            segmented.
        upper_threshold : float > lower_threshold
            Value of the upper segmentation criteria threshold for detecting
            syllable onsets. 
        lower_threshold : float < upper_threshold
            Value of the lower segmentation criteria threshold used for detecting 
            syllable offsets. 
        truth_table_suffix : str, optional
            This function requires that the truth table data be located in a 
            .csv file within folder_path\Bird_ID\ and begin with the Bird_ID
            followed by some descriptor. This is used to specify that final 
            part of the file name. The default is "_syll_table.csv".

        Returns
        -------
        segmentation_scores : pandas DataFrame
            DataFrame with columns `F1`, `precision`, `recall`, `upper_threshold`,
            `lower_threshold`, and `Bird_ID`, which contains the segmentation
            metrics for each bird. 
        segmentations_df : pandas DataFrame
            DataFrame with columns `onsets, `offsets`, `files`, and `Bird_ID` 
            which contains the onset and offset timestamps of every segmented 
            syllable in each file for each bird. 

        """
        #initialize empty dataframes to append to
        segmentation_scores = pd.DataFrame()
        
        segmentations_df = pd.DataFrame()
        
        #Loop through each bird
        for Bird_ID in Bird_IDs:
            #construct full path to folder containing song files. 
            song_folder = folder_path + Bird_ID + "/"
            
            #make segmentations
            seg_data = segmenter.make_segmentation_table(Bird_ID, song_folder, 
                                              upper_threshold = upper_threshold, 
                                              lower_threshold = lower_threshold)
            #load ground truth segmentations from memory
            seg_data = dataloading.Utils.add_ev_song_truth_table(seg_data, song_folder + Bird_ID + truth_table_suffix)
            
            #calculate segmentation accuracy metrics. 
            seg_data = Metrics.calc_F1(seg_data)
            
            #package metrics information into a dataframe
            segmentation_score = seg_data.seg_metrics
            segmentation_score['upper_threshold'] = upper_threshold
            segmentation_score['lower_threshold'] = lower_threshold
            segmentation_score['Bird_ID'] = Bird_ID
            segmentation_scores = segmentation_scores.append(segmentation_score)
            
            #package segmentation data into a dataframe
            segmentation_df = seg_data.seg_table
            segmentation_df["Bird_ID"] = Bird_ID
            segmentations_df = segmentations_df.append(segmentation_df)
            
        return segmentation_scores, segmentations_df
            
        
    def threshold_optimization_many_birds(segmenter, Bird_IDs, folder_path,
                                          threshold_range, threshold_step, lower_threshold, 
                                          truth_table_suffix = "_syll_table.csv"):
        """
        Finds the optimal segmentation threshold across multiple birds at once. 

        Parameters
        ----------
        segmenter : avn.segmentation.Segmenter child class type
            This determines the segmentation criteria used for threshold 
            segmentation. 
        Bird_IDs : List of strings
            List of unique bird identifiers. These should correspond to the 
            names of subfolders within the `folder_path` directory.
        folder_path : str
            Path to a local directory containing subdirectories named with the 
            Bird IDs in `Bird_IDs`, which in turn contain the .wav files to be
            segmented.
        threshold_range : tuple of floats
            Specifies the range thresholds to test. 
        threshold_step : float
            The size of the step between consequtive thresholds to be tested.
        lower_threshold : float
            Lower segmentation thrshold for determining syllable offsets (fixed).
        truth_table_suffix : str, optional
            This function requires that the truth table data be located in a 
            .csv file within folder_path\Bird_ID\ and begin with the Bird_ID
            followed by some descriptor. This is used to specify that final 
            part of the file name. The default is "_syll_table.csv".

        Returns
        -------
        optimal_threshold : float
            Value of the threshold which results in the best mean F1 score 
            across all birds.
        peak_mean_F1 : float
            Mean F1 score of segmentation using optimal threshold across all birds.
        segmentation_scores : pandas DataFrame
            DataFrame with columns `F1`, `precision`, `recall`, `upper_threshold`, 
            `lower_threshold`, and `Bird_ID` which contain the metrics of 
            segmentation at every segmentation threshold tested for every bird. 
            This can be useful for plotting the relationship between the 
            threshold value and metrics.
        
        Notes
        -----
        Selecting a very wide `threshold_range` and small `threshold_step` can 
        make this quite slow to run. It is recommended to test a wide 
        `threshold_range` with large `threshold_step` and plot F1 across 
        threshold values using `segmentation_scores` initially, then once 
        you have a better idea of the ballpark of the peak threshold try using 
        a smaller `threshold_range` with a finer `threshold_step`.
        
        """
        #initialize empty DataFrame to append to. 
        segmentation_scores = pd.DataFrame()
        
        #loop through each bird
        for Bird_ID in Bird_IDs:
            #create full path to song folder
            song_folder = folder_path + Bird_ID + "/"
            #create full path to ground truth segmentation data
            truth_table_path = song_folder + Bird_ID + truth_table_suffix
            
            #segmented bird with every threshold in the range and calculate 
                #metrics for each of those segmentations
            optimal_thresh, peak_F1, threshold_table = Utils.threshold_optimization(segmenter, Bird_ID, 
                                                                                    song_folder, truth_table_path,
                                                                                    threshold_range, 
                                                                                    threshold_step, 
                                                                                   lower_threshold)
            #package segmentation metrics into DataFrame
            threshold_table['Bird_ID'] = Bird_ID
            segmentation_scores = segmentation_scores.append(threshold_table)
        #calculate the mean F1 score for every upper_threshold value across birds.     
        mean_F1s = segmentation_scores.groupby("upper_threshold").mean()
        #find best mean F1 score. 
        peak_mean_F1 = mean_F1s.F1.max()
        
        #Determine the threshold which resulted in the peak mean F1 score.
        mean_F1s.reset_index(inplace = True)
        optimal_threshold = mean_F1s.upper_threshold.iloc[mean_F1s.F1.argmax()]
        
        return optimal_threshold, peak_mean_F1, segmentation_scores
    
    def plot_segmentations_many_birds(segmenter, Bird_IDs, folder_path, seg_label,
                                      upper_threshold, lower_threshold,
                                      plot_ground_truth = False,
                                      files_per_bird = 3, random_seed = 2021, 
                                      true_label = "Ground Truth", figsize = (20, 5), 
                                      seg_attribute = 'onsets', truth_table_suffix = "_syll_table.csv"):
        """
        Plots `files_per_bird` number of random example song spectrograms 
        with automatically generated segmentations (and optionally ground truth
        segmentations) overlaid for each bird in Bird_IDs. 


        Parameters
        ----------
        segmenter : avn.segmentation.Segmenter child class type
            This determines the segmentation criteria used for threshold 
            segmentation. 
        Bird_IDs : List of strings
            List of unique bird identifiers. These should correspond to the 
            names of subfolders within the `folder_path` directory.
        folder_path : str
            Path to a local directory containing subdirectories named with the 
            Bird IDs in `Bird_IDs`, which in turn contain the .wav files to be
            segmented.
        seg_label : str
            Label for automatic syllable segmentations to be displayed in legend.
        upper_threshold : float > lower_threshold
            Value of the upper segmentation criteria threshold for detecting
            syllable onsets. 
        lower_threshold : float < upper_threshold
            Value of the lower segmentation criteria threshold used for detecting 
            syllable offsets. 
        plot_ground_truth : bool, optional
            If True, both the automatically generated and ground truth syllable
            segmentations will be plotted. If False, only automatically 
            generated syllable segmentations will be plotted. The default is 
            False.
        files_per_bird : int>=1, optional
            The number of files to plot from each bird. The default is 3.
        random_seed : optional
            Any object that can be converted to an integer. This ensures that 
            the same set of randomly selected files will be plotted every time 
            this function is run with the same random_seed value. 
            The default is 2021.
        true_label : str, optional
            Label for ground truth syllable segmentations to be displayed in 
            legend. Only used if `plot_ground_truth` == True. 
            The default is 'Ground Truth'.
        figsize : tuple of floats, optional
            Specifies the dimensions of the output plot. The default is (20, 5).
        seg_attribute : {'onsets', 'offsets'}, optional
            Specifies whether syllable onset times or offset times should be 
            displayed. The default is 'onsets'.
        truth_table_suffix : str, optional
            This function requires that the truth table data be located in a 
            .csv file within folder_path\Bird_ID\ and begin with the Bird_ID
            followed by some descriptor. This is used to specify that final 
            part of the file name. The default is "_syll_table.csv".


        Returns
        -------
        None.

        """
        for Bird_ID in Bird_IDs:
            song_folder = folder_path + Bird_ID + "/"
            
            truth_table_path = song_folder + Bird_ID + truth_table_suffix
            
            seg_data = segmenter.make_segmentation_table(Bird_ID, song_folder, 
                                                         upper_threshold, 
                                                         lower_threshold)
            
            if plot_ground_truth:
                seg_data = dataloading.Utils.add_ev_song_truth_table(seg_data, truth_table_path)
            
            all_files = np.unique(seg_data.seg_table.files)
            
            random.seed(random_seed)
            indices_to_plot = random.sample(range(0, len(all_files)), files_per_bird)
            
            for rand_index in indices_to_plot:
                
                Plot.plot_segmentations(seg_data, seg_label = seg_label, 
                                        plot_ground_truth = plot_ground_truth,
                                    true_label = true_label, figsize = figsize, 
                                    seg_attribute = seg_attribute, file_idx = rand_index, 
                                    plot_title = all_files[rand_index])
    
    def make_segmentation_table_many_birds(segmenter, Bird_IDs, folder_path, 
                                           upper_threshold, lower_threshold, 
                                           save_to_csv = False, out_file_dir = None):
        """
        Generates syllable segmentations for many files across many birds.

        Parameters
        ----------
        segmenter : avn.segmentation.Segmenter child class type
            This determines the segmentation criteria used for threshold 
            segmentation. 
        Bird_IDs : List of strings
            List of unique bird identifiers. These should correspond to the 
            names of subfolders within the `folder_path` directory.
        folder_path : str
            Path to a local directory containing subdirectories named with the 
            Bird IDs in `Bird_IDs`, which in turn contain the .wav files to be
            segmented.
        upper_threshold : float > lower_threshold
            Value of the upper segmentation criteria threshold for detecting
            syllable onsets. 
        lower_threshold : float < upper_threshold
            Value of the lower segmentation criteria threshold used for detecting 
            syllable offsets.
        save_to_csv : bool, optional
            If True, segmentation table and metrics table .csv files will be saved 
            for each bird in the `out_file_dir` directory. These will have 
            the Bird_ID in the file name. The default is False.
        out_file_dir : str, optional
            Path to a local directory in which to save segmentation table 
            and metrics tables for each bird. This will only be used if 
            `save_to_csv` == True. The default is None.

        Returns
        -------
        segmentations_df : pandas DataFrame
            DataFrame with columns `onsets`, `offsets`, `files` and `Bird_ID` 
            which contains syllable onset and offset timestamps in seconds 
            for every automatically segmented syllable in every file for every 
            bird.

        """
        
        #initialize empty dataframe to append to
        segmentations_df = pd.DataFrame()
        
        #loop over each bird
        for Bird_ID in Bird_IDs:
            
            #determine complete path to song folder
            song_folder = folder_path + Bird_ID + "/"
            
            #generate segmentations
            seg_data = segmenter.make_segmentation_table(Bird_ID, song_folder, 
                                              upper_threshold = upper_threshold, 
                                              lower_threshold = lower_threshold)
            
            #package segmentations into DataFrame
            segmentation_df = seg_data.seg_table
            segmentation_df["Bird_ID"] = Bird_ID
            segmentations_df = segmentations_df.append(segmentation_df)
            
            #if save_to_scv = True, check that the `out_file_dir` exists, and 
                #if it does same the seg data to a csv. 
            if save_to_csv:
                if not os.path.isdir(out_file_dir):
                    raise ValueError("The supplied out file directory does not exist. out_file_dir: " + out_file_dir)
                    
                seg_data.save_as_csv(out_file_dir)
                
        return segmentations_df
                