# coding=utf-8

#    National Oceanic and Atmospheric Administration
#    Alaskan Fisheries Science Center
#    Resource Assessment and Conservation Engineering
#    Midwater Assessment and Conservation Engineering

# THIS SOFTWARE AND ITS DOCUMENTATION ARE CONSIDERED TO BE IN THE PUBLIC DOMAIN
# AND THUS ARE AVAILABLE FOR UNRESTRICTED PUBLIC USE. THEY ARE FURNISHED "AS
# IS." THE AUTHORS, THE UNITED STATES GOVERNMENT, ITS INSTRUMENTALITIES,
# OFFICERS, EMPLOYEES, AND AGENTS MAKE NO WARRANTY, EXPRESS OR IMPLIED, AS TO
# THE USEFULNESS OF THE SOFTWARE AND DOCUMENTATION FOR ANY PURPOSE.  THEY
# ASSUME NO RESPONSIBILITY (1) FOR THE USE OF THE SOFTWARE AND DOCUMENTATION;
# OR (2) TO PROVIDE TECHNICAL SUPPORT TO USERS.

'''
.. module:: echolab2.instruments.EK60

    :synopsis:  A high-level interface for reading SIMRAD ".raw" formatted
                files written by the Simrad EK60, ES60, ES70, ME70 sonar
                systems


| Developed by:  Rick Towler   <rick.towler@noaa.gov>
| National Oceanic and Atmospheric Administration (NOAA)
| Alaska Fisheries Science Center (AFSC)
| Midwater Assesment and Conservation Engineering Group (MACE)
|
| Authors:
|       Zac Berkowitz <zac.berkowitz@gmail.com>
|       Rick Towler   <rick.towler@noaa.gov>
| Maintained by:
|       Rick Towler   <rick.towler@noaa.gov>

$Id$
'''

import os
import numpy as np
from scipy.interpolate import interp1d
from .util.simrad_calibration import calibration
from .util.simrad_raw_file import RawSimradFile, SimradEOF
from .util.nmea_data import nmea_data
from .util.motion_data import motion_data
from .util.annotation_data import annotation_data
from .util import simrad_parsers
from .util import date_conversion
from ..ping_data import ping_data
from ..processing.processed_data import processed_data
from ..processing import line


class EK60(object):
    """This class is the 'file reader' class for Simrad EK60 instrument files.

    The EK60 class can read in one or more ER60 files and generates a raw_data
    object for each unique channel ID and data type in the files.

    Attributes:
        start_time: Start_time is a datetime object that defines the start
            time of the data within the EK60 class
        end_time: End_time is a datetime object that defines the end time of
            the data within the EK60 class
        start_ping: Start_ping is an integer that defines the first ping of the
            data within the EK60 class.
        end_ping: End_ping is an integer that defines the last ping of
            the data within the EK60 class.
        n_pings: Integer value representing the total number of pings.
        frequencies: List of frequencies read from raw data files.
        channel_ids: List of stings identifying the unique channel IDs read
            from the instrument files.
        n_channels: Integer value representing the total number of
            channels in the class instance.
        raw_data: A dictionary that stores the raw_data objects, one for each
            unique channel in the files. The dictionary keys are the channel
            numbers.
        nmea_data: reference to a NmeaData class instance that will contain
            the NMEA data from the data files.
        read_incremental; Boolean value controlling whether files are read
            incrementally or all at once. The default value is False.
        store_angles: Boolean control variable to set whether or not to store
            angle data generated by EK60 GPTs.
        store_power: Boolean control variable to set whether or not to store
            the power data generated by EK60 GPTs.
        read_max_sample_count: Integer value to specify the max sample count
            to read. This property can be used to limit the number of samples
            read (and memory used) when your data of interest is less than
            the total number of samples contained in the EK60 files.
        read_start_time: Datetime object containing timestamp of the first
            ping to read if not reading all pings based on time.
        read_end_time: Datetime object containing timestamp of last ping to
            read if not reading all ping based on time.
        read_start_ping: Integer ping number of first ping to read if not
            reading all pings based on ping number.
        read_end_ping: Integer ping number of last ping to read if not
            reading all pings based on ping number.
        read_start_sample: Integer sample number of first sample to read if not
            reading all samples based on sample number.
        read_end_sample: Integer sample number of last sample to read if not
            reading all samples based on sample number.
        read_frequencies: List of floats (i.e. 18000.0) specifying the
            frequencies to read. An empty list will result in all frequencies
            being read.
        read_channel_ids = List of strings specifying the channel_ids
            (i,e, 'GPT  38 kHz 009072033fa2 1-1 ES38B') of the channels to
            read. An empty list will result in all channels being read.
    """

    def __init__(self):
        """Initializes EK60 class object.

        Creates and sets several internal properties used to store information
        about data and control operation of file reading with EK60 object
        instance. Code is heavily commented to facilitate use.
        """

        # Define EK60 properties.

        # These properties control what data are stored when reading
        # ER60 raw files. These properties can be manipulated directly or by
        # specifying them when calling the read and append methods.

        # Set store_angles to true to store angle data if any of the data files
        # contain data recorded from EK60 GPTs or WBT data saved in the reduced
        # power/angle format.
        self.store_angles = True

        # Set store_power to true to store power data if any of the data files
        # contain data recorded from EK60 GPTs or WBT data saved in the reduced
        # power/angle format.
        self.store_power = True

        # Specify the maximum sample count to read.  This property can be used
        # to limit the number of samples read (and memory used) when your data
        # of interest is large.
        self.read_max_sample_count = None

        # These params store any limits set on storing of data contained in
        # the raw files read. These parameters can be set directly, or through
        # many of the methods where you can specify start/end time or ping
        # and sample number.
        self.read_start_time = None
        self.read_end_time = None
        self.read_start_ping = None
        self.read_end_ping = None
        self.read_start_sample = None
        self.read_end_sample = None

        # read_frequencies can be set to a list of floats specifying the
        # frequencies to read. An empty list will result in all frequencies
        # being read.
        self.read_frequencies = []

        # read_channel_ids can be set to a list of strings specifying the
        # channel_ids of the channels to read. An empty list will result in all
        # channels being read.
        self.read_channel_ids = []

        #  initialize the data arrays
        self._init()


    def _init(self):
        '''_init is an internal method that initializes the data fields of
        the EK60 class.
        '''

        # The start_time and end_time will define the time span of the
        # data within the EK60 class.
        self.start_time = None
        self.end_time = None

        # The start_ping and end_ping will define the ping span of the data
        # within the EK60 class.
        self.start_ping = None
        self.end_ping = None

        # n_pings stores the total number of pings read.
        self.n_pings = 0

        # n_files stores the total number of raw files read.
        self.n_files = 0

        # A list of stings identifying the channel IDs that have been read.
        # Channel IDs are used as keys into the raw_data attribute.
        self.channel_ids = []

        # frequency_map maps frequency in Hz to channel ID.
        self.frequency_map = {}

        # channel_map maps channel number to channel ID. Channel numbers
        # are based on the order each channel is added to the ER60 object.
        self.channel_map = {}

        # n_channels stores the total number of channels in the object.
        self.n_channels = 0

        # A dictionary to store the raw_data objects. The dictionary is keyed
        # by channel ID and each value is a list of raw_data objects associated
        # with that channel.
        self.raw_data = {}

        #  nmea_data stores the util.nmea_data object which will contain
        #  the NMEA data read from the raw file. This object has methods
        #  to extract, parse, and interpolate the NMEA data.
        self.nmea_data = nmea_data()

        #  motion_data is the util.motion_data object that will
        #  contain data from motion fields in the ER60 datagrams.
        #  This object has methods to extract and interpolate the motion data.
        self.motion_data = motion_data()

        # annotations stores the contents of the TAG0 aka "annotation"
        # datagrams.
        self.annotations = annotation_data()

        # data_array_dims contains the dimensions of the sample and angle or
        # complex data arrays specified as [n_pings, n_samples].  Values of
        # -1 specifythat the arrays will resize appropriately to contain all
        # pings and all samples read from the data source.  Setting values > 0
        # will create arrays that are fixed to that size.  Any samples beyond
        # the number specified as n_samples will be dropped.  When a ping is
        # added and the data arrays are full, the sample data will be rolled
        # such that the oldest data is dropped and the new ping is added.
        self._data_array_dims = [-1, -1]

        #  initialize the ping time - used to group "pings"
        self._this_ping_time = np.datetime64('1000-02')

        #  _last_progress stores the last reported progess through the current
        #  file in percent as integer.
        self._last_progress = -1

        # _config stores a reference to the current file's configuration datagram.
        self._config = None

        # file_channel_map maps channel number to channel ID for the current file.
        self._file_channel_map = {}



    def read_out(self, read_nmea=False,  *args, **kwargs):
        """Reads .out formatted bottom detection files. ER60 Mk 1 systems
        and ES60 and ES70 systems output .out files that contain bottom detections
        and a copy of the NMEA data that is also wrtten to the .raw files.

        Because the NMEA data is replicated at it is assumed that the user will
        also be reading .raw data, the read_nmea keyword by default is set to
        False. If you will only be reading .out file data, you will probably
        want to set read_nmea to True.

        This method will read these files and insert the bottom detection data in
        the appropriate raw_data object based on channel ID and data type. You can
        use the raw_data.get_bottom() method to get a pyEcholab2 line object
        representing the bottom detections. If you work with the bottom detection
        data directly, remember that the depths are computed using the sound speed
        at the time of collection. If you are using a different sound speed for
        processing, you will need to adjust the raw bottom depths accordingly.

        Args:
            bot_files (list): A list of .bot/.out files to be read.
            read_nmea (bool): Set this keyword to read NMEA data from .out files.
                .out files contain a copy of NMEA datagrams that are also written
                to the .raw file. By default these data are not read since they
                are duplicates. If you are not reading .raw data files and do
                want access to the NMEA data, set this leyword to True.
        """

        # Update the kwargs with the read_nmea argument.
        kwargs['nmea'] = read_nmea

        # .bot and .out files share the same format as .raw files and are read
        # using the same methods.
        self.append_raw(*args, **kwargs)


    def read_bot(self, *args, **kwargs):
        """Reads .bot formatted bottom detection files created by ER60 Mk II systems.

        This method will read .bot files and insert the bottom detection data in
        the appropriate raw_data object based on channel ID and data type. You can
        use the raw_data.get_bottom() method to get a pyEcholab2 line object
        representing the bottom detections. If you work with the bottom detection
        data directly, remember that the depths are computed using the sound speed
        at the time of collection. If you are using a different sound speed for
        processing, you will need to adjust the raw bottom depths accordingly.

        Args:
            bot_files (list): A list of .bot files to be read.

        """

        # .bot files don't have NMEA data but it can't hurt to set this
        kwargs['nmea'] = False

        # .bot and .out files share the same format as .raw files and are read
        # using the same methods.
        self.append_raw(*args, **kwargs)


    def read_raw(self, *args, **kwargs):
        """Reads one or more Simrad ER60 .raw files into memory. Overwrites
        existing data (if any).

        Args:
            raw_files (list): List containing full paths to data files to be
                read.
            power (bool): Controls whether power data is stored
            angles (bool): Controls whether angle data is stored
            max_sample_count (int): Specify the max sample count to read
                if your data of interest is less than the total number of
                samples contained in the instrument files.
            start_time (str): Specify a start time if you do not want to read
                from the first ping. The format of the time string must
                match the format specified in time_format_string.
            end_time (str): Specify an end time if you do not want to read
                to the last ping. The format of the time string must
                match the format specified in time_format_string.
            start_ping (int): Specify starting ping number if you do not want
                to start reading at first ping.
            end_ping (int): Specify end ping number if you do not want
                to read all pings.
            frequencies (list): List of floats (i.e. 18000.0) if you
                only want to read specific frequencies.
            channel_ids (list): A list of strings that contain the unique
                channel IDs to read. If no list is supplied, all channels are
                read.
            time_format_string (str): String containing the format of the
                start and end time arguments. Format is used to create datetime
                objects from the start and end time strings
            start_sample (int): Specify starting sample number if not
                reading from first sample.
            end_sample (int): Specify ending sample number if not
                reading to last sample.
        """

        #  initialize the data arrays - this discards any previously read data.
        self._init()

        #  now call append_raw, passing on all arguments
        self.append_raw(*args, **kwargs)


    def append_raw(self, raw_files, power=None, angles=None,
                nmea=True, max_sample_count=None, start_time=None,
                end_time=None, start_ping=None, end_ping=None, frequencies=None,
                channel_ids=None, time_format_string='%Y-%m-%d %H:%M:%S',
                incremental=None, start_sample=None, end_sample=None,
                progress_callback=None):

        """Reads one or more Simrad Ex60/ES70/ME70 .raw files and appends the data to any
        existing data. The data are ordered as read.

        Args:
            raw_files (list): List containing full paths to data files to be
                read.
            power (bool): Controls whether power data is stored
            angles (bool): Controls whether angle data is stored
            nmea (bool): Controls whether NMEA data is stored
            max_sample_count (int): Specify the max sample count to read
                if your data of interest is less than the total number of
                samples contained in the instrument files.
            start_time (datetime64): Specify a start time if you do not want
                to start reading from the first ping.
            end_time (datetime64): Specify an end time if you do not want to read
                to the last ping.
            start_ping (int): Specify starting ping number if you do not want
                to start reading at first ping.
            end_ping (int): Specify end ping number if you do not want
                to read all pings.
            frequencies (list): List of floats (i.e. 18000.0) if you
                only want to read specific frequencies.
            channel_ids (list): A list of strings that contain the unique
                channel IDs to read. If no list is supplied, all channels are
                read.
            start_sample (int): Specify starting sample number if not
                reading from first sample.
            end_sample (int): Specify ending sample number if not
                reading to last sample.
            progress_callback (function reference): Pass a reference to a
                function that accepts three arguments:
                    (filename, cumulative_pct, cumulative_bytes)
                This function will be periodicaly called, passing the
                current file name that is being read, the percent read,
                and the bytes read.
        """

        # Update the reader state variables.
        if start_time:
            self.read_start_time = start_time
        if end_time:
            self.read_end_time = end_time
        if start_ping:
            self.read_start_ping = start_ping
        if end_ping:
            self.read_end_ping = end_ping
        if start_sample:
            self.read_start_sample = start_sample
        if end_sample:
            self.read_end_sample = end_sample
        if power:
            self.store_power = power
        if angles:
            self.store_angles = angles
        if max_sample_count:
            self.read_max_sample_count = max_sample_count
        if frequencies:
            if not isinstance(frequencies, list):
                frequencies = [frequencies]
            self.read_frequencies = frequencies
        if channel_ids:
            self.read_channel_ids = channel_ids

        # Ensure that the raw_files argument is a list.
        if isinstance(raw_files, str):
            raw_files = [raw_files]

        # Iterate through the list of .raw files to read.
        for filename in raw_files:
            #  reset attributes that store the most recent configuration datagram
            self._config = None
            self._file_channel_map = {}
            self._file_channels = 0

            #  get the total file size
            total_bytes = os.stat(filename).st_size

            #  normalize the file path and split out the parts
            filename = os.path.normpath(filename)
            file_parts = filename.split(os.path.sep)
            current_filename = file_parts[-1]
            current_filepath = os.path.sep.join(file_parts[0:-1])

            # open the raw file
            fid = RawSimradFile(filename, 'r')
            self.n_files += 1

            #  read the configuration datagram - this will always be the
            #  first datagram in an EK60 raw file.
            _config = self._read_config(fid)

            #  extract the per channel data into our internal config attribute
            #  and add some additional fields to each channel
            self._config = _config['configuration']
            for channel in self._config:
                self._config[channel]['file_name'] = current_filename
                self._config[channel]['file_path'] = current_filepath
                self._config[channel]['file_bytes'] = total_bytes
                self._config[channel]['start_time'] = _config['timestamp']
                self._config[channel]['start_ping'] = self.n_pings

            #  set the cumulative_bytes var and start time
            cumulative_bytes = _config['bytes_read']

            #  create a variable to track when we're done reading.
            finished = False

            #  and read datagrams until we're done
            while not finished:
                #  read a datagram - method returns some basic info
                dg_info = self._read_datagram(fid, nmea=nmea)

                #  call progress callback if supplied
                if (progress_callback):
                    #  determine the progress as an integer percent.
                    cumulative_bytes += dg_info['bytes_read']
                    cumulative_pct = round(cumulative_bytes / total_bytes * 100.)

                    #  call the callback when the percent changes
                    if cumulative_pct != self._last_progress:
                        #  call the provided callback - the callback has 3 args,
                        #  the first is the full path to the current file and the
                        #  second is the percent read and the third is the bytes read.
                        progress_callback(filename, cumulative_pct, cumulative_bytes)
                        self._last_progress = cumulative_pct

                #  update our finished state var - the file will be finished when
                #  the reader hits the end of the file or if end time/ping are
                #  set and we hit that point.
                finished = dg_info['finished']

            #  close the file
            fid.close()

        # Trim excess data from arrays after reading.
        for channel_id in self.channel_ids:
            for raw in self.raw_data[channel_id]:
                raw.trim()
        self.nmea_data.trim()
        self.motion_data.trim()
        self.annotations.trim()


    def _read_config(self, fid):
        '''
        _read_config reads the raw file configuration datagram. It then checks if
        if there are any new channels it should be storing and if so, creates
        raw_data objects for those channels and updates some attributes.
        '''

        #  read the configuration datagram - this is the first datagram
        #  in an EK60 file.
        config_datagram = fid.read(1)
        config_datagram['timestamp'] = \
                np.datetime64(config_datagram['timestamp'], '[ms]')

        # Check for any new channels and add them if required
        file_channels = list(config_datagram['configuration'].keys())
        for channel_id in file_channels:
            # Increment the per file channel total
            self._file_channels += 1

            # Check if we're reading this channel
            if (not self.read_channel_ids or channel_id in self.read_channel_ids):
                # Check if we're reading this frequency
                frequency = config_datagram['configuration'][channel_id]['frequency']
                if (self.read_frequencies and frequency not in self.read_frequencies):
                    # There are specific frequencies specified and this
                    # is NOT one of them. Remove this channel from the config_datagram
                    # dictionary and continue.
                    config_datagram['configuration'].pop(channel_id)
                    continue

                # We're reading this channel - Check if we have seen this channel in
                # *this* file. If not, add it.
                if channel_id not in self._file_channel_map:

                    #  populate the _file_channel_map which maps channel
                    #  number to channel id *for this file only*.
                    self._file_channel_map[self._file_channels] = channel_id

                    # Set the file channel number in the configuration - we need this
                    # in certain cases when writing data.
                    config_datagram['configuration'][channel_id]['channel_number'] = self._file_channels

                # Check if we've seen this channel in any of the data
                if channel_id not in self.raw_data:
                    # This is a new channel, create a list to store this channel's raw_data objects.
                    # The raw_data objects are stored in a list because with the introduction of the
                    # EK80 different types of data can be stored in a raw file and the raw_data object
                    # was designed to contain a single data type. While this is not required for EK60
                    # data, the API was changed to match the EK80.
                    self.raw_data[channel_id] = []

                    #  add the channel to our list of channel IDs
                    self.channel_ids.append(channel_id)

                    #  increment the global channel total
                    self.n_channels += 1

                    # and populate the frequency and channel maps
                    this_freq = config_datagram['configuration'][channel_id]['frequency']
                    if this_freq in self.frequency_map:
                        self.frequency_map[this_freq].append(channel_id)
                    else:
                        self.frequency_map[this_freq] = [channel_id]
                    self.channel_map[self.n_channels] = channel_id

        # Return the configuration datagram dict
        return config_datagram


    def _read_datagram(self, fid, nmea=True):
        """Reads the next raw file datagram

        This method reads the next datagram from the file, storing the
        data contained in the datagram (if applicable), and returns a dict
        containing the number of bytes read, the datagramn timestamp, datagram
        type, and whether the reader is "finished". The finished flag will be set
        if end_time and/or end_ping is set in the reader and the datagram time
        is after the end time or the internal "ping" counter matches the
        specified end ping.

        Args:
            fid (file object): Pointer to currently open file object. This is a
                RawSimradFile file object and not the standard Python file
                object.
        """
        #  create the return dict that provides feedback on progress
        result = {'bytes_read':0, 'timestamp':None, 'type':None, 'finished':False}

        #  attempt to read the next datagram
        try:
            new_datagram = fid.read(1)
        except SimradEOF:
            #  we're at the end of the file
            result['finished'] = True
            return result

        # Convert the timestamp to a datetime64 object.
        # Check for NULL DG date which is returned as datetime.datetime(1601, 1, 1, 0, 0)
        if new_datagram['timestamp'].year < 1900:
            # This datagram has NULL date/time values
            new_datagram['timestamp'] = np.datetime64("NaT")
        else:
            # We have a plausible date/time value
            new_datagram['timestamp'] = \
                    np.datetime64(new_datagram['timestamp'], '[ms]')

        #  update the return dict properties
        result['timestamp'] = new_datagram['timestamp']
        result['bytes_read'] = new_datagram['bytes_read']
        result['type'] = new_datagram['type']

        # If this is a NMEA datagram and we're not storing them, bail
        if not nmea and new_datagram['type'].startswith('NME'):
            # This is a NMEA datagram and we're skipping them
            return result

        # Check if data should be stored based on time bounds.
        if self.read_start_time is not None:
            if new_datagram['timestamp'] < self.read_start_time:
                #  we have a start time but this data comes before it
                #  so we return without doing anything else
                return result
        if self.read_end_time is not None:
            if new_datagram['timestamp'] > self.read_end_time:
                #  we have a end time and this data comes after it
                #  so we are actually done reading - set the finished
                #  field in our return dict and return
                result['finished'] = True
                return result

        #  update the ping counter
        if new_datagram['type'].startswith('RAW'):
            #  make sure this is a unique time
            if self._this_ping_time != new_datagram['timestamp']:
                self.n_pings += 1
                self._this_ping_time = new_datagram['timestamp']

            # check if we're storing this channel
            if new_datagram['channel'] not in self._file_channel_map.keys():
                #  no, it's not in the list - just return
                return result
            else:
                #  add the channel_id to our datagram
                new_datagram['channel_id'] = self._file_channel_map[new_datagram['channel']]

        # Check if we should store this data based on ping bounds.
        if self.read_start_ping is not None:
            if self.n_pings < self.read_start_ping:
                #  we have a start ping but this data comes before it
                #  so we return without doing anything else
                return result
        if self.read_end_ping is not None:
            if self.n_pings > self.read_end_ping:
                #  we have a end ping and this data comes after it
                #  so we are actually done reading - set the finished
                #  field in our return dict and return
                result['finished'] = True
                return result

        # Update the end_time property.
        if self.end_time is not None:
            # We can't assume data will be read in time order.
            if self.end_time < new_datagram['timestamp']:
                self.end_time = new_datagram['timestamp']
        else:
            self.end_time = new_datagram['timestamp']

        # Process and store the datagrams by type.

        # RAW datagrams store raw acoustic data for a channel.
        if new_datagram['type'].startswith('RAW'):

            #  check if we should set our start time property
            if not self.start_time:
                self.start_time = new_datagram['timestamp']
            # Set the first ping number we read.
            if not self.start_ping:
                self.start_ping = self.n_pings
            # Update the last ping number.
            self.end_ping = self.n_pings

            #  loop through the raw_data objects to find the raw_data object
            #  to store this data.
            this_raw_data = None
            for raw_obj in self.raw_data[new_datagram['channel_id']]:
                if raw_obj.data_type == '':
                    #  This raw_data object has no type so is empty and can store anything
                    this_raw_data = raw_obj
                    break
                elif (new_datagram['mode'] == 1 and raw_obj.data_type == 'power' or
                      new_datagram['mode'] == 2 and raw_obj.data_type == 'angle' or
                      new_datagram['mode'] == 3 and raw_obj.data_type == 'power/angle'):
                    # This raw_data object contains a matching data type
                    this_raw_data = raw_obj
                    break

            #  check if we need to create a new raw_data object
            if this_raw_data is None:
                #  create the new raw_data object
                this_raw_data = raw_data(new_datagram['channel_id'],
                        store_power=self.store_power,
                        store_angles=self.store_angles,
                        max_sample_number=self.read_max_sample_count)
                # Set the transceiver type
                this_raw_data.transceiver_type = 'GPT'
                #  and add it to this channel's list of raw_data objects
                self.raw_data[new_datagram['channel_id']].append(this_raw_data)

            # Call this channel's append_ping method.
            this_raw_data.append_ping(new_datagram,
                    self._config[new_datagram['channel_id']],
                    start_sample=self.read_start_sample,
                    end_sample=self.read_end_sample)

            # Store the motion data in the motion data object
            self.motion_data.add_datagram(new_datagram['timestamp'],
                    new_datagram['heave'], new_datagram['pitch'],
                    new_datagram['roll'], new_datagram['heading'])

        # NME datagrams store ancillary data as NMEA-0183 style ASCII data.
        elif new_datagram['type'].startswith('NME'):
            # Add the datagram to our nmea_data object.
            self.nmea_data.add_datagram(new_datagram['timestamp'],
                    new_datagram['nmea_string'])

        # TAG datagrams contain time-stamped annotations inserted via the
        # recording software. They are not associated with a specific channel
        elif new_datagram['type'].startswith('TAG'):
            # Add this datagram to our annotation_data object
            self.annotations.add_datagram(new_datagram['timestamp'],
                    new_datagram['text'])

        # BOT datagrams contain bottom detections - these data are synchronous
        # with the RAW datagrams
        elif new_datagram['type'].startswith('BOT'):
            # iterate through the channels in this .bot file
            for i in range(self._file_channels):
                # Get this channel's ID
                this_chan = self._file_channel_map.get(i+1, None)

                # Check if we have data for this channel
                if this_chan in self.raw_data:
                    # This is hokey, but there is no way to know which data object
                    # this depth applies to so we just try for all data objects.
                    for data in self.raw_data[this_chan]:
                        # In order to avoid checking the index array below for every
                        # chan/datatype, we'll check for and add if needed the
                        # bottom detection attribute to alldata objects if any
                        # bottom data is read.
                        if not hasattr(data, 'detected_bottom'):
                            # This data object doesn't have the detected_bottom attribute.
                            # Create and add it.
                            new_attr = np.full((data.ping_time.shape), np.nan, np.float32)
                            data.add_data_attribute('detected_bottom', new_attr)

                        # Get the index of this detection and insert into data object
                        idx = data.ping_time == new_datagram['timestamp']
                        data.detected_bottom[idx] = new_datagram['depth'][i]

        # XML datagrams contain contain data encoded as an XML string.
        elif new_datagram['type'].startswith('XML'):
            #  XML datagrams have a subtype field identifying what
            #  kind of data they contain.
            if new_datagram['subtype'] == 'parameter':
                #  update the most recent parameter attribute for this channel
                self._tx_params[new_datagram[new_datagram['subtype']]['channel_id']] = \
                        new_datagram[new_datagram['subtype']]
            elif new_datagram['subtype'] == 'environment':
                #  update the most recent environment attribute
                self._environment = new_datagram[new_datagram['subtype']]

        # MRU datagrams contain vessel motion data
        elif new_datagram['type'].startswith('MRU'):
            # append this motion datagram to the motion_data object
            self.motion_data.add_datagram(new_datagram)

        # DEP datagrams contain bottom detection and "reflectivity" data
        elif new_datagram['type'].startswith('DEP'):
            # iterate through the channels in this .out file
            for i in range(self._file_channels):
                # Get this channel's ID
                this_chan = self._file_channel_map[i + 1]

                # Check if we have data for this channel
                if this_chan in self.raw_data:
                    # This is hokey, but there is no way to know which data object
                    # this depth applies to so we just try for all data objects.
                    for data in self.raw_data[this_chan]:
                        # In order to avoid checking the index array below for every
                        # chan/datatype, we'll check for and add if needed the
                        # bottom detection and reflectivity attributes to all
                        # data objects if any bottom data is read.
                        if not hasattr(data, 'detected_bottom'):
                            # This data object doesn't have the detected_bottom
                            # and reflectivity attributes. Create and add them.
                            new_attr = np.full((data.ping_time.shape), np.nan, np.float32)
                            data.add_data_attribute('detected_bottom', new_attr)
                            new_attr = np.full((data.ping_time.shape), np.nan, np.float32)
                            data.add_data_attribute('reflectivity', new_attr)

                        # Get the index of this detection and insert into data object
                        idx = data.ping_time == new_datagram['timestamp']
                        data.detected_bottom[idx] = new_datagram['depth'][i]
                        data.reflectivity[idx] = new_datagram['reflectivity'][i]

        else:
            #  report an unknown datagram type
            print("Unknown datagram type: " + str(new_datagram['type']))

        #  return our result dict which contains some basic info about
        #  the datagram we just read
        return result


    def get_channel_data(self, frequencies=None, channel_numbers=None, channel_ids=None):
        """returns a dict containing lists of raw_data objects for the specified channel IDs,
        or frequencies, or channel numbers.

        This method returns a dictionary of lists that contain references to one or
        more raw_data objects containing the data that has been read. The dictionary keys
        and the raw_data object(s) returned depend on the argument(s) passed. You can
        specify one or more frequencies, channel numbers, and/or channel IDs and the
        dictionary will be keyed by frequency, channel number, and/or channel ID and
        the key values will be a list containing the raw_data objects that correspond
        to that frequency, channel number, or channel ID.

        If you specify one or more frequencies, this method will return a dict, keyed
        by frequency where the values are a lists containing one or more raw_data objects
        associated with the specified frequency. The length of the list is equal to the
        number of unique channel + datatype combinations sharing that transmit frequency.

        If you specify one or more channel IDs, this method will return a dict, keyed
        by channel ID where the values are a lists containing one or more raw_data objects
        associated with the specified channel ID. The length of the list is equal to the
        number of unique channel + datatype combinations sharing that channel ID.

        And, as you've already guessed, if you specify one or more channel numbers, this
        method will return a dict, keyed by channel number where the values are a lists
        containing one or more raw_data objects associated with the specified channel
        number. The length of the list is equal to the number of unique channel + datatype
        combinations sharing that channel number.

        If called without arguments, it returns a dictionary, keyed by channel ID, that
        contains all channels read. The values are a lists containing one or more raw_data
        objects associated with the specified channel ID just as if you called the method
        specifying all channel IDs.

        You can specify multiple keyword arguments to return references to the data mapped
        in multiple ways if your analysis requires that flexibility. This method only returns
        references, it does not copy the data, so there is no real penalty for doing this.
        Do remember that if there is a valid reference to an object in memory it will continue
        to exist and consume RAM. If memory management is important for your application
        you will want to manage these and not just grab a bunch of references willy-nilly.

        Args:
            frequencies (float, list): Specify one or more frequencies in a list to return
                    data objects associated with that frequency.
            channel_id (str, list): Specify one or more channel IDs in a list to return
                    data objects associated with that channel ID.
            channel_number (int, list): Specify one or more channel numbers in a list to
                    return data objects associated with that channel number.
        Returns:
            Dictionary keyed by frequency/channel_id/channel_number. Values are lists
            of raw_data objects associated with the specified frequency/channel_id/
            channel_number.
        """

        # create the return dict
        channel_data = {}

        if channel_ids is not None:
            # Channel id is specified.

            # if we're passed a string, make it a list
            if isinstance(channel_ids, str):
                channel_ids = [channel_ids]

            #  work thru the channel ids and add if they exist
            for id in channel_ids:
                channel_data[id] = self.raw_data.get(id, None)

        elif frequencies is not None:
            # frequency is specified

            # if we're passed a number, make it a list
            if not isinstance(frequencies, list):
                frequencies = [frequencies]

            # and work through the freqs, adding if they exist
            for freq in frequencies:
                # There is not a 1:1 mapping of freqs to data so we have to
                # create a list and then extend it for each matching frequency.
                channel_data[freq] = []

                data_objs = self.frequency_map.get(freq, [None])
                for id in data_objs:
                    channel_data[freq].extend(self.raw_data[id])

        elif channel_numbers is not None:
            # channel_number is specified.

            # if we're passed a number, make it a list
            if not isinstance(channel_numbers, list):
                channel_numbers = [channel_numbers]

            # and work through the numbers, adding if they exist
            for num in channel_numbers:
                id = self.channel_map.get(num, None)
                channel_data[num] = self.raw_data.get(id, None)

        else:
            # No keywords specified - return all in a dict keyed by channel ID
            channel_data = self.raw_data

        return channel_data


    def write_raw(self, output_filenames, power=True, angles=True,
                 max_sample_count=None, start_time=None, end_time=None,
                 start_ping=None, end_ping=None, frequencies=None,
                 channel_ids=None,start_sample=None, end_sample=None, progress_callback=None,
                 overwrite=False, async_window=5, raw_index_array=None,
                 nmea_index_array=None, annotation_index_array=None,
                 strip_padding=True):
        """Writes one or more Simrad Ex60/ES70/ME70 .raw files.


        Args:
            output_filenames (str, dict): A string specifying the full path and
                output filename header that will be used to generate all of the
                output filenames OR a dict, keyed by *input* filename where the
                values specify the full path and *full* output filename that will
                be used when writing data associated with that input file.

                Providing a string will result in behavior similar to ER60 software
                where you provide the folder and filename header and it generates
                data in that folder naming the files with the header and appending
                the date and time:

                    output_filenames = 'C:\temp\test_ek60_file'

                would result in files named similar to:

                    C:\temp\test_ek60_file-D202020101-T120000.raw
                    C:\temp\test_ek60_file-D202020101-T122513.raw

                If, in the off chance that you have read data from two files that
                both start at the same time, the subsequent files will have a
                "-#" appended to the time, to ensure that the file names are
                unique.

                Providing a dict, keyed by input filename will allow full control
                of the output file naming. It is your responsibility to provide a
                dict where the keys map 1:1 to the input file names. If they do
                not, an error will be raised.

            power (bool): Controls whether power data is written. Has no effect if
                there is no power data to write.

            angles (bool): Controls whether angle data is written. Has no effect if
                there is no angle data to write.

            start_time (datetime64): Specify a start time when defining a range
                of data to write. When start_time is not provided, the start
                time will be the time of the first ping.

            end_time (datetime64): Specify a end time when defining a range
                of data to write. When end_time is not provided, the end
                time will be the time of the last ping.

            start_ping (int): Specify the starting ping number when defining a
                range of pings to write. If start_ping is not specified,
                the start ping is 1.

            end_ping (int): Specify the end ping number when defining a range
                of pings to write. If end_ping is not provided, the last
                ping is used.

            frequencies (list): List of floats specifying the frequencies in
                Hz you want to write. If frequencies is not specified, all
                frequencies will be written.

            channel_ids (list): A list of strings that contain the unique
                channel IDs to write. If no list is supplied, all channels are
                written.

            start_sample (int): Specify starting sample number if not
                writing from first sample.

            end_sample (int): Specify ending sample number if not
                writing to last sample.

            strip_padding (bool): Set to True to remove any NaN padding at the
                beginning and/or end of the sample data. Echolab stores sample data in
                rectangular arrays. When the sample number increases within or
                between files, these arrays are resized and pings with fewer samples
                are padded with NaNs as appropriate. When True, these samples are
                omitted when writing. This DOES NOT affect padded pings (entire pings
                that contain no data.) Padded pings are always removed before writing
                because they lack not only sample data, but all synchronous ping
                attributes. The default value is True.

            The following keywords are for advanced indexing and writing of the
            data. This can allow you to easily write subsets of data where the
            pings do not have to be contiguous.

            raw_index_array (dict): Set this to a dictionary, keyed by
                raw_data object reference, where the values are index arrays that
                specify the pings to write for each raw_data object. If you specify
                this keyword, the start/stop time/ping and time_order keywords will
                be ignored.

            THE FOLLOWING KEYWORDS ARE NOT IMPLEMENTED YET

            nmea_index_array (np.array):Set this to an index array that specifies
                the NMEA data that will be written to the raw file. This will
                override the default behavior of using the data start and end times
                (including when raw_index_array is defined.)

            annotation_index_array (np.array):Set this to an index array that specifies
                the annotation data that will be written to the raw file. This will
                override the default behavior of using the data start and end times
                (including when raw_index_array is defined.)
        """

        def new_datagram(dg_time, dg_type, dg_version):
            '''
            new_datagram builds a dict containing the keys common to all datagram
            dictionaries.
            '''

            # build the base datagram dict. These fields are common to all datagrams
            dgram = {}
            low_date, high_date = date_conversion.dt64_to_nt(dg_time)
            dgram['low_date'] = low_date
            dgram['high_date'] = high_date
            dgram['type'] = dg_type + str(dg_version)
            dgram['dg_version'] = dg_version

            return dgram


        def remove_sample_padding(data, sample_offset):
            '''
            remove_sample_padding will strip NaN padding from the beginning and
            end of a "ping". It adjusts the sample_offset if required. It does
            not alter data between the first and last non NaN values.
            '''

            # check for any nan data
            if data.ndim == 1:
                nan_idx = np.isnan(data)
            else:
                nan_idx = np.isnan(data[:,0])

            # Check if there are any NaNs at all
            if np.any(nan_idx):

                # find the data start and data end indexes
                start_idx = np.argmin(nan_idx)
                end_idx = np.argmax(nan_idx)

                # update the sample_offset if needed
                if start_idx != sample_offset:
                    sample_offset = start_idx

                # get data with padding removed
                if data.ndim == 1:
                    data = data[start_idx:end_idx]
                else:
                    data = data[start_idx:end_idx,:]

            return (data, sample_offset)


        #  initialize some vars to hold the current vessel attitude data
        this_heave = 0.0
        this_roll = 0.0
        this_pitch = 0.0
        this_heading = 0.0

        files_written = []

        # Process the args
        if start_time:
            start_time = start_time
        else:
            start_time = self.start_time

        if end_time:
            end_time = end_time
        else:
            end_time = self.end_time

        if channel_ids is None:
            channel_ids = self.channel_ids

        if start_sample is None:
            start_sample = 0

        if start_sample < 0:
            start_sample = 0

        if end_sample:
            if end_sample <= start_sample:
                end_sample = start_sample

        # Create the async data time delta - used when extracting async data
        # like NMEA or TAG for writing
        async_window_secs = np.timedelta64(async_window, 's')

        # create the inverse of the channel_map so we can look up the number
        # using the channel ID
        channel_number_map = {}
        for number in self.channel_map:
            channel_number_map[self.channel_map[number]] = number

        #  map dg_type to its parser
        DGRAM_PARSE_KEY = {'RAW': simrad_parsers.SimradRawParser(),
                           'CON': simrad_parsers.SimradConfigParser(),
                           'TAG': simrad_parsers.SimradAnnotationParser(),
                           'NME': simrad_parsers.SimradNMEAParser(),
                           'BOT': simrad_parsers.SimradBottomParser(),
                           'DEP': simrad_parsers.SimradDepthParser()}

        # map dg_type to version for this class/raw file spec
        DGRAM_VERSION_KEY = {'RAW':0,
                             'NME':0,
                             'TAG':0,
                             'CON':0,
                             'MRU':0}

        # Because a user can load multiple .raw files into an EK80 object, and those
        # .raw files can contain data from various system configurations, we need to
        # write potentially different configurations as separate files to ensure the
        # new data are interpreted correctly when read.
        #
        # Each ping in a raw_data object references a dict containing the system
        # configuration data for that channel as written in the .raw file header. That
        # dict also contains the input raw file name. The unique raw file names can be
        # used to determine how many files to write and the channel data that will be
        # in them.
        #
        # This is important to remember if you are trying to combine data from two raw
        # files into one. It is your responsibility to only combine data from files
        # recorded using the the same system parameters. After combining data from
        # two or more files, you need to replicate the first configuration dict in each
        # raw_data object across all pings:
        #
        # raw_data.configuration[:] = raw_data.configuration[0]


        # The first thing we need to do is identify all of the data that meets our
        # time/ping number, channel/frequency constraints. Then we group that by input
        # file name and again by channel. The result is a dict, keyed by file name,
        # with values that are dicts keyed by the channel ID

        # Work thru the specified channels
        data_by_file = {}
        n_datagrams_by_file = {}
        for channel in channel_ids:
            # and then this channel's raw_data objects

            for data in self.raw_data[channel]:
                # Check if we've been provided with an index array
                if raw_index_array is not None:
                    if isinstance(raw_index_array, dict) and data in raw_index_array:
                        # this is a dict mapping the index to theraw_data objects
                        this_idx = raw_index_array[data]
                    else:
                        # Assume that if we're not passed a dict, that this must be
                        # an index array that will be applied to all channels
                        this_idx = raw_index_array
                else:
                    # No index array provided. Get the indices of data from this
                    # channel/data that fall within the time span provided.
                    this_idx = data.get_indices(start_time=start_time,
                            end_time=end_time, start_ping=start_ping,
                            end_ping=end_ping)

                # check if we have any data from this channel/data
                if this_idx.size > 0:
                    # Find the unique configuration objects for this data/time range
                    # The config objects are different for each file read and we use
                    # that to determine what data goes in what output file.
                    #
                    # We can't use np.unique on dicts and the dict == operator does
                    # a deep comparison which fails when the values are numpy arrays.
                    #
                    # Instead we have to brute force this using the dict object ID.
                    # We still need a reference to the unique dicts so we have to
                    # keep a list of both the ID and dict. As we're doing the comparison
                    # we'll build an array of object IDs which we use below when
                    # identifying pings with each unique configuration.
                    unique_configs = []
                    unique_ids = []
                    conf_ids = []
                    for c in data.configuration:
                        # make sure this ping has a valid config. Padded pings will
                        # have NaNs so we'll ignore those. They will be removed later.
                        if isinstance(c, dict):
                            this_id = id(c)
                            conf_ids.append(this_id)
                            is_unique = True
                            for uc in unique_configs:
                                if uc['file_name'] == c['file_name']:
                                    is_unique = False
                        else:
                            is_unique = False
                            this_id = None
                            conf_ids.append(this_id)

                        if is_unique:
                            # this is a convienient place to filter by frequency...
                            if frequencies:
                                #  freqs specified, check if this is one of them
                                if c['frequency'] in frequencies:
                                    unique_ids.append(this_id)
                                    unique_configs.append(c)
                            else:
                                # writing all freqs
                                unique_ids.append(this_id)
                                unique_configs.append(c)

                    # Convert the list of cinfiguration IDs to a numpy array
                    conf_ids = np.array(conf_ids)

                    # Determine the indices for each config
                    for idx in range(len(unique_configs)):
                        # Get this config object and object ID
                        conf = unique_configs[idx]
                        conf_id = unique_ids[idx]

                        # Get the indices for each unique config - if we are passed a
                        # boolean array, we have to convert it to an index array.
                        if this_idx.dtype == np.bool_:
                            conf_idx = np.nonzero(np.logical_and(conf_ids == conf_id,
                                    this_idx))[0]
                        else:
                            conf_idx = this_idx[conf_ids[this_idx] == conf_id]

                        # To track progress, we'll total up the number of raw
                        # datagrams that we will ultimately write for this file.
                        n_datagrams = conf_idx.size

                        # Store the channel ID, config dict, reference to the
                        # raw data, and the index to the data by filename
                        this_data = {'channel_id':channel, 'configuration':conf,
                                'data':data, 'index':conf_idx}

                        if conf['file_name'] not in data_by_file:
                            # this is a new file, add it
                            data_by_file[conf['file_name']] = {channel:[this_data]}
                            # and set the initial datagram count
                            n_datagrams_by_file[conf['file_name']] = n_datagrams
                        else:
                            # this file is already on the list, check if the channel exists (this
                            # is the rare, but possible, multiple data types per channel case.)
                            if channel in data_by_file[conf['file_name']]:
                                data_by_file[conf['file_name']][channel].append(this_data)
                            else:
                                data_by_file[conf['file_name']][channel] = [this_data]
                            n_datagrams_by_file[conf['file_name']] += n_datagrams

        # Now that we have references to the data we need to write, grouped by input
        # filename, we can iterate through the this dict and write the data to disk.
        for infile in data_by_file:

            outfile_name = ''

            # check if we've been given a dict of filenames and if we have one for
            # this input file.
            if isinstance(output_filenames, dict):
                if infile in output_filenames:
                    outfile_name = output_filenames[infile]
                else:
                    raise ValueError("Input filename '" + infile + "' not found in " +
                        "output_filenames dictionary. No output filename available to " +
                        "map to.")

            # .raw files are always written in time order. We need to create a vector
            # of timestamps from all of our data sources (raw data, NMEA data, annotations,
            # and motion data) and use the sort indices to create a our datagram roadmap.

            # create the map arrays
            dg_times = np.array([], dtype='datetime64[ms]')
            dg_objects = np.array([], dtype='object')
            dg_obj_idx = np.array([], dtype=np.uint32)
            dg_type = np.array([], dtype='S3')

            # Add the raw data
            for channel in data_by_file[infile]:
                for data in data_by_file[infile][channel]:
                    # First, remove any empty pings from the index.
                    data['index'] = data['index'][np.isfinite(data['data'].transmit_power[data['index']])]
                    # Then extract the times and references to the data we're writing
                    times = data['data'].ping_time[data['index']]
                    dg_times = np.concatenate((dg_times, times))
                    dg_objects = np.concatenate((dg_objects, np.repeat(data['data'], times.size)))
                    dg_obj_idx = np.concatenate((dg_obj_idx, data['index']))
                    dg_type = np.concatenate((dg_type, np.repeat('RAW', times.size)))

            # Determine the data time window
            raw_start_time = dg_times[0]
            async_start_time = dg_times[0] - async_window_secs
            async_end_time = dg_times[-1] - async_window_secs

            # Next, add the annotation data
            annotation_idx = self.annotations.get_indices(start_time=async_start_time,
                end_time=async_end_time)
            times = self.annotations.times[annotation_idx]
            dg_times = np.insert(dg_times, 0, times)
            dg_objects = np.insert(dg_objects, 0, np.repeat(self.annotations, times.size))
            dg_obj_idx = np.insert(dg_obj_idx, 0, annotation_idx)
            dg_type = np.insert(dg_type, 0, np.repeat('TAG', times.size))

            # Then the motion data using our new time window
            motion_idx = self.motion_data.get_indices(start_time=async_start_time,
                        end_time=async_end_time)
            times = self.motion_data.times[motion_idx]
            dg_times = np.insert(dg_times, 0, times)
            dg_objects = np.insert(dg_objects, 0, np.repeat(self.motion_data, times.size))
            dg_obj_idx = np.insert(dg_obj_idx, 0, motion_idx)
            dg_type = np.insert(dg_type, 0, np.repeat('MRU', times.size))

            # Lastly, insert the NMEA data using our new time window
            nmea_idx = self.nmea_data.get_indices(start_time=async_start_time,
                        end_time=async_end_time)
            times = self.nmea_data.nmea_times[nmea_idx]
            dg_times = np.insert(dg_times, 0, times)
            dg_objects = np.insert(dg_objects, 0, np.repeat(self.nmea_data, times.size))
            dg_obj_idx = np.insert(dg_obj_idx, 0, nmea_idx)
            dg_type = np.insert(dg_type, 0, np.repeat('NME', times.size))

            # Now sort the time vector - must use stable sort to ensure proper dg order
            sorted_time_idx = np.argsort(dg_times, kind='stable')

            # Sort the map arrays using the sorted time index
            dg_times = dg_times[sorted_time_idx]
            dg_objects = dg_objects[sorted_time_idx]
            dg_obj_idx = dg_obj_idx[sorted_time_idx]
            dg_type = dg_type[sorted_time_idx]
            n_datagrams = dg_times.size

            # we are finally ready to write...

            # Generate output filename if needed - we assume we're passed
            # a valid path and filename prefix. e.g. 'c:/test/DY2020'
            if outfile_name == '':
                timestamp = date_conversion.dt64_to_datetime(raw_start_time)
                timestamp = timestamp.strftime("D%Y%m%d-T%H%M%S")
                outfile_name = output_filenames + '-' + timestamp + '.raw'

            # open the raw file
            outfile_name = os.path.normpath(outfile_name)
            if not overwrite and os.path.exists(outfile_name):
                raise IOError('File %s already exists and overwrite == False.' %(outfile_name))
            raw_fid = open(outfile_name, 'wb')

            # Initialize some vars to track our progress
            bytes_written = 0
            cumulative_pct = -1
            datagrams_processed = 0

            #  keep track of the files we're writing to return to caller
            files_written.append(outfile_name)

            # build the configuration dict
            config_dict = new_datagram(dg_times[0], 'CON', DGRAM_VERSION_KEY['CON'])
            config_dict['configuration'] = {}
            for channel in data_by_file[infile]:
                config_dict['configuration'][channel] = \
                    data_by_file[infile][channel][0]['configuration']

            #  pack config dict into raw byte stream and write
            bytes_written += raw_fid.write(DGRAM_PARSE_KEY['CON'].to_string(config_dict))

            #  now write out all the rest of the datagrams
            for idx in range(n_datagrams):

                #  create the base datagram dict
                dgram_dict = new_datagram(dg_times[idx], dg_type[idx],
                        DGRAM_VERSION_KEY[dg_type[idx]])

                #  now assemble the datagram dict based on the type
                if dg_type[idx] == 'RAW':

                    #  set the raw datagram values
                    dgram_dict['channel'] = data_by_file[infile][dg_objects[idx].channel_id][0]['configuration']['channel_number']
                    dgram_dict['transducer_depth'] = dg_objects[idx].transducer_depth[dg_obj_idx[idx]]
                    dgram_dict['frequency'] = dg_objects[idx].frequency[dg_obj_idx[idx]]
                    dgram_dict['transmit_power'] = dg_objects[idx].transmit_power[dg_obj_idx[idx]]
                    dgram_dict['pulse_length'] = dg_objects[idx].pulse_length[dg_obj_idx[idx]]
                    dgram_dict['bandwidth'] = dg_objects[idx].bandwidth[dg_obj_idx[idx]]
                    dgram_dict['sample_interval'] = dg_objects[idx].sample_interval[dg_obj_idx[idx]]
                    dgram_dict['sound_velocity'] = dg_objects[idx].sound_velocity[dg_obj_idx[idx]]
                    dgram_dict['absorption_coefficient'] = dg_objects[idx].absorption_coefficient[dg_obj_idx[idx]]
                    dgram_dict['heave'] = this_heave
                    dgram_dict['roll'] = this_roll
                    dgram_dict['pitch'] = this_pitch
                    dgram_dict['temperature'] = dg_objects[idx].temperature[dg_obj_idx[idx]]
                    dgram_dict['heading'] = this_heading
                    dgram_dict['transmit_mode'] = dg_objects[idx].transmit_mode[dg_obj_idx[idx]]
                    dgram_dict['spare0'] = ''

                    # adjust the sample offset if we're creating an additional offset during writing
                    sample_offset = start_sample + dg_objects[idx].sample_offset[dg_obj_idx[idx]]
                    dgram_dict['offset'] = sample_offset

                    # Extract the data and update the sample count
                    if hasattr(dg_objects[idx], 'power'):
                        power_data = dg_objects[idx].power[dg_obj_idx[idx],start_sample:end_sample]
                        if strip_padding:
                            power_data, sample_offset = remove_sample_padding(power_data, sample_offset)
                        sample_count = power_data.shape[0]
                    if hasattr(dg_objects[idx], 'angles_athwartship_e'):
                        angles_athwart_data = dg_objects[idx].angles_athwartship_e[dg_obj_idx[idx],start_sample:end_sample]
                        angles_along_data = dg_objects[idx].angles_alongship_e[dg_obj_idx[idx],start_sample:end_sample]
                        if strip_padding:
                            angles_athwart_data, sample_offset = remove_sample_padding(angles_athwart_data, sample_offset)
                            angles_along_data, sample_offset = remove_sample_padding(angles_along_data, sample_offset)
                        sample_count = angles_athwart_data.shape[0]
                    dgram_dict['count'] = sample_count

                    #  initialize the beam mode
                    mode = 0

                    # If we have angle data, we have to convert it back to indexed and combine.
                    if hasattr(dg_objects[idx], 'angles_athwartship_e'):
                        # Convert from electrical angles to indexed.
                        angles_athwart_data = (angles_athwart_data / raw_data.INDEX2ELEC).astype('int8')
                        angles_along_data = (angles_along_data / raw_data.INDEX2ELEC).astype('int8')
                        dgram_dict['angle'] = np.column_stack((angles_athwart_data,angles_along_data))

                        #  set the angles bit in the beam mode
                        mode = mode | (1 << 1)

                    # If we have power data, we have to convert it back to indexed
                    if hasattr(dg_objects[idx], 'power'):
                        dgram_dict['power'] = (power_data / raw_data.INDEX2POWER).astype('int16')

                        # Set the power bit in the beam mode
                        mode = mode | (1 << 0)

                    #  set the mode
                    dgram_dict['mode'] = mode

                    #  write it
                    bytes_written += raw_fid.write(DGRAM_PARSE_KEY[dg_type[idx]].to_string(dgram_dict))

                elif dg_type[idx] == 'TAG':
                    # Set the annotation text datagram value
                    dgram_dict['text'] = dg_objects[idx].text[dg_obj_idx[idx]]

                    #  write it
                    bytes_written += raw_fid.write(DGRAM_PARSE_KEY[dg_type[idx]].to_string(dgram_dict))

                elif dg_type[idx] == 'NME':
                    # Set the NME0 datagram values
                    dgram_dict['nmea_string'] = dg_objects[idx].raw_datagrams[dg_obj_idx[idx]]

                    #  write it
                    bytes_written += raw_fid.write(DGRAM_PARSE_KEY[dg_type[idx]].to_string(dgram_dict))

                elif dg_type[idx] == 'MRU':
                    # We don't actually write MRU datagrams for EK60 type raw files but since
                    # we store the data as if it were asyncronous, we process it like it is.
                    this_heave = dg_objects[idx].heave[dg_obj_idx[idx]]
                    this_roll = dg_objects[idx].roll[dg_obj_idx[idx]]
                    this_pitch = dg_objects[idx].pitch[dg_obj_idx[idx]]
                    this_heading = dg_objects[idx].heading[dg_obj_idx[idx]]


                else:
                    #  we should never encounter an unknown type, but raise exception if so
                    raise ValueError('Unknown datagram type encountered: ' + dg_type[idx] +
                            '. This should never happen...')

                #  call progress callback if supplied
                if (progress_callback):
                    #  determine the progress as percent.
                    pct_progress = round(datagrams_processed / n_datagrams * 100.)

                    #  call the callback when the percent changes
                    if cumulative_pct != pct_progress:
                        # Call the provided callback - the callback has 3 args,
                        # the first is the full path to the current file and the
                        # second is the percent written and the third is the bytes
                        # written.
                        progress_callback(outfile_name, pct_progress, bytes_written)
                        cumulative_pct = pct_progress

                # Increment our datagram counter - do this after the progress callback
                # block to ensure we emit a 0 percent callback
                datagrams_processed += 1

            # Done writing this file - close it
            raw_fid.close()

        # Writing complete, return a list of files written
        return files_written


    def __str__(self):
        """
        Reimplemented string method that provides some basic info about the
        EK60 class instance's contents.

        Returns: Message to print when calling print method on EK60 class
        instance object.
        """

        # Print the class and address.
        msg = str(self.__class__) + " at " + str(hex(id(self))) + "\n"

        # Print some more info about the EK60 instance.
        if self.channel_ids:
            n_channels = len(self.channel_ids)
            if n_channels > 1:
                msg = msg + ("    EK60 object contains data from " + str(
                    n_channels) + " channels:\n")
            else:
                msg = msg + ("    EK60 object contains data from 1 channel:\n")

            for channel_num in self.channel_map.keys():
                channel_id = self.channel_map[channel_num]
                for raw in self.raw_data[channel_id]:
                    msg = msg + ("        " + str(channel_num) + " :: " + channel_id + " :: " +
                            raw.data_type + " " + str(raw.shape) + "\n")
            msg = msg + ("    data start time: " + str(self.start_time) + "\n")
            msg = msg + ("      data end time: " + str(self.end_time) + "\n")
            msg = msg + ("    number of pings: " + str(self.end_ping -
                                                       self.start_ping + 1) + "\n")
        else:
            msg = msg + ("  EK60 object contains no data\n")

        return msg


class raw_data(ping_data):
    """
    the raw_data class contains a single channel's data extracted from a
    Simrad ER60 raw file.
    """

    # Define some instrument specific constants.

    # Simrad recommends a TVG correction factor of 2 samples to compensate
    # for receiver delay and TVG start time delay in EK60 and related
    # hardware.  Note that this correction factor is only applied when
    # computing Sv/sv and not Sp/sp.
    TVG_CORRECTION = 2

    # Define constants used to specify the target resampling interval for the
    # power and angle conversion functions.  These values represent the
    # standard sampling intervals for EK60 hardware when operated with the
    # ER60 software as well as ES60/70 systems and the ME70(?).
    RESAMPLE_SHORTEST = 0
    RESAMPLE_16   = 0.000016
    RESAMPLE_32  = 0.000032
    RESAMPLE_64  = 0.000064
    RESAMPLE_128  = 0.000128
    RESAMPLE_256 = 0.000256
    RESAMPLE_512 = 0.000512
    RESAMPLE_1024 = 0.001024
    RESAMPLE_2048 = 0.002048
    RESAMPLE_LONGEST = 1

    # Create a constant to convert indexed power to power.
    INDEX2POWER = (10.0 * np.log10(2.0) / 256.0)

    # Create a constant to convert from indexed angles to electrical angles.
    INDEX2ELEC = 180.0 / 128.0


    def __init__(self, channel_id, n_pings=500, n_samples=-1,
                 rolling=False, store_power=True,
                 store_angles=True, max_sample_number=None):
        """Creates a new, empty raw_data object.

        The raw_data class stores raw echosounder data from a single channel
        recorded using the Simrad ER60 application.

        NOTE: power is *always* stored in log form. If you manipulate power
              values directly, make sure they are stored in log form.

        The data arrays are not created upon instantiation. They will be created
        when the first ping is appended using the append_ping method.

        If rolling is True, and both the n_pings and n_samples arguments are
        provided, arrays of size (n_pings, n_samples) are created upon
        instantiation. Otherwise, the data arrays are created


        Args:
            channel_id (str): The channel ID of channel whose data are stored
                in this raw_data instance.
            n_pings (int): Sets the fixed "width" of the arrays for rolling
                arrays and if the object is not using rolling arrays it defines
                the "width" of the allocation chunks when appending data.
            n_samples (int): Sets the number of samples (rows) of the
                arrays. Default value is -1 samples which means the arrays
                will automatically resize to hold all of the sample data.
            rolling (bool): True = arrays have fixed sizes of n_pings set when
                the class is instantiated. When the n_pings + 1 ping is added
                the array is rolled left, dropping oldest ping and adding newest.
            store_power (bool): Boolean to control whether power data are
                stored in this raw_data object.
            store_angles (bool): Boolean to control whether angle data are
                stored in this raw_data object.
            max_sample_number (int): Integer specifying the maximum number of
                samples that will be stored in this instance's data arrays.
        """

        # Initialize the superclass
        super(raw_data, self).__init__()

        # Specify if data array size is fixed and the array data is rolled left
        # if the array fills up (True) or if the arrays are expanded when
        # necessary to hold additional data (False).
        self.rolling_array = bool(rolling)

        # If rolling is set, ensure we have been passed n_samples
        if self.rolling_array and n_samples < 1:
            raise ValueError('The n_samples argument must be defined and ' +
                    'greater than 0 when rolling == True.')

        # The channel ID is the unique identifier of the channel stored in
        # the object.
        self.channel_id = channel_id

        # Specify the horizontal size (columns) of the array allocation size.
        self.chunk_width = n_pings

        # Keep note if we should store the power, angle, or complex data.
        self.store_power = bool(store_power)
        self.store_angles = bool(store_angles)

        # Max_sample_number can be set to an integer specifying the maximum
        # number of samples that will be stored in the sample data arrays.
        # Samples beyond this will be discarded.
        self.max_sample_number = max_sample_number

        # data_type will be set to a string describing the type of sample data
        # stored. This value is an empty string until the first raw datagram
        # is added. At that point the data_type is set. A raw_data object can
        # only store 1 type of data. The data types are:
        #
        #   angle - contains angle data only
        #   power - contains power data only
        #   power/angle - contains power and angle data
        self.data_type = ''

        # transceiver_type stores the hardware identifier of the transceiver
        # used to collect the data. For EK60 on ER60 collection, this is always
        # GPT.
        self.transceiver_type = 'GPT'

        # Data_attributes is an internal list that contains the names of all
        # of the class's "data" properties. The echolab2 package uses this
        # attribute to generalize various functions that manipulate these
        # data.  Here we *extend* the list that is defined in the parent class.
        self._data_attributes += ['configuration',
                                  'transducer_depth',
                                  'frequency',
                                  'transmit_power',
                                  'pulse_length',
                                  'bandwidth',
                                  'sample_interval',
                                  'sound_velocity',
                                  'absorption_coefficient',
                                  'temperature',
                                  'transmit_mode',
                                  'sample_offset',
                                  'sample_count']


        #  create a list that stores the scalar object attributes
        self._obj_attributes = ['rolling_array',
                                'chunk_width',
                                'store_power',
                                'store_angles',
                                'max_sample_number',
                                'data_type',
                                'transceiver_type']


    def empty_like(self, n_pings, empty_times=True, no_data=False):
        """Returns raw_data object with data arrays filled with NaNs.

        The raw_data object has the same general characteristics of "this"
        object, but with data arrays filled with NaNs.

        Args:
            n_pings (int): Set n_pings to an integer specifying the number of
                pings in the new object. The vertical axis (both number of
                samples and depth/range values) will be the same as this object.
            empty_times (bool) Set to True to return an object with ping times
                set to NaT. Set to False to copy the ping times from this object.
        """

        # Create an instance of echolab2.EK60.raw_data and set the same basic
        # properties as this object.  Return the empty raw_data object.
        empty_obj = raw_data(self.channel_id, n_pings=n_pings,
                n_samples=self.n_samples, rolling=self.rolling_array,
                store_power=self.store_power, store_angles=self.store_angles,
                max_sample_number=self.max_sample_number)

        empty_obj.data_type = self.data_type

        return self._like(empty_obj, n_pings, np.nan, empty_times=empty_times,
                no_data=no_data)


    def copy(self):
        """creates a deep copy of this object."""

        # Create an empty raw_data object with the same basic props as this object.
        rd_copy = raw_data(self.channel_id)

        # Call the parent _copy helper method and return the result.
        return self._copy(rd_copy)



    def append_ping(self, sample_datagram, config_params, start_sample=None,
            end_sample=None):
        """Adds a "pings" worth of data to the object.

        This method extracts data from the provided sample_datagram dict and
        inserts it into the data arrays. Managing the data array sizes is the
        bulk of what this method does:

        If the raw_data.rolling_array attribute is false, columns are added to
        the data arrays as needed. To reduce overhead, the arrays are extended in
        chunks, not on a ping by ping basis. If the recording range or the pulse
        length changes requiring additional rows to be added, the data arrays will be
        resized to accomodate the maximum number of samples being stored. Existing
        samples are padded with NaNs as required. This vertical resize does not
        occur in chunks and the data are copied each time samples are added.
        This can have significant performance impacts in very specific cases.

        If raw_data.rolling_array is true, the data arrays are not resized but
        the data within the arrays is "rolled" or shifted left and the column at
        index 0 is moved to index n_pings - 1. Additional samples are discarded
        and pings with fewer samples are padded with NaNs as required.

        This method is typically only called by the EK60 class when reading a raw file.

        Args:
            sample_datagram (dict): The dictionary containing the parsed sample datagram.
            config_datagram (dict): The dictionary containing the parsed XML configuration
                                    datagram that goes with this sample data.
            start_sample (int):     The first sample to extract. Samples less than start_sample
                                    will be discarded. Default (None) starts with the first
                                    sample.
            end_sample (int):       The last sample to extract. Samples greater than end_sample
                                    will be discarded. Default (None) ends with the last
                                    sample.
        """

        # Set some defaults
        power_samps = -1
        angle_samps = -1
        max_data_samples = []

        # The contents of the first ping appended to a raw_data object defines
        # the data_type and determines how the data arrays will be created. Also,
        # since a raw_data object can only store one data_type, we disable saving
        # of the other types.
        if self.n_pings == -1:

            # Set defaults
            create_power = False
            create_angles = False

            # This must be a power, angle, or power/angle datagram
            # Check if either or both are provided
            if  sample_datagram['power'] is not None:
                create_power = True
                power_samps = sample_datagram['power'].shape[0]
                self.data_type = 'power'

            if sample_datagram['angle'] is not None:
                create_angles = True
                angle_samps = sample_datagram['angle'].shape[0]
                if create_power:
                    self.data_type = 'power/angle'
                else:
                    self.store_power = False
                    self.data_type = 'angle'

            # At this point if we're power, we're not storing angle data
            if self.data_type == 'power':
                self.store_angles = False

            #  determine the initial number of samples in our arrays
            if self.max_sample_number:
                n_samples = self.max_sample_number
            else:
                n_samples = max([angle_samps, power_samps])

            #  set the initial sample offset
            self.sample_offset = sample_datagram['offset']

            #  Initialize the data arrays
            self._create_arrays(self.chunk_width, n_samples, initialize=False,
                    create_power=create_power, create_angles=create_angles)

            # Initialize the ping counter to indicate that our data arrays
            # have been allocated.
            self.n_pings  = 0

        #  determine the number of samples in this datagram as well
        # as the number of samples in our data array(s).
        if self.store_angles:
            angle_samps = sample_datagram['angle'].shape[0]
            max_data_samples.append(self.angles_alongship_e.shape[1])
            max_data_samples.append(self.angles_athwartship_e.shape[1])
        if self.store_power:
            power_samps = sample_datagram['power'].shape[0]
            max_data_samples.append(self.power.shape[1])

        max_data_samples = max(max_data_samples)
        max_new_samples = max([power_samps, angle_samps])

        # Check if we need to truncate the sample data.
        if self.max_sample_number and (max_new_samples > self.max_sample_number):
            max_new_samples = self.max_sample_number
            if self.store_angles:
                sample_datagram['angle'] = \
                        sample_datagram['angle'][0:self.max_sample_number]
            if self.store_power:
                sample_datagram['power'] = \
                        sample_datagram['power'][0:self.max_sample_number]

        # Create 2 variables to store our current array size.
        ping_dims = self.ping_time.size
        sample_dims = max_data_samples

        # Check if we need to re-size or roll our data arrays.
        if self.rolling_array == False:
            # Check if we need to resize our data arrays.
            ping_resize = False
            sample_resize = False

            # Check the ping dimension.
            if self.n_pings == ping_dims:
                # Need to resize the ping dimension.
                ping_resize = True
                # Calculate the new ping dimension.
                ping_dims = ping_dims + self.chunk_width

            # Check the samples dimension.
            if max_new_samples > max_data_samples:
                # Need to resize the samples dimension.
                sample_resize = True
                # Calculate the new samples dimension.
                sample_dims = max_new_samples

            # Determine if we resize.
            if ping_resize or sample_resize:
                # We call the parent method's resize here to avoid updating
                # our n_pings attribute while reading data.
                super(raw_data, self).resize(ping_dims, sample_dims)
                #elf.resize(ping_dims, sample_dims)

            # Get an index into the data arrays for this ping and increment
            # our ping counter.
            this_ping = self.n_pings
            self.n_pings += 1

        else:
            # Check if we need to roll.
            if self.n_pings == ping_dims - 1:
                # When a rolling array is "filled" we stop incrementing the
                # ping counter and repeatedly append pings to the last ping
                # index in the array.
                this_ping = self.n_pings

                # Roll our array 1 ping.
                self._roll_arrays(1)

        # Update the channel configuration dict's end_* values
        config_params['end_ping'] = self.n_pings
        config_params['end_time'] = sample_datagram['timestamp']

        # Insert the config, environment, and filter object references for this ping.
        self.configuration[this_ping] = config_params

        # Now insert the data into our numpy arrays.
        self.ping_time[this_ping] = sample_datagram['timestamp']
        self.transducer_depth[this_ping] = sample_datagram['transducer_depth']
        self.frequency[this_ping] = sample_datagram['frequency']
        self.transmit_power[this_ping] = sample_datagram['transmit_power']
        self.pulse_length[this_ping] = sample_datagram['pulse_length']
        self.bandwidth[this_ping] = sample_datagram['bandwidth']
        self.sample_interval[this_ping] = sample_datagram['sample_interval']
        self.sound_velocity[this_ping] = sample_datagram['sound_velocity']
        self.absorption_coefficient[this_ping] = sample_datagram['absorption_coefficient']
        self.temperature[this_ping] = sample_datagram['temperature']
        self.transmit_mode[this_ping] = sample_datagram['transmit_mode']

        # Update sample count and sample offset values
        if start_sample:
            self.sample_offset[this_ping] = start_sample + sample_datagram['offset']

            if end_sample:
                self.sample_count[this_ping] = end_sample - start_sample + 1
            else:
                self.sample_count[this_ping] = sample_datagram['count'] - \
                                               start_sample
        else:
            self.sample_offset[this_ping] = sample_datagram['offset']
            start_sample = 0
            if end_sample:
                self.sample_count[this_ping] = end_sample + 1
            else:
                self.sample_count[this_ping] = sample_datagram['count']
                end_sample = sample_datagram['count']

        # Now store the "sample" data.

        # Check if we need to store power data.
        if self.store_power:
            if sample_datagram['power'].size > 0:
                # Get the subset of samples we're storing.
                power = sample_datagram['power'][start_sample:self.sample_count[this_ping]]

                # Convert the indexed power data to power dB.
                power = power.astype(self.sample_dtype) * self.INDEX2POWER

                # Check if we need to pad or trim our sample data.
                sample_pad = sample_dims - power.shape[0]
                if sample_pad > 0:
                    # The data array has more samples than this datagram - we
                    # need to pad the datagram.
                    self.power[this_ping,:] = np.pad(power,(0,sample_pad),
                            'constant', constant_values=np.nan)
                elif sample_pad < 0:
                    # The data array has fewer samples than this datagram - we
                    # need to trim the datagram.
                    self.power[this_ping,:] = power[0:sample_pad]
                else:
                    # The array has the same number of samples.
                    self.power[this_ping,:] = power
            else:
                # This is an empty ping
                self.power[this_ping,:] = np.nan

        # Check if we need to store angle data.
        if self.store_angles:
            if sample_datagram['angle'].size > 0:
                # Convert from indexed to electrical angles.
                alongship_e = sample_datagram['angle'][start_sample:self.sample_count[this_ping], 1].astype(self.sample_dtype)
                alongship_e *= self.INDEX2ELEC
                athwartship_e = sample_datagram['angle'][start_sample:self.sample_count[this_ping], 0].astype(self.sample_dtype)
                athwartship_e *= self.INDEX2ELEC

                # Check if we need to pad or trim our sample data.
                sample_pad = sample_dims - athwartship_e.shape[0]
                if sample_pad > 0:
                    # The data array has more samples than this datagram - we
                    # need to pad the datagram
                    self.angles_alongship_e[this_ping,:] = np.pad(alongship_e,
                            (0,sample_pad), 'constant', constant_values=np.nan)
                    self.angles_athwartship_e[this_ping,:] = np.pad(
                        athwartship_e,(0,sample_pad), 'constant', constant_values=np.nan)
                elif sample_pad < 0:
                    # The data array has fewer samples than this datagram - we
                    # need to trim the datagram.
                    self.angles_alongship_e[this_ping,:] = alongship_e[0:sample_pad]
                    self.angles_athwartship_e[this_ping,:] = athwartship_e[0:sample_pad]
                else:
                    # The array has the same number of samples.
                    self.angles_alongship_e[this_ping,:] = alongship_e
                    self.angles_athwartship_e[this_ping,:] = athwartship_e

            else:
                # This is an empty ping
                self.angles_alongship_e[this_ping,:] = np.nan
                self.angles_athwartship_e[this_ping,:] = np.nan


    def get_calibration(self, **kwargs):
        """Returns a calibration object populated from the data contained in this
        raw_data object.

        Calibration objects are passed to methods that transform the raw data
        to more useful forms and provide those methods the parameters they require
        such as sound speed, salinity, sampling interval, etc. The

        Args:

            absorption_method (str): Set to 'F&G' to use the method published by
                Francois and Garrison to compute absorption. This is the default
                in Echoview. You can also specify ‘A&M’ to use the method
                published by Ainslie and McColm.
                Default: 'F&G'

        Returns:
            A ek60_calibration object containing values from the raw data.
        """

        # Create an empty ek60 calibration object
        cal_obj = ek60_calibration(**kwargs)

        # Populate it using the data from "this" object
        cal_obj.from_raw_data(self)

        return cal_obj


    def get_power(self, calibration=None, **kwargs):
        """Returns a processed data object that contains the power data.

        This method performs all of the required transformations to place the
        raw power data into a rectangular array where all samples share the same
        thickness and are correctly arranged relative to each other.

        This process happens in 3 steps:

                Data are resampled so all samples have the same thickness.
                Data are shifted vertically to account for the sample offsets.
                Data are then regridded to a fixed time, range grid.

        Each step is performed only when required. Calls to this method will
        return much faster if the raw data share the same sample thickness,
        offset and sound speed.

        If calibration is set to an instance of EK60.ek60_ calibration the
        values in that object (if set) will be used when performing the
        transformations required to return the results. If the required
        parameters are not set in the calibration object or if no object is
        provided, this method will extract these parameters from the raw file
        data.

        Args:

            resample_interval (float): Set this to a float specifying the
                sampling interval (in seconds) used when generating the
                vertical axis for the return data. If the raw data sampling
                interval is different than the specified interval, the raw
                data will be resampled at the specified rate. 0 and 1 have
                special meaning. If set to 0 or 1, the data will only be resampled
                if the sampling interval changes. If it does change, when
                set to 0, the data will be resampled to the shortest sampling
                interval present in the data. If set to 1, it will be resampled
                to the longest interval present in the data.

                The following constants are defined in the class:

                    RESAMPLE_SHORTEST = 0
                    RESAMPLE_16   = 0.000016
                    RESAMPLE_32  = 0.000032
                    RESAMPLE_64  = 0.000064
                    RESAMPLE_128  = 0.000128
                    RESAMPLE_256 = 0.000256
                    RESAMPLE_512 = 0.000512
                    RESAMPLE_1024 = 0.001024
                    RESAMPLE_2048 = 0.002048
                    RESAMPLE_LONGEST = 1

                Default: RESAMPLE_SHORTEST

            return_indices (np.array uint32): Set this to a numpy array that contains
                the index values to return in the processed data object. This can be
                used for more advanced anipulations where start/end ping/time are
                inadequate.

            calibration (EK60.ek60_calibration): Set to an instance of
                EK60.ek60_calibration containing the calibration parameters
                you want to use when transforming to Sv/sv. If no calibration
                object is provided, the values will be extracted from the raw
                data.

            start_time (datetime64): Set to a numpy datetime64 oject specifying
                the start time of the data to convert. All data between the start
                and end time will be returned. If set to None, the start time is
                the first ping.
                Default: None

            end_time (datetime64): Set to a numpy datetime64 oject specifying
                the end time of the data to convert. All data between the start
                and end time will be returned. If set to None, the end time is
                the last ping.
                Default: None

            start_ping (int): Set to an integer specifying the first ping number
                to return. All pings between the start and end ping will be
                returned. If set to None, the first ping is set as the start ping.
                Default: None

            end_ping (int): Set to an integer specifying the end ping number
                to return. All pings between the start and end ping will be
                returned. If set to None, the last ping is set as the end ping.
                Default: None

            Note that you can set a start/end time OR a start/end ping. If you
            set both, one will be ignored.

            time_order (bool): Set to True to return data in time order. If
                False, data will be returned in the order it was read.
                Default: True

        Returns:
            A processed_data object containing power data.

        """

        # Check if user provided a cal object
        if calibration is None:
            # No - get one populated from raw data
            calibration = self.get_calibration()

        # Call the _get_sample_data method requesting the appropriate sample attribute.
        if hasattr(self, 'power'):
            p_data, return_indices = self._get_sample_data('power', calibration=calibration,
                    **kwargs)
        else:
            raise AttributeError('raw_data object does not contain power ' +
                    'data required to return, er, power.')

        # Set the data type.
        p_data.data_type = 'power'

        # Set the is_log attribute and return it.
        p_data.is_log = True

        return p_data


    def _get_power(self, calibration=None, **kwargs):
        """Returns a processed data object that contains the power data and
        an index array.

        This method is identical to get_power except that it also returns an
        index array that maps the pings in the processed_data object to the
        same pings in the "this" object. This is used internally.
        """

        # Check if user provided a cal object
        if calibration is None:
            # No - get one populated from raw data
            calibration = self.get_calibration()

        # Call the _get_sample_data method requesting the appropriate sample attribute.
        if hasattr(self, 'power'):
            p_data, return_indices = self._get_sample_data('power', calibration=calibration,
                    **kwargs)
        else:
            raise AttributeError('Raw data object does not contain power ' +
                    'data required to return, er, power.')

        # Set the data type.
        p_data.data_type = 'power'

        # Set the is_log attribute and return it.
        p_data.is_log = True

        return p_data, return_indices


    def get_sv(self, calibration=None, **kwargs):
        """Gets sv data.

        This is a convenience method which simply calls get_Sv and forces
        the linear keyword to True.

        Args:
            See getSv for arguments.

        Returns:
            A processed_data object containing sv.

        """

        # Check if user provided a cal object
        if calibration is None:
            # No - get one populated from raw data
            calibration = self.get_calibration()

        # Remove the linear keyword.
        kwargs.pop('linear', None)

        # Call get_Sp forcing linear to True.
        return self.get_Sv(linear=True, calibration=calibration, **kwargs)


    def get_Sv(self, calibration=None, linear=False, tvg_correction=True,
               return_depth=False, **kwargs):
        """Gets Sv data

        This method returns a processed_data object containing Sv or sv data.

        This method performs all of the required transformations to place the
        raw power data into a rectangular array where all samples share the same
        thickness and are correctly arranged relative to each other. It then
        computes Sv/sv as follows:

            Sv = power + 20 log10(Range) + (2 *  alpha * Range) - (10 * ...
                log10((TransmitPower * (10^(Gain/10))^2 * lambda^2 * ...
                c * tau * 10^(psi/10)) / (32 * pi^2)) - (2 * SaCorrection)

        Args:
            resample_interval (float): Set this to a float specifying the
                sampling interval (in seconds) used when generating the
                vertical axis for the return data. If the raw data sampling
                interval is different than the specified interval, the raw
                data will be resampled at the specified rate. 0 and 1 have
                special meaning. If set to 0 or 1, the data will only be resampled
                if the sampling interval changes. If it does change, when
                set to 0, the data will be resampled to the shortest sampling
                interval present in the data. If set to 1, it will be resampled
                to the longest interval present in the data.

                The following constants are defined in the class:

                    RESAMPLE_SHORTEST = 0
                    RESAMPLE_16   = 0.000016
                    RESAMPLE_32  = 0.000032
                    RESAMPLE_64  = 0.000064
                    RESAMPLE_128  = 0.000128
                    RESAMPLE_256 = 0.000256
                    RESAMPLE_512 = 0.000512
                    RESAMPLE_1024 = 0.001024
                    RESAMPLE_2048 = 0.002048
                    RESAMPLE_LONGEST = 1

                Default: RESAMPLE_SHORTEST

            return_indices (np.array uint32): Set this to a numpy array that contains
                the index values to return in the processed data object. This can be
                used for more advanced anipulations where start/end ping/time are
                inadequate.

            calibration (EK60.ek60_calibration): Set to an instance of
                EK60.ek60_calibration containing the calibration parameters
                you want to use when transforming to Sv/sv. If no calibration
                object is provided, the values will be extracted from the raw
                data.

            linear (bool): Set to True if getting "sv" data
                Default: False

            tvg_correction (bool): Set to True to apply TVG range correction.
                Typically you want to leave this at True.
                Default: True

            return_depth (bool): Set to True to return a processed_data object
                with a depth axis. When False, the processed_data object has
                a range axis.
                Default: False

            start_time (datetime64): Set to a numpy datetime64 oject specifying
                the start time of the data to convert. All data between the start
                and end time will be returned. If set to None, the start time is
                the first ping.
                Default: None

            end_time (datetime64): Set to a numpy datetime64 oject specifying
                the end time of the data to convert. All data between the start
                and end time will be returned. If set to None, the end time is
                the last ping.
                Default: None

            start_ping (int): Set to an integer specifying the first ping number
                to return. All pings between the start and end ping will be
                returned. If set to None, the first ping is set as the start ping.
                Default: None

            end_ping (int): Set to an integer specifying the end ping number
                to return. All pings between the start and end ping will be
                returned. If set to None, the last ping is set as the end ping.
                Default: None

            Note that you can set a start/end time OR a start/end ping. If you
            set both, one will be ignored.

            time_order (bool): Set to True to return data in time order. If
                False, data will be returned in the order it was read.
                Default: True

        Returns:
            A processed_data object containing Sv (or sv if linear is True).

        """
        # If we're not given a cal object, create an empty one
        if calibration is None:
            calibration = self.get_calibration() #ek60_calibration()

        # Get the power data - this step also resamples and arranges the raw data.
        p_data, return_indices = self._get_power(calibration=calibration, **kwargs)

        # Set the data type and is_log attribute.
        if linear:
            attribute_name = 'sv'
            p_data.is_log = False

        else:
            attribute_name = 'Sv'
            p_data.is_log = True
        p_data.data_type = attribute_name

        # Convert power to Sv/sv.
        sv_data = self._convert_power(p_data, calibration, attribute_name,
                linear, return_indices, tvg_correction)

        # Set the data attribute in the processed_data object.
        p_data.data = sv_data

        # Check if we need to convert to depth
        if return_depth:
            p_data.to_depth(calibration)

        return p_data


    def get_sp(self, **kwargs):
        """Gets sp data.

        This method returns a processed_data object containing sp data. This is
        a convenience method which simply calls get_Sp and forces the linear
        keyword to True.

        See get_Sp for more detail.

        Args:
            See get_Sp for argument descriptions.

        Returns:
            returns a processed_data object containing sp
        """

        # Remove the linear keyword.
        kwargs.pop('linear', None)

        # Call get_Sp, forcing linear to True.
        return self.get_Sp(linear=True, **kwargs)


    def get_Sp(self,  calibration=None, linear=False, tvg_correction=False,
            return_depth=False, **kwargs):
        """Gets Sp data.

        This method returns a processed_data object containing Sp or sp data.

        This method performs all of the required transformations to place the
        raw power data into a rectangular array where all samples share the same
        thickness and are correctly arranged relative to each other. It then
        computes Sp as follows:

             Sp = power + 40 * log10(Range) + (2 *  alpha * Range) - (10
             * ... log10((TransmitPower * (10^(gain/10))^2 * lambda^2) / (16 *
             pi^2)))

        Args:
            resample_interval (float): Set this to a float specifying the
                sampling interval (in seconds) used when generating the
                vertical axis for the return data. If the raw data sampling
                interval is different than the specified interval, the raw
                data will be resampled at the specified rate. 0 and 1 have
                special meaning. If set to 0 or 1, the data will only be resampled
                if the sampling interval changes. If it does change, when
                set to 0, the data will be resampled to the shortest sampling
                interval present in the data. If set to 1, it will be resampled
                to the longest interval present in the data.

                The following constants are defined in the class:

                    RESAMPLE_SHORTEST = 0
                    RESAMPLE_16   = 0.000016
                    RESAMPLE_32  = 0.000032
                    RESAMPLE_64  = 0.000064
                    RESAMPLE_128  = 0.000128
                    RESAMPLE_256 = 0.000256
                    RESAMPLE_512 = 0.000512
                    RESAMPLE_1024 = 0.001024
                    RESAMPLE_2048 = 0.002048
                    RESAMPLE_LONGEST = 1

                Default: RESAMPLE_SHORTEST

            return_indices (np.array uint32): Set this to a numpy array that contains
                the index values to return in the processed data object. This can be
                used for more advanced anipulations where start/end ping/time are
                inadequate.

            calibration (EK60.ek60_calibration): Set to an instance of
                EK60.ek60_calibration containing the calibration parameters
                you want to use when transforming to Sv/sv. If no calibration
                object is provided, the values will be extracted from the raw
                data.

            linear (bool): Set to True if getting "sv" data
                Default: False

            tvg_correction (bool): Set to True to apply TVG range correction.
                By default, TVG range correction is not applied to the data. This
                results in output that is consistent with the Simrad "P" telegram
                and TS data exported from Echoview version 4.3 and later (prior
                versions applied the correction by default).

                If you intend to perform single target detections you must apply
                the TVG range correction at some point in your process. This can
                be done by either setting the tvgCorrection keyword of this function
                or it can be done as part of your single target detection routine.
                Default: False

            return_depth (bool): Set to True to return a processed_data object
                with a depth axis. When False, the processed_data object has
                a range axis.
                Default: False

            start_time (datetime64): Set to a numpy datetime64 oject specifying
                the start time of the data to convert. All data between the start
                and end time will be returned. If set to None, the start time is
                the first ping.
                Default: None

            end_time (datetime64): Set to a numpy datetime64 oject specifying
                the end time of the data to convert. All data between the start
                and end time will be returned. If set to None, the end time is
                the last ping.
                Default: None

            start_ping (int): Set to an integer specifying the first ping number
                to return. All pings between the start and end ping will be
                returned. If set to None, the first ping is set as the start ping.
                Default: None

            end_ping (int): Set to an integer specifying the end ping number
                to return. All pings between the start and end ping will be
                returned. If set to None, the last ping is set as the end ping.
                Default: None

            Note that you can set a start/end time OR a start/end ping. If you
            set both, one will be ignored.

            time_order (bool): Set to True to return data in time order. If
                False, data will be returned in the order it was read.
                Default: True

        Returns:
            A processed_data object containing Sp (or sp if linear is True).

        """
        # Check if user provided a cal object
        if calibration is None:
            # No - get one populated from raw data
            calibration = self.get_calibration()

        # Get the power data - this step also resamples and arranges the raw data.
        p_data, return_indices = self._get_power(calibration=calibration, **kwargs)

        # Set the data type.
        if linear:
            attribute_name = 'sp'
            p_data.is_log = False
        else:
            attribute_name = 'Sp'
            p_data.is_log = True
        p_data.data_type = attribute_name

        # Convert
        sp_data = self._convert_power(p_data, calibration, attribute_name,
                linear, return_indices, tvg_correction)

        # Set the data attribute in the processed_data object.
        p_data.data = sp_data

        # Check if we need to convert to depth or heave correct.
        if return_depth:
            p_data.to_depth(calibration)

        return p_data


    def get_bottom(self, calibration=None, return_indices=None,
            return_depth=False, **kwargs):
        """Gets a echolab2 line object containing the sounder detected bottom
        depths.

        The sounder detected bottom depths are computed using the sound speed
        setting at the time of recording. If you are applying a different sound
        speed setting via the calibration argument when getting the converted
        sample data, you must also pass the same calibration object to this method
        to ensure that the sounder detected bottom depths align with your sample
        data.

        Also, if you request sample data on a depth grid, you should set the
        return_depth keyword here too.

        Args:
            return_indices (np.array uint32): Set this to a numpy array that contains
                the index values to return in the processed data object. This can be
                used for more advanced anipulations where start/end ping/time are
                inadequate.

            calibration (EK60.ek60_calibration): Set to an instance of
                EK60.ek60_calibration containing the calibration parameters
                you want to use when transforming to Sv/sv. If no calibration
                object is provided, the values will be extracted from the raw
                data.

            return_depth (bool): Set to True to return a line object with a
                with a depth axis. When False, the line object has a range axis.
                Default: False

            start_time (datetime64): Set to a numpy datetime64 oject specifying
                the start time of the data to convert. All data between the start
                and end time will be returned. If set to None, the start time is
                the first ping.
                Default: None

            end_time (datetime64): Set to a numpy datetime64 oject specifying
                the end time of the data to convert. All data between the start
                and end time will be returned. If set to None, the end time is
                the last ping.
                Default: None

            start_ping (int): Set to an integer specifying the first ping number
                to return. All pings between the start and end ping will be
                returned. If set to None, the first ping is set as the start ping.
                Default: None

            end_ping (int): Set to an integer specifying the end ping number
                to return. All pings between the start and end ping will be
                returned. If set to None, the last ping is set as the end ping.
                Default: None

            Note that you can set a start/end time OR a start/end ping. If you
            set both, one will be ignored.

            time_order (bool): Set to True to return data in time order. If
                False, data will be returned in the order it was read.
                Default: True

        Raises:
            ValueError: The return indices exceed the number of pings in
                the raw data object

        Returns:
            A line object containing the sounder detected bottom depths.

        """

        # Check if the user supplied an explicit list of indices to return.
        if isinstance(return_indices, np.ndarray):
            if max(return_indices) > self.ping_time.shape[0]:
                raise ValueError("One or more of the return indices provided " +
                        "exceeds the number of pings in the raw_data object")
        else:
            # Get an array of index values to return.
            return_indices = self.get_indices(**kwargs)

        # Check if user provided a cal object
        if calibration is None:
            # No - create an empty one - all cal values will come from the raw data
            calibration = ek60_calibration()

        # Extract the recorded sound velocity.
        sv_recorded = self.sound_velocity[return_indices]

        # Get the calibration params required for detected depth conversion.
        cal_parms = {'sound_velocity':None,
                     'transducer_depth':None}

        # Next, iterate through the dict, calling the method to extract the
        # values for each parameter.
        for key in cal_parms:
            cal_parms[key] = calibration.get_parameter(self, key,
                    return_indices)

        # Check if we have to adjust the depth due to a change in sound speed.
        if np.all(np.isclose(sv_recorded, cal_parms['sound_velocity'])):
            converted_depths = self.detected_bottom[return_indices]
        else:
            cf = cal_parms['sound_velocity'].astype('float') / sv_recorded
            converted_depths = cf * self.detected_bottom[return_indices]

        # Check if we're returning range by subtracting a transducer offset.
        if return_depth == False:
            # Data is recorded as depth - convert to range.
            converted_depths -= cal_parms['transducer_depth'][return_indices]

        # Create a line object to return with our adjusted data.
        bottom_line = line.line(ping_time=self.ping_time[return_indices],
                data=converted_depths, **kwargs)

        return bottom_line


    def get_physical_angles(self, calibration=None, **kwargs):
        """Gets the alongship and athwartship angle data.

        This method returns an tuple of processed data objects (alongship,
        athwartship) containing the physical angle data.

        This method performs all of the required transformations to place the
        raw electrical angle data into rectangular arrays where all samples
        share the same thickness and are correctly arranged relative to each other.
        It then transform the electrical angles into physical angles.

        Args:
            return_indices (np.array uint32): Set this to a numpy array that contains
                the index values to return in the processed data object. This can be
                used for more advanced anipulations where start/end ping/time are
                inadequate.

            calibration (EK60.ek60_calibration): Set to an instance of
                EK60.ek60_calibration containing the calibration parameters
                you want to use when transforming to Sv/sv. If no calibration
                object is provided, the values will be extracted from the raw
                data.

            return_depth (bool): Set to True to return a line object with a
                with a depth axis. When False, the line object has a range axis.
                Default: False

            start_time (datetime64): Set to a numpy datetime64 oject specifying
                the start time of the data to convert. All data between the start
                and end time will be returned. If set to None, the start time is
                the first ping.
                Default: None

            end_time (datetime64): Set to a numpy datetime64 oject specifying
                the end time of the data to convert. All data between the start
                and end time will be returned. If set to None, the end time is
                the last ping.
                Default: None

            start_ping (int): Set to an integer specifying the first ping number
                to return. All pings between the start and end ping will be
                returned. If set to None, the first ping is set as the start ping.
                Default: None

            end_ping (int): Set to an integer specifying the end ping number
                to return. All pings between the start and end ping will be
                returned. If set to None, the last ping is set as the end ping.
                Default: None

            Note that you can set a start/end time OR a start/end ping. If you
            set both, one will be ignored. If you provide an index array, both
            ping and time bounds will be ignored.

            time_order (bool): Set to True to return data in time order. If
                False, data will be returned in the order it was read.
                Default: True

        Returns:
            Two rocessed_data objects with alongship and athwartship angle data.
        """
        # Check if user provided a cal object
        if calibration is None:
            # No - get one populated from raw data
            calibration = self.get_calibration()

        # Get the electrical angles.
        alongship, athwartship, return_indices = \
                self._get_electrical_angles(calibration=calibration, **kwargs)

        # Get the calibration params required for angle conversion.
        cal_parms = {'angle_sensitivity_alongship':None,
                     'angle_sensitivity_athwartship':None,
                     'angle_offset_alongship':None,
                     'angle_offset_athwartship':None}

        # Next, iterate through the dict, calling the method to extract the
        # values for each parameter.
        for key in cal_parms:
            cal_parms[key] = calibration.get_parameter(self, key,
                    return_indices)

        # Compute the physical angles.
        alongship.data /= cal_parms['angle_sensitivity_alongship'][:, np.newaxis]
        alongship.data -= cal_parms['angle_offset_alongship'][:, np.newaxis]
        athwartship.data /= cal_parms['angle_sensitivity_athwartship'][:, np.newaxis]
        athwartship.data -= cal_parms['angle_offset_athwartship'][:, np.newaxis]

        # Set the data types.
        alongship.data_type = 'angles_alongship'
        athwartship.data_type = 'angles_athwartship'

        # We do not need to convert to depth here since the electrical_angle
        # data will already have been converted to depth if requested.

        return alongship, athwartship


    def get_electrical_angles(self, return_depth=False, calibration=None, **kwargs):
        """Gets unconverted angle data.

        This method returns a tuple of processed_data objects containing angle data
        as electrical angles.

        See get_physical_angles for more detail.

        Args:
            See get_physical_angles for argument descriptions.

        Returns:
            Two processed data objects containing the unconverted
            angles_alongship_e and angles_athwartship_e data.
        """
        # Check if user provided a cal object
        if calibration is None:
            # No - get one populated from raw data
            calibration = self.get_calibration()

        # Call the _get_sample_data method requesting the 'angles_alongship_e'
        # sample attribute. The method will return a reference to a newly created
        # processed_data object.
        alongship, return_indices = self._get_sample_data('angles_alongship_e',
                calibration=calibration, **kwargs)

        # Repeat for the athwartship data.
        athwartship, return_indices = self._get_sample_data('angles_athwartship_e',
                calibration=calibration, **kwargs)

        # Set the data type.
        alongship.data_type = 'angles_alongship_e'
        athwartship.data_type = 'angles_athwartship_e'

        # Covert to depth
        if return_depth:
            alongship.to_depth(calibration)
            athwartship.to_depth(calibration)

        return alongship, athwartship


    def resize(self, new_ping_dim, new_sample_dim):
        """Resizes raw_data object.

        This method re-implements ping_data.resize, updating n_pings
        attribute after resizing.

        Args:
            new_ping_dim (int): Used to resize the sample data array
                (horizontal axis).
            new_sample_dim (int): Used to resize the sample data array
                (vertical axis).
        """
        # Call the parent method to resize the arrays (n_samples is updated here).
        super(raw_data, self).resize(new_ping_dim, new_sample_dim)

        # Update n_pings.
        self.n_pings = self.ping_time.shape[0]


    def match_pings(self, other_data, match_to='cs'):
        """Matches the ping times in this object to the ping times in the EK60.raw_data
        object provided. It does this by matching times, inserting and/or deleting
        pings as needed. It does not interpolate. Ping times in the other object
        that aren't in this object are inserted. Ping times in this object that
        aren't in the other object are deleted. If the time axes do not intersect
        at all, all of the data in this object will be deleted and replaced with
        empty pings for the ping times in the other object.


        Args:
            other_data (ping_data): A ping_data type object that this object
            will be matched to.

            match_to (str): Set to a string defining the precision of the match.

                cs : Match to a 100th of a second
                ds : Match to a 10th of a second
                s  : Match to the second

        Returns:
            A dictionary with the keys 'inserted' and 'removed' containing the
            indices of the pings inserted and removed.
        """
        return super(raw_data, self).match_pings(other_data, match_to='cs')


    def _get_electrical_angles(self, return_depth=False, calibration=None,
            **kwargs):
        """Retrieves unconverted angles_alongship_e and angles_athwartship_e
        data and creates an index array mapping pings.

        _get_electrical_angles is identical to get_electrical_angles except
        that it also returns an index array that maps the pings in the
        processed_data object to the same pings in the "this" object. This is
        used internally.

        Args:
            return_depth (bool): If true, return the vertical axis of the
                data as depth.  Otherwise, return as range.
            calibration (calibration object): The data calibration object where
                calibration data will be retrieved.
            **kwargs

        Returns:
            Two processed data objects containing the unconverted
            angles_alongship_e and angles_athwartship_e data and an index array
            mapping pings to this object.
        """
        # Check if user provided a cal object
        if calibration is None:
            # No - get one populated from raw data
            calibration = self.get_calibration()

        # Call the _get_sample_data method requesting the 'angles_alongship_e'
        # sample attribute. The method will return a reference to a newly created
        # processed_data object.
        if hasattr(self, 'angles_alongship_e') or hasattr(self, 'angles_athwartship_e'):
            if hasattr(self, 'angles_alongship_e'):
                alongship, return_indices = self._get_sample_data('angles_alongship_e',
                        calibration=calibration, **kwargs)
            else:
                raise AttributeError('Raw data object does not contain the ' +
                        'angles_alongship_e attribute required to return angle data.')

            # use the already computed return_indices from the first
            # call to _get_sample_data to get the athwartship data
            if hasattr(self, 'angles_athwartship_e'):
                kwargs.pop('return_indices', None)
                athwartship, ri = self._get_sample_data('angles_athwartship_e',
                        return_indices=return_indices, calibration=calibration, **kwargs)
            else:
                raise AttributeError('Raw data object does not contain the ' +
                        'angles_athwartship_e attribute required to return angle data.')

        else:
            # We don't have complex nor electrical angle data required so
            # we can't do anything.
            raise AttributeError('Raw data object does not contain the ' +
                        'data required to return angle data.')

        # Set the data type.
        alongship.data_type = 'angles_alongship_e'
        athwartship.data_type = 'angles_athwartship_e'

        # Apply depth and/or heave correction
        if return_depth:
            alongship.to_depth(calibration)
            athwartship.to_depth(calibration)

        return alongship, athwartship, return_indices


    def is_fm(self):
        '''Convenience method that returns True if the raw data contains FM pings.
        This method exists for API parity with the EK80 class.
        '''

        # EK60 hardware is CW only
        return False


    def is_cw(self):
        '''Convenience method that returns True if the raw data contains CW pings.
        This method exists for API parity with the EK80 class.
        '''
        # EK60 hardware is CW only
        return True


    def get_frequency(self, unique=False):
        '''Convenience method that returns the frequency of the transmit signals
        of the data stored in the raw_data object. For the EK60 this simply
        returns the frequency attribute.

        This method has the same interface as the EK80.get_frequency() method.

        '''

        if unique:
            return np.unique(self.frequency)
        else:
            return self.frequency


    def _get_sample_data(self, property_name, calibration=None,
            resample_interval=RESAMPLE_SHORTEST, return_indices=None,
            **kwargs):
        """Retrieves sample data.

        This method returns a processed data object that contains the
        sample data from the property name provided. It performs all of the
        required transformations to place the data into a rectangular
        array where all samples in all pings share the same thickness and are
        correctly arranged relative to each other.

        This process happens in 3 steps:

                Data are resampled so all samples have the same sampling interval.
                Data are shifted vertically to account for the sample offsets.
                Data are then interpolated to a common sound velocity

        Each step is performed only when required. Calls to this method will
        return much faster if the raw data share the same sample thickness,
        offset and sound speed.

        If calibration is set to an instance of an ek60_calibration object
        the values in that object (if set) will be used when performing the
        transformations required to return the results. If the required
        parameters are not set in the calibration object or if no object is
        provided, this method will extract these parameters from the raw data.

        Args:
            property_name (str): The attribute name of the sample data
                to be returned. For the EK60 the sample data attributes
                are 'complex', 'power', 'angles_alongship_e', and
                'angles_athwartship_e' and the available attributes will
                depend on how the data were collected and stored.

            calibration (ek60_calibration object): The calibration object where
                calibration data will be retrieved. If this is set to None,
                calibration data will be directly extracted from the raw data.

            resample_interval (float): The echosounder sampling interval used to
                define the vertical position of the samples. If data was collected
                with a different sampling interval it will be resampled to
                the specified interval. The default behavior is to resample
                to the shortest sampling interval in the data. This value has
                no effect when all data share the same sampling interval.

            return_indices (array): A numpy array of indices to return.

        Raises:
            ValueError: Return indices exceeds the number of pings.
            AttributeError: The attribute name doesn't exist.

        Returns:
            The processed data object containing the sample data.
        """

        def get_range_vector(num_samples, sample_interval, sound_speed,
                             sample_offset):
            """
            get_range_vector returns a NON-CORRECTED range vector.
            """
            # Calculate the thickness of samples with this sound speed.
            thickness = sample_interval * sound_speed / 2.0
            # Calculate the range vector.
            #range = (np.arange(0, num_samples) + sample_offset) * thickness
            range = (np.arange(-1, num_samples-1) + sample_offset) * thickness

            return range

        # Check if the user supplied an explicit list of indices to return.
        if isinstance(return_indices, np.ndarray):
            if max(return_indices) > self.ping_time.shape[0]:
                raise ValueError("One or more of the return indices provided "
                        "exceeds the " + "number of pings in the " + "raw_data object")
        else:
            # Get an array of index values to return.
            return_indices = self.get_indices(**kwargs)

        # Create the processed_data object we will return.
        p_data = processed_data(self.channel_id,
                self.get_frequency(unique=True)[0], None)

        # Populate it with time and ping number.
        p_data.ping_time = self.ping_time[return_indices].copy()

        # Get a reference to the data we're operating on.
        if hasattr(self, property_name):
            data = getattr(self, property_name)
        else:
            raise AttributeError("The attribute name " + property_name +
                    " does not exist.")

        # Populate the calibration parameters required for this method.
        # First, create a dict with key names that match the attributes names
        # of the calibration parameters we require for this method.
        cal_parms = {'sample_interval':None,
                     'sound_velocity':None,
                     'sample_offset':None,
                     'pos_z':None,
                     'transducer_depth':None}

        # Next, iterate through the dictionary calling the method to extract
        # the values for each parameter.
        for key in cal_parms:
            cal_parms[key] = calibration.get_parameter(self, key,
                    return_indices)

        # Check if we have multiple sample offset values and get the minimum.
        unique_sample_offsets = np.unique(
            cal_parms['sample_offset'][~np.isnan(cal_parms['sample_offset'])])
        min_sample_offset = min(unique_sample_offsets)

        # Check if we need to resample our sample data.
        unique_sample_interval = np.unique(
            cal_parms['sample_interval'][~np.isnan(cal_parms['sample_interval'])])
        if unique_sample_interval.shape[0] > 1:
            # There are at least 2 different sample intervals in the data.  We
            # must resample the data.  We'll deal with adjusting sample offsets
            # here too.
            (output, sample_interval) = self._vertical_resample(data[return_indices],
                    cal_parms['sample_interval'], unique_sample_interval, resample_interval,
                    cal_parms['sample_offset'], min_sample_offset,
                    is_power=property_name == 'power')
        else:
            # We don't have to resample, but check if we need to shift any
            # samples based on their sample offsets.
            if unique_sample_offsets.shape[0] > 1:
                # We have multiple sample offsets so we need to shift some of
                # the samples.
                output = self._vertical_shift(data[return_indices],
                        cal_parms['sample_offset'], unique_sample_offsets,
                        min_sample_offset)
            else:
                # The data all have the same sample intervals and sample
                # offsets.  Simply copy the data as is.
                output = data[return_indices].copy()

            # Get the sample interval value to use for range conversion below.
            sample_interval = unique_sample_interval[0]

        # Check if we have a fixed sound speed.
        unique_sound_velocity = np.unique(cal_parms['sound_velocity'][~np.isnan(cal_parms['sound_velocity'])])
        if unique_sound_velocity.shape[0] > 1:
            # There are at least 2 different sound speeds in the data or
            # provided calibration data.  Interpolate all data to the most
            # common range (which is the most common sound speed).
            sound_velocity = None
            n = 0
            for speed in unique_sound_velocity:
                # Determine the sound speed with the most pings.
                if np.count_nonzero(cal_parms['sound_velocity'] == speed) > n:
                   sound_velocity = speed

            # Calculate the target range.
            range = get_range_vector(output.shape[1], sample_interval,
                    sound_velocity, min_sample_offset)

            # Now interpolate the samples for pings with different sound speeds
            for speed in unique_sound_velocity:
                # Only interpolate the "other" speeds
                if speed != sound_velocity:

                    # Get an array of indexes in the output array to interpolate.
                    pings_to_interp = np.where(cal_parms['sound_velocity'] == speed)[0]

                    # Compute this data's range
                    resample_range = get_range_vector(output.shape[1], sample_interval,
                        cal_parms['sound_velocity'][pings_to_interp[0]], min_sample_offset)

                    # Interpolate samples to target range
                    interp_f = interp1d(resample_range, output[pings_to_interp,:], axis=1,
                            bounds_error=False, fill_value=np.nan, assume_sorted=True)
                    output[pings_to_interp,:] = interp_f(resample_range)

#            # Get an array of indexes in the output array to interpolate.
#            pings_to_interp = np.where(cal_parms['sound_velocity'] != sound_velocity)[0]
#
#            # Iterate through this list of pings to change.  Interpolating each
#            # ping.
#            for ping in pings_to_interp:
#                # Resample using the provided sound speed.
#                resample_range = get_range_vector(output.shape[1], sample_interval,
#                        cal_parms['sound_velocity'][ping], min_sample_offset)
#                output[ping,:] = np.interp(range, resample_range, output[ping, :])

        else:
            # We have a fixed sound speed and only need to calculate a single
            # range vector.
            sound_velocity = unique_sound_velocity[0]
            range = get_range_vector(output.shape[1], sample_interval,
                    sound_velocity, min_sample_offset)

        # Assign the results to the "data" processed_data object.
        p_data.add_data_attribute('data', output)

        # Calculate the sample thickness.
        sample_thickness = sample_interval * sound_velocity / 2.0

        # Now assign range, sound_velocity, sample thickness and offset to
        # the processed_data object.
        p_data.add_data_attribute('range', range)
        p_data.sound_velocity = sound_velocity
        p_data.sample_thickness = sample_thickness
        p_data.sample_offset = min_sample_offset
        p_data.sample_interval = sample_interval

        # Add the transducer draft attribute - We're going to assume that
        # if pos_z is set, that transducer_depth is relative to that. This
        # may not always be true. Also making the assumption that the Z
        # axis is positive out of the water.

        # Compute the draft and add the attribute
        xdcr_draft =  -cal_parms['pos_z'] + cal_parms['transducer_depth']
        p_data.add_data_attribute('transducer_draft', xdcr_draft)

        # Return the processed_data object containing the requested data.
        return p_data, return_indices


    def _convert_power(
            self, power_data, calibration, convert_to, linear,
            return_indices, tvg_correction):
        """Converts power to Sv/sv/Sp/sp

        Args:
            power_data (processed_data): A processed_data object with the raw power
                data read from the file.
            calibration (calibration object): The data calibration object where
                calibration data will be retrieved.
            convert_to (str):  A string that specifies what to convert the
                power to.  Possible values are: Sv, sv, Sp, or sp.
            linear (bool):  Set to True to return linear values.
            return_indices (array): A numpy array of indices to return.
            tvg_correction (bool): Set to True to apply a correction to the
                range of 2 * sample thickness.

        Returns:
            An array with the converted data.
        """

        # Populate the calibration parameters required for this method.
        # First, create a dictionary with key names that match the attribute
        # names of the calibration parameters we require for this method.
        cal_parms = {'gain':None,
                     'transmit_power':None,
                     'equivalent_beam_angle':None,
                     'pulse_length':None,
                     'absorption_coefficient':None,
                     'sa_correction':None}

        # Next, iterate through the dictionary, calling the method to extract
        # the values for each parameter.
        for key in cal_parms:
            cal_parms[key] = calibration.get_parameter(self, key,
                    return_indices)

        # Get sound_velocity from the power data since get_power might have
        # manipulated this value. Remember that we're operating on a
        # processed_data object so all pings share the same sound speed.
        cal_parms['sound_velocity'] = np.empty((return_indices.shape[0]),
                dtype=self.sample_dtype)
        cal_parms['sound_velocity'].fill(power_data.sound_velocity)

        # For EK60 hardware use pulse duration when computing gains
        effective_pulse_duration = cal_parms['pulse_length']

        # Calculate the system gains.
        wavelength = cal_parms['sound_velocity'] / power_data.frequency
        if convert_to in ['sv','Sv']:
            gains = 10 * np.log10((cal_parms['transmit_power'] * (10**( cal_parms['gain'] / 10.0))**2 *
                    wavelength**2 * cal_parms['sound_velocity'] * effective_pulse_duration *
                10**(cal_parms['equivalent_beam_angle']/10.0)) / (32 * np.pi**2))
        else:
            gains = 10 * np.log10((cal_parms['transmit_power'] * (10**(
                cal_parms['gain']/10.0))**2 * wavelength**2) / (16 * np.pi**2))

        # Get the range for TVG calculation.
        c_range = np.zeros(power_data.shape, dtype=power_data.sample_dtype)
        c_range += power_data.range
        c_range += power_data.sample_thickness

        if tvg_correction:
            # For the Ex60 hardware, the corrected range is computed as:
            #   c_range = range - (2 * sample_thickness)
            c_range -= (2.0 * power_data.sample_thickness)

            #  zero out negative ranges
            c_range[c_range < 0] = 0

        # Calculate time varied gain.
        tvg = c_range.copy()
        tvg[tvg < 1] = 1
        if convert_to in ['sv','Sv']:
            tvg = 20.0 * np.log10(tvg)
        else:
            tvg = 40.0 * np.log10(tvg)

        # Calculate absorption - our starting point.
        data = (2.0 * cal_parms['absorption_coefficient'])[:,np.newaxis] * c_range

        # Add in power and TVG.
        data += power_data.data
        data += tvg

        # Subtract the applied gains.
        data -= gains[:, np.newaxis]

        # Apply sa correction for Sv/sv.
        if convert_to in ['sv','Sv']:
            data -= (2.0 * cal_parms['sa_correction'])[:, np.newaxis]

        # Check if we're returning linear or log values.
        if linear:
            # Convert to linear units
            data /= 10.00
            data = 10.0 ** data

        # Return the result.
        return data


    def _create_arrays(self, n_pings, n_samples, initialize=False, create_power=False,
            create_angles=False):
        """Initializes raw_data data arrays.

        This is an internal method. Note that all arrays must be numpy arrays.

        Args:
            n_pings (int): Number of pings.
            n_samples (int): Number of samples.
            initialize (bool): Set to True to initialize arrays.
            create_power (bool): Set to True to create the power attribute.
            create_angles (bool): Set to True to create the angles_alongship_e
                                  angles_athwartship_e attributes.
        """

        # First, create uninitialized arrays.
        self.ping_time = np.empty((n_pings), dtype='datetime64[ms]')

        #  create the arrays that contain references to the async data objects
        self.configuration = np.empty((n_pings), dtype='object')

        #  the rest of the arrays store syncronous ping data
        self.transducer_depth = np.empty((n_pings), np.float32)
        self.frequency = np.empty((n_pings), np.float32)
        self.transmit_power = np.empty((n_pings), np.float32)
        self.pulse_length = np.empty((n_pings), np.float32)
        self.bandwidth = np.empty((n_pings), np.float32)
        self.sample_interval = np.empty((n_pings), np.float32)
        self.sound_velocity = np.empty((n_pings), np.float32)
        self.absorption_coefficient = np.empty((n_pings), np.float32)
        self.temperature = np.empty((n_pings), np.float32)
        self.transmit_mode = np.empty((n_pings), np.uint8)
        self.sample_offset =  np.empty((n_pings), np.uint32)
        self.sample_count = np.empty((n_pings), np.uint32)


        #  and the 2d sample data arrays
        if create_power and self.store_power:
            self.power = np.empty((n_pings, n_samples),
                dtype=self.sample_dtype, order='C')
            self.n_samples = n_samples
            self._data_attributes.append('power')
        if create_angles and self.store_angles:
            self.angles_alongship_e = np.empty((n_pings, n_samples),
                dtype=self.sample_dtype, order='C')
            self._data_attributes.append('angles_alongship_e')
            self.angles_athwartship_e = np.empty((n_pings, n_samples),
                dtype=self.sample_dtype, order='C')
            self._data_attributes.append('angles_athwartship_e')
            self.n_samples = n_samples

        #  update our shape attribute
        self._shape()

        # Check if we should initialize them.
        if initialize:
            self.ping_time.fill(np.datetime64('NaT'))
            self.transducer_depth.fill(np.nan)
            self.frequency.fill(np.nan)
            self.transmit_power.fill(np.nan)
            self.pulse_length.fill(np.nan)
            self.bandwidth.fill(np.nan)
            self.sample_interval.fill(np.nan)
            self.sound_velocity.fill(np.nan)
            self.absorption_coefficient.fill(np.nan)
            self.temperature.fill(np.nan)
            self.transmit_mode.fill(0)
            self.sample_offset.fill(0)
            self.sample_count.fill(0)

            if create_angles and self.store_power:
                self.power.fill(np.nan)
            if create_angles and self.store_angles:
                self.angles_alongship_e.fill(np.nan)
                self.angles_athwartship_e.fill(np.nan)


    def __str__(self):
        """
        Reimplemented string method that provides some basic info about the
        raw_data object.
        """

        # Print the class and address.
        msg = str(self.__class__) + " at " + str(hex(id(self))) + "\n"

        # Print some more info about the EK60.raw_data instance.
        n_pings = len(self.ping_time)
        if n_pings > 0:
            msg = msg + "                channel(s): " + self.channel_id + "\n"
            msg = msg + "    frequency (first ping): " + str(
                self.frequency[0]) + " Hz\n"
            msg = msg + " pulse length (first ping): " + str(
                self.pulse_length[0] * 1000) + " ms\n"
            msg = msg + "           data start time: " + str(
                self.ping_time[0]) + "\n"
            msg = msg + "             data end time: " + str(
                self.ping_time[n_pings-1]) + "\n"
            msg = msg + "           number of pings: " + str(n_pings) + "\n"
            if hasattr(self, 'power'):
                n_pings,n_samples = self.power.shape
                msg = msg + ("    power array dimensions: (" + str(n_pings) +
                             "," + str(n_samples) + ")\n")
            if hasattr(self, 'angles_alongship_e'):
                n_pings,n_samples = self.angles_alongship_e.shape
                msg = msg + ("    angle array dimensions: (" + str(n_pings) +
                             "," + str(n_samples) + ")\n")
        else:
            msg = msg + "  raw_data object contains no data\n"

        return msg


class ek60_calibration(calibration):
    """
    The calibration class contains parameters required for transforming power
    and electrical angle data to Sv/sv, Sp/sp and physical angles.

    When converting raw data to power, Sv/sv, Sp/sp, or to physical angles
    you have the option of passing a calibration object containing the data
    you want used during these conversions. To use this object you create an
    instance and populate the attributes with your calibration data.

    You can provide the data in 2 forms:
        As a scalar - the single value will be used for all pings.
        As a vector - a vector of values as long as the number of pings
            in the raw_data object where the first value will be used
            with the first ping, the second with the second, and so on.

    If you set any attribute to None, that attribute's values will be obtained
    from the raw_data object which contains the value at the time of recording.
    If you do not pass a calibration object to the conversion methods *all*
    of the cal parameter values will be extracted from the raw_data object.
    """

    def __init__(self, absorption_method='F&G'):
        '''Create an instance of an ek60_calibration object.

        '''

        #  call the parent init
        super(ek60_calibration, self).__init__()

        # Set up the attributes that are specific to the EK60 system. In general
        # these attribute names will match the parameter names found within the
        # EK60 formatted .raw file.

        #  these attributes are properties of the raw_data class
        self._raw_attributes = ['transducer_depth', 'frequency', 'transmit_power', 'pulse_length',
                'bandwidth' ,'sample_interval' ,'sound_velocity' ,'absorption_coefficient', 'temperature',
                'transmit_mode','sample_offset','sample_count']
        self._init_attributes(self._raw_attributes)

        #  these attributes are found in the configuration datagram
        self._config_attributes = ['transect_name', 'version' ,'survey_name', 'sounder_name',
                'spare0', 'channel_id', 'beam_type', 'frequency', 'gain', 'equivalent_beam_angle',
                'beamwidth_alongship', 'beamwidth_athwartship', 'angle_sensitivity_alongship',
                'angle_sensitivity_athwartship', 'angle_offset_alongship', 'angle_offset_athwartship',
                'pos_x', 'pos_y', 'pos_z', 'dir_x', 'dir_y','dir_z','spare2','sa_correction',
                'gpt_software_version', 'pulse_length_table','spare1','gain_table','spare3','spare4']
        self._init_attributes(self._config_attributes)

        # Add "special" attributes - These are attributes that require specific handling


    def from_raw_data(self, raw_data, return_indices=None):
        """Populates the calibration object.

        This method uses the values extracted from a raw_data object to
        populate the calibration object.

        Args:
            raw_data (raw_data): The object where parameters will be extracted
                from and used to populate the calibration object.
            return_indices (array): A numpy array of indices to return.
        """

        #  call the parent method
        super(ek60_calibration, self).from_raw_data(raw_data,
                return_indices=return_indices)

        # Handle our special attributes
        #  None for ek60_calibration


    def get_attribute_from_raw(self, raw_data, param_name, return_indices=None):
        """get_attribute_from_raw gets an individual attribute using the data
        within the provided raw_data object.
        """

        #  first call the parent method to get the "standard" attributes
        param_data, return_indices = super(ek60_calibration, self).get_attribute_from_raw(
                raw_data, param_name, return_indices=return_indices)

        # Now handle attributes that need some special handling

        # sa_correction is't explicitly defined in the calibration object. It
        # has to be pulled from the sa_correction_table.
        if param_name == 'sa_correction':
            param_data = np.empty((return_indices.shape[0]), dtype=np.float32)
            for i in range(return_indices.shape[0]):
                if not np.isnan(raw_data.pulse_length[return_indices[i]]):
                    config_obj = raw_data.configuration[return_indices[i]]
                    table_mask = np.isclose(config_obj['pulse_length_table'],
                            [raw_data.pulse_length[return_indices[i]]])
                    param_data[i] = config_obj['sa_correction_table'][table_mask][0]
                else:
                    param_data[i] = np.nan

        return param_data


    def __str__(self):
        """Re-implements string method that provides some basic info about
        the ek60_calibration object

        Returns:
            A string with information about the ek60_calibration instance.
        """

        #  print the class and address
        msg = str(self.__class__) + " at " + str(hex(id(self))) + "\n"

        # Create a list of attributes to print out - I'm just adding them
        # all right now. Can set specific attributes if this is too much.
        attr_to_display = []
        attr_to_display.extend(self._raw_attributes)
        attr_to_display.extend(self._config_attributes)

        # And assemble the string
        for param_name in attr_to_display:
            n_spaces = 32 - len(param_name)
            msg += n_spaces * ' ' + param_name
            # Extract data from raw_data attribues
            if hasattr(self, param_name):
                attr = getattr(self, param_name)
                if isinstance(attr, np.ndarray):
                    msg += ' :: array ' + str(attr.size) + ' :: First value: ' + str(attr[0]) + '\n'
                else:
                    if attr:
                            msg += ' :: scalar :: Value: ' + str(attr) + '\n'
                    else:
                        msg += ' :: No value set\n'

        return msg
