# ----------------------------------------------------------------------------
# Copyright (c) 2017 Massachusetts Institute of Technology (MIT)
# All rights reserved.
#
# Distributed under the terms of the BSD 3-clause license.
#
# The full license is in the LICENSE file, distributed with this software.
# ----------------------------------------------------------------------------
"""digital_rf_deprecated_hdf5.py is a module that allows python to read deprecated digital rf version 1 data.

It uses h5py to read old format.  Maintain to support upconversion to digital rf 2 only. No ability to write Drf 1.


$Id$
"""

# standard python imports
import os, os.path, sys
import types
import glob
import datetime, time
import warnings

# third party imports
import numpy
import h5py


class read_hdf5:
    """The class read_hdf5 is an object used to read digital rf v1 Hdf5 files as specified
    in the http://www.haystack.mit.edu/pipermail/rapid-dev/2014-February/000273.html email thread.

    This class allows random access to the rf data.

    """
    def __init__(self, top_level_directory_arg, load_all_metadata=False):
        """__init__ will verify the data in top_level_directory_arg is as expected.  It will analyze metadata
        to the degree specified in the load_all_metadata flag so that other methods can return more quickly

        Inputs:
            top_level_directory_arg - either a single top level, directory, or a list.  A directory can be a file system path or a url,
                where the url points to a top level directory.
            load_all_metadata - if True, loads all possible metadata at init.  If False, only loads high
                level metadata for faster __init__ speed.   A basic rule of thumb:
                    **** use load_all_metadata=False to make __init__ faster   ****
                    **** use load_all_metadata=True to make read_vector faster, at the cost of slower __init__ ****

        A top level directory must contain <channel_name>/<YYYY-MM-DDTHH-MM-SS/rf@<unix_seconds>.<%03i milliseconds>.h5

        If more than one top level directory contains the same channel_name subdirectory, this is considered the same channel.  An error
        is raised if their sample rates differ, or if their time periods overlap.

        This method will create the following attributes:

        self._top_level_dir_dict - a dictionary with keys = top_level_directory string, value = access mode (eg, 'local', 'file', or 'http')
            This attribute is static, that is, it is not updated when self.reload() called

        self._channel_dict - a dictionary with keys = channel_name, and value is a _channel_metadata object.

        self._load_all_metadata - True if full metadata search required by default, False if minimal metadata.

        self._last_update_has_full_metadata - True if last update got full metadata, False is last update got minimal
            metadata.  At init will equal self._load_all_metadata, but will be set to the load_all_metadata in reload
            when that method is called later.
        """

        # first, make top_level_directory_arg a list if a string
        if type(top_level_directory_arg) == types.StringType:
            top_level_arg = [top_level_directory_arg]
        else:
            top_level_arg = top_level_directory_arg

        # create static attribute self._top_level_dir_dict
        self._top_level_dir_dict = {}
        for top_level_directory in top_level_arg:
            if top_level_directory[0:7] == 'file://':
                self._top_level_dir_dict[top_level_directory] = 'file'
            elif top_level_directory[0:7] == 'http://':
                self._top_level_dir_dict[top_level_directory] = 'http'
            else:
                # make sure absolute path used
                if top_level_directory[0] != '/':
                    this_top_level_dir = os.path.join(os.getcwd(), top_level_directory)
                else:
                    this_top_level_dir = top_level_directory
                self._top_level_dir_dict[this_top_level_dir] = 'local'

        self._channel_dict = {}

        self._load_all_metadata = load_all_metadata

        self.reload()


    def reload(self, load_all_metadata=None):
        """reload updates the attribute self._channel_dict.

            Inputs:
                load_all_metadata - If load_all_metadata is True, then get complete metadata.  If
                    load_all_metadata is False, only get high level metadata. Default is None, in which
                    case load_all_metadata=self._load_all_metadata, as set in init.
        """
        if load_all_metadata is None:
            load_all_metadata = self._load_all_metadata

        self._last_update_has_full_metadata = load_all_metadata

        # first update the channel list
        channel_dict = {} # a temporary dict with key = channels, value = list of top level directories where found
        for top_level_dir in self._top_level_dir_dict.keys():
            channels_found = self._get_channels_in_dir(top_level_dir)
            for channel in channels_found:
                channel_name = os.path.basename(channel)
                if channel_dict.has_key(channel_name):
                    channel_dict[channel_name].append(top_level_dir)
                else:
                    channel_dict[channel_name] = [top_level_dir]

        # next throw away any metadata where the entire channel not longer exists
        remove_keys = []
        for channel_name in self._channel_dict.keys():
            if channel_name not in channel_dict.keys():
                # channel no longer exists
                remove_keys.append(channel_name)
        if len(remove_keys):
            for remove_key in remove_keys:
                del self._channel_dict[remove_key]


        # update all channels
        for channel_name in channel_dict.keys():
            if not self._channel_dict.has_key(channel_name):
                # a new channel is found - create it
                top_level_dir_metadata_list = []
                for top_level_dir in channel_dict[channel_name]:
                    new_top_level_metaddata = _top_level_dir_metadata(top_level_dir, channel_name,
                                                                      self._top_level_dir_dict[top_level_dir])
                    top_level_dir_metadata_list.append(new_top_level_metaddata)
                top_level_dir_metadata_list.sort()
                new_channel_metadata = _channel_metadata(channel_name, top_level_dir_meta_list = top_level_dir_metadata_list)
                new_channel_metadata.update(complete_update=load_all_metadata)
                self._channel_dict[channel_name] = new_channel_metadata

            else:
                # handle any changes to the top_level_dir list
                chan_obj = self._channel_dict[channel_name] # just to shorten the following code
                found_dirs = [] # a list of directories in _channel_metadata.top_level_dir_meta_list found
                for top_level_dir in channel_dict[channel_name]:
                    found = False
                    for chan_top_dir in chan_obj.top_level_dir_meta_list:
                        if top_level_dir == chan_top_dir.top_level_dir:
                            found_dirs.append(chan_top_dir.top_level_dir)
                            found = True
                            break
                    if not found:
                        # this is a new top level
                        new_top_level_meta = _top_level_dir_metadata(top_level_dir, channel_name,
                                                                     self._top_level_dir_dict[top_level_dir])
                        chan_obj.add_top_level(new_top_level_meta)
                        found_dirs.append(top_level_dir)

                # make sure all top level dirs in chan metadata still exist
                for chan_top_dir in chan_obj.top_level_dir_meta_list:
                    if chan_top_dir.top_level_dir not in found_dirs:
                        # this top level dir no longer has data
                        chan_obj.remove_top_level_metadata(chan_top_dir.top_level_dir)

                chan_obj.update(complete_update=load_all_metadata)


    def get_channels(self):
        """get_channels returns a alphabetically sorted list of channels in this read_hdf5 object
        """
        channels = self._channel_dict.keys()
        channels.sort()
        return(channels)


    def get_bounds(self, channel_name):
        """get_bounds returns a tuple of (first_unix_sample, last_unix_sample) for a given channel name
        """
        channel_metadata = self._channel_dict[channel_name]
        return((int(channel_metadata.unix_start_sample),
                int(channel_metadata.unix_start_sample + channel_metadata.sample_extent)))



    def get_rf_file_metadata(self, channel_name):
        """get_rf_file_metadata returns a dictionary of metadata found as attributes in the Hdf5 file /rf_data
        dataset for the given channel name.
        """
        return(self._channel_dict[channel_name].metadata_dict)



    def get_metadata(self, channel_name, timestamp=None):
        """get_metadata returns a h5py.File object pointing to the metadata*.h5 file at the top level of the
        channel directory.  The user is responsible for closing that file when done.  If timestamp is None,
        the latest metadata*.h5 will be returned.  Otherwise it will open the earliest metadata file with timestamp
        greater than or equal to timestamp

        Use of this old metadata scheme is now strongly discouraged. Use get_rf_file_metadata for metadata within the
        Digital RF standard, or Digital Metdata for other metadata.
        """
        warnings.warn("get_metadata is now deprecated.  Use get_rf_file_metadata for metadata within the Digital RF standard, or Digital Metdata for other metadata.",
                      DeprecationWarning)
        # first, get a sorted list of all metadata*.hf files
        metadata_file_list = []
        metadata_basename_list = [] # to make sure there are no repeated basenames
        channel = self._channel_dict[channel_name]
        for top_level_dir_obj in channel.top_level_dir_meta_list:
            metadata_files = glob.glob(os.path.join(top_level_dir_obj.top_level_dir,
                                                    top_level_dir_obj.channel_name, 'metadata@*.h5'))
            metadata_files.sort()
            for metadata_file in metadata_files:
                basename = os.path.basename(metadata_file)
                if basename in metadata_basename_list:
                    raise IOError, 'found repeated metadata file names in channel %s' % (channel_name)
                # verify its a good file
                try:
                    f = h5py.File(metadata_file, 'r')
                    f.close()
                except:
                    continue
                metadata_basename_list.append(basename)
                metadata_file_list.append(metadata_file)

        if len(metadata_file_list) == 0:
            raise IOError, 'No metadata files found in channel %s' % (channel_name)

        # open right metadata file
        if timestamp is None:
            return(h5py.File(metadata_file_list[-1], 'r'))
        else:
            rightFile = None
            for metadata_file in metadata_file_list:
                basename = os.path.basename(metadata_file)
                this_timestamp = int(basename[len('metadata@'):basename.find('.')])
                if this_timestamp <= timestamp:
                    rightFile = metadata_file
                else:
                    break
            if rightFile is None:
                raise IOError, 'All metadata files found in channel %s after timestamp' % (channel_name, timestamp)
            return(h5py.File(rightFile, 'r'))





    def get_continuous_blocks(self, start_unix_sample, stop_unix_sample, channel_name):
        """get_continuous_blocks returns a numpy array of dtype u64 and shape (N,2) where the first
        column represents the unix_sample of a continuous block of data, and the second column represents the
        number of samples in that continuous block.  Only samples between (start_unix_sample, stop_unix_sample)
        inclusive will be returned.

        Calls the private method _get_continuous_blocks.  If that raises a _MissingMetadata exception, calls
        reload with load_all_metadata == True to get missing metadata, and then retries _get_continuous_blocks.

        Returns IOError if no blocks found

        Inputs:
            start_unix_sample, stop_unix_sample - only samples between (start_unix_sample, stop_unix_sample)
                inclusive will be returned.  Value of both are samples since 1970-01-01

            channel_name - channel to examine
        """
        try:
            return(self._get_continuous_blocks(start_unix_sample, stop_unix_sample, channel_name))
        except _MissingMetadata:
            self.reload(True)
            # try again
            return(self._get_continuous_blocks(start_unix_sample, stop_unix_sample, channel_name))



    def read_vector(self, unix_sample, vector_length, channel_name):
        """read_vector returns a numpy vector of complex8 type, no matter the dtype of the Hdf5 file
        or the number of channels. Shape is (vector_length, num_subchannels). Single value (real) files will
        have the imaginary part set to zero.

        Calls read_vector_raw, then converts result.

        Inputs:
            unix_sample - the number of samples since 1970-01-01 at start of data

            vector_length - the number of continuous samples to include

            channel_name - the channel name to use

        This method will raise an IOError error if the returned vector would include any missing data.
        It will also raise an IOError is any of the files needed to read the data have been deleted.
        This is possible because metadata on which this call is based might be out of date.
        """
        z = self.read_vector_raw(unix_sample, vector_length, channel_name)

        if z.dtype == numpy.complex64:
            return(z)
        elif z.dtype in (numpy.complex128, numpy.complex256):
            return(numpy.array(z, dtype=numpy.complex64))

        if not hasattr(z.dtype, 'names'):
            return(numpy.array(z, dtype=numpy.complex64))
        elif z.dtype.names is None:
            return(numpy.array(z, dtype=numpy.complex64))
        z = numpy.array(z['r'] + z['i']*1.0j, dtype=numpy.complex64)
        return(z)



    def read_vector_raw(self, unix_sample, vector_length, channel_name):
        """read_vector_raw returns a numpy array of dim(up to num_samples, num_subchannels) of the dtype in the Hdf5 files.

        If complex data, real and imag data will have names 'r' and 'i' if underlying data are integers
        or be numpy complex data type if underlying data floats.

        Inputs:
            unix_sample - the number of samples since 1970-01-01 at start of data

            vector_length - the number of continuous samples to include

            channel_name - the channel name to use

        This method will raise an IOError error if the returned vector would include any missing data.
        It will also raise an IOError is any of the files needed to read the data have been deleted.
        This is possible because metadata on which this call is based might be out of date.
        """
        if vector_length < 1:
            raise IOError, 'Number of samples requested must be greater than 0, not %i' % (vector_length)

        # make sure everything is a long
        unix_sample = int(unix_sample)
        vector_length = int(vector_length)

        channel_metadata = self._channel_dict[channel_name]

        ret_array = None
        first_unix_sample = None

        if self._last_update_has_full_metadata:
            # make sure we don't request beyond the metadata
            last_top_level_metadata = channel_metadata.top_level_dir_meta_list[-1]
            if last_top_level_metadata.unix_start_sample + last_top_level_metadata.sample_extent < unix_sample + vector_length:
                raise IOError, 'request in _read_vector beyond existing Metadata using full metadata'

        for top_level_dir in channel_metadata.top_level_dir_meta_list:
            # note - even with partial metadata, the edges of top_level_dir are absolutely correct
            if top_level_dir.unix_start_sample + top_level_dir.sample_extent < unix_sample:
                # this top level dir is too early
                continue
            if unix_sample + vector_length < top_level_dir.unix_start_sample:
                # this top level dir is too late
                continue
            # run the faster version if self._last_update_has_full_metadata == True, version with searching if False
            this_array, this_unix_sample = top_level_dir.get_continuous_vector(max(unix_sample, top_level_dir.unix_start_sample),
                                                                               min(unix_sample + vector_length,
                                                                                   top_level_dir.unix_start_sample + top_level_dir.sample_extent),
                                                                               self._last_update_has_full_metadata)
            ret_array = self._combine_continuous_vectors(ret_array, this_array, first_unix_sample, this_unix_sample)
            if first_unix_sample is None:
                first_unix_sample = unix_sample

        if ret_array is None:
            raise IOError, 'No data found for channel %s between %i and %i' % (channel_name, unix_sample,
                                                                               unix_sample + vector_length)

        if len(ret_array) != vector_length:
            raise IOError, 'Requested %i samples, but only found %i' % (vector_length, len(ret_array))

        return(ret_array)


    def read_vector_c81d(self, unix_sample, vector_length, channel_name, subchannel=0):
        """read_vector_c81d returns a numpy vector of complex8 type, no matter the dtype of the Hdf5 file
        or the number of channels. Error thrown if subchannel doesn't exist.

        Inputs:
            unix_sample - the number of samples since 1970-01-01 at start of data

            vector_length - the number of continuous samples to include

            channel_name - the channel name to use

            subchannel - which subchannel to use.  Default is 0 (first)

        This method will raise an IOError error if the returned vector would include any missing data.
        It will also raise an IOError is any of the files needed to read the data have been deleted.
        This is possible because metadata on which this call is based might be out of date.
        """
        z = self.read_vector_raw(unix_sample, vector_length, channel_name)

        if z.shape[1] < subchannel + 1:
            raise ValueError, 'Returned data has only %i subchannels, does not have subchannel %i' % (z.shape[1], subchannel)

        if z.dtype == numpy.complex64:
            return(z[:,subchannel])
        elif z.dtype in (numpy.complex128, numpy.complex256):
            return(numpy.array(z[:,subchannel], dtype=numpy.complex64))

        slice = (z[:,subchannel])
        if not hasattr(slice.dtype, 'names'):
            raise ValueError, 'Single valued channels cannot be cast to complex'
        elif slice.dtype.names is None:
            raise ValueError, 'Single valued channels cannot be cast to complex'
        slice = numpy.array(slice['r'] + slice['i']*1.0j, dtype=numpy.complex64)
        return(slice)





    def _get_continuous_blocks(self, start_unix_sample, stop_unix_sample, channel_name):
        """_get_continuous_blocks is a private method that returns a numpy array of dtype u64 and shape (N,2) where the first
        column represents the unix_sample of a continuous block of data, and the second column represents the
        number of samples in that continuous block.  Only samples between (start_unix_sample, stop_unix_sample)
        inclusive will be returned.

        Raises _MissingMetadata exception if reload needs to be called.

        Returns IOError if no blocks found

        Inputs:
            start_unix_sample, stop_unix_sample - only samples between (start_unix_sample, stop_unix_sample)
                inclusive will be returned.  Value of both are samples since 1970-01-01

            channel_name - channel to examine
        """
        channel_metadata = self._channel_dict[channel_name]

        ret_array = numpy.array([], dtype=numpy.uint64)
        for top_level_dir in channel_metadata.top_level_dir_meta_list:
            if top_level_dir.unix_start_sample + top_level_dir.sample_extent < start_unix_sample:
                # this top level dir is too early
                continue
            if stop_unix_sample < top_level_dir.unix_start_sample:
                # this top level dir is too late
                continue
            this_array = top_level_dir.get_continuous_blocks(max(start_unix_sample, top_level_dir.unix_start_sample),
                                                             min(stop_unix_sample, top_level_dir.unix_start_sample + top_level_dir.sample_extent))
            ret_array = self._combine_blocks(ret_array, this_array, top_level_dir.samples_per_file)

        if len(ret_array) == 0:
            raise IOError, 'No data found for channel %s between %i and %i' % (channel_name, start_unix_sample,
                                                                               stop_unix_sample)

        return(ret_array)



    def _combine_continuous_vectors(self, first_array, second_array, first_start_sample, second_start_sample):
        """_combine_continuous_vectors returns the concatenation of first_array and second_array,  Raises error
        if two vectors are not continuous.

        Inputs:
            first_array - first array to combine.  If None, just return second_array
            second_array - second_array to merge at end of first
            first_start_sample - unix_sample of first sample in first_array.  None if first_array is None
            second_start_sample - unix_sample of first sample in second_array
        """

        if first_array is None:
            return(second_array)

        if len(first_array) != second_start_sample - first_start_sample:
            raise IOError, '_combine_continuous_vectors trying to combine two non-continuous vectors'

        return(numpy.concatenate((first_array, second_array)))



    def _combine_blocks(self, first_array, second_array, samples_per_file):
        """_combine_blocks combines two numpy array of dtype u64 and shape (N,2) where the first
        column represents the unix_sample of a continuous block of data, and the second column represents the
        number of samples in that continuous block. The first row of the second array may or may not be contiguous
        with the last row of the first array.  If it is contiguous, that row will not be included, and the
        number of samples in that first row will instead be added to the last row of first_array. If not contiguous,
        the two arrays are simply concatenated
        """
        if len(first_array) == 0:
            return(second_array)
        is_contiguous = False
        if first_array[-1][0] + first_array[-1][1] > second_array[0][0]:
            raise IOError, 'overlapping data found in top level directories %i %i' % \
                (first_array[-1][0] + first_array[-1][1], second_array[0][0])
        if first_array[-1][0] + first_array[-1][1] == second_array[0][0]:
            is_contiguous = True
        if is_contiguous:
            first_array[-1][1] += second_array[0][1]
            if len(second_array) == 1:
                return(first_array)
            else:
                return(numpy.concatenate([first_array, second_array[1:]]))
        else:
            return(numpy.concatenate([first_array, second_array]))



    def _get_channels_in_dir(self, top_level_dir):
        """_get_channels_in_dir returns a list of channel names found in top_level_dir

        Inputs:
            top_level_dir - string indicating top_level_dir
        """
        # define glob string for sub_directories in form YYYY-MM-DDTHH-MM-SS
        sub_directory_glob = '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]-[0-9][0-9]-[0-9][0-9]'

        retList = []
        access_mode = self._top_level_dir_dict[top_level_dir]
        # for now only local access
        if access_mode not in ('local'):
            raise ValueError, 'access_mode %s not yet implemented' % (access_mode)

        if access_mode == 'local':
            potential_channels = glob.glob(os.path.join(top_level_dir, '*', sub_directory_glob))
            for potential_channel in potential_channels:
                channel_name = os.path.dirname(potential_channel)
                if channel_name not in retList:
                    retList.append(channel_name)

        return(retList)




class _channel_metadata:
    """The _channel_metadata is a private class to hold and access metadata about a particular digital_rf channel.
    A channel can extend over one of more top level directories.
    """

    def __init__(self, channel_name, unix_start_sample = 0, sample_extent = 0, top_level_dir_meta_list = []):
        """__init__ creates a new _channel_metadata object

        Inputs:
            channel_name - channel name (name of subdirectory defining this channel)
            unix_start_sample - unix start sample - first sample time in unix timeseconds * sample rate. If default
                0, then unknown
            sample_extent - number of samples between first and last in data
            top_level_dir_meta_list - a time ordered list of _top_level_dir_metadata objects.  Default is empty list

        Affects:
            create four input arguments as attributes.  Also creates self.metadata_dict, which is a dictionary of atributes
            found in the Hdf5 files (eg, sample_rate)

        """
        self.channel_name = channel_name
        self.unix_start_sample = int(unix_start_sample)
        self.sample_extent = int(sample_extent)
        self.top_level_dir_meta_list = top_level_dir_meta_list
        self.top_level_dir_meta_list.sort()
        self.metadata_dict = {} # stores all metadata for this _channel_metadata


    def update(self, complete_update=False):
        """update will cause this _channel_metadata object to update itself.

        If complete_update == False, then only get high level metadata.  If complete_update,
        update all possible metadata.

        Inputs:
            complete_update - if True, then update all metadata.  If False,
                    the default, only update high level metadata.
        """
        for top_level_meta in self.top_level_dir_meta_list:
            top_level_meta.update(complete_update)
            if not self.metadata_dict.has_key('uuid_str'):
                for key in top_level_meta.metadata_dict.keys():
                    self.metadata_dict[key] = top_level_meta.metadata_dict[key]
        self.reset_indices()


    def add_top_level(self, top_level_dir_meta):
        """add_top_level will add a new _top_level_dir_metadata object to self.top_level_dir_meta_list

        Inputs:
            top_level_dir_meta - new _top_level_dir_metadata object to add
        """
        self.top_level_dir_meta_list.append(top_level_dir_meta)
        self.top_level_dir_meta_list.sort()
        self.update()


    def remove_top_level_metadata(self, top_level_dir):
        """remove_top_level_metadata removes all metadata associated with this channel and one top_level_directory.  Raise
        ValueError is top_level_dir not found
        """
        remove_list = []
        for i, top_level_dir_meta in enumerate(self.top_level_dir_meta_list):
            if top_level_dir == top_level_dir_meta.top_level_dir:
                remove_list.append(i)

        if len(remove_list) == 0:
            raise ValueError, 'No directory %s found in this channel' % (top_level_dir)
        elif len(remove_list) > 1:
            raise ValueError, 'More than one directory %s found in this channel' % (top_level_dir)

        self.top_level_dir_meta_list.pop(remove_list[0])

        self.reset_indicies()


    def reset_indices(self):
        """reset_indices recalculates self.unix_start_sample and self.sample_extent based on self.top_level_dir_meta_list
        """
        if len(self.top_level_dir_meta_list) == 0:
            # set to unknown
            self.unix_start_sample = int(0)
            self.sample_extent = int(0)
            return

        self._verify_non_overlapping_data()
        self.unix_start_sample = int(self.top_level_dir_meta_list[0].unix_start_sample)
        self.sample_extent = int(self.top_level_dir_meta_list[-1].unix_start_sample + self.top_level_dir_meta_list[-1].sample_extent - self.unix_start_sample)


    def _verify_non_overlapping_data(self):
        """_verify_non_overlapping_data raises an error if any overlapping top level directories found
        """
        for i, record in enumerate(self.top_level_dir_meta_list):
            if i == 0:
                last_unix_start_sample = record.unix_start_sample
                last_sample_extent = record.sample_extent
            else:
                this_unix_start_sample = record.unix_start_sample
                this_sample_extent = record.sample_extent
                if last_unix_start_sample + last_sample_extent > this_unix_start_sample:
                    raise IOError, 'Overlapping samples found in top level dir %s' % (record.top_level_dir)
                last_unix_start_sample = this_unix_start_sample
                last_sample_extent = this_sample_extent



class _top_level_dir_metadata:
    """The _top_level_dir_metadata is a private class to hold and access metadata about a particular digital_rf channel in
    a particular top level directory.
    """

    def __init__(self, top_level_dir, channel_name, access_mode, unix_start_sample = 0, sample_extent = 0,
                 samples_per_file=0, sub_directory_recarray=None, sub_directory_dict=None):
        """__init__ creates a new _top_level_dir_metadata

        Inputs:
            top_level_dir - full path the top level directory that contains the parent channel_name
            channel_name - the channel_name subdirectory name
            access_mode - string giving access mode (eg, 'local', 'file', or 'http')
            unix_start_sample - unix start sample - first sample time in unix timeseconds * sample rate. If default
                0, then unknown
            sample_extent - number of samples between first and last in data. If default 0, then unknown
            samples_per_file - number of samples per file. If default 0, then unknown
            sub_directory_recarray - a ordered numpy recarray with one row describing summary information about a single
                sub_directory in that channel/top_level_dir named YYYY-MM-DDTHH-MM-SS with the following columns:
                1) 'subdirectory' - in form YYYY-MM-DDTHH-MM-SS,
                2) 'unix_start_sample' (for that subdirectory).  May be zero in no detailed metadata for this directory yet
                3) 'sample_extent' eg, number of samples to last sample in that subdirectory.  May be zero if no detailed
                    metadata yet for this subdirectory.
                4) 'file_count' number of Hdf5 data files in directory.  Will be zero if no detailed
                    metadata yet for this subdirectory.
                5) 'last_timestamp' UTC time stamp of latest data file in directory. .  Will be zero if no detailed
                    metadata yet for this subdirectory
                Order is by subdirectory and/or unix_start_sample
            sub_directory_dict - a dictionary with key = sub_directory, value = _sub_directory_metadata object

        Affects: creates an attribute for each input argument

        Also creates cached attributes to speed reads with sparse metadata:
            self._last_file - open H5py file last read
            self._last_start_sample - sample start index of cached file
            This file must be gap free, or it is never cached

        """
        self.top_level_dir = top_level_dir
        self.channel_name = channel_name
        self.access_mode = access_mode
        self.unix_start_sample = int(unix_start_sample)
        self.sample_extent = int(sample_extent)
        self.samples_per_file = int(samples_per_file)
        self.sub_directory_recarray = sub_directory_recarray
        self.sub_directory_dict = sub_directory_dict
        self.metadata_dict = {} # to be populated by rf file metadata

        # data type of sub_directory_array
        self.data_t = numpy.dtype([('subdirectory', numpy.str_, 512), ('unix_start_sample', numpy.uint64, 1), ('sample_extent', numpy.uint64, 1),
                                   ('file_count', numpy.uint64, 1), ('last_timestamp', numpy.double, 1)])

        if self.sub_directory_recarray is None:
            # create empty array
            self.sub_directory_recarray = numpy.array([], dtype=self.data_t)

        # define glob strings for sub_directories in form YYYY-MM-DDTHH-MM-SS and rf files in form rf@*.*.h5
        self._sub_directory_glob = '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]-[0-9][0-9]-[0-9][0-9]'
        self._rf_file_glob = 'rf@[0-9]*.[0-9][0-9][0-9].h5'

        # attributes to allow caching
        self._last_file = None
        self._last_start_sample = None


    def update(self, complete_update=False):
        """update will cause this _top_level_dir_metadata object to update itself.

        If complete_update == False, then only get high level metadata.
        If complete_update, update all possible metadata.

        Inputs:
            complete_update - if True, then update all metadata, not matter the other arguments.  If False,
                    the default, limited update according to the other arguments.
        """
        if complete_update:
            self._full_update()
        else:
            self._high_level_reload()



    def get_continuous_blocks(self, start_unix_sample, stop_unix_sample):
        """get_continuous_blocks returns a numpy array of dtype u64 and shape (N,2) where the first
        column represents the unix_sample of a continuous block of data, and the second column represents the
        number of samples in that continuous block.  Only samples between (start_unix_sample, stop_unix_sample)
        inclusive will be returned.

        Returns IOError if no blocks found

        Inputs:
            start_unix_sample, stop_unix_sample - only samples between (start_unix_sample, stop_unix_sample)
                inclusive will be returned.
        """
        # to improve speed, do searchsorted to get first index to look into
        first_index = numpy.searchsorted(self.sub_directory_recarray['unix_start_sample'],
                                         numpy.array([start_unix_sample]))
        first_index = first_index[0]
        if first_index > 0:
            first_index -= 1
        ret_array = numpy.array([], dtype=numpy.uint64)
        for i in range(first_index, len(self.sub_directory_recarray)):
            this_start_sample = int(self.sub_directory_recarray['unix_start_sample'][i])
            if this_start_sample > stop_unix_sample:
                break
            if stop_unix_sample < this_start_sample:
                continue
            this_extent = int(self.sub_directory_recarray['sample_extent'][i])
            if this_extent == 0:
                raise _MissingMetadata, 'this_extent == 0'

            # now check that subdirectories with metadata are still up to date
            base_subdirectory = self.sub_directory_recarray['subdirectory'][i]
            file_count, last_timestamp = self._get_subdirectory_file_info(base_subdirectory)
            self.sub_directory_dict[base_subdirectory].update_if_needed(file_count, last_timestamp)

            sub_dir_metadata = self.sub_directory_dict[self.sub_directory_recarray['subdirectory'][i]]
            this_array = sub_dir_metadata.get_continuous_blocks(max(start_unix_sample, this_start_sample),
                                                                min(stop_unix_sample, (this_start_sample + this_extent)-1))
            ret_array = self._combine_blocks(ret_array, this_array, self.samples_per_file)

        return(ret_array)



    def get_continuous_vector(self, start_unix_sample, stop_unix_sample, last_update_has_full_metadata=False):
        """get_continuous_vector returns a tuple of (numpy array of data, first unix_sample in returned data)
        Only samples between (start_unix_sample, stop_unix_sample) (excludes stop_unix_sample) will be returned.

        Returns (None, None) if no blocks found

        Inputs:
            start_unix_sample, stop_unix_sample - only samples between (start_unix_sample, stop_unix_sample)
                (excludes stop_unix_sample) will be returned.

            last_update_has_full_metadata - if True, use standard metadata to access data.  If False (the default),
                use glob to search for right file without using detailed metadata (slower performance, but less metadata
                discovery time).
        """
        if last_update_has_full_metadata:
            # to improve speed, do searchsorted to get first index to look into
            first_index = numpy.searchsorted(self.sub_directory_recarray['unix_start_sample'],
                                             numpy.array([start_unix_sample]))
            first_index = first_index[0]
            if first_index > 0:
                first_index -= 1
            ret_array = None
            first_unix_sample = None
            for i in range(first_index, len(self.sub_directory_recarray)):
                this_start_sample = int(self.sub_directory_recarray['unix_start_sample'][i])
                if this_start_sample >= stop_unix_sample:
                    break
                if stop_unix_sample < this_start_sample:
                    continue
                this_extent = int(self.sub_directory_recarray['sample_extent'][i])
                if this_extent == 0:
                    raise IOError, 'Metadata not found in recarray'
                if this_start_sample + this_extent <= start_unix_sample:
                    continue

                sub_dir_metadata = self.sub_directory_dict[self.sub_directory_recarray['subdirectory'][i]]
                this_array, unix_sample = sub_dir_metadata.get_continuous_vector(max(start_unix_sample, this_start_sample),
                                                                                  min(stop_unix_sample, this_start_sample + this_extent))

                ret_array = self._combine_continuous_vectors(ret_array, this_array, first_unix_sample, unix_sample)
                if first_unix_sample is None:
                    first_unix_sample = unix_sample

            return((ret_array, first_unix_sample))

        else:
            # check if we can use cached file
            if self._last_start_sample:
                if start_unix_sample > self._last_start_sample and \
                    stop_unix_sample <= self._last_start_sample + self.samples_per_file:
                    return(self._get_data_from_cache(start_unix_sample, stop_unix_sample))
                else:
                    # cache has expired
                    try:
                        self._last_file.close()
                    except:
                        pass
                    self._last_file = None
                    self._last_start_sample = None

            # only partial metadata
            files_to_search = self._get_files_to_search(start_unix_sample, stop_unix_sample)
            ret_array = None
            for file_to_search in files_to_search:
                arr = self._read_data_from_file(file_to_search, start_unix_sample, stop_unix_sample, ret_array)
                if (not arr is None) and ret_array is None:
                    ret_array = arr
                elif arr is None:
                    continue
                else:
                    ret_array = numpy.concatenate((ret_array, arr))
                if not ret_array is None:
                    if len(ret_array) == stop_unix_sample - start_unix_sample:
                        break

            # verify success
            if ret_array is None:
                raise IOError, 'No valid data found'
            if len(ret_array) != stop_unix_sample - start_unix_sample:
                raise IOError, 'Wanted len %i, but got len %i' % (stop_unix_sample - start_unix_sample, len(ret_array))

            return((ret_array, start_unix_sample))




    def _high_level_reload(self):
        """_high_level_reload updates only high level metadata.  Basically this is only the first and last sample
        """
        base_subdirectory_list = self._get_subdirectories()
        dt1970 = datetime.datetime(1970,1,1)

        # first pass is to remove any subdirectories that have disappeared
        rows_to_delete_arr = []
        for i, subdirectory in enumerate(self.sub_directory_recarray['subdirectory']):
            if subdirectory not in base_subdirectory_list:
                rows_to_delete_arr.append(i)
                del self.sub_directory_dict[subdirectory]
        if len(rows_to_delete_arr) > 0:
            rows_to_delete_arr = numpy.array(rows_to_delete_arr, numpy.int64)
            self.sub_directory_recarray = numpy.delete(self.sub_directory_recarray, rows_to_delete_arr)

        # next pass creates any new rows in _sub_directory_metadata
        for base_subdirectory in base_subdirectory_list:
            try:
                file_count, last_timestamp = self._get_subdirectory_file_info(base_subdirectory)
            except IOError:
                new_sub_dir_meta = _sub_directory_metadata(self.top_level_dir, self.channel_name,
                                                           self.access_mode, base_subdirectory)
                if not self.sub_directory_dict is None:
                    self.sub_directory_dict[base_subdirectory] = new_sub_dir_meta
                else:
                    self.sub_directory_dict = {base_subdirectory: new_sub_dir_meta}
                # extend self.sub_directory_recarray by one
                self.sub_directory_recarray.resize(len(self.sub_directory_recarray) + 1)

                # get estimate of first sample without IO
                subDirDT = datetime.datetime.strptime(os.path.basename(base_subdirectory), '%Y-%m-%dT%H-%M-%S')
                total_secs = (subDirDT - dt1970).total_seconds()
                if len(self.metadata_dict.keys()) == 0:
                    # force it to be created
                    new_sub_dir_meta.get_last_sample()
                    self.metadata_dict = new_sub_dir_meta.metadata_dict
                    self.samples_per_file = self.metadata_dict['samples_per_file']
                sample_index = total_secs * self.metadata_dict['sample_rate']
                self.sub_directory_recarray[-1] = (base_subdirectory, sample_index, 0, 0, 0) # use default values

        # update first and last only
        self.sub_directory_dict[base_subdirectory_list[0]].update()
        if len(base_subdirectory_list) > 1:
            self.sub_directory_dict[base_subdirectory_list[-1]].update()

        self.unix_start_sample = int(self.sub_directory_dict[base_subdirectory_list[0]].get_first_sample())
        last_sample = int(self.sub_directory_dict[base_subdirectory_list[-1]].get_last_sample())
        self.sample_extent = int(last_sample - self.unix_start_sample)




    def _full_update(self):
        """_full_update will cause this _top_level_dir_metadata object to update all possible metadata
        """
        update_needed = False # will be set to True if any subdirectory updated
        base_subdirectory_list = self._get_subdirectories(verify_files=True)

        # first pass is to remove any subdirectories that have disappeared
        rows_to_delete_arr = []
        for i, subdirectory in enumerate(self.sub_directory_recarray['subdirectory']):
            if subdirectory not in base_subdirectory_list:
                rows_to_delete_arr.append(i)
                del self.sub_directory_dict[subdirectory]
        if len(rows_to_delete_arr) > 0:
            rows_to_delete_arr = numpy.array(rows_to_delete_arr, numpy.int64)
            self.sub_directory_recarray = numpy.delete(self.sub_directory_recarray, rows_to_delete_arr)

        # next pass
        for i, base_subdirectory in enumerate(base_subdirectory_list):
            try:
                file_count, last_timestamp = self._get_subdirectory_file_info(base_subdirectory)
                # call update_if_needed to make faster
                if self.sub_directory_dict[base_subdirectory].update_if_needed(file_count, last_timestamp):
                    first_unix_sample, sample_extent, file_count, samples_per_file, last_timestamp = \
                        self.sub_directory_dict[base_subdirectory].get_summary_metadata()
                    self.sub_directory_recarray[i] = (base_subdirectory, first_unix_sample, sample_extent,
                                                           file_count, last_timestamp)
                    update_needed = True
            except IOError:
                update_needed = True
                new_sub_dir_meta = _sub_directory_metadata(self.top_level_dir, self.channel_name,
                                                           self.access_mode, base_subdirectory)
                new_sub_dir_meta.update()
                if len(self.metadata_dict.keys()) == 0:
                    self.metadata_dict = new_sub_dir_meta.metadata_dict
                if not self.sub_directory_dict is None:
                    self.sub_directory_dict[base_subdirectory] = new_sub_dir_meta
                else:
                    self.sub_directory_dict = {base_subdirectory: new_sub_dir_meta}
                first_unix_sample, sample_extent, file_count, samples_per_file, last_timestamp = \
                    new_sub_dir_meta.get_summary_metadata()
                # extend self.sub_directory_recarray by one
                self.sub_directory_recarray.resize(len(self.sub_directory_recarray) + 1)
                self.sub_directory_recarray[-1] = (base_subdirectory, first_unix_sample, sample_extent, file_count,
                                                   last_timestamp)

                # handle samples_per_file
                if self.samples_per_file == 0:
                    self.samples_per_file = int(samples_per_file)
                elif self.samples_per_file != int(samples_per_file):
                    raise IOError, 'Samples per file changed from %i to %i with subdirectory %s' % \
                        (self.samples_per_file, samples_per_file, base_subdirectory)
                continue



        if update_needed:
            self._verify_non_overlapping_data()
            # update summary metadata
            self.unix_start_sample = int(self.sub_directory_recarray['unix_start_sample'][0])
            last_sample = int(self.sub_directory_recarray['unix_start_sample'][-1]) + \
                int(self.sub_directory_recarray['sample_extent'][-1])
            self.sample_extent = int(last_sample - self.unix_start_sample)




    def _verify_non_overlapping_data(self):
        """_verify_non_overlapping_data raises an error if any overlapping subdirectories found
        """
        for i, record in enumerate(self.sub_directory_recarray):
            if i == 0:
                last_unix_start_sample = record['unix_start_sample']
                last_sample_extent = record['sample_extent']
            else:
                this_unix_start_sample = record['unix_start_sample']
                this_sample_extent = record['sample_extent']
                if last_unix_start_sample + last_sample_extent > this_unix_start_sample:
                    if this_sample_extent != 0: # otherwise not trustworthy
                        raise IOError, 'Overlapping samples found in subdirectory %s - last_unix_start_sample=%i, last_sample_extent=%i, this_unix_start_sample=%i' \
                            % (record['subdirectory'], last_unix_start_sample, last_sample_extent, this_unix_start_sample)
                last_unix_start_sample = this_unix_start_sample
                last_sample_extent = this_sample_extent



    def _get_subdirectory_file_info(self, subdirectory):
        """_get_subdirectory_file_info returns a tuple ot (num_files, last_timestamp) for a given
        subdirectory using the self.sub_directory_recarray recarray.  Raises IOError if subdirectory
        not found in recarray.
        """
        result = numpy.argwhere(self.sub_directory_recarray['subdirectory'] == subdirectory)
        if len(result) == 0:
            raise IOError, 'subdirectory %s not found' % (subdirectory)
        if len(result) > 1:
            raise ValueError, 'got unexpected result %s' % (str(result))
        return((self.sub_directory_recarray['file_count'][result[0][0]],
                self.sub_directory_recarray['last_timestamp'][result[0][0]]))



    def _combine_continuous_vectors(self, first_array, second_array, first_start_sample, second_start_sample):
        """_combine_continuous_vectors returns the concatenation of first_array and second_array,  Raises error
        if two vectors are not continuous.

        Inputs:
            first_array - first array to combine.  If None, just return second_array
            second_array - second_array to merge at end of first
            first_start_sample - unix_sample of first sample in first_array.  None if first_array is None
            second_start_sample - unix_sample of first sample in second_array
        """

        if first_array is None:
            return(second_array)

        if len(first_array) != second_start_sample - first_start_sample:
            raise IOError, '_combine_continuous_vectors trying to combine two non-continuous vectors'

        return(numpy.concatenate((first_array, second_array)))



    def _get_subdirectories(self, verify_files=False):
        """_get_subdirectories returns a sorted list of base subdirectory names

        Inputs:
            verify_files - If True, only return subdirectories with h5 files.  If False (the default),
            return any subdirectory that matches the format, independent of whether it has files
        """
        # for now only local access
        if self.access_mode not in ('local'):
            raise ValueError, 'access_mode %s not yet implemented' % (access_mode)
        subdirectory_list = glob.glob(os.path.join(self.top_level_dir, self.channel_name, self._sub_directory_glob))
        subdirectory_list.sort()
        if not verify_files:
            return(subdirectory_list)
        retList = [] # only return those with files
        for subdirectory in subdirectory_list:
            if len(glob.glob(os.path.join(subdirectory, '*.h5'))) > 0:
                retList.append(subdirectory)
        return(retList)



    def _combine_blocks(self, first_array, second_array, samples_per_file):
        """_combine_blocks combines two numpy array of dtype u64 and shape (N,2) where the first
        column represents the unix_sample of a continuous block of data, and the second column represents the
        number of samples in that continuous block. The first row of the second array may or may not be contiguous
        with the last row of the first array.  If it is contiguous, that row will not be included, and the
        number of samples in that first row will instead be added to the last row of first_array. If not contiguous,
        the two arrays are simply concatenated
        """
        if len(first_array) == 0:
            return(second_array)
        is_contiguous = False
        if first_array[-1][0] + first_array[-1][1] > second_array[0][0]:
            raise IOError, 'overlapping data found in top level directories %i %i' % \
                (first_array[-1][0] + first_array[-1][1], second_array[0][0])
        if first_array[-1][0] + first_array[-1][1] == second_array[0][0]:
            is_contiguous = True
        if is_contiguous:
            first_array[-1][1] += second_array[0][1]
            if len(second_array) == 1:
                return(first_array)
            else:
                return(numpy.concatenate([first_array, second_array[1:]]))
        else:
            return(numpy.concatenate([first_array, second_array]))


    def _get_files_to_search(self, start_unix_sample, stop_unix_sample):
        """_get_files_to_search is a private method designed to return a subset of data files that might contain
        data to return.  Used only when complete metadata not available.

        Inputs:
            start_unix_sample, stop_unix_sample - only samples between (start_unix_sample, stop_unix_sample)
                (excludes stop_unix_sample) will be returned, so only return files that might contain that range
        """
        seconds_per_file =  1 + int(self.metadata_dict['samples_per_file'][0] / self.metadata_dict['sample_rate'][0])
        start_integer_sec = int(start_unix_sample/self.metadata_dict['sample_rate'][0])
        stop_integer_sec = int(stop_unix_sample/self.metadata_dict['sample_rate'][0])
        seconds_list = range(start_integer_sec - seconds_per_file, stop_integer_sec + 1)

        # now glob for these files in all directories until no more found
        files_to_search = []
        first_index = numpy.searchsorted(self.sub_directory_recarray['unix_start_sample'],
                                         numpy.array([start_unix_sample]))
        first_index = first_index[0]
        if first_index == len(self.sub_directory_recarray['unix_start_sample']):
            first_index -= 1
        first_index_sec = int(self.sub_directory_recarray['unix_start_sample'][first_index]/self.metadata_dict['sample_rate'][0])
        if first_index > 0 and start_integer_sec - seconds_per_file < first_index_sec:
            first_index -= 1
        for i in range(first_index, len(self.sub_directory_recarray)):
            files_this_subdir = 0 # one method to determine when to break
            this_dir_start_sec = int(self.sub_directory_recarray['unix_start_sample'][i]/self.metadata_dict['sample_rate'][0])
            if this_dir_start_sec > stop_integer_sec + seconds_per_file:
                # another break test
                break
            for second in seconds_list:
                glob_str = os.path.join(self.sub_directory_recarray['subdirectory'][i],
                                        'rf@%i.???.h5' % (second))
                new_files = glob.glob(glob_str)
                files_this_subdir += len(new_files)
                files_to_search += new_files
            # break if none found after second subdirectory
            if i > first_index and files_this_subdir == 0:
                break

        files_to_search.sort()
        return(files_to_search)


    def _read_data_from_file(self, file_to_search, start_unix_sample, stop_unix_sample, ret_array):
        """_read_data_from_file reads data (if any) from file.  Used with minimal metadata
        """
        # make sure cache is clear if this called
        if self._last_start_sample:
            try:
                self._last_file.close()
            except:
                pass
            self._last_file = None
            self._last_start_sample = None

        samples_per_file = int(self.metadata_dict['samples_per_file'][0])
        f = h5py.File(file_to_search, 'r')
        rf_data_index = f['/rf_data_index']

        if ret_array is None:
            # see if this is the first file with data
            file_start_index = None
            # walk through rf_data_index until right index found (if any)
            for i in range(len(rf_data_index)):
                this_file_index = int(rf_data_index[i,1])
                this_sample_index = int(rf_data_index[i,0])
                if i < len(rf_data_index)-1:
                    samples_left = int(rf_data_index[i+1,1]) - this_file_index
                else:
                    samples_left = int(samples_per_file - this_file_index)
                if this_sample_index + samples_left > start_unix_sample:
                    # the starting read point for the first file was found
                    file_start_index = int(this_file_index + (start_unix_sample-this_sample_index))
                    # make sure there are no gaps in this file to be read
                    if i < len(rf_data_index)-1:
                        samples_left_to_read = min(int(rf_data_index[i+1,1]) - file_start_index,
                                                   stop_unix_sample - start_unix_sample)
                        if samples_left_to_read < stop_unix_sample - start_unix_sample:
                            f.close()
                            raise IOError, 'Gap found in first file %s read' % (file_to_search)
                    else:
                        samples_left_to_read = min(samples_per_file - file_start_index, stop_unix_sample - start_unix_sample)
                    rf_data = f['/rf_data'][file_start_index:file_start_index + samples_left_to_read]
                    # see if we can cache this file
                    if len(rf_data_index) == 1:
                        self._last_start_sample = this_sample_index
                        if not self._last_file is None:
                            try:
                                self._last_file.close()
                            except:
                                pass
                        self._last_file = f
                    else:
                        f.close()
                    return(rf_data)

            # no data found
            try:
                f.close()
            except:
                pass
            return(None)

        else:
            # make sure cache is clear if muliple file read
            if self._last_start_sample:
                try:
                    self._last_file.close()
                except:
                    pass
                self._last_file = None
                self._last_start_sample = None

            # append all needed data from this file
            # first, verify this file begins where we expect
            first_file_sample = rf_data_index[0,0]
            if first_file_sample != start_unix_sample + len(ret_array):
                f.close()
                raise IOError, 'gap found at file %s -expected index %i, got %i' % (file_to_search, start_unix_sample + len(ret_array),
                                                                                    first_file_sample)
            # verify no gaps over this read
            if len(rf_data_index) > 1:
                samples_in_this_file = rf_data_index[1,1] - rf_data_index[0,1]
                if samples_in_this_file < (stop_unix_sample - start_unix_sample) - len(ret_array):
                    f.close()
                    raise IOError, 'not enough samples in file %s before data gap' % (file_to_search)

            samples_to_read = min(samples_per_file, (stop_unix_sample - start_unix_sample) - len(ret_array))
            rf_data = f['/rf_data'][0:samples_to_read]
            f.close()
            return(rf_data)




    def _get_data_from_cache(self, start_unix_sample, stop_unix_sample):
        """_get_data_from_cache simple returns the desired data from the cached Hdf5 file

        Inputs: start_unix_sample, stop_unix_sample - only samples between (start_unix_sample, stop_unix_sample)
                (excludes stop_unix_sample) will be returned.

        Calling method tested that this read is possible entirely within this file
        """
        start_index = start_unix_sample - self._last_start_sample
        samples_to_read = stop_unix_sample - start_unix_sample
        return((self._last_file['/rf_data'][start_index: start_index+samples_to_read], start_unix_sample))


    def __cmp__(self, other):
        """__cmp__ compares two _top_level_dir_metadata objects
        """
        # only the same channel can be compared
        if self.channel_name != other.channel_name:
            raise ValueError, 'Cannot compare mismatched channel names %s and %s' % (self.channel_name, other.channel_name)

        if self.unix_start_sample != 0 and other.unix_start_sample != 0:
            return(cmp(self.unix_start_sample, other.unix_start_sample))

        # use subdirectory names instead
        # for now only local access
        if self.access_mode not in ('local'):
            raise ValueError, 'access_mode %s not yet implemented' % (access_mode)

        first_subdirectory_list = glob.glob(os.path.join(self.top_level_dir, self.channel_name, self._sub_directory_glob))
        first_subdirectory_list.sort()
        if len(first_subdirectory_list) == 0:
            raise ValueError, 'Cannot compare top level directory because it has no data' % (self.top_level_dir)
        first_subdirectory = os.path.basename(first_subdirectory_list[0])

        second_subdirectory_list = glob.glob(os.path.join(other.top_level_dir, other.channel_name, self._sub_directory_glob))
        second_subdirectory_list.sort()
        if len(second_subdirectory_list) == 0:
            raise ValueError, 'Cannot compare top level directory because it has no data' % (other.top_level_dir)
        second_subdirectory = os.path.basename(second_subdirectory_list[0])

        return(cmp(first_subdirectory, second_subdirectory))


    def __del__(self):
        """__del__makes sure self._last_file is closed.  Does not happen automatically.
        """
        if not self._last_file is None:
            try:
                self._last_file.close()
            except:
                pass



class _sub_directory_metadata:
    """The _sub_directory_metadata is a private class to hold and access metadata about a particular digital_rf channel in
    a particular subdirectory.
    """

    def __init__(self, top_level_dir, channel_name, access_mode, subdirectory):
        """__init__ creates a new _sub_directory_metadata object

        Inputs:
            top_level_dir - full path the top level directory that contains the parent channel_name
            channel_name - the channel_name subdirectory name
            access_mode - string giving access mode (eg, 'local', 'file', or 'http')
            subdirectory - subdirectory name in form YYYY-MM-DDTHH-MM-SS

        Affects:
            Sets self.metadata to None.  When update called, self.metadata will be set to a numpy.recarray
            with columns:
                1. unix_sample_index - number of samples since 1970-01-01 to the start of a contiguous data block (uint64_t)
                2. file_index - where in the file this contiguous block of data begins
                3. rf_basename (25 char string)

            Also sets self.cont_metadata to None.  When update called, self.cont_metadata will be set
            to a numpy.recarray about block of contiguous data with columns:
                1. unix_sample_index - number of samples since 1970-01-01 to the start of a contiguous data block (uint64_t)
                2. sample_extent - number of continuous samples

            Also sets self.samples_per_file and self.file_count and self.last_timestamp to None.  Will be set with
            first call to update

            Also creates cached attributes to speed reads with detailed metadata:
            self._last_file - open H5py file last read
            self._last_start_sample - sample start index of cached file
            This file must be gap free, or it is never cached

        """
        self.top_level_dir = top_level_dir
        self.channel_name = channel_name
        self.access_mode = access_mode
        self.subdirectory = subdirectory
        self.metadata = None
        self.cont_metadata = None
        self.samples_per_file = None
        self.file_count = None
        self.last_timestamp = None # timestamp of last file in UTC
        self.metadata_dict = {} # to be populated by rf file metadata

        self._rf_file_glob = 'rf@[0-9]*.[0-9][0-9][0-9].h5'

        # data type of sub_directory_array
        self.data_t = numpy.dtype([('unix_sample_index', numpy.uint64, 1), ('file_index', numpy.uint64, 1),
                                   ('rf_basename', numpy.str_, 25)])
        self.cont_data_t = numpy.dtype([('unix_sample_index', numpy.uint64, 1), ('sample_extent', numpy.uint64, 1)])

        # set to an empty recarray
        if self.metadata is None:
            self.metadata = numpy.array([], dtype=self.data_t)
        if self.cont_metadata is None:
            self.cont_metadata = numpy.array([], dtype=self.cont_data_t)

        # attributes to allow caching
        self._last_file = None
        self._last_start_sample = None


    def get_summary_metadata(self):
        """get_summary_metadata returns a tuple of (first_unix_sample, sample_extent, file_count, samples_per_file,
        last_timestamp) for this _sub_directory_metadata object.

        Raises IOError if no self.metadata
        """
        if len(self.metadata) == 0:
            raise IOError, 'Must call update before calling get_summary_metadata, or subdirectory %s empty' % (self.subdirectory)

        first_unix_sample = int(self.metadata['unix_sample_index'][0])
        last_unix_sample = int(self.metadata['unix_sample_index'][-1]) + ((self.samples_per_file - int(self.metadata['file_index'][-1])) - 1)
        return((first_unix_sample, 1+(last_unix_sample-first_unix_sample), self.file_count, self.samples_per_file,
                self.last_timestamp))


    def update_if_needed(self, file_count, last_timestamp):
        """update_if_needed calls update only if input file_count or last_timestamp indicate an update
        is needed. Returns True if update actually called, False otherwise

        Inputs:
            file_count - number of file in subdirectory when last checked
            last_timestamp - UTC timestamp of last file in subdirectory when last checked
        """

        # for now only local access
        if self.access_mode not in ('local'):
            raise ValueError, 'access_mode %s not yet implemented' % (access_mode)

        rf_file_list = glob.glob(os.path.join(self.top_level_dir, self.channel_name, self.subdirectory,
                                              self._rf_file_glob))
        if len(rf_file_list) == 0:
            raise IOError, 'subdirectory %s empty' % (self.subdirectory)

        rf_file_list.sort()
        if len(rf_file_list) != file_count:
            self.update()
            return(True)
        elif abs(self._get_utc_timestamp(rf_file_list[-1]) - last_timestamp) > 2.0:
            # leave margin for error
            self.update()
            return(True)

        return(False)



    def update(self):
        """update updates self.metadata.  If it was a file name, it reads that data into memory, and then updates it
        """
        # for now only local access
        if self.access_mode not in ('local'):
            raise ValueError, 'access_mode %s not yet implemented' % (access_mode)

        rf_file_list = glob.glob(os.path.join(self.top_level_dir, self.channel_name, self.subdirectory,
                                              self._rf_file_glob))
        rf_file_list.sort()
        rf_file_basename_list = [os.path.basename(rf_file) for rf_file in rf_file_list]

        # first check to see if we can update things quickly if the data is continuous
        if self._update_continuous_data(rf_file_basename_list, rf_file_list):
            return

        unique_rf_basenames = numpy.unique(self.metadata['rf_basename'])

        # first step is to delete all lines where the rf file has been deleted
        rows_to_delete_arr = numpy.array([], dtype=numpy.int64)
        for rf_basename in unique_rf_basenames:
            if rf_basename not in rf_file_basename_list:
                # get a list of all rows with that file
                result = numpy.argwhere(self.metadata['rf_basename'] == rf_basename)
                result = result.flatten()
                rows_to_delete_arr = numpy.concatenate((rows_to_delete_arr, result))
        if len(rows_to_delete_arr) > 0:
            self.metadata = numpy.delete(self.metadata, rows_to_delete_arr)
            unique_rf_basenames = numpy.unique(self.metadata['rf_basename'])

        # the next step is to add rows from each file where it does not yet exist in self.metadata
        if len(self.metadata['rf_basename']) > 0:
            first_file_index = rf_file_basename_list.index(self.metadata['rf_basename'][-1]) + 1
        else:
            first_file_index = 0
        # we are only looping over files not already in self.metadata, and all data will be appended
        for i, rf_file_basename in enumerate(rf_file_basename_list[first_file_index:]):
            # verify the last file is not still being written
            if rf_file_basename == rf_file_basename_list[-1]:
                if self._file_is_open(rf_file_list[-1]):
                    self.file_count = len(rf_file_basename_list) - 1 # last file not counted
                    if self.file_count > 0:
                        self.last_timestamp = self._get_utc_timestamp(rf_file_list[-2])
                    continue
                else:
                    self.file_count = len(rf_file_basename_list)
                    self.last_timestamp = self._get_utc_timestamp(rf_file_list[-1])
            added_rows = self._get_new_rows(rf_file_basename)
            if added_rows is None:
                continue
            if len(self.metadata_dict.keys()) == 0:
                self.metadata_dict = self._get_rf_metadata(rf_file_basename)
            self.metadata = numpy.concatenate((self.metadata, added_rows))

        self._update_cont_metadata()


    def get_continuous_blocks(self, start_unix_sample, stop_unix_sample):
        """get_continuous_blocks returns a numpy array of dtype u64 and shape (N,2) where the first
        column represents the unix_sample of a continuous block of data, and the second column represents the
        number of samples in that continuous block.  Only samples between (start_unix_sample, stop_unix_sample)
        inclusive will be returned.

        Returns IOError if no blocks found

        Inputs:
            start_unix_sample, stop_unix_sample - only samples between (start_unix_sample, stop_unix_sample)
                inclusive will be returned.
        """
        # to improve speed, do searchsorted to get first index to look into
        first_index = numpy.searchsorted(self.cont_metadata['unix_sample_index'],
                                         numpy.array([start_unix_sample]))
        first_index = first_index[0]
        if first_index > 0:
            first_index -= 1

        # for now, deal with two edges later
        bool_arr = self.cont_metadata['unix_sample_index'] >= start_unix_sample
        bool_arr1 = self.cont_metadata['unix_sample_index'] <= stop_unix_sample
        bool_arr = numpy.logical_and(bool_arr, bool_arr1)
        metadata_slice = self.cont_metadata[bool_arr]

        # handle front edge
        ones = numpy.ones((len(bool_arr),))
        zeros = numpy.zeros((len(bool_arr),))
        indices = numpy.nonzero(numpy.where(bool_arr, ones, zeros))
        if len(indices[0]):
            first_index = indices[0][0]
            if first_index != 0:
                # in this case, we may need to add one line before, but modify it
                previous_sample_index = self.cont_metadata[first_index-1]['unix_sample_index']
                previous_sample_extent = self.cont_metadata[first_index-1]['sample_extent']
                if previous_sample_index + previous_sample_extent >= start_unix_sample:
                    metadata_slice = numpy.concatenate((self.cont_metadata[first_index-1:first_index],
                                                       metadata_slice))
                    metadata_slice[0] = (start_unix_sample, previous_sample_extent - \
                        (start_unix_sample - previous_sample_index))
            # else there is no need to fix front edge
        else:
            previous_sample_index = self.cont_metadata[0]['unix_sample_index']
            previous_sample_extent = self.cont_metadata[0]['sample_extent']
            if previous_sample_index + previous_sample_extent >= start_unix_sample:
                metadata_slice = numpy.zeros((1,), dtype=self.cont_data_t)
                metadata_slice[0] = (start_unix_sample, previous_sample_extent - \
                    (start_unix_sample - previous_sample_index))

        # fix end if need
        last_sample_index = metadata_slice[-1]['unix_sample_index']
        last_sample_extent = metadata_slice[-1]['sample_extent']
        real_last_sample_extent = 1 + (stop_unix_sample - last_sample_index)
        if real_last_sample_extent < last_sample_extent:
            metadata_slice[-1]['sample_extent'] = real_last_sample_extent

        ret_arr = numpy.zeros((len(metadata_slice), 2), dtype=numpy.uint64)
        ret_arr[:,0] = metadata_slice['unix_sample_index']
        ret_arr[:,1] = metadata_slice['sample_extent']

        return(ret_arr)


    def get_continuous_vector(self, start_unix_sample, stop_unix_sample):
        """get_continuous_vector returns a tuple of (numpy array of data, first unix_sample in returned data)
        Only samples between (start_unix_sample, stop_unix_sample) (excludes stop_unix_sample) will be returned.

        Returns IOError if no blocks found

        Inputs:
            start_unix_sample, stop_unix_sample - only samples between (start_unix_sample, stop_unix_sample)
                (excludes stop_unix_sample) will be returned.
        """
        # check if we can use cached file
        if self._last_start_sample:
            if start_unix_sample > self._last_start_sample and \
                stop_unix_sample <= self._last_start_sample + self.samples_per_file:
                return(self._get_data_from_cache(start_unix_sample, stop_unix_sample))
            else:
                # cache has expired
                try:
                    self._last_file.close()
                except:
                    pass
                self._last_file = None
                self._last_start_sample = None

        # first verify no gaps in this range
        self._verify_no_gaps(start_unix_sample, stop_unix_sample)

        # to improve speed, do searchsorted to get first index to look into
        first_index = numpy.searchsorted(self.metadata['unix_sample_index'],
                                         numpy.array([start_unix_sample]))
        first_index = int(first_index[0])
        if first_index == len(self.metadata):
            first_index -= 1
        elif self.metadata['unix_sample_index'][first_index] > start_unix_sample:
            first_index -= 1
        ret_array = None
        first_unix_sample = None
        this_hdf5_file = None
        samples_read = 0
        samples_to_read = stop_unix_sample - start_unix_sample
        for i in range(first_index, len(self.metadata)):
            if self.metadata['unix_sample_index'][i] >= stop_unix_sample:
                raise IOError, 'Did not get expected read - debug'

            this_hdf5_file = self.metadata['rf_basename'][i]
            full_hdf5_file = os.path.join(self.top_level_dir, self.channel_name, self.subdirectory, this_hdf5_file)

            # get max possible length of this read as block_len
            if i == len(self.metadata) - 1:
                # last index
                block_len = self.samples_per_file - int(self.metadata['file_index'][i])
            elif self.metadata['rf_basename'][i+1] == this_hdf5_file:
                block_len = int(self.metadata['file_index'][i+1]) - int(self.metadata['file_index'][i])
            else:
                block_len = self.samples_per_file - int(self.metadata['file_index'][i])

            # next get file start index
            if i == first_index:
                offset = int(start_unix_sample) - int(self.metadata['unix_sample_index'][first_index])
                block_len -= offset # we will not get the full read predicted above
                start_file_index = int(self.metadata['file_index'][i]) + offset
            else:
                start_file_index = int(self.metadata['file_index'][i])

            # next get read len
            if samples_read + block_len > samples_to_read:
                read_len = samples_to_read - samples_read
            else:
                read_len = block_len

            # finally - read it!!!
            f = h5py.File(full_hdf5_file, 'r')
            rf_data = f['/rf_data'][start_file_index:start_file_index + read_len]

            if ret_array is None:
                ret_array = rf_data
            else:
                ret_array = numpy.concatenate((ret_array, rf_data))
            samples_read += read_len
            if samples_read == samples_to_read:
                # check whether we can cache it
                if i == first_index and len(f['/rf_data_index']) == 1:
                    self._last_start_sample = int(self.metadata['unix_sample_index'][i])
                    if not self._last_file is None:
                        try:
                            self._last_file.close()
                        except:
                            pass
                    self._last_file = f
                else:
                    f.close()
                break
            else:
                f.close()

        return((ret_array, start_unix_sample))



    def get_first_sample(self):
        """get_first_sample returns the first sample index in this subdirectory.  May be exact (if self.metadata
        not is None) or an estimate based one subdirectory naming convention.
        """
        if len(self.metadata) > 0:
            return(self.metadata['unix_sample_index'][0])

        rf_file_list = glob.glob(os.path.join(self.top_level_dir, self.channel_name, self.subdirectory,
                                              self._rf_file_glob))

        if len(rf_file_list) == 0:
            raise IOError, 'No valid rf files found in subdirectory %s' % \
                (os.path.join(self.top_level_dir, self.channel_name, self.subdirectory))

        rf_file_list.sort()

        new_rows = self._get_new_rows(os.path.basename(rf_file_list[0]))
        return(new_rows['unix_sample_index'][0])


    def get_last_sample(self):
        """get_last_sample returns the last sample in a subdirectory.
        """
        if len(self.metadata) > 0 and len(self.metadata_dict.keys()):
            return(self.metadata['unix_sample_index'][-1] + self.metadata_dict['samples_per_file'] - self.metadata['file_index'][-1])

        rf_file_list = glob.glob(os.path.join(self.top_level_dir, self.channel_name, self.subdirectory,
                                              self._rf_file_glob))

        if len(rf_file_list) == 0:
            raise IOError, 'No valid rf files found in subdirectory %s' % \
                (os.path.join(self.top_level_dir, self.channel_name, self.subdirectory))

        rf_file_list.sort()

        if len(self.metadata_dict.keys()) == 0:
            self.metadata_dict = self._get_rf_metadata(os.path.basename(rf_file_list[0]))

        index = -1
        if self._file_is_open(rf_file_list[-1]) and len(rf_file_list) > 1:
            index = -2

        new_rows = self._get_new_rows(os.path.basename(rf_file_list[index]))
        if (new_rows is None and index == -2) or (new_rows is None and len(rf_file_list) == 1):
            raise IOError, 'Unable to read rf file %s - one possible error is an empty file, which can be removed via <find . -size 0 -exec rm {} \;>' % \
                (rf_file_list[index])
        elif new_rows is None and index == -1:
            index = -2
            new_rows = self._get_new_rows(os.path.basename(rf_file_list[index]))
            if new_rows is None:
                raise IOError, 'Unable to read rf file %s - one possible error is an empty file, which can be removed via <find . -size 0 -exec rm {} \;>' % \
                    (rf_file_list[index])
        try:
            return(new_rows['unix_sample_index'][-1] + self.metadata_dict['samples_per_file'] - new_rows['file_index'][-1])
        except:
            if index == -1 and len(rf_file_list) > 1:
                # try earlier file
                index = -2
                new_rows = self._get_new_rows(os.path.basename(rf_file_list[index]))
                if new_rows is None:
                    raise IOError, 'Unable to read rf file %s - one possible error is an empty file, which can be removed via <find . -size 0 -exec rm {} \;>' % \
                        (rf_file_list[index])
                return(new_rows['unix_sample_index'][-1] + self.metadata_dict['samples_per_file'] - new_rows['file_index'][-1])
            else:
                raise



    def _update_continuous_data(self, rf_file_basename_list, rf_file_list):
        """_update_continuous_data updates all metadata if data in subdirectory is continuous, then return True.  Does nothing
        and returns False if not continuous data.

        Determines if continuous by looking only at first and last file in rf_file_basename_list.

        Inputs:
            rf_file_basename_list - sorted list of basenames in subdirectory
            rf_file_list - sorted list of full names  in subdirectory
        """
        first_index = None
        last_index = None

        # get info from first file
        for i, rf_file_basename in enumerate(rf_file_basename_list):
            if first_index is None:
                first_row = self._get_new_rows(rf_file_basename)
                if not first_row is None:
                    if len(first_row) > 1:
                        return(False)
                    first_index = i
                    first_sample = first_row['unix_sample_index'][0]
                    break
                else:
                    continue

        if first_index is None:
            return(False)

        # get info from last file
        if not self._file_is_open(rf_file_list[-1]):
            last_row = self._get_new_rows(rf_file_basename_list[-1])
            last_index = len(rf_file_basename_list)
            self.last_timestamp = self._get_utc_timestamp(rf_file_list[-1])
        else:
            last_row = self._get_new_rows(rf_file_basename_list[-2])
            last_index = len(rf_file_basename_list) - 1
            self.last_timestamp = self._get_utc_timestamp(rf_file_list[-2])
        if last_row is None:
            return(False)
        if len(last_row) > 1:
            return(False)
        last_sample = last_row['unix_sample_index']
        self.file_count = last_index - first_index
        if (self.file_count - 1) * self.samples_per_file < last_sample - first_sample:
            # data gaps detected
            return(False)

        # create all metadata
        self.metadata = numpy.recarray((self.file_count,), dtype = self.data_t)
        sample_data = numpy.arange(0,self.file_count*self.samples_per_file, self.samples_per_file,dtype=numpy.int64)
        sample_data += first_sample
        self.metadata['unix_sample_index'] = sample_data
        self.metadata['file_index'][:] = 0
        self.metadata['rf_basename'] = rf_file_basename_list[first_index:last_index]

        self._update_cont_metadata()
        return(True)




    def _verify_no_gaps(self, start_unix_sample, stop_unix_sample):
        """_verify_no_gaps raises an IOError if there is a gap between start_unix_sample, stop_unix_sample
        """
         # to improve speed, do searchsorted to get first index to look into
        first_index = numpy.searchsorted(self.cont_metadata['unix_sample_index'],
                                         numpy.array([start_unix_sample]))
        first_index = first_index[0]
        if first_index == len(self.cont_metadata):
            first_index -= 1
        elif self.cont_metadata['unix_sample_index'][first_index] > start_unix_sample:
            first_index -= 1
        offset = start_unix_sample - int(self.cont_metadata['unix_sample_index'][first_index])
        if self.cont_metadata['sample_extent'][first_index] - offset < stop_unix_sample - start_unix_sample:
            raise IOError, 'gap found between samples %i and %i' % (start_unix_sample, stop_unix_sample)



    def _get_new_rows(self, rf_file_basename):
        """_get_new_rows is a private method that returns all needed rows for self.metadata in the correct recarray
        format for rf_file_basename, or None if that file has disappeared

        Inputs:
            rf_file_basename - rf file to examine

        Throws IOError if global indices overlap with previous metadata
        """
        # read data from /rf_data_index
        fullname = os.path.join(self.top_level_dir, self.channel_name, self.subdirectory, rf_file_basename)
        try:
            f = h5py.File(fullname, 'r')
        except IOError:
            # presumably file deleted
            return(None)
        rf_data_index = f['/rf_data_index']
        samples_per_file = f['rf_data'].attrs['samples_per_file'][0]
        if self.samples_per_file is None:
            self.samples_per_file = int(samples_per_file)
        elif self.samples_per_file != int(samples_per_file):
            raise IOError, 'Illegal change in samples_per_file from %i to %i in file %s' % (self.samples_per_file, int(samples_per_file),
                                                                                            fullname)

        # create recarray
        new_rows = numpy.zeros((len(rf_data_index),),dtype=self.data_t)
        new_rows['unix_sample_index'] = rf_data_index[:,0]
        new_rows['file_index'] = rf_data_index[:,1]
        new_rows['rf_basename'] = rf_file_basename

        f.close()

        return(new_rows)


    def _get_rf_metadata(self, rf_file_basename):
        """_get_rf_metadata is a private method that returns a dictionary of all metadata stored in each rf file,
        or empty dict if that file has disappeared

        Inputs:
            rf_file_basename - rf file to examine

        Returns dictionary with string keys:
            sample_rate
            samples_per_file
            uuid_str
        """
        ret_dict = {}
        fullname = os.path.join(self.top_level_dir, self.channel_name, self.subdirectory, rf_file_basename)
        try:
            f = h5py.File(fullname, 'r')
        except IOError:
            return({})
        dataset = f['/rf_data']
        for attr in dataset.attrs:
            ret_dict[str(attr)] = dataset.attrs[attr]

        f.close()
        return(ret_dict)



    def _combine_blocks(self, first_array, second_array, samples_per_file):
        """_combine_blocks combines two numpy array of dtype u64 and shape (N,2) where the first
        column represents the unix_sample of a continuous block of data, and the second column represents the
        number of samples in that continuous block. The first row of the second array may or may not be contiguous
        with the last row of the first array.  If it is contiguous, that row will not be included, and the
        number of samples in that first row will instead be added to the last row of first_array. If not contiguous,
        the two arrays are simply concatenated
        """
        if len(first_array) == 0:
            return(second_array)
        is_contiguous = False
        if first_array[-1][0] + first_array[-1][1] > second_array[0][0]:
            raise IOError, 'overlapping data found in top level directories %i, %i' % \
                (first_array[-1][0] + first_array[-1][1], second_array[0][0])
        if first_array[-1][0] + first_array[-1][1] == second_array[0][0]:
            is_contiguous = True
        if is_contiguous:
            first_array[-1][1] += second_array[0][1]
            if len(second_array) == 1:
                return(first_array)
            else:
                return(numpy.concatenate([first_array, second_array[1:]]))
        else:
            return(numpy.concatenate([first_array, second_array]))


    def _update_cont_metadata(self):
        """_update_cont_metadata completely rebuilds self.cont_metadata
        """
        cont_meta = []
        # handle empty dir case
        if len(self.metadata) == 0:
            self.cont_metadata = numpy.zeros((len(cont_meta),), dtype=self.cont_data_t)
            return
        for i in range(len(self.metadata)):
            if i == 0:
                cont_meta.append([self.metadata['unix_sample_index'][0], 0])
                last_sample = self.metadata['unix_sample_index'][0]
                last_index = 0
                continue
            this_sample = self.metadata['unix_sample_index'][i]
            this_index = self.metadata['file_index'][i]
            if this_index == 0:
                num_samples = self.samples_per_file - last_index
            else:
                num_samples = this_index - last_index
                if num_samples < 1:
                    raise ValueError, 'bug in self.metadata'
            cont_meta[-1][1] += num_samples
            if this_sample - last_sample == num_samples:
                if this_index != 0:
                    raise ValueError, 'bug 2 in self.metadata'
            else:
                cont_meta.append([this_sample, 0])
            last_sample = this_sample
            last_index = this_index

        # handle end of last file
        edge_samples = self.samples_per_file - last_index

        cont_meta[-1][1] += edge_samples

        cont_meta = numpy.array(cont_meta)

        # create self.cont_metadata
        self.cont_metadata = numpy.zeros((len(cont_meta),), dtype=self.cont_data_t)
        self.cont_metadata['unix_sample_index'] = cont_meta[:,0]
        self.cont_metadata['sample_extent'] = cont_meta[:,1]



    def _file_is_open(self, rf_file):
        """_file_is_open returns True if rf_file might be open (or corrupt), False otherwise
        """
        if time.time() - os.path.getmtime(rf_file) < 3:
            return(True)
        else:
            try:
                f = h5py.File(rf_file)
                f['/rf_data'].attrs['digital_rf_version']
                f.close()
                return(False)
            except:
                try:
                    f.close()
                except:
                    pass
                return(True)


    def _get_utc_timestamp(self, fullfile):
        """_get_utc_timestamp returns the last modification timestamp of fullfile in UTC
        """
        # for now only local access
        if self.access_mode not in ('local'):
            raise ValueError, 'access_mode %s not yet implemented' % (access_mode)

        return(os.path.getmtime(fullfile) - time.timezone)


    def _get_data_from_cache(self, start_unix_sample, stop_unix_sample):
        """_get_data_from_cache simple returns the desired data from the cached Hdf5 file

        Inputs: start_unix_sample, stop_unix_sample - only samples between (start_unix_sample, stop_unix_sample)
                (excludes stop_unix_sample) will be returned.

        Calling method tested that this read is possible entirely within this file
        """
        start_index = start_unix_sample - self._last_start_sample
        samples_to_read = stop_unix_sample - start_unix_sample
        return((self._last_file['/rf_data'][start_index: start_index+samples_to_read], start_unix_sample))


class _MissingMetadata(Exception):
    """_MissingMetadata is a Exception that will be raised when metadata needs to be updated
    """
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)
