#!python
# ----------------------------------------------------------------------------
# Copyright (c) 2017 Massachusetts Institute of Technology (MIT)
# All rights reserved.
#
# Distributed under the terms of the BSD 3-clause license.
#
# The full license is in the LICENSE file, distributed with this software.
# ----------------------------------------------------------------------------
"""digital_rf_archive.py is a tool for archiving Digital RF data.

$Id$
"""
# standard python imports
import datetime
import time
import glob
import argparse
import os
import sys
import socket
import urlparse
import subprocess
import shutil
import multiprocessing
import traceback


def archive_subdirectory_local_local(args):
    """archive_subdirectory_local_local is the method called by a pool of multiprocessing process to archive
    one subdirectory from a local machine to a local machine.

    Inputs: args, a list of:
        source - path to high level source directory
        channel - channel name
        subdir - basename of subdirectory to archive
        dt - datetime of subdirectory name
        nextDT - datetime of next subdirectory.  None if no next subdirectory.
        is_metadata - True if metadata, False if RF data
        channel_dest - path to where channel will be backed up
        startDT - start datetime of archive
        endDT - end datetime of archive
        gzip - gzip (0-9) 0 is no compression
        verbose - True for verbose output, False otherwise

    Returns: the number of files backed up
    """
    # constants
    _rf_file_glob = 'rf@[0-9]*.[0-9][0-9][0-9].h5'
    _metadata_file_glob = '*@[0-9]*.h5'

    file_count = 0

    source, channel, subdir, dt, nextDT, is_metadata, channel_dest, startDT, endDT, gzip, verbose = args

    # determine if this channel is compressed
    if is_metadata:
        archive_files = glob.glob(os.path.join(
            source, channel, subdir, _metadata_file_glob))
    else:
        archive_files = glob.glob(os.path.join(
            source, channel, subdir, _rf_file_glob))
    if len(archive_files) == 0:
        if verbose:
            print('skipping because no files found in %s' % (subdir))
        return(0)

    # decide if this is a full copy, partial copy, or skip
    if dt > endDT:
        if verbose:
            print('skipping %s because after period %s - %s' % (subdir, str(startDT),
                                                                str(endDT)))
        return(0)

    if dt < startDT and nextDT != None:
        if nextDT < startDT:
            if verbose:
                print('skipping %s because entirely before period %s - %s' % (subdir, str(startDT),
                                                                              str(endDT)))
            return(0)

    if gzip > 0:
        is_compressed = _is_h5_file_compressed(archive_files[0])
    else:
        is_compressed = False

    # the only options left now are complete or partial copies
    # check if complete copy needed
    if startDT < dt and nextDT != None:
        if nextDT < endDT:
            if (is_compressed or gzip == 0) or is_metadata:
                if verbose:
                    print('%s - complete copy' % (subdir))
                shutil.copytree(os.path.join(source, channel, subdir),
                                os.path.join(channel_dest, subdir))
                file_count += len(glob.glob(os.path.join(source,
                                                         channel, subdir, '*')))
                return(file_count)
    if verbose:
        print('%s partial copy or copy with added compression' % (subdir))
    target_dir = os.path.join(channel_dest, subdir)
    os.mkdir(target_dir)
    if is_metadata:
        archive_files = glob.glob(os.path.join(
            source, channel, subdir, _metadata_file_glob))
    else:
        archive_files = glob.glob(os.path.join(
            source, channel, subdir, _rf_file_glob))
    archive_files.sort()  # allows us to stop checking when a file moves out of time range
    for archive_file in archive_files:
        basename = os.path.basename(archive_file)
        ts = datetime.datetime.utcfromtimestamp(
            float(basename[basename.find('@') + 1:-3]))
        if ts >= startDT and ts <= endDT:
            # check if we need to compress it
            if not is_compressed and gzip > 0 and not is_metadata:
                src_file = os.path.join(target_dir, basename)
                cmd = 'h5repack -i %s -o %s -f rf_data:GZIP=%i' % (
                    archive_file, src_file, gzip)
                try:
                    subprocess.check_call(cmd.split())
                    file_count += 1
                except:
                    traceback.print_exc()
                    print('cmd <%s> failed' % (cmd))
            else:
                shutil.copyfile(archive_file, os.path.join(
                    target_dir, os.path.basename(archive_file)))
                file_count += 1
        elif ts > endDT:
            return(file_count)

    return(file_count)


def archive_subdirectory_local_remote(args):
    """archive_subdirectory_local_remote is the method called by a pool of multiprocessing process to archive
    one subdirectory from a local machine to a remote machine.

    Inputs: args, a list of:
        source - path to high level source directory
        channel - channel name
        subdir - basename of subdirectory to archive
        dt - datetime of subdirectory name
        nextDT - datetime of next subdirectory.  None if no next subdirectory.
        is_metadata - True if metadata, False if RF data
        channel_dest - path to where channel will be backed up
        startDT - start datetime of archive
        endDT - end datetime of archive
        gzip - gzip (0-9) ) is no compression
        verbose - True for verbose output, False otherwise

    Returns: the number of files backed up
    """
    # constants
    _rf_file_glob = 'rf@[0-9]*.[0-9][0-9][0-9].h5'
    _metadata_file_glob = '*@[0-9]*.h5'

    file_count = 0

    source, channel, subdir, dt, nextDT, is_metadata, channel_dest, startDT, endDT, gzip, verbose = args

    # determine if this channel is compressed
    if is_metadata:
        archive_files = glob.glob(os.path.join(
            source, channel, subdir, _metadata_file_glob))
    else:
        archive_files = glob.glob(os.path.join(
            source, channel, subdir, _rf_file_glob))
    if len(archive_files) == 0:
        if verbose:
            print('skipping because no files found in %s' % (subdir))
        return(0)

    # decide if this is a full copy, partial copy, or skip
    if dt > endDT:
        if verbose:
            print('skipping %s because after period %s - %s' % (subdir, str(startDT),
                                                                str(endDT)))
        return(0)

    if dt < startDT and nextDT != None:
        if nextDT < startDT:
            if verbose:
                print('skipping %s because entirely before period %s - %s' % (subdir, str(startDT),
                                                                              str(endDT)))
            return(0)

    if gzip > 0:
        is_compressed = _is_h5_file_compressed(archive_files[0])
    else:
        is_compressed = False

    # the only options left now are complete or partial copies
    # check if complete copy needed
    if startDT < dt and nextDT != None:
        if nextDT < endDT:
            if (is_compressed or gzip == 0) or is_metadata:
                if verbose:
                    print('%s - complete copy' % (subdir))
                cmd = 'scp -q -r %s %s' % (os.path.join(source, channel, subdir),
                                           os.path.join(channel_dest, subdir))
                try:
                    subprocess.check_call(cmd.split())
                    file_count += len(glob.glob(os.path.join(source,
                                                             channel, subdir, '*')))
                except subprocess.CalledProcessError:
                    raise IOError, 'Failed to copy entire subdirectory with cmd <%s>' % (
                        cmd)
                return(file_count)

    if verbose:
        print('%s partial copy or copy with added compression' % (subdir))
    target_dir = os.path.join(channel_dest, subdir)
    # make channel dir locally, then copy
    subdir_local = os.path.join('/tmp', subdir)
    try:
        os.system('rm -rf %s' % (subdir_local))
    except:
        pass
    os.mkdir(subdir_local)
    cmd = 'scp -q -r %s %s' % (subdir_local, channel_dest)
    try:
        subprocess.check_call(cmd.split())
    except subprocess.CalledProcessError:
        raise IOError, 'Failed to create remote subdirectory dir with cmd <%s>' % (
            cmd)
    try:
        os.system('rm -rf %s' % (subdir_local))
    except:
        pass

    if is_metadata:
        archive_files = glob.glob(os.path.join(
            source, channel, subdir, _metadata_file_glob))
    else:
        archive_files = glob.glob(os.path.join(
            source, channel, subdir, _rf_file_glob))
    archive_files.sort()  # allows us to stop checking when a file moves out of time range
    for archive_file in archive_files:
        basename = os.path.basename(archive_file)
        ts = datetime.datetime.utcfromtimestamp(
            float(basename[basename.find('@') + 1:-3]))
        if ts >= startDT and ts <= endDT:
            # if we need to compress it, make a copy first
            if not is_compressed and gzip > 0 and not is_metadata:
                src_file = os.path.join('/tmp', basename)
                try:
                    os.remove(src_file)
                except:
                    pass
                cmd = 'h5repack -i %s -o %s -f rf_data:GZIP=%i' % (
                    archive_file, src_file, gzip)
                try:
                    subprocess.check_call(cmd.split())
                except:
                    traceback.print_exc()
                    print('cmd <%s> failed' % (cmd))
                    continue
            else:
                src_file = archive_file
            cmd = 'scp -q %s %s' % (src_file, target_dir)
            try:
                subprocess.check_output(cmd.split())
                file_count += 0
            except subprocess.CalledProcessError:
                raise IOError, 'Failed to copy archive_file with cmd <%s>' % (
                    cmd)
            if src_file != archive_file:
                os.remove(src_file)
        elif ts > endDT:
            break

    return(file_count)


def _is_h5_file_compressed(filename):
    """_is_h5_file_compressed returns True if Hdf5 file gzip compresses, False otherwise
    """
    cmd = 'h5stat %s' % (filename)
    output = subprocess.check_output(cmd.split())
    for line in output.split('\n'):
        if line.find('GZIP') != -1:
            items = line.split()
            level = int(items[-1])
            if level > 0 and level < 10:
                return(True)
    return(False)


class archive:
    """archive is a class to archive a digital rf data set
    """

    def __init__(self, startDT, endDT, source, dest, pool=1, channels=None, is_metadata=False,
                 gzip=1, verbose=False, check_only=False):
        """
        __init__ will create a Digital RF archive

        Inputs:
            startDT - start datetime for the data archive
            endDT end datetime for the data archive
            source - string defining data source. Either a path, or a url.
                Empty string is defined as $PWD
            dest - where archive is to be created.  May be a path, or a scp
                type path (user@host:/path) If scp-type, keypair must allow
                scp without passwords
            pool - number of multiprocessing processes to use. Default is 1
            channels - a list of channels to archive.  If None (the default),
                archive all channels
            is_metadata - False if standard rf data, True if metadata with modified file
                naming convention.  That convention is *@<timestamp>*h5.  Metadata is stored
                under metadata subdirectory, rf_data under rf_data subdirectory
            gzip - integer specifying level of gzip compression to apply if file not already compressed.
                Default is 1.  0 for no compression.
            verbose - if True, print one line for each subdirectory.  If False (the default),
                no output except for errors.
            check_only - if True, simply check if enough space. Raises error if not local source and local dest.

        Attributes:
            self._source_type - local if local input data, remote if remote
            self._chan_dist - dictionary with keys = channel found, values = tuple of
                1: ordered list of tuples of
                    1. paths or urls to subdirectories,
                    2. datetime of that subdirectory name. and
                2: list of metadata files in channel directory
        """
        # constants
        self._sub_directory_glob = '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]-[0-9][0-9]-[0-9][0-9]'
        self._rf_file_glob = 'rf@[0-9]*.[0-9][0-9][0-9].h5'
        self._properties_glob = '*_properties.h5'
        self._metadata_file_glob = '*@[0-9]*.h5'
        self._metadata_dir = 'metadata'
        self._rf_dir = 'rf_data'

        # verify arguments
        if endDT <= startDT:
            print('endDT <%s> must be after startDT <%s>' %
                  (str(endDT), str(startDT)))
            sys.exit(-1)

        # save args
        self.startDT = startDT
        self.endDT = endDT
        self.source = source
        self.dest = dest
        self.pool_count = pool
        self.channels = channels
        self.is_metadata = bool(is_metadata)
        self.gzip = int(gzip)
        self.verbose = bool(verbose)
        self.check_only = bool(check_only)

        self._source_type = 'local'
        if len(self.source) > 5:
            if self.source[0:5] == 'http:':
                self._source_type = 'remote'

        if self._source_type == 'local':
            # make sure source is full path
            if self.source[0] != '/':
                self.source = os.path.join(os.getcwd(), self.source)

        self._dest_type = 'local'
        hyphen = self.dest.find(':')
        slash = self.dest.find('/')
        if hyphen != -1:
            if slash == -1:
                self._dest_type = 'remote'
            elif hyphen < slash:
                self._dest_type = 'remote'

        if self.check_only:
            if self._dest_type != 'local' or self._source_type != 'local':
                raise ValueError, 'check_only flag only valid for local source and destination'

        if self.verbose:
            # set up a timer
            t = time.time()

        if self._source_type == 'local':
            self._chan_dict = self.get_chan_dict_local()
        else:
            self._chan_dict = self.get_chan_dict_remote()

        # verify all channels exist if specified
        if self.channels:
            for channel in self.channels:
                if not self._chan_dict.has_key(channel):
                    raise IOError, 'Requested channel %s not found' % (channel)
        else:
            self.channels = self._chan_dict.keys()  # archive all channels

        self._hostname = self._get_hostname()
        self._top_level_dest_dir = self.get_top_level_dest_dir()

        # set _next_level_dir to rf_data or metadata
        if self.is_metadata:
            self._next_level_dir = self._metadata_dir
        else:
            self._next_level_dir = self._rf_dir

        if self._dest_type == 'local' and self._source_type == 'local':
            # verify enough disk space
            needed_GB, avail_GB = self.estimate_disk_space()
            if self.verbose or self.check_only:
                print('Estimated GB to use = %f GB, available = %f GB' %
                      (needed_GB, avail_GB))
                if self.check_only:
                    sys.exit(-1)
            if needed_GB > avail_GB:
                raise IOError, "aborting because needed GB %f greater than available GB %f" % (
                    needed_GB, avail_GB)

        self._create_top_level_dest_dirs()

        # create a pool of workers
        self._pool = multiprocessing.Pool(pool)

        for channel in self.channels:
            if self.verbose:
                print('archiving channel %s' % (channel))
            result = self._archive_channel(channel)

        if self.verbose:
            print('digital_rf_archive took %f seconds' % (time.time() - t))

    def _archive_channel(self, channel):
        """_archive_channel will archive the channel.  Returns True if any data archived, False otherwise
        """
        # to simplify the code, break into for subroutines depending on
        # self._source_type self._dest_type
        if self._source_type == 'local':
            if self._dest_type == 'local':
                return(self._archive_channel_local_local(channel))
            else:
                return(self._archive_channel_local_remote(channel))

        else:
            raise ValueError, 'nyi'

    def _archive_channel_local_local(self, channel):
        """_archive_channel_local_local archives local input data to a local destination
        """
        is_compressed = None  # not yet known
        subdir_list = self._chan_dict[channel]
        # make channel dir in self._top_level_dest_dir/self._next_level_dir
        channel_dest = os.path.join(self.dest, self._top_level_dest_dir, self._next_level_dir,
                                    channel)
        os.mkdir(channel_dest)

        # first copy all the properties
        for properties_file in subdir_list[1]:
            if self.verbose:
                print('archiving properties file %s' % (properties_file))
            shutil.copyfile(os.path.join(self.source, channel, properties_file),
                            os.path.join(channel_dest, os.path.basename(properties_file)))

        # build iterable of arguments
        args_list = []
        for i, items in enumerate(subdir_list[0]):
            subdir, dt = items
            if i < len(subdir_list[0]) - 1:
                nextDT = subdir_list[0][i + 1][1]
            else:
                nextDT = None
            # distribute the subdirectories to the pool
            args_list.append([self.source, channel, subdir, dt, nextDT, self.is_metadata, channel_dest,
                              self.startDT, self.endDT, self.gzip, self.verbose])

        file_count_list = self._pool.map(
            archive_subdirectory_local_local, args_list)

        if self.verbose:
            print('%i files backed up in channel %s' %
                  (sum(file_count_list), channel))
        if sum(file_count_list) == 0:
            print("WARNING: No files backed up in channel %s" % (channel))

    def _archive_channel_local_remote(self, channel):
        """_archive_channel_local_remote archives local input data to a remote destination
        """
        is_compressed = None  # not yet known
        subdir_list = self._chan_dict[channel]
        file_count = 0

        # make channel dir in self._top_level_dest_dir
        top_level_dest = os.path.join(
            self.dest, self._top_level_dest_dir, self._next_level_dir)
        channel_dest = os.path.join(
            self.dest, self._top_level_dest_dir, self._next_level_dir, channel)
        # make channel dir locally, then copy
        channel_local = os.path.join('/tmp', channel)
        try:
            os.system('rm -rf %s' % (channel_local))
        except:
            pass
        os.mkdir(channel_local)
        cmd = 'scp -q -r %s %s' % (channel_local, top_level_dest)
        try:
            subprocess.check_call(cmd.split())
        except subprocess.CalledProcessError:
            raise IOError, 'Failed to create remote channel dir with cmd <%s>' % (
                cmd)
        try:
            os.system('rm -rf %s' % (channel_local))
        except:
            pass

        # first copy all the properties
        for properties_file in subdir_list[1]:
            if self.verbose:
                print('archiving properties file %s' % (properties_file))
            cmd = 'scp -q %s %s' % (os.path.join(self.source,
                                                 channel, properties_file), channel_dest)
            try:
                subprocess.check_call(cmd.split())
            except subprocess.CalledProcessError:
                raise IOError, 'Failed to copy properties file with cmd <%s>' % (
                    cmd)

        # build iterable of arguments
        args_list = []
        for i, items in enumerate(subdir_list[0]):
            subdir, dt = items
            if i < len(subdir_list[0]) - 1:
                nextDT = subdir_list[0][i + 1][1]
            else:
                nextDT = None
            # distribute the subdirectories to the pool
            args_list.append([self.source, channel, subdir, dt, nextDT, self.is_metadata, channel_dest,
                              self.startDT, self.endDT, self.gzip, self.verbose])

        file_count_list = self._pool.map(
            archive_subdirectory_local_remote, args_list)

        if self.verbose:
            print('%i files backed up in channel %s' %
                  (sum(file_count_list), channel))
        if sum(file_count_list) == 0:
            print("WARNING: No files backed up in channel %s" % (channel))

    def _get_hostname(self):
        """_get_hostname returns the hostname of the local system or the remote system of the source data
        """
        if self._source_type == 'local':
            return(socket.gethostname())
        else:
            o = urlparse.urlparse(self.source)
            return(o.netloc)

    def _create_top_level_dest_dirs(self):
        """_create_top_level_dest_dirs will create the dest top level directory if needed. Raise IOError
        if it can't be created. Also creates subdirectories <metadata> and <rf_data>
        under that directory.
        """
        full_path = os.path.join(self.dest, self._top_level_dest_dir)
        if self._dest_type == 'local':
            if not os.path.isdir(self.dest):
                raise IOError, '%s does not exist or is not a directory' % (
                    self.dest)
            if not os.access(full_path, os.W_OK):
                os.mkdir(full_path)
                os.mkdir(os.path.join(full_path, self._rf_dir))
                os.mkdir(os.path.join(full_path, self._metadata_dir))
        else:
            # try to create the three remote directories - even if they already
            # exist, should succeed
            for i, this_dir in enumerate((self._top_level_dest_dir, self._metadata_dir, self._rf_dir)):
                localDir = os.path.join('/tmp', this_dir)
                try:
                    os.system('rm -rf %s' % (localDir))
                except:
                    pass
                os.mkdir(localDir)
                # now scp better succeed
                if i == 0:
                    cmd = 'scp -q -r %s %s' % (localDir, self.dest)
                else:
                    cmd = 'scp -q -r %s %s' % (localDir, os.path.join(
                        self.dest, self._top_level_dest_dir))
                try:
                    subprocess.check_call(cmd.split())
                except subprocess.CalledProcessError:
                    raise IOError, 'Failed to create remote top level dir with cmd <%s>' % (
                        cmd)
                try:
                    os.system('rm -rf %s' % (localDir))
                except:
                    pass

    def _is_h5_file_compressed(self, filename):
        cmd = 'h5stat %s' % (filename)
        output = subprocess.check_output(cmd.split())
        for line in output.split('\n'):
            if line.find('GZIP') != -1:
                items = line.split()
                level = int(items[-1])
                if level > 0 and level < 10:
                    return(True)
        return(False)

    def estimate_disk_space(self):
        """estimate_disk_space returns a tuple of (needed_GB, avail_GB) as floats for local dest"""
        # constants
        _rf_file_glob = 'rf@[0-9]*.[0-9][0-9][0-9].h5'
        _metadata_file_glob = '*@[0-9]*.h5'

        # estimate available disk space
        result = os.statvfs(self.dest)
        avail_GB = (result.f_bavail * result.f_frsize) / (1024 * 1024 * 1024.0)

        # estimate needed space
        needed_GB = 0.0
        compress = None  # will be estimated with test file
        for channel in self.channels:
            subdir_list = self._chan_dict[channel]

            # properties
            for properties_file in subdir_list[1]:
                needed_GB += os.path.getsize(os.path.join(
                    self.source, channel, properties_file))

            # rf data
            for i, items in enumerate(subdir_list[0]):
                subdir, dt = items
                if i < len(subdir_list[0]) - 1:
                    nextDT = subdir_list[0][i + 1][1]
                else:
                    nextDT = None

                if dt > self.endDT:
                    break
                if nextDT != None:
                    if nextDT <= self.startDT:
                        # too early
                        continue

                if self.is_metadata:
                    archive_files = glob.glob(os.path.join(
                        self.source, channel, subdir, _metadata_file_glob))
                    compress = 1.0
                else:
                    archive_files = glob.glob(os.path.join(
                        self.source, channel, subdir, _rf_file_glob))
                    if compress == None and len(archive_files) > 0:
                        compress = self._test_compression(archive_files[0])

                for archive_file in archive_files:
                    needed_GB += compress * os.path.getsize(archive_file)

        return((needed_GB / (1024 * 1024 * 1024.0), avail_GB))

    def _test_compression(self, testHdf5File):
        """_test_compression returns the gzip compression ratio for test file testHdf5File
        """
        if self.gzip == 0:
            return(1.0)
        resultFile = '/tmp/test.h5'
        try:
            os.remove(resultFile)
        except:
            pass
        cmd = 'h5repack -i %s -o %s -f rf_data:GZIP=%i' % (
            testHdf5File, resultFile, self.gzip)
        subprocess.check_call(cmd.split())
        result = float(os.path.getsize(resultFile)) / \
            os.path.getsize(testHdf5File)
        os.remove(resultFile)
        return(result)

    def get_top_level_dest_dir(self):
        """returns the top level dest directory to write to in format
        <hostname>_YYYY-MM-DDTHH:MM:SS_YYYY-MM-DDTHH:MM:SS
        """
        startDTStr = self.startDT.strftime('%Y%m%dT%H%M%S')
        endDTStr = self.endDT.strftime('%Y%m%dT%H%M%S')
        return('%s_%s_%s' % (self._hostname, startDTStr, endDTStr))

    def get_chan_dict_local(self):
        """get_chan_dict_local returns a dictionary with keys = channels found,
            values = list of two lists - 1: list of
            subdirectory basenames, 2: list of properties files in channel directory
        """
        ret_dict = {}
        subdirectories = glob.glob(os.path.join(
            self.source, '*', self._sub_directory_glob))
        subdirectories.sort()
        for subdirectory in subdirectories:
            basename = os.path.basename(subdirectory)
            dt = datetime.datetime.strptime(basename, '%Y-%m-%dT%H-%M-%S')
            channel_path = os.path.dirname(subdirectory)
            channel = os.path.basename(channel_path)
            if not ret_dict.has_key(channel):
                ret_dict[channel] = [[], []]
            ret_dict[channel][0].append((basename, dt))

        # now get properties files
        properties_files = glob.glob(os.path.join(
            self.source, '*', self._properties_glob))
        for properties_file in properties_files:
            basename = os.path.basename(properties_file)
            channel_path = os.path.dirname(properties_file)
            channel = os.path.basename(channel_path)
            if ret_dict.has_key(channel):
                ret_dict[channel][1].append(basename)

        return(ret_dict)

    def get_chan_dict_remote(self):
        """get_chan_dict_local returns a dictionary with keys = channels found,
            values = list of two lists - 1: list of
            subdirectory basenames, 2: list of metadata files in channel directory
        """
        return('nyi')


### main begins here ###
if __name__ == '__main__':

    # command line interface
    parser = argparse.ArgumentParser(
        description='digital_rf_archive.py is a tool for archiving Digital RF data..')
    parser.add_argument('--startDT', metavar='start datetime string',
                        help='Start UT datetime for archiving in format YYYY-MM-DDTHH:MM:SS', required=True)
    parser.add_argument('--endDT', metavar='end datetime string',
                        help='End UT datetime for archiving in format YYYY-MM-DDTHH:MM:SS', required=True)
    parser.add_argument('--source', metavar='Top level directory or url',
                        help='Top level directory or url containing the Digital RF channels or metadata channels to archive',
                        required=True)
    parser.add_argument('--dest', metavar='Full path to destination',
                        help='Full path to destination directory.  May be local, or remote in scp form (user@host:)',
                        required=True)
    parser.add_argument('--pool', metavar='Pool multiprocessing count.',
                        help='Number of multiprocesses processes to use in the pool.  Default=1.',
                        required=False, type=int, default=1)
    parser.add_argument('--channel', metavar='Optional channel name', action='append',
                        help='Optional channel name to backup.  May be given more than once.  Default is all channels.')
    parser.add_argument('--is_metadata', action='store_true', default=False,
                        help='Set this flag if metadata, as opposed to rf_data, being archived.')
    parser.add_argument('--gzip', metavar='GZIP compression 1-9.',
                        help='Level of GZIP compression 1-9.  Default=1. 0 for no compression',
                        required=False, type=int, default=1)
    parser.add_argument('-v',  '--verbose', action='store_true', default=False,
                        help='Get one line of output for each subdirectory archived.')
    parser.add_argument('--check_only', action='store_true', default=False,
                        help='Set this flag to simply check if enough space. Only possible for local source and local dest.')
    args = parser.parse_args()

    # create datetimes
    try:
        startDT = datetime.datetime.strptime(args.startDT, '%Y-%m-%dT%H:%M:%S')
    except:
        print('startDT <%s> not in expected YYYY-MM-DDTHH:MM:SS format' %
              (args.startDT))
        sys.exit(-1)
    try:
        endDT = datetime.datetime.strptime(args.endDT, '%Y-%m-%dT%H:%M:%S')
    except:
        print('endDT <%s> not in expected YYYY-MM-DDTHH:MM:SS format' %
              (args.endDT))
        sys.exit(-1)

    # verify gzip in range
    if args.gzip < 0 or args.gzip > 9:
        raise ValueError, 'gzip must be 0 (no compression), or 1-9, not %i' % (gzip)

    # call main class
    archive(startDT, endDT, args.source, args.dest, args.pool, args.channel, args.is_metadata,
            args.gzip, args.verbose, args.check_only)
