# Copyright (c) Microsoft Corporation
#
# All rights reserved.
#
# MIT License
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

# compat imports
from __future__ import (
    absolute_import, division, print_function, unicode_literals
)
from builtins import (  # noqa
    bytes, dict, int, list, object, range, ascii, chr, hex, input,
    next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import enum
import logging
import math
try:
    import pathlib2 as pathlib
except ImportError:  # noqa
    import pathlib
try:
    import queue
except ImportError:  # noqa
    import Queue as queue
import threading
import time
# non-stdlib imports
# local imports
import blobxfer.models.crypto
import blobxfer.models.metadata
import blobxfer.operations.azure.blob
import blobxfer.operations.azure.blob.append
import blobxfer.operations.azure.blob.block
import blobxfer.operations.azure.blob.page
import blobxfer.operations.azure.file
import blobxfer.operations.crypto
import blobxfer.operations.md5
import blobxfer.operations.progress
import blobxfer.operations.resume
import blobxfer.util

# create logger
logger = logging.getLogger(__name__)


class UploadAction(enum.Enum):
    Skip = 1
    CheckMd5 = 2
    Upload = 3


class Uploader(object):
    """Uploader"""
    def __init__(self, general_options, creds, spec):
        # type: (Uploader, blobxfer.models.options.General,
        #        blobxfer.operations.azure.StorageCredentials,
        #        blobxfer.models.upload.Specification) -> None
        """Ctor for Uploader
        :param Uploader self: this
        :param blobxfer.models.options.General general_options: general opts
        :param blobxfer.operations.azure.StorageCredentials creds: creds
        :param blobxfer.models.uplaod.Specification spec: upload spec
        """
        self._all_files_processed = False
        self._crypto_offload = None
        self._md5_meta_lock = threading.Lock()
        self._md5_map = {}
        self._md5_offload = None
        self._upload_lock = threading.Lock()
        self._upload_queue = queue.Queue()
        self._upload_set = set()
        self._upload_start_time = None
        self._disk_threads = []
        self._upload_total = 0
        self._upload_sofar = 0
        self._upload_bytes_total = 0
        self._upload_bytes_sofar = 0
        self._upload_terminate = False
        self._transfer_lock = threading.Lock()
        self._transfer_queue = queue.Queue()
        self._transfer_set = set()
        self._transfer_threads = []
        self._start_time = None
        self._delete_exclude = set()
        self._ud_map = {}
        self._containers_created = set()
        self._fileshare_dir_lock = threading.Lock()
        self._dirs_created = {}
        self._general_options = general_options
        self._creds = creds
        self._spec = spec
        self._resume = None
        self._exceptions = []

    @property
    def termination_check(self):
        # type: (Uploader) -> bool
        """Check if terminated
        :param Uploader self: this
        :rtype: bool
        :return: if terminated
        """
        with self._upload_lock:
            with self._transfer_lock:
                return (self._upload_terminate or
                        len(self._exceptions) > 0 or
                        (self._all_files_processed and
                         len(self._upload_set) == 0 and
                         len(self._transfer_set) == 0))

    @property
    def termination_check_md5(self):
        # type: (Uploader) -> bool
        """Check if terminated from MD5 context
        :param Uploader self: this
        :rtype: bool
        :return: if terminated from MD5 context
        """
        with self._md5_meta_lock:
            with self._upload_lock:
                return (self._upload_terminate or
                        (self._all_files_processed and
                         len(self._md5_map) == 0 and
                         len(self._upload_set) == 0))

    @staticmethod
    def create_unique_id(src, ase):
        # type: (blobxfer.models.upload.LocalPath,
        #        blobxfer.models.azure.StorageEntity) -> str
        """Create a unique id given a LocalPath and StorageEntity
        :param blobxfer.models.upload.LocalPath src: local path
        :param blobxfer.models.azure.StorageEntity ase: azure storage entity
        :rtype: str
        :return: unique id for pair
        """
        return ';'.join(
            (str(src.absolute_path), ase._client.primary_endpoint, ase.path)
        )

    @staticmethod
    def create_unique_transfer_id(local_path, ase, offsets):
        # type: (blobxfer.models.upload.LocalPath,
        #        blobxfer.models.azure.StorageEntity) -> str
        """Create a unique transfer id given a offsets
        :param blobxfer.models.upload.LocalPath local_path: local path
        :param blobxfer.models.azure.StorageEntity ase: azure storage entity
        :param blobxfer.models.upload.Offsets offsets: upload offsets
        :rtype: str
        :return: unique id for transfer
        """
        return ';'.join(
            (str(local_path.absolute_path), ase._client.primary_endpoint,
             ase.path, str(local_path.view.fd_start), str(offsets.range_start))
        )

    @staticmethod
    def create_destination_id(client, container, name):
        # type: (azure.storage.StorageClient, str, str) -> str
        """Create a unique destination id
        :param azure.storage.StorageClient client: storage client
        :param str container: container name
        :param str name: entity name
        :rtype: str
        :return: unique id for the destination
        """
        path = str(pathlib.PurePath(name))
        return ';'.join((client.primary_endpoint, container, path))

    @staticmethod
    def append_slice_suffix_to_name(name, slice):
        # type: (str, int) -> str
        """Append a vectored io (stripe) slice suffix to a given name
        :param str name: entity name
        :param int slice: slice num
        :rtype: str
        :return: name with appended suffix
        """
        return '{}.bxslice-{}'.format(name, slice)

    def _update_progress_bar(self, stdin=False):
        # type: (Uploader, bool) -> None
        """Update progress bar
        :param Uploader self: this
        :param bool stdin: stdin upload
        """
        if not self._all_files_processed:
            return
        blobxfer.operations.progress.update_progress_bar(
            self._general_options,
            'upload',
            self._upload_start_time,
            self._upload_total,
            self._upload_sofar,
            self._upload_bytes_total,
            self._upload_bytes_sofar,
            stdin_upload=stdin,
        )

    def _pre_md5_skip_on_check(self, src, rfile):
        # type: (Uploader, blobxfer.models.upload.LocalPath,
        #        blobxfer.models.azure.StorageEntity) -> None
        """Perform pre MD5 skip on check
        :param Uploader self: this
        :param blobxfer.models.upload.LocalPath src: local path
        :param blobxfer.models.azure.StorageEntity rfile: remote file
        """
        md5 = blobxfer.models.metadata.get_md5_from_metadata(rfile)
        key = blobxfer.operations.upload.Uploader.create_unique_id(src, rfile)
        with self._md5_meta_lock:
            self._md5_map[key] = (src, rfile, md5)
        self._md5_offload.add_localfile_for_md5_check(
            key, None, str(src.absolute_path), md5, rfile.mode, src.view)

    def _post_md5_skip_on_check(self, key, local_md5, md5_match):
        # type: (Uploader, str, str, bool) -> None
        """Perform post MD5 skip on check
        :param Uploader self: this
        :param str key: md5 map key
        :param str local_md5: local md5
        :param bool md5_match: if MD5 matches
        """
        with self._md5_meta_lock:
            src, rfile, remote_md5 = self._md5_map.pop(key)
        uid = blobxfer.operations.upload.Uploader.create_unique_id(src, rfile)
        if self._general_options.verbose:
            logger.debug('pre-transfer MD5 check: {} <L..R> {} {}'.format(
                local_md5, remote_md5, src.relative_path))
        if md5_match:
            with self._upload_lock:
                self._upload_set.remove(uid)
                self._upload_total -= 1
            if self._general_options.dry_run:
                logger.info('[DRY RUN] MD5 match, skipping: {} -> {}'.format(
                    src.absolute_path, rfile.path))
        else:
            if self._general_options.dry_run:
                with self._upload_lock:
                    self._upload_set.remove(uid)
                    self._upload_total -= 1
                logger.info('[DRY RUN] MD5 mismatch, upload: {} -> {}'.format(
                    src.absolute_path, rfile.path))
            else:
                self._add_to_upload_queue(src, rfile, uid)

    def _check_for_uploads_from_md5(self):
        # type: (Uploader) -> None
        """Check queue for a file to upload
        :param Uploader self: this
        """
        cv = self._md5_offload.done_cv
        while not self.termination_check_md5:
            result = None
            cv.acquire()
            while True:
                result = self._md5_offload.pop_done_queue()
                if result is None:
                    # use cv timeout due to possible non-wake while running
                    cv.wait(1)
                    # check for terminating conditions
                    if self.termination_check_md5:
                        break
                else:
                    break
            cv.release()
            if result is not None:
                self._post_md5_skip_on_check(result[0], result[3], result[4])

    def _add_to_upload_queue(self, src, rfile, uid):
        # type: (Uploader, blobxfer.models.upload.LocalPath,
        #        blobxfer.models.azure.StorageEntity, str) -> None
        """Add remote file to download queue
        :param Uploader self: this
        :param blobxfer.models.upload.LocalPath src: local path
        :param blobxfer.models.azure.StorageEntity rfile: remote file
        :param str uid: unique id
        """
        # prepare local file for upload
        ud = blobxfer.models.upload.Descriptor(
            src, rfile, uid, self._spec.options, self._general_options,
            self._resume)
        if ud.entity.is_encrypted:
            with self._upload_lock:
                self._ud_map[uid] = ud
        # add download descriptor to queue
        self._upload_queue.put(ud)
        if self._upload_start_time is None:
            with self._upload_lock:
                if self._upload_start_time is None:
                    self._upload_start_time = blobxfer.util.datetime_now()

    def _initialize_disk_threads(self):
        # type: (Uploader) -> None
        """Initialize disk threads
        :param Uploader self: this
        """
        logger.debug('spawning {} disk threads'.format(
            self._general_options.concurrency.disk_threads))
        for _ in range(self._general_options.concurrency.disk_threads):
            thr = threading.Thread(target=self._worker_thread_upload)
            self._disk_threads.append(thr)
            thr.start()

    def _initialize_transfer_threads(self):
        # type: (Uploader) -> None
        """Initialize transfer threads
        :param Uploader self: this
        """
        logger.debug('spawning {} transfer threads'.format(
            self._general_options.concurrency.transfer_threads))
        for _ in range(self._general_options.concurrency.transfer_threads):
            thr = threading.Thread(target=self._worker_thread_transfer)
            self._transfer_threads.append(thr)
            thr.start()

    def _wait_for_disk_threads(self, terminate):
        # type: (Uploader, bool) -> None
        """Wait for disk threads
        :param Uploader self: this
        :param bool terminate: terminate threads
        """
        if terminate:
            self._upload_terminate = terminate
        for thr in self._disk_threads:
            thr.join()

    def _wait_for_transfer_threads(self, terminate):
        # type: (Uploader, bool) -> None
        """Wait for transfer threads
        :param Uploader self: this
        :param bool terminate: terminate threads
        """
        if terminate:
            self._upload_terminate = terminate
        for thr in self._transfer_threads:
            thr.join()

    def _worker_thread_transfer(self):
        # type: (Uploader) -> None
        """Worker thread transfer
        :param Uploader self: this
        """
        while not self.termination_check:
            try:
                ud, ase, offsets, data = self._transfer_queue.get(
                    block=False, timeout=0.1)
            except queue.Empty:
                continue
            try:
                self._process_transfer(ud, ase, offsets, data)
            except Exception as e:
                with self._upload_lock:
                    self._exceptions.append(e)

    def _process_transfer(self, ud, ase, offsets, data):
        # type: (Uploader, blobxfer.models.upload.Descriptor,
        #        blobxfer.models.azure.StorageEntity,
        #        blobxfer.models.upload.Offsets, bytes) -> None
        """Process transfer instructions
        :param Uploader self: this
        :param blobxfer.models.upload.Descriptor ud: upload descriptor
        :param blobxfer.models.azure.StorageEntity ase: Storage entity
        :param blobxfer.models.upload.Offsets offsets: offsets
        :param bytes data: data to upload
        """
        # issue put range
        self._put_data(ud, ase, offsets, data)
        # accounting
        with self._transfer_lock:
            if ud.local_path.use_stdin:
                self._upload_bytes_total += offsets.num_bytes
            elif offsets.chunk_num == 0:
                self._upload_bytes_total += ase.size
            self._upload_bytes_sofar += offsets.num_bytes
            self._transfer_set.remove(
                blobxfer.operations.upload.Uploader.create_unique_transfer_id(
                    ud.local_path, ase, offsets))
        ud.complete_offset_upload(offsets.chunk_num)
        # add descriptor back to upload queue only for append blobs
        if ud.entity.mode == blobxfer.models.azure.StorageModes.Append:
            self._upload_queue.put(ud)
        # update progress bar
        self._update_progress_bar(stdin=ud.local_path.use_stdin)

    def _put_data(self, ud, ase, offsets, data):
        # type: (Uploader, blobxfer.models.upload.Descriptor,
        #        blobxfer.models.azure.StorageEntity,
        #        blobxfer.models.upload.Offsets, bytes) -> None
        """Put data in Azure
        :param Uploader self: this
        :param blobxfer.models.upload.Descriptor ud: upload descriptor
        :param blobxfer.models.azure.StorageEntity ase: Storage entity
        :param blobxfer.models.upload.Offsets offsets: offsets
        :param bytes data: data to upload
        """
        if ase.mode == blobxfer.models.azure.StorageModes.Append:
            # append block
            if data is not None:
                blobxfer.operations.azure.blob.append.append_block(ase, data)
        elif ase.mode == blobxfer.models.azure.StorageModes.Block:
            # handle one-shot uploads
            if ud.is_one_shot_block_blob:
                metadata = ud.generate_metadata()
                if not ud.entity.is_encrypted and ud.must_compute_md5:
                    digest = blobxfer.util.base64_encode_as_string(
                        ud.md5.digest())
                else:
                    digest = None
                blobxfer.operations.azure.blob.block.create_blob(
                    ase, data, digest, metadata)
                return
            # upload block
            if data is not None:
                blobxfer.operations.azure.blob.block.put_block(
                    ase, offsets, data)
        elif ase.mode == blobxfer.models.azure.StorageModes.File:
            # upload range
            if data is not None:
                blobxfer.operations.azure.file.put_file_range(
                    ase, offsets, data)
        elif ase.mode == blobxfer.models.azure.StorageModes.Page:
            if data is None:
                return
            # compute aligned size
            aligned = blobxfer.util.page_align_content_length(
                offsets.num_bytes)
            # align page
            if aligned != offsets.num_bytes:
                data = data.ljust(aligned, b'\0')
            if blobxfer.operations.md5.check_data_is_empty(data):
                return
            # upload page
            blobxfer.operations.azure.blob.page.put_page(
                ase, offsets.range_start, offsets.range_start + aligned - 1,
                data)

    def _worker_thread_upload(self):
        # type: (Uploader) -> None
        """Worker thread upload
        :param Uploader self: this
        """
        max_set_len = self._general_options.concurrency.transfer_threads << 2
        while not self.termination_check:
            try:
                if len(self._transfer_set) > max_set_len:
                    time.sleep(0.1)
                    continue
                else:
                    ud = self._upload_queue.get(block=False, timeout=0.1)
            except queue.Empty:
                continue
            try:
                self._process_upload_descriptor(ud)
            except Exception as e:
                with self._upload_lock:
                    self._exceptions.append(e)

    def _prepare_upload(self, ase):
        # type: (Uploader, blobxfer.models.azure.StorageEntity) -> None
        """Prepare upload
        :param Uploader self: this
        :param blobxfer.models.azure.StorageEntity ase: Storage entity
        """
        if ase.mode == blobxfer.models.azure.StorageModes.Append:
            if ase.append_create:
                # create container if necessary
                blobxfer.operations.azure.blob.create_container(
                    ase, self._containers_created)
                # create remote blob
                blobxfer.operations.azure.blob.append.create_blob(ase)
        elif ase.mode == blobxfer.models.azure.StorageModes.Block:
            # create container if necessary
            blobxfer.operations.azure.blob.create_container(
                ase, self._containers_created)
        elif ase.mode == blobxfer.models.azure.StorageModes.File:
            # create share directory structure
            with self._fileshare_dir_lock:
                # create container if necessary
                blobxfer.operations.azure.file.create_share(
                    ase, self._containers_created)
                # create parent directories
                blobxfer.operations.azure.file.create_all_parent_directories(
                    ase, self._dirs_created)
            # create remote file
            blobxfer.operations.azure.file.create_file(ase)
        elif ase.mode == blobxfer.models.azure.StorageModes.Page:
            # create container if necessary
            blobxfer.operations.azure.blob.create_container(
                ase, self._containers_created)
            # create remote blob
            blobxfer.operations.azure.blob.page.create_blob(ase)

    def _process_upload_descriptor(self, ud):
        # type: (Uploader, blobxfer.models.upload.Descriptor) -> None
        """Process upload descriptor
        :param Uploader self: this
        :param blobxfer.models.upload.Descriptor: upload descriptor
        """
        # get upload offsets
        offsets, resume_bytes = ud.next_offsets()
        # add resume bytes to counter
        if resume_bytes is not None:
            with self._transfer_lock:
                self._upload_bytes_total += ud.entity.size
                self._upload_bytes_sofar += resume_bytes
                logger.debug('adding {} sofar {} from {}'.format(
                    resume_bytes, self._upload_bytes_sofar, ud.entity.name))
            del resume_bytes
        # check if all operations completed
        if offsets is None and ud.all_operations_completed:
            # finalize file
            self._finalize_upload(ud)
            # accounting
            with self._upload_lock:
                if ud.entity.is_encrypted:
                    self._ud_map.pop(ud.unique_id)
                self._upload_set.remove(ud.unique_id)
                self._upload_sofar += 1
            return
        # if nothing to upload, re-enqueue for finalization
        if offsets is None:
            self._upload_queue.put(ud)
            return
        # prepare upload
        if offsets.chunk_num == 0:
            self._prepare_upload(ud.entity)
        # prepare replicae targets
        if blobxfer.util.is_not_empty(ud.entity.replica_targets):
            for ase in ud.entity.replica_targets:
                if offsets.chunk_num == 0:
                    self._prepare_upload(ase)
        # encrypt if necessary
        if ud.entity.is_encrypted and ud.entity.size > 0:
            # send iv through hmac if first chunk
            if offsets.chunk_num == 0:
                ud.hmac_data(ud.current_iv)
            # encrypt data
            if self._crypto_offload is None:
                # read data from file and encrypt
                data, _ = ud.read_data(offsets)
                encdata = blobxfer.operations.crypto.aes_cbc_encrypt_data(
                    ud.entity.encryption_metadata.symmetric_key,
                    ud.current_iv, data, offsets.pad)
                # send encrypted data through hmac
                ud.hmac_data(encdata)
                data = encdata
                # save last 16 encrypted bytes for next IV
                ud.current_iv = \
                    encdata[-blobxfer.models.crypto.AES256_BLOCKSIZE_BYTES:]
            else:  # noqa
                # crypto offload is not supported with AES256-CBC FullBlob
                raise NotImplementedError()
                # self._crypto_offload.add_encrypt_chunk(
                #     str(ud.local_path.absolute_path), offsets,
                #     ud.entity.encryption_metadata.symmetric_key,
                #     ud.current_iv)
                # encrypted data will be retrieved from a temp file once
                # retrieved from crypto queue
                # return_early = True
        else:
            data, newoffset = ud.read_data(offsets)
            # set new offset if stdin
            if newoffset is not None:
                offsets = newoffset
        # re-enqueue for other threads to upload if not append
        if ud.entity.mode != blobxfer.models.azure.StorageModes.Append:
            self._upload_queue.put(ud)
        # no data can be returned on stdin uploads
        if ud.local_path.use_stdin and not data:
            return
        # add data to transfer queue
        with self._transfer_lock:
            self._transfer_set.add(
                blobxfer.operations.upload.Uploader.create_unique_transfer_id(
                    ud.local_path, ud.entity, offsets))
        self._transfer_queue.put((ud, ud.entity, offsets, data))
        # iterate replicas
        if blobxfer.util.is_not_empty(ud.entity.replica_targets):
            for ase in ud.entity.replica_targets:
                with self._transfer_lock:
                    self._transfer_set.add(
                        blobxfer.operations.upload.Uploader.
                        create_unique_transfer_id(ud.local_path, ase, offsets)
                    )
                self._transfer_queue.put((ud, ase, offsets, data))

    def _finalize_block_blob(self, ud, metadata):
        # type: (Uploader, blobxfer.models.upload.Descriptor, dict) -> None
        """Finalize Block blob
        :param Uploader self: this
        :param blobxfer.models.upload.Descriptor ud: upload descriptor
        :param dict metadata: metadata dict
        """
        if not ud.entity.is_encrypted and ud.must_compute_md5:
            digest = blobxfer.util.base64_encode_as_string(ud.md5.digest())
        else:
            digest = None
        blobxfer.operations.azure.blob.block.put_block_list(
            ud.entity, ud.last_block_num, digest, metadata)
        if blobxfer.util.is_not_empty(ud.entity.replica_targets):
            for ase in ud.entity.replica_targets:
                blobxfer.operations.azure.blob.block.put_block_list(
                    ase, ud.last_block_num, digest, metadata)

    def _set_blob_properties(self, ud):
        # type: (Uploader, blobxfer.models.upload.Descriptor) -> None
        """Set blob properties (md5, cache control)
        :param Uploader self: this
        :param blobxfer.models.upload.Descriptor ud: upload descriptor
        """
        if ud.requires_non_encrypted_md5_put:
            digest = blobxfer.util.base64_encode_as_string(ud.md5.digest())
        else:
            digest = None
        blobxfer.operations.azure.blob.set_blob_properties(ud.entity, digest)
        if blobxfer.util.is_not_empty(ud.entity.replica_targets):
            for ase in ud.entity.replica_targets:
                blobxfer.operations.azure.blob.set_blob_properties(ase, digest)

    def _set_blob_metadata(self, ud, metadata):
        # type: (Uploader, blobxfer.models.upload.Descriptor, dict) -> None
        """Set blob metadata
        :param Uploader self: this
        :param blobxfer.models.upload.Descriptor ud: upload descriptor
        :param dict metadata: metadata dict
        """
        blobxfer.operations.azure.blob.set_blob_metadata(ud.entity, metadata)
        if blobxfer.util.is_not_empty(ud.entity.replica_targets):
            for ase in ud.entity.replica_targets:
                blobxfer.operations.azure.blob.set_blob_metadata(ase, metadata)

    def _resize_blob(self, ud, size):
        # type: (Uploader, blobxfer.models.upload.Descriptor, int) -> None
        """Resize page blob
        :param Uploader self: this
        :param blobxfer.models.upload.Descriptor ud: upload descriptor
        :param int size: content length
        """
        blobxfer.operations.azure.blob.page.resize_blob(ud.entity, size)
        if blobxfer.util.is_not_empty(ud.entity.replica_targets):
            for ase in ud.entity.replica_targets:
                blobxfer.operations.azure.blob.page.resize_blob(ase, size)

    def _finalize_nonblock_blob(self, ud, metadata):
        # type: (Uploader, blobxfer.models.upload.Descriptor, dict) -> None
        """Finalize Non-Block blob
        :param Uploader self: this
        :param blobxfer.models.upload.Descriptor ud: upload descriptor
        :param dict metadata: metadata dict
        """
        # resize blobs to final size if required
        needs_resize, final_size = ud.requires_resize()
        if needs_resize:
            self._resize_blob(ud, final_size)
        # set md5 page blob property if required
        if (ud.requires_non_encrypted_md5_put or
                ud.entity.cache_control is not None):
            self._set_blob_properties(ud)
        # set metadata if needed
        if blobxfer.util.is_not_empty(metadata):
            self._set_blob_metadata(ud, metadata)

    def _finalize_azure_file(self, ud, metadata):
        # type: (Uploader, blobxfer.models.upload.Descriptor, dict) -> None
        """Finalize Azure File
        :param Uploader self: this
        :param blobxfer.models.upload.Descriptor ud: upload descriptor
        :param dict metadata: metadata dict
        """
        # set md5 file property if required
        if ud.requires_non_encrypted_md5_put:
            digest = blobxfer.util.base64_encode_as_string(ud.md5.digest())
        else:
            digest = None
        if digest is not None or ud.entity.cache_control is not None:
            blobxfer.operations.azure.file.set_file_properties(
                ud.entity, digest)
            if blobxfer.util.is_not_empty(ud.entity.replica_targets):
                for ase in ud.entity.replica_targets:
                    blobxfer.operations.azure.file.set_file_properties(
                        ase, digest)
        # set file metadata if needed
        if blobxfer.util.is_not_empty(metadata):
            blobxfer.operations.azure.file.set_file_metadata(
                ud.entity, metadata)
            if blobxfer.util.is_not_empty(ud.entity.replica_targets):
                for ase in ud.entity.replica_targets:
                    blobxfer.operations.azure.file.set_file_metadata(
                        ase, metadata)

    def _finalize_upload(self, ud):
        # type: (Uploader, blobxfer.models.upload.Descriptor) -> None
        """Finalize file upload
        :param Uploader self: this
        :param blobxfer.models.upload.Descriptor ud: upload descriptor
        """
        metadata = ud.generate_metadata()
        if ud.requires_put_block_list:
            # put block list for non one-shot block blobs
            self._finalize_block_blob(ud, metadata)
        elif ud.remote_is_page_blob or ud.remote_is_append_blob:
            # append and page blob finalization
            self._finalize_nonblock_blob(ud, metadata)
        elif ud.remote_is_file:
            # azure file finalization
            self._finalize_azure_file(ud, metadata)
        # set access tier
        if ud.requires_access_tier_set:
            blobxfer.operations.azure.blob.block.set_blob_access_tier(
                ud.entity)

    def _get_destination_paths(self):
        # type: (Uploader) ->
        #        Tuple[blobxfer.operations.azure.StorageAccount, str, str, str]
        """Get destination paths
        :param Uploader self: this
        :rtype: tuple
        :return: (storage account, container, name, dpath)
        """
        for dst in self._spec.destinations:
            for dpath in dst.paths:
                sdpath = str(dpath)
                cont, dir = blobxfer.util.explode_azure_path(sdpath)
                sa = self._creds.get_storage_account(
                    dst.lookup_storage_account(sdpath))
                yield sa, cont, dir, dpath

    def _delete_extraneous_files(self):
        # type: (Uploader) -> None
        """Delete extraneous files on the remote
        :param Uploader self: this
        """
        if not self._spec.options.delete_extraneous_destination:
            return
        # list blobs for all destinations
        checked = set()
        deleted = 0
        for sa, container, vpath, dpath in self._get_destination_paths():
            key = ';'.join((sa.name, sa.endpoint, str(dpath)))
            if key in checked:
                continue
            logger.debug(
                'attempting to delete extraneous blobs/files from: {}'.format(
                    key))
            if (self._spec.options.mode ==
                    blobxfer.models.azure.StorageModes.File):
                files = blobxfer.operations.azure.file.list_all_files(
                    sa.file_client, container)
                for file in files:
                    try:
                        pathlib.Path(file).relative_to(vpath)
                    except ValueError:
                        continue
                    id = blobxfer.operations.upload.Uploader.\
                        create_destination_id(sa.file_client, container, file)
                    if id not in self._delete_exclude:
                        if self._general_options.dry_run:
                            logger.info('[DRY RUN] deleting file: {}'.format(
                                file))
                        else:
                            if self._general_options.verbose:
                                logger.debug('deleting file: {}'.format(file))
                            blobxfer.operations.azure.file.delete_file(
                                sa.file_client, container, file)
                            deleted += 1
            else:
                blobs = blobxfer.operations.azure.blob.list_all_blobs(
                    sa.block_blob_client, container)
                for blob in blobs:
                    try:
                        pathlib.Path(blob.name).relative_to(vpath)
                    except ValueError:
                        continue
                    id = blobxfer.operations.upload.Uploader.\
                        create_destination_id(
                            sa.block_blob_client, container, blob.name)
                    if id not in self._delete_exclude:
                        if self._general_options.dry_run:
                            logger.info('[DRY RUN] deleting blob: {}'.format(
                                blob.name))
                        else:
                            if self._general_options.verbose:
                                logger.debug('deleting blob: {}'.format(
                                    blob.name))
                            blobxfer.operations.azure.blob.delete_blob(
                                sa.block_blob_client, container, blob.name)
                            deleted += 1
            checked.add(key)
        logger.info('deleted {} extraneous blobs/files'.format(deleted))

    def _check_upload_conditions(self, local_path, rfile):
        # type: (Uploader, blobxfer.models.upload.LocalPath,
        #        blobxfer.models.azure.StorageEntity) -> UploadAction
        """Check for upload conditions
        :param Uploader self: this
        :param blobxfer.models.LocalPath local_path: local path
        :param blobxfer.models.azure.StorageEntity rfile: remote file
        :rtype: UploadAction
        :return: upload action
        """
        lpath = local_path.absolute_path
        # check if local file still exists
        if not local_path.use_stdin and not lpath.exists():
            if self._general_options.verbose:
                logger.warning(
                    'skipping file that no longer exists: {}'.format(lpath))
            return UploadAction.Skip
        # if remote file doesn't exist, upload
        if rfile is None or rfile.from_local:
            return UploadAction.Upload
        # check overwrite option
        if not self._spec.options.overwrite:
            if rfile.mode == blobxfer.models.azure.StorageModes.Append:
                rfile.append_create = False
                return UploadAction.Upload
            logger.info(
                'not overwriting remote file: {} (local: {})'.format(
                    rfile.path, lpath))
            return UploadAction.Skip
        # check skip on options, MD5 match takes priority
        md5 = blobxfer.models.metadata.get_md5_from_metadata(rfile)
        if self._spec.skip_on.md5_match and blobxfer.util.is_not_empty(md5):
            return UploadAction.CheckMd5
        # if neither of the remaining skip on actions are activated, upload
        if (not self._spec.skip_on.filesize_match and
                not self._spec.skip_on.lmt_ge):
            return UploadAction.Upload
        # check skip on file size match
        ul_fs = None
        if self._spec.skip_on.filesize_match:
            lsize = local_path.size
            if rfile.mode == blobxfer.models.azure.StorageModes.Page:
                lsize = blobxfer.util.page_align_content_length(lsize)
            if rfile.size == lsize:
                ul_fs = False
                if self._general_options.verbose:
                    logger.debug('filesize match: {} == {} size={}'.format(
                        local_path.absolute_path, rfile.path, lsize))
            else:
                ul_fs = True
        # check skip on lmt ge
        ul_lmt = None
        if self._spec.skip_on.lmt_ge:
            mtime = blobxfer.util.datetime_from_timestamp(
                local_path.lmt, as_utc=True)
            if rfile.lmt >= mtime:
                ul_lmt = False
                if self._general_options.verbose:
                    logger.debug('lmt ge match: {} lmt={} >= {} lmt={}'.format(
                        rfile.path, rfile.lmt, local_path.absolute_path,
                        mtime))
            else:
                ul_lmt = True
        # upload if either skip on mismatch is True
        if ul_fs or ul_lmt:
            return UploadAction.Upload
        else:
            return UploadAction.Skip

    def _check_for_existing_remote(self, sa, cont, name):
        # type: (Uploader, blobxfer.operations.azure.StorageAccount,
        #        str, str) -> bobxfer.models.azure.StorageEntity
        """Check for an existing remote file
        :param Uploader self: this
        :param blobxfer.operations.azure.StorageAccount sa: storage account
        :param str cont: container
        :param str name: entity name
        :rtype: blobxfer.models.azure.StorageEntity
        :return: remote storage entity
        """
        ase = None
        if self._spec.options.overwrite or not sa.can_read_object:
            return ase
        if self._spec.options.mode == blobxfer.models.azure.StorageModes.File:
            fp = blobxfer.operations.azure.file.get_file_properties(
                sa.file_client, cont, name)
        else:
            fp = blobxfer.operations.azure.blob.get_blob_properties(
                sa.block_blob_client, cont, name, self._spec.options.mode)
        if fp is not None:
            if blobxfer.models.crypto.EncryptionMetadata.\
                    encryption_metadata_exists(fp.metadata):
                ed = blobxfer.models.crypto.EncryptionMetadata()
                ed.convert_from_json(fp.metadata, fp.name, None)
            else:
                ed = None
            ase = blobxfer.models.azure.StorageEntity(cont, ed)
            if (self._spec.options.mode ==
                    blobxfer.models.azure.StorageModes.File):
                dir, _, _ = blobxfer.operations.azure.file.parse_file_path(
                    name)
                ase.populate_from_file(sa, fp, dir)
            else:
                ase.populate_from_blob(sa, fp)
                # overwrite tier with specified storage tier
                if ase.mode == blobxfer.models.azure.StorageModes.Block:
                    ase.access_tier = self._spec.options.access_tier
            # always overwrite cache control/content type with option
            ase.cache_control = (
                self._spec.options.store_file_properties.cache_control
            )
            ase.content_type = (
                self._spec.options.store_file_properties.content_type or
                blobxfer.util.get_mime_type(ase.name)
            )
        return ase

    def _generate_destination_for_source(self, local_path):
        # type: (Uploader, blobxfer.models.upload.LocalSourcePath) ->
        #        Tuple[blobxfer.operations.azure.StorageAccount,
        #        blobxfer.models.azure.StorageEntity)
        """Generate entities for source path
        :param Uploader self: this
        :param blobxfer.models.upload.LocalSourcePath local_path: local path
        :rtype: tuple
        :return: storage account, storage entity
        """
        # construct stripped destination path
        spath = local_path.relative_path
        # apply strip components
        if self._spec.options.strip_components > 0:
            _rparts = local_path.relative_path.parts
            _strip = min(
                (len(_rparts) - 1, self._spec.options.strip_components)
            )
            if _strip > 0:
                spath = pathlib.Path(*_rparts[_strip:])
        # create a storage entity for each destination
        for sa, cont, name, dpath in self._get_destination_paths():
            # if not renaming, form name from with spath
            if not self._spec.options.rename and not local_path.use_stdin:
                name = str(name / spath)
            if blobxfer.util.is_none_or_empty(name):
                raise ValueError(
                    ('invalid destination, must specify a container or '
                     'fileshare and remote file name: {}').format(dpath))
            # do not check for existing remote right now if striped
            # vectored io mode
            if (self._spec.options.vectored_io.distribution_mode ==
                    blobxfer.models.upload.
                    VectoredIoDistributionMode.Stripe):
                ase = None
            else:
                ase = self._check_for_existing_remote(sa, cont, name)
            if ase is None:
                # encryption metadata will be populated later, if required
                ase = blobxfer.models.azure.StorageEntity(cont, ed=None)
                ase.populate_from_local(
                    sa, cont, name, self._spec.options.mode,
                    self._spec.options.store_file_properties.cache_control,
                    self._spec.options.store_file_properties.content_type,
                )
                if ase.mode == blobxfer.models.azure.StorageModes.Block:
                    ase.access_tier = self._spec.options.access_tier
            yield sa, ase

    def _vectorize_and_bind(self, local_path, dest):
        # type: (Uploader, blobxfer.models.upload.LocalPath,
        #        List[blobxfer.models.azure.StorageEntity]) ->
        #        Tuple[blobxfer.operations.upload.UploadAction,
        #        blobxfer.models.upload.LocalPath,
        #        blobxfer.models.azure.StorageEntity]
        """Vectorize local path to destinations, if necessary, and bind
        :param Uploader self: this
        :param blobxfer.models.LocalPath local_path: local path
        :param list dest: list of destination tuples (sa, ase)
        :rtype: tuple
        :return: action, LocalPath, ase
        """
        if (self._spec.options.vectored_io.distribution_mode ==
                blobxfer.models.upload.VectoredIoDistributionMode.Stripe and
                not local_path.use_stdin):
            # compute total number of slices
            slices = int(math.ceil(
                local_path.total_size /
                self._spec.options.vectored_io.stripe_chunk_size_bytes))
            # check if vectorization is possible
            if slices == 1:
                sa, ase = dest[0]
                if not sa.can_write_object:
                    raise RuntimeError(
                        'unable to write to remote path {} as credential '
                        'for storage account {} does not permit write '
                        'access'.format(ase.path, sa.name))
                action = self._check_upload_conditions(local_path, ase)
                yield action, local_path, ase
                return
            num_dest = len(dest)
            logger.debug(
                '{} slices for vectored out of {} to {} destinations'.format(
                    slices, local_path.absolute_path, num_dest))
            # pre-populate slice map for next pointers
            slice_map = {}
            for i in range(0, slices):
                sa, ase = dest[i % num_dest]
                if not sa.can_write_object:
                    raise RuntimeError(
                        'unable to write to remote path {} as credential '
                        'for storage account {} does not permit write '
                        'access'.format(ase.path, sa.name))
                name = blobxfer.operations.upload.Uploader.\
                    append_slice_suffix_to_name(ase.name, i)
                sase = self._check_for_existing_remote(sa, ase.container, name)
                if sase is None:
                    # encryption metadata will be populated later, if required
                    sase = blobxfer.models.azure.StorageEntity(
                        ase.container, ed=None)
                    sase.populate_from_local(
                        sa, ase.container, name, self._spec.options.mode,
                        self._spec.options.store_file_properties.cache_control,
                        self._spec.options.store_file_properties.content_type,
                    )
                    if sase.mode == blobxfer.models.azure.StorageModes.Block:
                        sase.access_tier = self._spec.options.access_tier
                slice_map[i] = sase
            # create new local path to ase mappings
            curr = 0
            for i in range(0, slices):
                start = curr
                end = (
                    curr +
                    self._spec.options.vectored_io.stripe_chunk_size_bytes
                )
                if end > local_path.total_size:
                    end = local_path.total_size
                ase = slice_map[i]
                if i < slices - 1:
                    next_entry = blobxfer.models.metadata.\
                        create_vectored_io_next_entry(slice_map[i + 1])
                else:
                    next_entry = None
                lp_slice = blobxfer.models.upload.LocalPath(
                    parent_path=local_path.parent_path,
                    relative_path=local_path.relative_path,
                    use_stdin=False,
                    view=blobxfer.models.upload.LocalPathView(
                        fd_start=start,
                        fd_end=end,
                        slice_num=i,
                        mode=self._spec.options.vectored_io.distribution_mode,
                        total_slices=slices,
                        next=next_entry,
                    )
                )
                action = self._check_upload_conditions(lp_slice, ase)
                yield action, lp_slice, ase
                curr = end
        elif (self._spec.options.vectored_io.distribution_mode ==
                blobxfer.models.upload.VectoredIoDistributionMode.Replica):
            action_map = {}
            for sa, ase in dest:
                if not sa.can_write_object:
                    raise RuntimeError(
                        'unable to write to remote path {} as credential '
                        'for storage account {} does not permit write '
                        'access'.format(ase.path, sa.name))
                action = self._check_upload_conditions(local_path, ase)
                if action not in action_map:
                    action_map[action] = []
                action_map[action].append(ase)
            for action in action_map:
                dst = action_map[action]
                if len(dst) == 1:
                    yield action, local_path, dst[0]
                else:
                    if (action == UploadAction.CheckMd5 or
                            action == UploadAction.Skip):
                        for ase in dst:
                            yield action, local_path, ase
                    else:
                        primary_ase = dst[0]
                        if primary_ase.replica_targets is None:
                            primary_ase.replica_targets = []
                        primary_ase.replica_targets.extend(dst[1:])
                        # add replica targets to deletion exclusion set
                        if self._spec.options.delete_extraneous_destination:
                            for rt in primary_ase.replica_targets:
                                self._delete_exclude.add(
                                    blobxfer.operations.upload.Uploader.
                                    create_destination_id(
                                        rt._client, rt.container, rt.name)
                                )
                        yield action, local_path, primary_ase
        else:
            for sa, ase in dest:
                if not sa.can_write_object:
                    raise RuntimeError(
                        'unable to write to remote path {} as credential '
                        'for storage account {} does not permit write '
                        'access'.format(ase.path, sa.name))
                action = self._check_upload_conditions(local_path, ase)
                yield action, local_path, ase

    def _run(self):
        # type: (Uploader) -> None
        """Execute Uploader
        :param Uploader self: this
        """
        # mark start
        self._start_time = blobxfer.util.datetime_now()
        logger.info('blobxfer start time: {0}'.format(self._start_time))
        # check renames
        if not self._spec.sources.can_rename() and self._spec.options.rename:
            raise RuntimeError(
                'cannot rename to specified destination with multiple sources')
        # initialize resume db if specified
        if self._general_options.resume_file is not None:
            self._resume = blobxfer.operations.resume.UploadResumeManager(
                self._general_options.resume_file)
        # initialize MD5 processes
        if ((self._spec.options.store_file_properties.md5 or
             self._spec.skip_on.md5_match) and
                self._general_options.concurrency.md5_processes > 0):
            self._md5_offload = blobxfer.operations.md5.LocalFileMd5Offload(
                num_workers=self._general_options.concurrency.md5_processes)
            self._md5_offload.initialize_check_thread(
                self._check_for_uploads_from_md5)
        # initialize crypto processes
        if (self._spec.options.rsa_public_key is not None and
                self._general_options.concurrency.crypto_processes > 0):
            logger.warning(
                'crypto offload for upload is not possible due to '
                'sequential nature of {} and FullBlob encryption mode'.format(
                    blobxfer.models.crypto.EncryptionMetadata.
                    _ENCRYPTION_ALGORITHM)
            )
        # initialize worker threads
        self._initialize_disk_threads()
        self._initialize_transfer_threads()
        # initialize local counters
        files_processed = 0
        skipped_files = 0
        skipped_size = 0
        approx_total_bytes = 0
        # iterate through source paths to upload
        seen = set()
        for src in self._spec.sources.files(self._general_options.dry_run):
            # create a destination array for the source
            dest = [
                (sa, ase) for sa, ase in
                self._generate_destination_for_source(src)
            ]
            for action, lp, ase in self._vectorize_and_bind(src, dest):
                dest_id = blobxfer.operations.upload.Uploader.\
                    create_destination_id(ase._client, ase.container, ase.name)
                if dest_id in seen:
                    raise RuntimeError(
                        'duplicate destination entity detected: {}/{}'.format(
                            ase._client.primary_endpoint, ase.path))
                seen.add(dest_id)
                if self._spec.options.delete_extraneous_destination:
                    self._delete_exclude.add(dest_id)
                if self._spec.options.delete_only:
                    continue
                files_processed += 1
                if action == UploadAction.Skip:
                    skipped_files += 1
                    skipped_size += ase.size if ase.size is not None else 0
                    if self._general_options.dry_run:
                        logger.info('[DRY RUN] skipping: {} -> {}'.format(
                            lp.absolute_path, ase.path))
                    continue
                approx_total_bytes += lp.size
                if blobxfer.util.is_not_empty(ase.replica_targets):
                    approx_total_bytes += lp.size * len(ase.replica_targets)
                # add to potential upload set
                uid = blobxfer.operations.upload.Uploader.create_unique_id(
                    lp, ase)
                with self._upload_lock:
                    self._upload_set.add(uid)
                    self._upload_total += 1
                if action == UploadAction.CheckMd5:
                    self._pre_md5_skip_on_check(lp, ase)
                elif action == UploadAction.Upload:
                    if self._general_options.dry_run:
                        logger.info('[DRY RUN] upload: {} -> {}'.format(
                            lp.absolute_path, ase.path))
                        with self._upload_lock:
                            self._upload_set.remove(uid)
                            self._upload_total -= 1
                    else:
                        self._add_to_upload_queue(lp, ase, uid)
        del seen
        # set remote files processed
        with self._md5_meta_lock:
            self._all_files_processed = True
        with self._upload_lock:
            upload_size_mib = approx_total_bytes / blobxfer.util.MEGABYTE
            if self._spec.options.delete_only:
                logger.info(
                    'skipping upload transfers as only deletion option '
                    'was specified')
            else:
                logger.debug(
                    ('{0} files {1:.4f} MiB filesize, lmt_ge, or no overwrite '
                     'skipped').format(
                        skipped_files, skipped_size / blobxfer.util.MEGABYTE))
                logger.debug(
                    ('{0} local files processed, waiting for upload '
                     'completion of approx. {1:.4f} MiB').format(
                         files_processed, upload_size_mib))
        del files_processed
        del skipped_files
        del skipped_size
        del upload_size_mib
        del approx_total_bytes
        # wait for uploads to complete
        self._wait_for_disk_threads(terminate=False)
        self._wait_for_transfer_threads(terminate=False)
        end_time = blobxfer.util.datetime_now()
        # update progress bar
        self._update_progress_bar()
        # check for exceptions
        if len(self._exceptions) > 0:
            logger.error('exceptions encountered while uploading')
            # raise the first one
            raise self._exceptions[0]
        # check for mismatches
        if (self._upload_sofar != self._upload_total or
                self._upload_bytes_sofar != self._upload_bytes_total):
            raise RuntimeError(
                'upload mismatch: [count={}/{} bytes={}/{}]'.format(
                    self._upload_sofar, self._upload_total,
                    self._upload_bytes_sofar, self._upload_bytes_total))
        # delete all remaining local files not accounted for if
        # delete extraneous enabled
        self._delete_extraneous_files()
        # delete resume file if we've gotten this far
        if self._resume is not None:
            self._resume.delete()
        # output throughput
        if self._upload_start_time is not None:
            ultime = (end_time - self._upload_start_time).total_seconds()
            if ultime == 0:  # noqa
                ultime = 1e-9
            mibup = self._upload_bytes_total / blobxfer.util.MEGABYTE
            mibps = mibup / ultime
            logger.info(
                ('elapsed upload + verify time and throughput of {0:.4f} '
                 'GiB: {1:.3f} sec, {2:.4f} Mbps ({3:.3f} MiB/s)').format(
                     mibup / 1024, ultime, mibps * 8, mibps))
        end_time = blobxfer.util.datetime_now()
        logger.info('blobxfer end time: {0} (elapsed: {1:.3f} sec)'.format(
            end_time, (end_time - self._start_time).total_seconds()))

    def start(self):
        # type: (Uploader) -> None
        """Start the Uploader
        :param Uploader self: this
        """
        try:
            blobxfer.operations.progress.output_parameters(
                self._general_options, self._spec)
            self._run()
        except (KeyboardInterrupt, Exception) as ex:
            logger.exception(ex)
            if isinstance(ex, KeyboardInterrupt):
                logger.info(
                    'KeyboardInterrupt detected, force terminating '
                    'processes and threads (this may take a while)...')
            try:
                self._wait_for_transfer_threads(terminate=True)
                self._wait_for_disk_threads(terminate=True)
            except Exception as tex:
                logger.exception(tex)
            finally:
                raise ex
        finally:
            # shutdown processes
            if self._md5_offload is not None:
                self._md5_offload.finalize_processes()
            if self._crypto_offload is not None:
                self._crypto_offload.finalize_processes()
            # close resume file
            if self._resume is not None:
                self._resume.close()
