#!/usr/bin/env python3
# vim: set ts=8 sw=4 sts=4 et ai:
from __future__ import print_function
"""
proxmove: Proxmox Node Migration -- migrate nodes from one proxmox
cluster to another

This is proxmove.  proxmove is free software: you can redistribute it
and/or modify it under the terms of the GNU General Public License as
published by the Free Software Foundation, version 3 or any later
version.
"""
import argparse
import configparser
import logging
import logging.config
import os
import random
import re
import string
import subprocess
import sys
import time
from collections import defaultdict
from datetime import datetime
from proxmoxer import ProxmoxAPI
from http.client import InvalidURL
from urllib.parse import urlparse

__author__ = 'Walter Doekes'
__copyright__ = 'Copyright (C) Walter Doekes, OSSO B.V. 2016'
__licence__ = 'GPLv3+'
__version__ = '0.0.6'

# TODO: see NotImplementedErrors
# TODO: split up into modules (storage, etc.)

log = logging.getLogger('proxmove')

# Drop LC_* from environment.
environ = os.environ
for key in list(environ.keys()):
    if key.startswith('LC_'):
        del environ[key]

SUFFIX_CLONING = '--CLONING'    # source, when cloning
SUFFIX_CREATING = '--CREATING'  # dest, while cloning
SUFFIX_MIGRATED = '--MIGRATED'  # source, when done
PROXMOX_VOLUME_TYPES = ('ide', 'sata', 'scsi', 'virtio')


def human_size_fmt(num, suffix='B'):
    for unit in ('', 'Ki', 'Mi', 'Gi', 'Ti'):
        if abs(num) < 1024.0:
            return '{:3.1f}{}{}'.format(num, unit, suffix)
        num /= 1024.0
    return '{:.1f}{}{}'.format(num, 'Ti', suffix)


class _HumanSizeScan(object):
    multipliers = {
        'T': 1024 * 1024 * 1024 * 1024,
        'G': 1024 * 1024 * 1024,
        'M': 1024 * 1024,
        'K': 1024,
        'B': 1,
    }

    def __call__(self, num_str):
        numeric, suffix = self.split(num_str)
        if '.' in numeric:
            numeric = float(numeric)
        else:
            numeric = int(numeric)
        multiplier = self.multipliers.get(suffix[0:1].upper())
        if not multiplier:
            if suffix:
                raise ValueError('unknown suffix found in {!r}'.format(
                    num_str))
            multiplier = 1
        return int(numeric * multiplier)

    def split(self, num_str):
        num_str = num_str.lstrip()
        num_parts = []
        suffix = ''
        for i, ch in enumerate(num_str):
            if ch in '0123456789.':
                num_parts.append(ch)
            elif ch == ',':
                pass
            else:
                suffix = num_str[i:].strip()
                break
        return ''.join(num_parts), suffix
human_size_scan = _HumanSizeScan()


class ProxmoveError(Exception):
    pass


class PrepareError(ProxmoveError):
    """
    Failure during preparation and requirements checking. This is only
    raised if we haven't mutated any data yet.
    """
    pass


class ArgumentParser14191(argparse.ArgumentParser):
    """ArgumentParser from argparse that handles out-of-order positional
    arguments.

    This is a workaround created by Glenn Linderman in July 2012. You
    can now do this:

        parser = ArgumentParser14191()
        parser.add_argument('-f', '--foo')
        parser.add_argument('cmd')
        parser.add_argument('rest', nargs='*')
        # some of these would fail with the regular parser:
        for args, res in (('-f1 cmd 1 2 3', 'ok'),
                          ('cmd -f1 1 2 3', 'would_fail'),
                          ('cmd 1 -f1 2 3', 'would_fail'),
                          ('cmd 1 2 3 -f1', 'ok')):
            try: out = parser.parse_args(args.split())
            except: print 'args', 'failed', res
            # out: Namespace(cmd='cmd', foo='1', rest=['1', '2', '3'])

    Bugs: http://bugs.python.org/issue14191
    Files: http://bugs.python.org/file26273/t18a.py
    Changes: renamed to ArgumentParser14191 ** PEP cleaned ** hidden
      ErrorParser inside ArgumentParser14191 ** documented ** used
      new-style classes super calls  (Walter Doekes, March 2015)
    """
    class ErrorParser(argparse.ArgumentParser):
        def __init__(self, *args, **kwargs):
            self.__errorobj = None
            super(ArgumentParser14191.ErrorParser, self).__init__(
                *args, add_help=False, **kwargs)

        def error(self, message):
            if self.__errorobj:
                self.__errorobj.error(message)
            else:
                argparse.ArgumentParser.error(self, message)

        def seterror(self, errorobj):
            self.__errorobj = errorobj

    def __init__(self, *args, **kwargs):
        self.__setup = False
        self.__opt = ArgumentParser14191.ErrorParser(*args, **kwargs)
        super(ArgumentParser14191, self).__init__(*args, **kwargs)
        self.__opt.seterror(self)
        self.__setup = True

    def add_argument(self, *args, **kwargs):
        super(ArgumentParser14191, self).add_argument(*args, **kwargs)
        if self.__setup:
            chars = self.prefix_chars
            if args and len(args[0]) and args[0][0] in chars:
                self.__opt.add_argument(*args, **kwargs)

    def parse_args(self, args=None, namespace=None):
        ns, remain = self.__opt.parse_known_args(args, namespace)
        ns = super(ArgumentParser14191, self).parse_args(remain, ns)
        return ns


class ProxmoxClusters(dict):
    @classmethod
    def from_filename(cls, filename):
        clusters = cls()
        parser = configparser.ConfigParser(
            interpolation=None, inline_comment_prefixes=('#', ';'),
            empty_lines_in_values=False)

        try:
            with open(filename) as fp:
                try:
                    parser.read_file
                except AttributeError:
                    parser.readfp(fp)
                else:
                    parser.read_file(fp)
        except FileNotFoundError:
            raise ValueError('cannot access config file: {}'.format(
                filename))

        for section in parser.sections():
            if section == configparser.DEFAULTSECT:
                raise ValueError(
                    'unexpected default section: {}'.format(section))

            section_split = section.split(':')
            type_ = section_split.pop(0)

            # [pve:CLUSTER_ALIAS]
            if type_ == 'pve' and len(section_split) == 1:
                cluster_alias = section_split[0]
                clusters[cluster_alias] = ProxmoxCluster.from_section(
                    cluster_alias, parser.items(section))
            # [storage:CLUSTER_ALIAS:STORAGE_NAME[@NODE]]
            elif type_ == 'storage' and len(section_split) == 2:
                cluster_alias = section_split[0]
                if cluster_alias not in clusters:
                    raise ValueError(
                        'storage describing unknown cluster {!r}: {}'.format(
                            cluster_alias, section))
                clusters[cluster_alias].add_storage(
                    section_split[1], parser.items(section))
            # [...]
            else:
                raise ValueError(
                    'unknown section type; use pve/storage: {}'.format(
                        section))

        return clusters


class ProxmoxCluster(object):
    @classmethod
    def from_section(cls, name, section):
        cluster = cls(name)

        for key, value in section:
            if key == 'api':
                if cluster.api_url:
                    raise ValueError(
                        'duplicate api key in pve section {!r}'.format(
                            name))
                cluster.api_url = value
            else:
                raise ValueError(
                    'unknown key {!r} in pve section {!r}'.format(
                        key, name))

        return cluster

    def __init__(self, name):
        self.name = name
        self.repoid = None
        self.api_url = None
        self._cache = {}
        self._storages = {}
        self._storages_nodespecific = defaultdict(dict)
        self._vms = {}

    def add_storage(self, name, section):
        try:
            name, node = name.split('@', 1)
        except ValueError:
            storage = ProxmoxStorage.from_section(name, section)
            self._storages[name] = storage
        else:
            storage = ProxmoxStorage.from_section(name, section)
            self._storages_nodespecific[node][name] = storage

    def get_nodes(self):
        if 'nodes' not in self._cache:
            tmp = self.api.nodes.get()
            nodes = [
                node['node'] for node in tmp
                if node.get('uptime') and node['type'] == 'node']
            self._cache['nodes'] = nodes
        return self._cache['nodes']

    def get_storage(self, node, storage):
        ret = None
        if node in self._storages_nodespecific:
            ret = self._storages_nodespecific[node].get(storage)
        if not ret:
            ret = self._storages.get(storage)
        if not ret:
            raise ValueError(
                'storage {!r} in {!r} cluster missing in config'.format(
                    storage, self.name))
        return ret

    @property
    def api(self):
        if not hasattr(self, '_api'):
            try:
                res = urlparse(self.api_url)
            except InvalidURL as e:
                raise ValueError(
                    'splitting {!r} api {!r} URL failed: {}'.format(
                        self.name, self.api_url, e))
            api = ProxmoxAPI(
                res.hostname, port=res.port, user=res.username,
                password=res.password, verify_ssl=True)
            self._api = api
        return self._api

    def create_vm(self, type_, config, nodeid):
        if not nodeid:
            nodeid = self.get_random_node()

        log.info('Creating new VM {!r} on {!r}, node {!r}'.format(
            config['name'], self.name, nodeid))

        # Create disks according to config. Temporarily set ide\d+ and
        # virtio\d+ devices to none until we can get them copied over.
        mutable_config = {}
        for key, value in config.items():
            if key == 'name':
                mutable_config[key] = value + SUFFIX_CREATING
            elif re.match(
                    '^({})\d+$'.format('|'.join(PROXMOX_VOLUME_TYPES)), key):
                # Wipe it. We'll add the disks manually later on.
                pass
            else:
                mutable_config[key] = value

        # Guess new VMID and create one.
        vmid = self.get_free_vmid()
        api_node = self.api.nodes(nodeid)
        vmhash = getattr(api_node, type_).create(vmid=vmid, **mutable_config)

        # Wait a while to ensure that we get the VM.
        log.info(
            '- created new VM {!r} as {}; waiting for it to show up'.format(
                mutable_config['name'], vmhash))
        for i in range(30):
            try:
                self._cache = {}  # purge cache, we expect changes
                vm = self.get_vm(mutable_config['name'])
            except ProxmoxVm.DoesNotExist:
                pass
            else:
                break
            time.sleep(1)
        else:
            raise ProxmoveError('Could not get newly created VM {!r}'.format(
                mutable_config['name']))

        log.info('- created new VM {!r}: {}'.format(vm.name, vm))
        return vm

    def get_vms_dict(self):
        if 'cluster.resources.type=vm' not in self._cache:
            self._cache['cluster.resources.type=vm'] = (
                self.api.cluster.resources.get(type='vm'))
        return self._cache['cluster.resources.type=vm']

    def get_free_vmid(self):
        """
        BEWARE: To get the numbers right, we need to have enough
        permissions to see all VMs. PVEVMAdmin permissions required?
        """
        self._cache = {}  # purge cache, so we don't get stale VMID lists
        vms = self.get_vms_dict()
        if not vms:
            return 100
        ordered_vms = [vm['vmid'] for vm in vms]
        ordered_vms.sort()
        if (ordered_vms[-1] - ordered_vms[0] + 1) == len(ordered_vms):
            return ordered_vms[-1] + 1
        prev = ordered_vms[0]
        for vmid in ordered_vms[1:]:
            if prev + 1 != vmid:
                return prev + 1
            prev = vmid
        raise NotImplementedError('this cannot happen: {}'.format(
            ordered_vms))

    def get_random_node(self):
        nodes = self.get_nodes()
        return random.choice(nodes)

    def get_vm(self, name):
        if name in self._vms:
            return self._vms[name]
        proxmox_vms = self.get_vms_dict()
        res = [vm for vm in proxmox_vms if vm.get('name') == name]
        if len(res) == 0:
            raise ProxmoxVm.DoesNotExist(
                'VM named {!r} not found in cluster {!r}'.format(
                    name, self.name))
        elif len(res) > 1:
            raise ProxmoveError(
                'VM named {!r} found multiple times in cluster {!r}'.format(
                    name, self.name))
        vm = self._vms[name] = ProxmoxVm.from_dict(
            res[0], api=self.api, cluster=self)
        return vm

    def ping(self):
        version = self.api.version.get()
        if not isinstance(version, dict) or 'release' not in version:
            raise ProxmoveError(
                'cluster {!r} did not return proper version: {!r}'.format(
                    version))
        self.repoid = version['repoid']

    def __str__(self):
        if self.repoid:
            return '{}<{}>'.format(self.name, self.repoid)
        return self.name


class ProxmoxStorage(object):
    @classmethod
    def from_section(cls, name, section):
        storages = []
        for model in (ProxmoxStoragePlain, ProxmoxStorageZfs):
            try:
                storage = model.from_section(name, section)
            except PrepareError as e:
                pass
            else:
                storages.append(storage)

        if len(storages) != 1:
            raise PrepareError(
                'storage section {!r} could not be parsed (requires ssh/path/'
                'keys)'.format(name))

        storage = storages[0]
        storage.set_from_section(section)
        return storage

    def __init__(self, name):
        # Set from inifile.
        self.name = name
        self.ssh = None
        self.path = None
        self.temp = None

        # Set through command line.
        self.bwlimit = None

    def check_prerequisites(self):
        self.check_prerequisite_config()
        self.check_prerequisite_commands()
        self.check_prerequisite_paths()

    def check_prerequisite_config(self):
        if not (self.ssh and self.path and self.temp):
            raise PrepareError(
                'missing one or more of ssh/path/temp in {!r} storage '
                'section'.format(self.name))

    def check_prerequisite_commands(self):
        raise NotImplementedError('subclasses need to implemented this')

    def check_prerequisite_paths(self):
        try:
            self.ssh_command(['test', '-d', self.temp])
        except subprocess.CalledProcessError:
            raise PrepareError(
                'temp dir {!r} of storage {!r} does not exist; '
                'please create it!'.format(self.temp, self.name))

    def copy(self, data_size, disk_size, src_location, src_format,
             dst_storage, dst_id, dst_name):
        dst_temp = self.copy_to_temp(src_location, dst_storage, dst_name)

        if dst_temp:
            log.info('Temp data {!r} on {}'.format(dst_temp, dst_storage))
            new_path, new_format = dst_storage.copy_from_temp(
                disk_size, dst_temp, src_format, dst_id, dst_name)
        else:
            assert not src_format, src_format
            new_path, new_format = self.copy_direct(
                data_size, src_location, dst_storage, dst_name)

        return new_path, new_format

    def copy_to_temp(self, src_location, dst_storage, dst_name):
        raise NotImplementedError('subclasses need to implemented this')

    def copy_from_temp(self, disk_size, src_temp, src_format, dst_id,
                       dst_name):
        raise NotImplementedError('subclasses need to implemented this')

    def copy_direct(self, data_size, src_location, dst_storage, dst_name):
        raise NotImplementedError('subclasses need to implemented this')

    def run_command(self, command, hide_stderr=False, tty=False):
        kwargs = {}
        if tty:
            kwargs['stdin'] = sys.stdin
            kwargs['stdout'] = sys.stdout
            kwargs['stderr'] = sys.stderr
        if hide_stderr:
            kwargs['stderr'] = subprocess.DEVNULL

        if tty:
            # We don't need to catch KeyboardInterrupt and SystemExit
            # now that we have ssh -t.
            proc = subprocess.Popen(command, **kwargs)
            status = proc.wait()
            if status != 0:
                raise subprocess.CalledProcessError(
                    returncode=status, cmd=command,
                    output='Failure with status {}'.format(status))
            return ''
        else:
            return subprocess.check_output(command, **kwargs)

    def has_commands(self, commands):
        try:
            result = self.ssh_command(['which'] + commands, hide_stderr=True)
        except subprocess.CalledProcessError:
            return False
        result = result.decode('ascii', 'replace').strip()
        found = set([i.rsplit('/', 1)[-1] for i in result.split('\n')])
        expected = set(commands)
        assert not (found - expected), (expected, found)
        return not (expected - found)  # should have them all

    def ssh_command(self, command, hide_stderr=False, tty=False):
        extra = []
        if tty:
            extra.append('-t')
        return self.run_command(
            # We could auto-add host key using StrictHostKeyChecking=no,
            # but I'm not sure we want to.
            ['ssh', '-A',  # '-o', 'StrictHostKeyChecking=no',
             self.ssh] + extra + command, hide_stderr=hide_stderr, tty=tty)

    def set_bandwidth_limit(self, bwlimit):
        self.bwlimit_mbps = bwlimit

    def get_volume(self, location, properties):
        return ProxmoxVolume(location, properties, storage=self)

    def set_from_section(self, section):
        for key, value in section:
            if key == 'ssh':
                self.set_ssh(value)
            elif key == 'path':
                self.set_path(value)
            elif key == 'temp':
                self.set_temp(value)

    def set_ssh(self, value):
        if self.ssh:
            raise ValueError(
                'duplicate ssh key in {!r} storage section: {}'.format(
                    self.name, value))
        if value.startswith('-'):  # TODO: nor spaces or quotes...
            raise ValueError(
                'ssh value cannot start with an option-dash in {!r} '
                'storage section: {!r}'.format(self.name, value))
        self.ssh = value

    def set_path(self, value):
        if self.path:
            raise ValueError(
                'duplicate path key in {!r} storage section: {}'.format(
                    self.name, value))
        self.path = value

    def set_temp(self, value):
        if self.temp:
            raise ValueError(
                'duplicate temp key in {!r} storage section: {}'.format(
                    self.name, value))
        if not value.startswith('/'):
            raise ValueError(
                'unexpected tokens in temp path in {!r} storage '
                'section: {}'.format(self.name, value))
        self.temp = value

    def __str__(self):
        return self.name


class ProxmoxStoragePlain(ProxmoxStorage):
    @classmethod
    def from_section(cls, name, section):
        paths = [value for key, value in section if key == 'path']
        if len(paths) != 1 or not paths[0].startswith('/'):
            raise PrepareError('not my kind of config')

        return cls(name)

    def get_image_size(self, image_location):
        path = os.path.join(self.path, image_location)
        # Use ls -l instead of stat because stat %s/%z is not
        # standardized across BSD/GNU.
        try:
            data = self.ssh_command(
                ['ls', '-l', path], hide_stderr=True)
        except subprocess.CalledProcessError:
            return None

        exact_size = data.split()[4].decode('ascii', 'replace')
        return int(exact_size)

    def check_prerequisite_commands(self):
        if not self.has_commands(['ssh', 'scp']):
            raise PrepareError(
                'storage {!r} lacks ssh(1) or scp(1)'.format(
                    self.name), file=sys.stderr)

    def copy_to_temp(self, src_location, dst_storage, dst_name):
        src_path = os.path.join(self.path, src_location)
        dst_temp = os.path.join(dst_storage.temp, 'temp-proxmove', dst_name)
        scp_dest = '{}:{}'.format(dst_storage.ssh, dst_temp)
        log.info('scp(1) copy from {!r} (on {}) to {!r}'.format(
            src_path, self, scp_dest))

        # mkdir
        dst_storage.ssh_command(['mkdir', '-p', os.path.dirname(dst_temp)])
        # scp, using ssh+scp instead of local-scp, so we can add our
        # beloved options.
        scp_command = ['scp', '-o', 'StrictHostKeyChecking=no']
        # Add bandwidth limits in kbit/s.
        if self.bwlimit_mbps:
            scp_command.extend(['-l', str(self.bwlimit_mbps * 1024)])
        # Source/destination.
        scp_command.extend([src_path, scp_dest])
        # Exec.
        self.ssh_command(scp_command, tty=True)

        return dst_temp

    def copy_from_temp(self, disk_size, src_temp, src_format, dst_id,
                       dst_name):
        if src_format != 'qcow2':
            raise NotImplementedError(
                'format conversion from {!r} not implemented'.format(
                    src_format))
        dst_format = 'qcow2'

        rel_path = os.path.join(
            str(dst_id), '{}.{}'.format(dst_name, dst_format))
        dst_path = os.path.join(self.path, rel_path)

        log.info('Moving data from {!r} to {!r}'.format(src_temp, dst_path))
        self.ssh_command(['mkdir', '-p', os.path.dirname(dst_path)])
        self.ssh_command(['mv', src_temp, dst_path])

        # In case old_format != new_format, we would need to update
        # properties. So the following must be true for now.
        assert src_format == dst_format, (src_format, dst_format)
        return rel_path, dst_format

    def set_path(self, value):
        if not value.startswith('/'):
            raise ValueError(
                'path should start with / in {!r} storage section: {}'.format(
                    self.name, value))
        return super().set_path(value)


class ProxmoxStorageZfs(ProxmoxStorage):
    @classmethod
    def from_section(cls, name, section):
        paths = [value for key, value in section if key == 'path']
        if len(paths) != 1 or not paths[0].startswith('zfs:'):
            raise PrepareError('not my kind of config')

        return cls(name)

    def check_prerequisite_commands(self):
        if not self.has_commands(['ssh', 'zfs', 'mbuffer']):
            raise PrepareError(
                'storage {!r} lacks ssh(1), zfs(1) or mbuffer(1)'.format(
                    self.name))

    def get_image_size(self, image_location):
        temp_snapname = '{}/{}@temp-{}'.format(
            self.path, image_location, random.random())
        data = self.ssh_command(
            ['zfs', 'snapshot', temp_snapname, ';'
             'zfs', 'send', '-Rnv', temp_snapname, ';',
             'zfs', 'destroy', temp_snapname], hide_stderr=True)
        parts = data.split()
        if not parts:
            return None

        human_size = parts[-1].decode('ascii', 'replace')
        # Add a bit of size since it's a guesstimate. You'd rather not
        # see the transfer go over 100%.
        return int(human_size_scan(human_size) * 1.02)

    def copy_to_temp(self, src_location, dst_storage, dst_name):
        if isinstance(dst_storage, ProxmoxStorageZfs):
            # ZFS->ZFS copying requires no temp files.
            return None
        else:
            # TODO: Simply cat /dev/zvol/.../... to the destfile?
            raise NotImplementedError(
                'ZFS->other copying is not implemented yet')

    def copy_from_temp(self, disk_size, src_temp, src_format, dst_id,
                       dst_name):
        dst_zfs = '{}/{}'.format(self.path, dst_name)
        self.ssh_command(['zfs', 'create', '-V', disk_size, dst_zfs])

        dst_path = os.path.join('/dev/zvol', self.path, dst_name)
        log.info('Writing data from temp {!r} to {!r} (on {})'.format(
            src_temp, dst_path, self))

        if src_format is None:
            src_format = 'raw'
        if src_format not in ('qcow2', 'raw'):
            raise NotImplementedError(
                'format conversion from {!r} not implemented'.format(
                    src_format))

        self.ssh_command(
            # -n = no create volume
            # -p = progress
            ['qemu-img', 'convert', '-n', '-p', '-f', src_format,
             '-O', 'raw', src_temp, dst_path], tty=True)
        log.info('Removing temp {!r} (on {})'.format(dst_path, self))
        self.ssh_command(['rm', src_temp])

        # Return name and format (dst_name is the filesystem name on the
        # ZFS pool known to belong to dst_storage).
        return dst_name, None

    def copy_direct(self, data_size, src_location, dst_storage, dst_name):
        src_zfs = '{}/{}@proxmove-{}'.format(
            self.path, src_location, datetime.now().strftime('%y%m%d-%H%M%S'))
        dst_zfs = '{}/{}'.format(dst_storage.path, dst_name)
        log.info('zfs(1) send/recv {} data from {!r} to {!r} (on {})'.format(
            human_size_fmt(data_size), src_zfs, dst_zfs, dst_storage))

        self.ssh_command(['zfs', 'snapshot', src_zfs])

        # mbuffer takes k-, M- or G-bytes
        mbuffer_write_limit = ''
        if self.bwlimit_mbps:
            mbuffer_write_limit = '-R {}k'.format(self.bwlimit_mbps * 128)

        # pv shows a nice progress bar. It's optional.
        optional_pv_pipe = ''
        if dst_storage.has_commands(['pv']) and data_size:
            log.debug(
                "Using pv(1) for progress bar because it's available")
            optional_pv_pipe = (
                # --fineta does not exist on all versions..
                # --force is required to make it display anything..
                'pv --force --eta --progress -s {} | '.format(data_size))

        # Older mbuffer(1) [v20140310-3] on the receiving end
        # may say: "mbuffer: warning: No controlling terminal
        # and no autoloader command specified." This is fixed
        # in newer versions [v20150412-3].
        self.ssh_command(
            ["zfs send -R {src_zfs} | "
             "mbuffer -q -s 128k -m 1G {src_bwlim} | "
             "ssh -o StrictHostKeyChecking=no {dst_ssh} "
             "'mbuffer {optional_quiet_mbuffer} -s 128k -m 1G | "
             " {dst_pv}"
             " zfs recv {dst_zfs}'".format(
                 src_zfs=src_zfs,
                 src_bwlim=mbuffer_write_limit,
                 dst_ssh=dst_storage.ssh,
                 optional_quiet_mbuffer=(
                     '-q' if optional_pv_pipe else ''),
                 dst_pv=optional_pv_pipe,
                 dst_zfs=dst_zfs)],
            tty=True)

        # Return name and format (dst_name is the filesystem name on the
        # ZFS pool known to belong to dst_storage).
        return dst_name, None

    def set_path(self, value):
        assert value.startswith('zfs:'), value
        pool_name = value[4:]
        valid_characters = string.ascii_letters + string.digits + '_-'
        if not all(i in valid_characters for i in pool_name):
            raise PrepareError(
                'invalid characters in zfs pool name found {!r} storage '
                'section: {}'.format(self.name, value))
        return super().set_path(pool_name)


class ProxmoxVm(object):
    class DoesNotExist(ProxmoveError):
        pass

    @classmethod
    def from_dict(cls, dict_, api, cluster):
        vm = cls(
            dict_['name'], dict_['node'], dict_['type'], dict_['vmid'],
            dict_['status'], api, cluster)
        vm.get_config()  # get config and check for pending changes at once
        return vm

    def __init__(self, name, node, type_, id_, status, api, cluster):
        self.name = name
        self.node = node
        self.type = type_  # qemu|lxc|...
        self.id = id_
        self.status = status  # "running"/"stopped"
        self.api = api
        self.api_vm = getattr(api.nodes(node), type_)(id_)
        self.cluster = cluster
        self._cache = {}

    def check_config(self):
        """
        Check whether we can move this VM.
        """
        config = self.get_config()

        # Check pending. We expect a list of dictionaries with a 'key'
        # key and 'value' and/or 'pending' keys.
        pending_config = self.api_vm.pending.get()  # may not exist for lxc
        pending = []
        for dict_ in pending_config:
            keys = dict_.keys()
            if keys == set(['key', 'value']):
                assert config.get(dict_['key']) == dict_['value']
            else:
                pending.append('{!r}: {!r} => {!r}'.format(
                    dict_['key'], dict_.get('value'),
                    dict_.get('pending')))

        if pending:
            # Contains 'pending' changes. Refuse to continue.
            pending.sort()
            raise PrepareError(
                'VM {!r} contains pending changes:\n  {}'.format(
                    self.name, '\n  '.join(pending)))

    def get_config(self):
        """
        Get current configuration and store name.
        """
        if 'config' not in self._cache:
            next_config = self.api_vm.config.get()
            self.name = next_config['name']
            self._cache['config'] = next_config
        return self._cache['config']

    def create_volume(self, key, source_volume, storage):
        """
        Create volume from source_volume.
        """
        if storage is None:
            # Take properties and set first argument to 'none'.
            # E.g. "san06:abc.iso,media=cdrom" => "none,media=cdrom"
            parts = source_volume.properties.split(',', 1)
            parts[0] = 'none'
            properties = ','.join(parts)
            self.api_vm.config.put(**{key: properties})
            self._cache = {}
            log.info('Ejected (cdrom?) volume {!r} ({}) added to {}'.format(
                key, properties, self))
        else:
            # We actually have to do copying.
            log.info('Begin copy of {!r} ({}) to {}'.format(
                key, source_volume, storage))
            new_volume = source_volume.clone(storage, self.id, key)
            self.api_vm.config.put(**{key: new_volume.as_properties()})

    def get_volumes(self):
        if 'volumes' not in self._cache:
            volumes = {}
            for key, value in self.get_config().items():
                if key.startswith(PROXMOX_VOLUME_TYPES):
                    location, properties = value.split(',', 1)
                    if location == 'none':
                        volume = ProxmoxVolume(None, properties)
                    else:
                        storage, location = location.split(':', 1)
                        storage = self.cluster.get_storage(self.node, storage)
                        volume = storage.get_volume(location, properties)
                    volumes[key] = volume
            self._cache['volumes'] = volumes
        return self._cache['volumes']

    def rename(self, new_name):
        self.api_vm.config.put(name=new_name)
        self.name = new_name

    def ensure_started(self, timeout=120):
        if self.status == 'running':
            log.debug('Skipping start, already running: {}'.format(self))
            return

        log.info('Starting VM {}'.format(self))

        self.api_vm.status.start.create()
        for i in range(timeout + 10):
            time.sleep(1)
            status = self.api_vm.status.current.get()
            if status['status'] == 'running':
                self.status = status['status']
                break
        else:
            self.status = status['status']
            raise ProxmoveError(
                'VM {!r} refuses to start: status = {!r}'.format(
                    self.name, self.status))

        log.info('- started VM {}'.format(self))

    def ensure_stopped(self, timeout=120):
        if self.status == 'stopped':
            log.debug('Skipping stop, already stopped: {}'.format(self))
            return

        log.info('Stopping VM {}; will forcibly kill after {} seconds'.format(
            self, timeout + 10))

        # forceStop takes a boolean, but proxmoxer won't pass True as
        # 'true', but as 'True' which is not valid JSON.
        self.api_vm.status.shutdown.create(forceStop='1', timeout=timeout)
        for i in range(timeout + 10):
            time.sleep(1)
            status = self.api_vm.status.current.get()
            if status['status'] == 'stopped':
                self.status = status['status']
                break
        else:
            self.status = status['status']
            raise ProxmoveError(
                'VM {!r} refuses to shut down: status = {!r}'.format(
                    self.name, self.status))

        log.info('- stopped VM {}'.format(self))

    def add_comment(self, comment):
        config = self.get_config()
        if 'description' in config:
            comment = config['description'].rstrip() + '\n' + comment.strip()
        else:
            comment = comment.strip()

        self.api_vm.config.put(description=comment)
        self._cache = {}  # drop cache

    def __str__(self):
        return '{}@{}<{}/{}/{}>'.format(
            self.name, self.node, self.type, self.id, self.status)


class ProxmoxVolume(object):
    def __init__(self, location, properties, storage=None):
        self.location = location
        self.properties = properties
        self.storage = storage

    def is_removable(self):
        parts = self.properties.split(',')
        return self.location is None or 'media=cdrom' in parts

    def get_format(self):
        """
        Return image file format. Use the format property, or the image
        file extension as fallback.
        """
        ret = self.get_property('format')
        if not ret and isinstance(self.storage, ProxmoxStoragePlain):  # hacks
            ret = self.location.rsplit('.', 1)[-1]
        return ret

    def get_property(self, what):
        search = '{}='.format(what)
        parts = self.properties.split(',')
        for part in parts:
            if part.startswith(search):
                return part[len(search):]
        return None

    def get_properties(self, **updates):
        """
        Get properties list (abc=def,ghi=jkl) but update properties from
        the updates argument or leave them out if the value is None.
        """
        search_list = tuple('{}='.format(i) for i in updates.keys())
        ret = []

        parts = self.properties.split(',')
        for part in parts:
            if not part.startswith(search_list):
                ret.append(part)

        for key, value in updates.items():
            if value:
                parts.append('{}={}'.format(key, value))

        return ','.join(ret)

    def get_size(self):
        if not hasattr(self, '_get_size'):
            if self.storage:
                self._get_size = self.storage.get_image_size(self.location)
            else:
                self._get_size = None
        return self._get_size

    def get_human_size(self):
        size = self.get_size()
        if not size:
            return '<unknown>'
        return human_size_fmt(size)

    def clone(self, new_storage, new_vmid, new_key):
        """
        new_key could be "virtio0" or "ide2" or ...
        """
        # Create new name and let the storage backend do the copying.
        new_name = 'vm-{}-{}'.format(new_vmid, new_key)
        new_path, new_format = self.storage.copy(
            self.get_size(), self.get_property('size'),
            self.location, self.get_format(),
            new_storage, new_vmid, new_name)

        # Log message so we know that any percentages shown arealy are
        # irrelevant. (pv(1) gets a guesstimate which may be off by a
        # few percent. We don't want the user to think the transfer
        # stopped at 98%.)
        log.info('Volume transferring/conversion 100% complete!')
        return ProxmoxVolume(
            new_path, self.get_properties(format=new_format),
            storage=new_storage)

    def as_properties(self):
        """
        Return something like: "san06:123/vm-123-disk.qcow2,size=50G,..."
        """
        if self.storage:
            location = '{}:{}'.format(self.storage.name, self.location)
        else:
            assert self.location == 'none', self.location
            location = self.location

        if self.properties:
            return '{},{}'.format(location, self.properties)
        return location

    def __str__(self):
        if self.storage:
            return '{}:{},{}'.format(
                self.storage.name, self.location, self.properties)
        return '{},{}'.format(self.location, self.properties)


class DefaultConfigTranslator(object):
    def config(self, old_config):
        new_config = {}
        for key, value in old_config.items():
            # The digest is used to prevent changes if the current
            # config doesn't match the digest. This blocks
            # concurrent updates.
            if key == 'digest':
                pass
            else:
                new_config[key] = value
        return new_config


class VmMover(object):
    def __init__(self, options):
        self.src_pve = options.source
        self.dst_pve = options.destination
        self.dst_node = options.nodeid
        self.dst_storage = options.storage
        self.bw_limit = options.bwlimit
        self.vms_requested = options.vm

        self.ignore_exists = options.ignore_exists
        self.skip_disks = options.skip_disks
        # Don't start if there are no disks..
        self.skip_start = options.skip_start or options.skip_disks

        self.vms_to_move = None
        self.storages = None

    def prepare(self):
        log.debug('Sanity checks and preparation')
        self.prepare_vm_list()
        self.prepare_vm_checks()
        self.prepare_storage_list()
        self.prepare_storage_prerequisites()
        self.prepare_storage_settings()

    def prepare_vm_list(self):
        log.debug('Checking VMs existence on source and destination')
        vms = []
        for vm_name in self.vms_requested:
            self.prepare_vm_check_source(vm_name)
            self.prepare_vm_check_destination(vm_name)
            vm = self.src_pve.get_vm(vm_name)
            vms.append(vm)

        self.vms_to_move = vms
        self.vms_requested = None

    def prepare_vm_check_source(self, vm_name):
        for suffix in (SUFFIX_CLONING, SUFFIX_MIGRATED):
            vm_name_with_suffix = '{}{}'.format(vm_name, suffix)
            try:
                self.src_pve.get_vm(vm_name_with_suffix)
            except ProxmoxVm.DoesNotExist:
                pass
            else:
                raise PrepareError(
                    'VM {!r} exists on source already'.format(
                        vm_name_with_suffix))

    def prepare_vm_check_destination(self, vm_name):
        for suffix in ('', SUFFIX_CREATING):
            vm_name_with_suffix = '{}{}'.format(vm_name, suffix)
            try:
                self.dst_pve.get_vm(vm_name_with_suffix)
            except ProxmoxVm.DoesNotExist:
                pass
            else:
                if not (self.ignore_exists and not suffix):
                    raise PrepareError(
                        'VM {!r} exists on destination already'.format(
                            vm_name_with_suffix))

    def prepare_vm_checks(self):
        log.debug('Checking for pending changes in {} VMs to move'.format(
            len(self.vms_to_move)))

        for vm in self.vms_to_move:
            vm.check_config()

    def prepare_storage_list(self):
        storages = set([self.dst_storage])
        for vm in self.vms_to_move:
            for volume in vm.get_volumes().values():
                if volume.storage:
                    storages.add(volume.storage)

        log.debug('Found {} relevant storages: {}'.format(
            len(storages), ', '.join(str(i) for i in storages)))
        self.storages = storages

    def prepare_storage_prerequisites(self):
        log.debug('Checking storage prerequisites')
        for storage in self.storages:
            storage.check_prerequisites()

    def prepare_storage_settings(self):
        log.debug('Set bandwidth limit on storage objects')
        for storage in self.storages:
            storage.set_bandwidth_limit(self.bw_limit)

    def run(self, dry_run):
        translator = DefaultConfigTranslator()
        for vm in self.vms_to_move:
            self.move_vm(vm, translator, dry_run)

    def move_vm(self, src_vm, translator, dry_run):
        # Config and notes:
        # {'bootdisk': 'virtio0',
        #  'cores': 8,
        #  'digest': 'XXXXXXXXXXXXXXXXcfa3e49c8568312e7d148505',
        #  # v-- we "eject" the cdrom, replacing it with "none"
        #  'ide2': 'san06:iso/debian-8.0.0-amd64-netinst.iso,media=cdrom',
        #  'memory': 4096,
        #  'name': 'vm-to-move.example.com',
        #  # v-- vmbr138 will work fine as long as cluster is in same location
        #  'net0': 'virtio=XX:XX:XX:XX:8B:22,bridge=vmbr138',
        #  # v-- is constant (apparently?)
        #  'ostype': 'l26',
        #  'smbios1': 'uuid=XXXXXXXX-XXXX-XXXX-8712-e68a063d993e,'
        #             'manufacturer=example,product=vm-to-move',
        #  'sockets': 1,
        #  # v-- disk moving is covered in ProxmoxVolume
        #  'virtio0': 'san8:520/vm-520-disk-1.qcow2,format=qcow2,iops_rd=5000,'
        #             'iops_wr=400,size=50G'}
        #
        # Steps:
        # - translate old config to new config
        # - create new config on nodeX on dest (no disks, "CREATING" name)
        # - stop old host, rename to "CLONING"
        # - move all disks, one by one
        # - rename dest to real name, rename source to "MIGRATED", add comment
        log.info('Attempt moving {} => {} (node {!r}): {}'.format(
            self.src_pve, self.dst_pve, self.dst_node, src_vm.name))

        log.info('- source VM {}'.format(src_vm))
        for key, volume in src_vm.get_volumes().items():
            # TODO: Check whether volume is accessible? Must be combined
            # with the is_removable() and skip_disks options below.
            log.info('- storage {!r}: {} (blobsize={})'.format(
                key, volume, volume.get_human_size()))

        if not dry_run:
            # Translate config, create new VM.
            src_vm_name = src_vm.name
            dst_config = translator.config(src_vm.get_config())
            dst_vm = self.dst_pve.create_vm(
                src_vm.type, dst_config, nodeid=self.dst_node)

            # Stop old VM, create storage, copy storage.
            src_vm.ensure_stopped()
            src_vm.rename(src_vm_name + SUFFIX_CLONING)
            for key, volume in src_vm.get_volumes().items():
                assert key not in dst_vm.get_volumes(), key

                if volume.is_removable():
                    storage = None  # eject
                elif self.skip_disks:
                    log.debug('Not copying volume {} as requested'.format(key))
                    storage = None
                else:
                    storage = self.dst_storage

                dst_vm.create_volume(key, volume, storage=storage)

            # Done? Rename both.
            dst_vm.rename(dst_config['name'])
            src_vm.rename(src_vm_name + SUFFIX_MIGRATED)
            src_vm.add_comment('{} UTC: Migrated to {}'.format(
                datetime.utcnow(), dst_vm))

            # Start VM.
            if self.skip_start:
                log.debug('Not starting VM as requested')
            else:
                dst_vm.ensure_started()

            log.info('Completed moving {} => {} (node {!r}): {}'.format(
                self.src_pve, self.dst_pve, self.dst_node, dst_vm.name))


class CommandLoader(object):
    def __init__(self):
        self.argparse()
        self.load_config()
        self.process()

    def argparse(self):
        self.argparse_create()
        self.argparse_add_options()
        self.argparse_add_arguments()
        self.argparse_run()

    def argparse_create(self):
        # CURRENT:
        # - move SRCCLUSTER DSTCLUSTER DSTNODE DSTSTORAGE VM..
        # FUTURE:
        # - ssh CLUSTER:STORAGE
        # - nodes CLUSTER (show nodes (and storage?))
        # - storages CLUSTER:NODE (show nodes)
        # - move SRC DST DSTNODE VM..
        # - dumpconfig (based with the storage devices..)
        self.parser = ArgumentParser14191(
            description=(
                'Migrate VMs from one Proxmox cluster to another.'),
            epilog=(
                'Cluster aliases and storage locations should be defined '
                'in ~/.proxmoverc (or see -c option). See the example '
                'proxmoverc.sample. It requires [pve:CLUSTER_ALIAS] sections '
                'for the proxmox "api" URL and '
                '[storage:CLUSTER_ALIAS:STORAGE_NAME] sections with "ssh", '
                '"path" and "temp" settings.'))

    def argparse_add_options(self):
        self.parser.add_argument(
            '-c', '--config', action='store', metavar='FILENAME',
            default='~/.proxmoverc', help=(
                'use alternate configuration inifile'))
        self.parser.add_argument(
            '-n', '--dry-run', action='store_true', help=(
                'stop before doing any writes'))
        self.parser.add_argument(
            '--bwlimit', action='store', metavar='MBPS', type=int, help=(
                'limit bandwidth in Mbit/s'))
        self.parser.add_argument(
            '--skip-disks', action='store_true', help=(
                'do the move, but skip copying of the disks; '
                'implies --skip-start'))
        self.parser.add_argument(
            '--skip-start', action='store_true', help=(
                'do the move, but do not start the new instance'))
        self.parser.add_argument(
            '--version', action='version', version=(
                'proxmove {}'.format(__version__)))

        # Debugging options. Not needed in full view at the moment.
        self.parser.add_argument(
            # Enables debug log.
            '--debug', action='store_true', help=argparse.SUPPRESS)
        self.parser.add_argument(
            # Ignores the fact that a VM exists already. Can be used to test
            # moves between the same platform.
            '--ignore-exists', action='store_true', help=argparse.SUPPRESS)

    def argparse_add_arguments(self):
        self.parser.add_argument(
            'source', action='store', help=(
                'alias of source cluster'))
        self.parser.add_argument(
            'destination', action='store', help=(
                'alias of destination cluster'))
        self.parser.add_argument(
            'nodeid', action='store', help=(
                'node on destination cluster'))
        self.parser.add_argument(
            'storage', action='store', help=(
                'storage on destination node'))
        self.parser.add_argument(
            'vm', action='store', nargs='+', help=(
                'one or more VMs (guests) to move'))

    def argparse_run(self):
        self.args = self.parser.parse_args()

        # Set debug mode as soon as possible.
        if self.args.debug:
            log.setLevel('DEBUG')

    def load_config(self):
        log.debug('Parsing config file: {}'.format(self.args.config))
        try:
            self.clusters = ProxmoxClusters.from_filename(
                os.path.expanduser(self.args.config))
        except ValueError as e:
            self.parser.error(e.args[0])

    def process(self):
        self.process_clusters()
        self.process_nodes()

    def process_clusters(self):
        if (not self.args.ignore_exists and
                self.args.source == self.args.destination):
            self.parser.error('source and destination arguments are the same')

        self.args.source = self.process_cluster(self.args.source)
        self.args.destination = self.process_cluster(self.args.destination)

    def process_cluster(self, cluster_name):
        try:
            cluster = self.clusters[cluster_name]
        except KeyError:
            found_clusters = ', '.join(sorted(self.clusters.keys()))
            self.parser.error(
                'cluster {!r} is not configured in config '
                '(expected one of: {})'.format(cluster_name, found_clusters))

        try:
            cluster.ping()
        except Exception as e:
            self.parser.error(
                'cluster {!r} is unavailable: {}: {}'.format(
                    cluster.name, type(e).__name__, e.args[0]))

        return cluster

    def process_nodes(self):
        if self.args.nodeid not in self.args.destination.get_nodes():
            self.parser.error(
                'destination node {!r} not found in cluster'.format(
                    self.args.nodeid))

        try:
            self.args.storage = self.args.destination.get_storage(
                self.args.nodeid, self.args.storage)
        except ValueError:
            self.parser.error(
                'destination storage {!r} not found in node {!r}'.format(
                    self.args.storage, self.args.nodeid))

    def get_options(self):
        return self.args


def main():
    # Set logging mode.
    logconfig = {
        'version': 1,
        'formatters': {
            'full': {
                'format': '%(asctime)-15s: %(levelname)s: %(message)s'}},
        'handlers': {
            'console': {
                'class': 'logging.StreamHandler', 'formatter': 'full'}},
        'loggers': {
            '': {'handlers': ['console'], 'level': 'WARNING'},
            'proxmove': {'handlers': ['console'], 'level': 'INFO',
                         'propagate': False}},
    }
    logging.config.dictConfig(logconfig)

    # Load up config.
    options = CommandLoader().get_options()

    # Create mover and start the move.
    vmmover = VmMover(options)
    try:
        vmmover.prepare()
    except PrepareError as e:
        print('error: {}'.format(e), file=sys.stderr)
        print('aborting', file=sys.stderr)
        sys.exit(1)
    else:
        vmmover.run(options.dry_run)


if __name__ == '__main__':
    main()
