#!/usr/bin/env python
# Copyright 2012-2018 CERN for the benefit of the ATLAS collaboration.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Authors:
# - Mario Lassnig, <mario.lassnig@cern.ch>, 2012-2018
# - Vincent Garonne, <vgaronne@gmail.com>, 2012-2018
# - Martin Barisits, <martin.barisits@cern.ch>, 2012-2018
# - Thomas Beermann, <thomas.beermann@cern.ch>, 2012-2017
# - Yun-Pin Sun, <winter0128@gmail.com>, 2012-2013
# - Cedric Serfon, <cedric.serfon@cern.ch>, 2013-2018
# - Ralph Vigne, <ralph.vigne@cern.ch>, 2013
# - David Cameron, <d.g.cameron@gmail.com>, 2014
# - Tomas Kouba, <tomas.kouba@cern.ch>, 2014
# - Wen Guan, <wguan.icedew@gmail.com>, 2014
# - Joaquin Bogado, <jbogado@linti.unlp.edu.ar>, 2014-2018
# - Evangelia Liotiri, <evangelia.liotiri@cern.ch>, 2015
# - Tobias Wegner, <twegner@cern.ch>, 2017
# - Brian Bockelman, <bbockelm@cse.unl.edu>, 2017-2018
# - Nicolo Magini, <Nicolo.Magini@cern.ch>, 2018
# - Frank Berghaus, <frank.berghaus@cern.ch>, 2018

from __future__ import print_function

import argparse
import json
import logging
import os
import random
import signal
import socket
import subprocess
import sys
import time
import traceback
import uuid


import argcomplete
import requests
import tabulate

try:
    from ConfigParser import NoOptionError, NoSectionError
except ImportError:
    from configparser import NoOptionError, NoSectionError

from copy import deepcopy
from functools import wraps
try:
    from Queue import Queue, Empty
except ImportError:
    from queue import Queue, Empty
from threading import Thread, Event
from xml.etree import ElementTree

from rucio.client.client import Client
from rucio import version
from rucio.common.config import config_get
from rucio.common.exception import (DataIdentifierAlreadyExists, Duplicate, FileAlreadyExists, AccessDenied, ResourceTemporaryUnavailable,
                                    DataIdentifierNotFound, InvalidObject, RSENotFound, InvalidRSEExpression, DuplicateContent, RSEProtocolNotSupported,
                                    RuleNotFound, CannotAuthenticate, MissingDependency, UnsupportedOperation, FileConsistencyMismatch,
                                    RucioException, DuplicateRule, NoFilesDownloaded, NotAllFilesDownloaded)
from rucio.common.utils import adler32, md5, generate_uuid, execute, chunks, sizefmt, Color, detect_client_location, make_valid_did
from rucio.rse import rsemanager as rsemgr

SUCCESS = 0
FAILURE = 1

DEFAULT_SECURE_PORT = 443
DEFAULT_PORT = 80

STOP_REQUEST = Event()

logger = logging.getLogger("user")
gfal2_logger = logging.getLogger("gfal2")
tablefmt = 'psql'


def setup_logger(logger):
    logger.setLevel(logging.INFO)
    hdlr = logging.StreamHandler()

    def emit_decorator(fnc):
        def func(*args):
            if 'RUCIO_LOGGING_FORMAT' not in os.environ:
                levelno = args[0].levelno
                if levelno >= logging.CRITICAL:
                    color = '\033[31;1m'
                elif levelno >= logging.ERROR:
                    color = '\033[31;1m'
                elif levelno >= logging.WARNING:
                    color = '\033[33;1m'
                elif levelno >= logging.INFO:
                    color = '\033[32;1m'
                elif levelno >= logging.DEBUG:
                    color = '\033[36;1m'
                else:
                    color = '\033[0m'
                formatter = logging.Formatter('{0}%(asctime)s\t%(levelname)s\t%(message)s\033[0m'.format(color))
            else:
                formatter = logging.Formatter(os.environ['RUCIO_LOGGING_FORMAT'])
            hdlr.setFormatter(formatter)
            return fnc(*args)
        return func
    hdlr.emit = emit_decorator(hdlr.emit)
    logger.addHandler(hdlr)


def setup_gfal2_logger(logger):
    logger.setLevel(logging.CRITICAL)
    logger.addHandler(logging.StreamHandler())


setup_logger(logger)
setup_gfal2_logger(gfal2_logger)


def signal_handler(sig, frame):
    logger.warning('You pressed Ctrl+C! Exiting gracefully')
    child_processes = subprocess.Popen('ps -o pid --ppid %s --noheaders' % os.getpid(), shell=True, stdout=subprocess.PIPE)
    child_processes = child_processes.stdout.read()
    for pid in child_processes.split("\n")[:-1]:
        try:
            os.kill(int(pid), signal.SIGTERM)
        except Exception:
            print('Cannot kill child process')
    STOP_REQUEST.set()
    sys.exit(1)


signal.signal(signal.SIGINT, signal_handler)


def extract_scope(did):
    # Try to extract the scope from the DSN
    if did.find(':') > -1:
        if len(did.split(':')) > 2:
            raise RucioException('Too many colons. Cannot extract scope and name')
        scope, name = did.split(':')[0], did.split(':')[1]
        if name.endswith('/'):
            name = name[:-1]
        return scope, name
    else:
        scope = did.split('.')[0]
        if did.startswith('user') or did.startswith('group'):
            scope = ".".join(did.split('.')[0:2])
        if did.endswith('/'):
            did = did[:-1]
        return scope, did


def send_trace(trace, trace_endpoint, user_agent, retries=5, threadnb=None, total_threads=None):
    if user_agent.startswith('pilot'):
        logger.debug('pilot detected - not sending trace')
        return 0
    else:
        if threadnb is not None and total_threads is not None:
            logger.debug('Thread %s/%s : sending trace' % (threadnb, total_threads))
        else:
            logger.debug('sending trace')

    for dummy in range(retries):
        try:
            requests.post(trace_endpoint + '/traces/', verify=False, data=json.dumps(trace))
            return 0
        except Exception as error:
            if threadnb is not None and total_threads is not None:
                logger.debug('Thread %s/%s : %s' % (threadnb, total_threads, error))
            else:
                logger.debug(error)
    return 1


def exception_handler(function):
    @wraps(function)
    def new_funct(*args, **kwargs):
        try:
            return function(*args, **kwargs)
        except InvalidObject as error:
            logger.error(error)
            return error.error_code
        except DataIdentifierNotFound as error:
            logger.error(error)
            logger.debug('This means that the Data IDentifier you provided is not known by Rucio.')
            return error.error_code
        except AccessDenied as error:
            logger.error(error)
            logger.debug('This error is a permission issue. You cannot run this command with your account.')
            return error.error_code
        except DataIdentifierAlreadyExists as error:
            logger.error(error)
            logger.debug('This means that the Data IDentifier you try to add is already registered in Rucio.')
            return error.error_code
        except RSENotFound as error:
            logger.error(error)
            logger.debug('This means that the Rucio Storage Element you provided is not known by Rucio.')
            return error.error_code
        except InvalidRSEExpression as error:
            logger.error(error)
            logger.debug('This means the RSE expression you provided is not syntactically correct.')
            return error.error_code
        except DuplicateContent as error:
            logger.error(error)
            logger.debug('This means that the DID you want to attach is already in the target DID.')
            return error.error_code
        except TypeError as error:
            logger.error(error)
            logger.debug('This means the parameter you passed has a wrong type.')
            return FAILURE
        except RuleNotFound as error:
            logger.error(error)
            logger.debug('This means the rule you specified does not exist.')
            return error.error_code
        except UnsupportedOperation as error:
            logger.error(error)
            logger.debug('This means you cannot change the status of the DID.')
            return error.error_code
        except MissingDependency as error:
            logger.error(error)
            logger.debug('This means one dependency is missing.')
            return error.error_code
        except NoFilesDownloaded as error:
            return error.error_code
        except NotAllFilesDownloaded as error:
            return error.error_code
        except KeyError as error:
            if 'x-rucio-auth-token' in error:
                used_account = None
                try:  # get the configured account from the configuration file
                    used_account = '%s (from rucio.cfg)' % config_get('client', 'account')
                except:
                    pass
                try:  # are we overriden by the environment?
                    used_account = '%s (from RUCIO_ACCOUNT)' % os.environ['RUCIO_ACCOUNT']
                except:
                    pass
                logger.error('Specified account %s does not have an associated identity.' % used_account)
            else:
                logger.error(error)
                logger.debug(traceback.format_exc())
                logger.error('\nThe object is missing this property: %s\n'
                             'This should never happen. Please rerun the last command with the "-v" option to gather more information.\n'
                             'Please followup with all relevant information at: %s' % (str(error), config_get('policy', 'support')))
            raise
        except RucioException as error:
            logger.error(error)
            return error.error_code
        except Exception as error:
            logger.debug(traceback.format_exc())
            logger.error(error)
            logger.error('\nRucio exited with an unexpected/unknown error.\n'
                         'Please rerun the last command with the "-v" option to gather more information.\n'
                         'If it is a problem concerning your experiment or if you unsure what to do, please followup at: %s\n'
                         'If you are sure that there is a problem with Rucio itself, please followup at: %s' % (config_get('policy', 'support'),
                                                                                                                config_get('policy', 'support_rucio')))
            return FAILURE
    return new_funct


def get_client(args):
    """
    Returns a new client object.
    """
    if args.auth_strategy == 'userpass':
        creds = {'username': args.username, 'password': args.password}
    else:
        creds = None

    try:
        client = Client(rucio_host=args.host, auth_host=args.auth_host,
                        account=args.account,
                        auth_type=args.auth_strategy, creds=creds,
                        ca_cert=args.ca_certificate, timeout=args.timeout,
                        user_agent=args.user_agent)
    except CannotAuthenticate as error:
        logger.error(error)
        if not args.auth_strategy:
            if 'RUCIO_AUTH_TYPE' in os.environ:
                auth_type = os.environ['RUCIO_AUTH_TYPE']
            else:
                try:
                    auth_type = config_get('client', 'auth_type')
                except (NoOptionError, NoSectionError):
                    logger.error('Cannot get AUTH_TYPE')
                    sys.exit(FAILURE)
        if auth_type == 'x509_proxy':
            logger.error('Please verify that your proxy is still valid and renew it if needed.')
        sys.exit(FAILURE)
    return client


@exception_handler
def ping(args):
    """
    Pings a Rucio server.
    """
    client = get_client(args)
    server_info = client.ping()
    if server_info:
        print(server_info['version'])
        return SUCCESS
    logger.error('Ping failed')
    return FAILURE


@exception_handler
def whoami_account(args):
    """
    %(prog)s show [options] <field1=value1 field2=value2 ...>

    Show extended information of a given account
    """
    client = get_client(args)
    info = client.whoami()
    for k in info:
        print(k.ljust(10) + ' : ' + str(info[k]))
    return SUCCESS


@exception_handler
def list_dataset_replicas(args):
    """
    %(prog)s list [options] <field1=value1 field2=value2 ...>

    List dataset replicas
    """
    client = get_client(args)
    datasets, result = {}, {}
    scope, name = extract_scope(args.did)
    meta = client.get_metadata(scope, name)
    if meta['did_type'] != 'DATASET':
        dids = client.scope_list(scope=scope, name=name, recursive=True)
        for did in dids:
            if did['type'] == 'FILE':
                dsn = '%s:%s' % (did['parent']['scope'], did['parent']['name'])
                if dsn not in datasets:
                    datasets[dsn] = 0
                datasets[dsn] += 1
    else:
        datasets['%s:%s' % (scope, name)] = 0
    for dsn in datasets:
        scope, name = extract_scope(dsn)
        result[dsn] = {}
        for rep in client.list_dataset_replicas(scope, name, args.deep):
            result[dsn][rep['rse']] = [rep['rse'], rep['available_length'], rep['length']]
    if args.csv:
        for dsn in result:
            for rse in list(result[dsn].values()):
                print("{0}, {1}, {2}".format(rse[0], rse[1], rse[2]))
    else:
        for dsn in result:
            print('\nDATASET: %s' % (dsn))
            print(tabulate.tabulate(list(result[dsn].values()), tablefmt=tablefmt, headers=['RSE', 'FOUND', 'TOTAL']))
    return SUCCESS


@exception_handler
def list_file_replicas(args):
    """
    %(prog)s list [options] <field1=value1 field2=value2 ...>

    List file replicas
    """
    client = get_client(args)
    protocols = None
    if args.protocols:
        protocols = args.protocols.split(',')

    table = []
    dids = []
    if args.missing and not args.selected_rse:
        print('Cannot use --missing without specifying a RSE')
        return FAILURE
    if args.link and ':' not in args.link:
        print('The substition parameter must equal --link="/pfn/dir:/dst/dir"')
        return FAILURE

    for did in args.dids:
        scope, name = extract_scope(did)
        client.get_metadata(scope=scope, name=name)  # break with Exception before streaming replicas if DID does not exist
        dids.append({'scope': scope, 'name': name})

    replicas = client.list_replicas(dids, schemes=protocols,
                                    all_states=args.all_states,
                                    rse_expression=args.rse_expression,
                                    metalink=args.metalink,
                                    client_location=detect_client_location(),
                                    sort=args.sort, domain=args.domain)

    if args.metalink:
        print(replicas[:-1])  # last character is newline, no need to print that
    else:
        if args.missing:
            for replica in replicas:
                rses = list(replica['rses'].keys())
                if args.selected_rse not in rses:
                    table.append([replica['scope'], replica['name']])
            print(tabulate.tabulate(table, tablefmt=tablefmt, headers=['SCOPE', 'NAME']))
        elif args.link:
            pfn_dir, dst_dir = args.link.split(':')
            for replica in replicas:
                if args.selected_rse:
                    if args.selected_rse in list(replica['rses'].keys()) and replica['rses'][args.selected_rse]:
                        for pfn in replica['rses'][args.selected_rse]:
                            os.symlink(dst_dir + pfn.rsplit(pfn_dir)[-1], replica['name'])
                else:
                    for rse in replica['rses']:
                        if replica['rses'][rse]:
                            for pfn in replica['rses'][rse]:
                                os.symlink(dst_dir + pfn.rsplit(pfn_dir)[-1], replica['name'])
        elif args.pfns:
            for replica in replicas:
                if args.selected_rse:
                    if args.selected_rse in list(replica['rses'].keys()) and replica['rses'][args.selected_rse]:
                        for pfn in replica['rses'][args.selected_rse]:
                            print(pfn)
                else:
                    for rse in replica['rses']:
                        if replica['rses'][rse]:
                            for pfn in replica['rses'][rse]:
                                print(pfn)
        else:
            for replica in replicas:
                if 'bytes' in replica:
                    for rse in replica['rses']:
                        for pfn in replica['rses'][rse]:
                            if args.selected_rse:
                                if rse == args.selected_rse:
                                    table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], '{0}: {1}'.format(rse, pfn)])
                            else:
                                table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], '{0}: {1}'.format(rse, pfn)])
            print(tabulate.tabulate(table, tablefmt=tablefmt, headers=['SCOPE', 'NAME', 'FILESIZE', 'ADLER32', 'RSE: REPLICA']))

    return SUCCESS


@exception_handler
def add_dataset(args):
    """
    %(prog)s add-dataset [options] <dsn>

    Add a dataset identifier.
    """
    client = get_client(args)
    scope, name = extract_scope(args.did)
    client.add_dataset(scope=scope, name=name, statuses={'monotonic': args.monotonic}, lifetime=args.lifetime)
    print('Added %s:%s' % (scope, name))
    return SUCCESS


@exception_handler
def add_container(args):
    """
    %(prog)s add-container [options] <dsn>

    Add a container identifier.
    """
    client = get_client(args)
    scope, name = extract_scope(args.did)
    client.add_container(scope=scope, name=name, statuses={'monotonic': args.monotonic}, lifetime=args.lifetime)
    print('Added %s:%s' % (scope, name))
    return SUCCESS


@exception_handler
def attach(args):
    """
    %(prog)s attach [options] <field1=value1 field2=value2 ...>

    Attach a data identifier.
    """
    client = get_client(args)
    scope, name = extract_scope(args.todid)
    dids = []
    if args.fromfile:
        if len(args.dids) > 1:
            logger.error("If --fromfile option is active, only one file is supported. The file should contain a list of dids, one per line.")
            return FAILURE
        try:
            f = open(args.dids[0], 'r')
            for did in f.readlines():
                cscope, cname = extract_scope(did.rstrip())
                dids.append({'scope': cscope, 'name': cname})
        except IOError:
            logger.error("Can't open file '" + args.dids[0] + "'.")
            return FAILURE
    else:
        for did in args.dids:
            cscope, cname = extract_scope(did)
            dids.append({'scope': cscope, 'name': cname})
    client.attach_dids(scope=scope, name=name, dids=dids)
    print('DIDs successfully attached to %s:%s' % (scope, name))
    return SUCCESS


@exception_handler
def detach(args):
    """
    %(prog)s detach [options] <field1=value1 field2=value2 ...>

    Detach data identifier.
    """
    client = get_client(args)
    scope, name = extract_scope(args.fromdid)
    dids = []
    for did in args.dids:
        cscope, cname = extract_scope(did)
        dids.append({'scope': cscope, 'name': cname})
    client.detach_dids(scope=scope, name=name, dids=dids)
    print('DIDs successfully detached from %s:%s' % (scope, name))
    return SUCCESS


@exception_handler
def list_dids(args):
    """
    %(prog)s list-dids scope[:*|:name] [--filter 'key=value' | --recursive]

    List the data identifiers for a given scope.
    """
    client = get_client(args)
    filters = {}
    type = 'all'
    table = []
    if args.filter:
        if args.recursive:
            logger.error('Option recursive and filter cannot be used together')
            return FAILURE
        else:
            try:
                for key, value in [(a.split('=')[0], a.split('=')[1]) for a in args.filter.split(',')]:
                    if key == 'type':
                        if value.upper() in ['ALL', 'COLLECTION', 'CONTAINER', 'DATASET', 'FILE']:
                            type = value.lower()
                        else:
                            logger.error('{0} is not a valid type. Valid types are {1}'.format(value, ['ALL', 'COLLECTION', 'CONTAINER', 'DATASET', 'FILE']))
                            return FAILURE
                    else:
                        if value.lower() == 'true':
                            value = '1'
                        elif value.lower() == 'false':
                            value = '0'
                        filters[key] = value
            except Exception:
                logger.error("Invalid Filter. Filter must be 'key=value'")
                return FAILURE

    try:
        scope, name = extract_scope(args.did[0])
        if name == '':
            name = '*'
    except InvalidObject:
        scope = args.did[0]
        name = '*'
    if scope not in client.list_scopes():
        logger.error('Scope not found')
        return FAILURE

    if args.recursive:
        logger.error('Option recursive cannot be used with wildcards')
        return FAILURE
    elif ('name' in filters) and (name != '*'):
        logger.error('You cannot use a wildcard query and a filter by name')
        return FAILURE
    filters['name'] = name

    for did in client.list_dids(scope, filters=filters, type=type, long=True):
        table.append(['%s:%s' % (did['scope'], did['name']), did['did_type']])

    if args.short:
        for did, dummy in table:
            print(did)
    else:
        print(tabulate.tabulate(table, tablefmt=tablefmt, headers=['SCOPE:NAME', '[DID TYPE]']))
    return SUCCESS


@exception_handler
def list_scopes(args):
    """
    %(prog)s list-scopes <scope>

    List scopes.
    """
    # For the moment..
    client = get_client(args)
    scopes = client.list_scopes()
    for scope in scopes:
        print(scope)
    return SUCCESS


@exception_handler
def list_files(args):
    """
    %(prog)s list-files [options] <field1=value1 field2=value2 ...>

    List data identifier contents.
    """
    client = get_client(args)
    if args.csv:
        for did in args.dids:
            scope, name = extract_scope(did)
            for f in client.list_files(scope=scope, name=name):
                guid = f['guid']
                if guid:
                    guid = '%s-%s-%s-%s-%s' % (guid[0:8], guid[8:12], guid[12:16], guid[16:20], guid[20:32])
                else:
                    guid = '(None)'
                print("{0}:{1},{2},{3},{4},{5}".format(f['scope'], f['name'], guid, f['adler32'], sizefmt(f['bytes'], args.human), f['events']))
        return SUCCESS
    elif args.LOCALPATH:

        print('''<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
<!DOCTYPE POOLFILECATALOG SYSTEM "InMemory">
<POOLFILECATALOG>''')

        file_str = ''' <File ID="%s">
  <physical>
   <pfn filetype="ROOT_All" name="%s/%s"/>
  </physical>
  <logical>
   <lfn name="%s"/>
  </logical>
 </File>'''

        for did in args.dids:
            scope, name = extract_scope(did)
            for f in client.list_files(scope=scope, name=name):
                guid = f['guid']
                if guid:
                    guid = '%s-%s-%s-%s-%s' % (guid[0:8], guid[8:12], guid[12:16], guid[16:20], guid[20:32])
                else:
                    guid = '(None)'
                print(file_str % (guid, args.LOCALPATH, f['name'], f['name']))

        print('</POOLFILECATALOG>')
        return SUCCESS
    else:
        table = []
        for did in args.dids:
            totfiles = 0
            totsize = 0
            totevents = 0
            scope, name = extract_scope(did)
            for file in client.list_files(scope=scope, name=name):
                totfiles += 1
                totsize += int(file['bytes'])
                if file['events']:
                    totevents += int(file.get('events', 0))
                guid = file['guid']
                if guid:
                    guid = '%s-%s-%s-%s-%s' % (guid[0:8], guid[8:12], guid[12:16], guid[16:20], guid[20:32])
                else:
                    guid = '(None)'
                table.append(['%s:%s' % (file['scope'], file['name']), guid, 'ad:%s' % file['adler32'], sizefmt(file['bytes'], args.human), file['events']])
            print(tabulate.tabulate(table, tablefmt=tablefmt, headers=['SCOPE:NAME', 'GUID', 'ADLER32', 'FILESIZE', 'EVENTS']))
            print('Total files : %s' % totfiles)
            print('Total size : %s' % sizefmt(totsize, args.human))
            if totevents:
                print('Total events : %s' % totevents)
        return SUCCESS


@exception_handler
def list_content(args):
    """
    %(prog)s list-content [options] <field1=value1 field2=value2 ...>

    List data identifier contents.
    """
    client = get_client(args)
    table = []
    for did in args.dids:
        scope, name = extract_scope(did)
        for content in client.list_content(scope=scope, name=name):
            table.append(['%s:%s' % (content['scope'], content['name']), content['type'].upper()])
    if args.short:
        for did, dummy in table:
            print(did)
    else:
        print(tabulate.tabulate(table, tablefmt=tablefmt, headers=['SCOPE:NAME', '[DID TYPE]']))
    return SUCCESS


@exception_handler
def list_content_history(args):
    """
    %(prog)s list-content-history [options] <field1=value1 field2=value2 ...>

    List data identifier contents.
    """
    client = get_client(args)
    table = []
    for did in args.dids:
        scope, name = extract_scope(did)
        for content in client.list_content_history(scope=scope, name=name):
            table.append(['%s:%s' % (content['scope'], content['name']), content['type'].upper()])
    print(tabulate.tabulate(table, tablefmt=tablefmt, headers=['SCOPE:NAME', '[DID TYPE]']))
    return SUCCESS


@exception_handler
def list_parent_dids(args):
    """
    %(prog)s list-parent-dids

    List parent data identifier.
    """
    client = get_client(args)
    if args.pfns:
        dict_datasets = {}
        for res in client.get_did_from_pfns(args.pfns):
            for key in res:
                if key not in dict_datasets:
                    dict_datasets[key] = []
                for rule in client.list_associated_rules_for_file(res[key]['scope'], res[key]['name']):
                    if '%s:%s' % (rule['scope'], rule['name']) not in dict_datasets[key]:
                        dict_datasets[key].append('%s:%s' % (rule['scope'], rule['name']))
        for pfn in dict_datasets:
            print('PFN: ', pfn)
            print('Parents: ', ','.join(dict_datasets[pfn]))
    elif args.guids:
        guids = []
        for input in args.guids:
            try:
                uuid.UUID(input)
            except ValueError:
                continue
        dict_datasets = {}
        for guid in guids:
            for did in client.get_dataset_by_guid(guid):
                if guid not in dict_datasets:
                    dict_datasets[guid] = []
                for rule in client.list_associated_rules_for_file(did['scope'], did['name']):
                    if '%s:%s' % (rule['scope'], rule['name']) not in dict_datasets[guid]:
                        dict_datasets[guid].append('%s:%s' % (rule['scope'], rule['name']))
        for guid in dict_datasets:
            print('GUID: ', guid)
            print('Parents : ', ','.join(dict_datasets[guid]))
    elif args.did:
        table = []
        scope, name = extract_scope(args.did)
        for dataset in client.list_parent_dids(scope=scope, name=name):
            table.append(['%s:%s' % (dataset['scope'], dataset['name']), dataset['type']])
        print(tabulate.tabulate(table, tablefmt=tablefmt, headers=['SCOPE:NAME', '[DID TYPE]']))
    else:
        print('At least one option has to be given. Use -h to list the options.')
        return FAILURE
    return SUCCESS


@exception_handler
def close(args):
    """
    %(prog)s close [options] <field1=value1 field2=value2 ...>

    Close a dataset or container.
    """
    client = get_client(args)
    for did in args.dids:
        scope, name = extract_scope(did)
        client.set_status(scope=scope, name=name, open=False)
        print('%(scope)s:%(name)s has been closed.' % locals())
    return SUCCESS


@exception_handler
def reopen(args):
    """
    %(prog)s reopen [options] <field1=value1 field2=value2 ...>

    Reopen a dataset or container (only for privileged users).
    """
    client = get_client(args)
    for did in args.dids:
        scope, name = extract_scope(did)
        client.set_status(scope=scope, name=name, open=True)
        print('%(scope)s:%(name)s has been reopened.' % locals())
    return SUCCESS


@exception_handler
def stat(args):
    """
    %(prog)s stat [options] <field1=value1 field2=value2 ...>

    List attributes and statuses about data identifiers..
    """
    client = get_client(args)
    for did in args.dids:
        scope, name = extract_scope(did)
        info = client.get_did(scope=scope, name=name)
        for key, val in info.items():
            print('%(key)s: %(val)s' % locals())
    return SUCCESS


def erase(args):
    """
    %(prog)s erase [options] <field1=value1 field2=value2 ...>

    Delete data identifier.
    """
    client = get_client(args)
    for did in args.dids:
        if '*' in did:
            logger.warning("This command doesn't support wildcards! Skipping DID: %s" % did)
            continue
        try:
            scope, name = extract_scope(did)
        except RucioException as error:
            logger.warning('DID is in wrong format: %s' % did)
            logger.debug('Error: %s' % error)
            continue

        if args.undo:
            try:
                client.set_metadata(scope=scope, name=name, key='lifetime', value=None)
                logger.info('Erase undo for DID: {0}:{1}'.format(scope, name))
            except Exception:
                logger.warning('Cannot undo erase operation on DID. DID not existent or grace period of 24 hours already expired.')
                logger.warning('    DID: {0}:{1}'.format(scope, name))
        else:
            try:
                # set lifetime to expire in 24 hours (value is in seconds).
                client.set_metadata(scope=scope, name=name, key='lifetime', value=86400)
                logger.info('CAUTION! erase operation is irreversible after 24 hours. To cancel this operation you can run the following command:')
                print("rucio erase --undo {0}:{1}".format(scope, name))
            except RucioException as error:
                logger.warning('Failed to erase DID: %s' % did)
                logger.debug('Error: %s' % error)
    return SUCCESS


def __get_dataset(args):
    '''Parse helper for upload'''
    dsscope = None
    dsname = None
    for item in args:
        if item.count(':') == 1:
            if dsscope:
                raise Exception("Only one dataset should be given")
            else:
                dsscope, dsname = item.split(':')
    return dsscope, dsname


def __get_files(args):
    '''Parse helper for upload'''
    files = []
    for item in args:
        # Skip things that look like a scope:datasetname
        if item.count(':') == 1:
            logger.warning("{0} cannot be distinguished from scope:datasetname. Skipping it.".format(item))
            continue
        if os.path.isdir(item):
            dname, subdirs, fnames = next(os.walk(item))
            # Check if there are files in the directory
            if fnames:
                for fname in fnames:
                    files.append(os.path.join(dname, fname))
            # No files, but subdirectories. Needed to be added one-by-one
            # Maybe change so we look through the subdirs and add those files?
            elif subdirs:
                raise Exception("Directory ({directory}) has no files in it. Please add subdirectories individually.".format(directory=dname))
            else:
                raise Exception("Directory ({directory}) is empty.".format(directory=dname))
        elif os.path.isfile(item):
            files.append(item)
        else:
            logger.warning('{0} is not a directory or file or does not exist'.format(item))
    return files


@exception_handler
def upload(args):
    """
    rucio upload [scope:datasetname] [folder/] [files1 file2 file3]
    %(prog)s upload [options] <field1=value1 field2=value2 ...>

    Upload files into Rucio
    """

    rse_settings = rsemgr.get_rse_info(args.rse)
    if rse_settings['availability_write'] != 1:
        logger.critical('RSE is not available for writing right now. Aborting.')
        return FAILURE

    client = get_client(args)
    try:
        dsscope, dsname = __get_dataset(args.args)    # None, None if no dataset given
    except Exception:
        logger.error('rucio upload only allows to upload files to one dataset, more than one provided.')
        return FAILURE

    files = __get_files(args.args)               # a list of file names (even if a directory is given)
    if not files:
        return FAILURE
    list_files = []
    files_to_list = []
    revert_dict = {}
    if args.scope:
        fscope = args.scope
    else:
        fscope = 'user.' + client.account

    if args.pfn and not args.no_register:
        logger.error('You must specify --no-register if you use the --pfn option. Aborting.')
        return FAILURE
    elif args.pfn:
        selected_pfn_protocol = args.pfn.split(':')[0]
        logger.debug('PFN option specified, extracting protocol from given PFN: %s' % selected_pfn_protocol)
        try:
            rse_settings['hostname'] = ''
            rse_settings['prefix'] = ''
            protocol = rsemgr.select_protocol(rse_settings, operation='read', scheme=selected_pfn_protocol)
        except RSEProtocolNotSupported as error:
            logger.error('The specified protocol (%s) is not supported by %s' % (args.protocol, args.rse))
            logger.debug(error)
            return FAILURE

    summary = []
    trace = {}
    trace['hostname'] = socket.getfqdn()
    trace['scope'] = fscope
    trace['uuid'] = generate_uuid()
    if len(files) > 1 and args.guid:
        logger.error("A single GUID was specified on the command line, but there are multiple files to upload.")
        logger.error("If GUID auto-detection is not used, only one file may be uploaded at a time")
        return FAILURE
    if len(files) > 1 and args.name:
        logger.error("A single LFN was specified on the command line, but there are multiple files to upload.")
        logger.error("If LFN auto-detection is not used, only one file may be uploaded at a time")
        return FAILURE
    for filename in files:
        try:
            size = os.stat(filename).st_size
            a32_checksum = adler32(filename)
            md5_checksum = md5(filename)
            base_name = os.path.basename(filename)
            dir_name = os.path.dirname(filename)
            name = base_name
            if args.name:
                name = args.name
            logger.debug('Extracting filesize (%s) and checksum (%s) for file %s:%s' % (str(size), a32_checksum, fscope, name))
            files_to_list.append({'scope': fscope, 'name': name, 'filename': base_name})
            current_file_info = {'scope': fscope, 'filename': base_name, 'name': name, 'bytes': size, 'adler32': a32_checksum, 'md5': md5_checksum, 'state': 'C', 'meta': {}}
            if not args.guid and 'pool.root' in filename.lower() and not args.no_register:  # is a root file, getting the GUID
                status, output, err = execute('pool_extractFileIdentifier {0}'.format(filename))
                if status != 0:
                    logger.error('Trying to upload ROOT files but pool_extractFileIdentifier tool can not be found.')
                    logger.error('Setup your ATHENA environment and try again.')
                    return FAILURE
                try:
                    logger.debug('Extracting GUID from POOL file: {0}'.format(output.splitlines()[-1].split()[0].replace('-', '').lower()))
                    guid = output.splitlines()[-1].split()[0].replace('-', '').lower()
                except Exception:
                    logger.error('Error during GUID extraction. Failing. None of the files will be uploaded.')
                    return FAILURE
                current_file_info['meta']['guid'] = guid
            elif args.guid:
                logger.info('Manually set GUID: %s' % args.guid.replace('-', ''))
                current_file_info['meta']['guid'] = args.guid.replace('-', '')
            else:
                logger.debug('Automatically setting new GUID')
                current_file_info['meta']['guid'] = generate_uuid()
            list_files.append(current_file_info)
            revert_dict[fscope, name] = dir_name

        except OSError as error:
            logger.error(error)
            logger.error("No operation will be performed. Exiting!")
            return FAILURE

    if args.protocol:
        try:
            logger.debug('Forcing protocol : %s' % args.protocol)
            logger.debug('Forcing RSE : %s' % args.rse)
            protocol = rsemgr.select_protocol(rse_settings,
                                              operation='read',
                                              scheme=args.protocol)
        except RSEProtocolNotSupported as error:
            logger.error('The specified protocol (%s) is not supported by %s' % (args.protocol, args.rse))
            logger.debug(error)
            return FAILURE
        rse_settings['protocols'] = [protocol, ]

    if args.account is None:
        account = client.whoami()['account']
    else:
        account = args.account
    logger.debug('Using account %s' % (account))

    trace['account'] = client.account
    trace['dataset'] = ''
    trace['datasetScope'] = ''
    trace['eventType'] = 'upload'
    trace['eventVersion'] = version.RUCIO_VERSION[0]
    if dsscope and dsname and args.no_register is False:
        if files_to_list.count({'scope': dsscope, 'name': dsname}) > 0:
            # There is a file with the name of the destination dataset.
            logger.error('scope:name for the files must be different from scope:name for the destination dataset. {0}:{1}'.format(dsscope, dsname))
            return FAILURE
        try:
            client.add_dataset(scope=dsscope, name=dsname, rules=[{'account': client.account, 'copies': 1, 'rse_expression': args.rse, 'grouping': 'DATASET', 'lifetime': args.lifetime}])
            logger.info('Dataset successfully created')
            trace['dataset'] = dsname
            trace['datasetScope'] = dsscope
        except DataIdentifierAlreadyExists:
            # TODO: Need to check the rules thing!!
            logger.warning("The dataset name already exist")
    else:
        logger.debug('Skipping dataset registration')

    # Adding files to the catalog
    for f in list_files:
        logger.debug("Processing file %s:%s for upload" % (f['scope'], f['name']))
        try:  # If the did already exist in the catalog, only should be upload if the checksum is the same
            meta = client.get_metadata(f['scope'], f['name'])
            replicastate = [rep for rep in client.list_replicas([{'scope': f['scope'], 'name': f['name']}], all_states=True)]
            if args.rse not in replicastate[0]['rses']:
                try:
                    client.add_replicas(files=[f], rse=args.rse)
                except RucioException:
                    logger.error("A Rucio exception occurred when registering a file replica in Rucio.")
                    raise
            # logger.warning("The file {0}:{1} already exist in the catalog and will not be added.".format(f['scope'], f['name']))
            if rsemgr.exists(rse_settings=rse_settings, files={'name': f['name'], 'scope': f['scope']}):
                logger.warning('File {0}:{1} already exists on RSE. Will not try to reupload'.format(f['scope'], f['name']))
            else:
                if meta['adler32'] == f['adler32']:
                    logger.info('Local files and file %s:%s recorded in Rucio have the same checksum. Will try the upload' % (f['scope'], f['name']))
                    directory = revert_dict[f['scope'], f['name']]
                    trace['remoteSite'] = rse_settings['rse']
                    trace['protocol'] = rse_settings['protocols'][0]['scheme']
                    trace['filesize'] = f['bytes']
                    trace['transferStart'] = time.time()
                    f['upstate'] = rsemgr.upload(rse_settings=rse_settings,
                                                 lfns=[{'filename': f['filename'],
                                                        'name': f['name'],
                                                        'scope': f['scope'],
                                                        'adler32': f['adler32'],
                                                        'md5': f['md5'],
                                                        'filesize': f['bytes']}],
                                                 source_dir=directory,
                                                 force_pfn=args.pfn,
                                                 transfer_timeout=args.transfer_timeout)
                    trace['transferEnd'] = time.time()
                    trace['clientState'] = 'DONE'
                    logger.info('File %s:%s successfully uploaded on the storage' % (f['scope'], f['name']))
                    send_trace(trace, client.host, args.user_agent)
                    summary.append(deepcopy(f))
                    f.pop('upstate', None)
                else:
                    raise DataIdentifierAlreadyExists
        except NotImplementedError as error:
            for proto in rse_settings['protocols']:
                if proto['domains']['wan']['read'] == 1:
                    prot = proto['scheme']
            logger.error('Protocol {0} for RSE {1} not supported!'.format(prot, args.rse))
            return FAILURE
        except DataIdentifierNotFound:
            try:
                if args.no_register is False:
                    # Skipping registration for pilot
                    logger.info('Adding replicas in Rucio catalog')
                    try:
                        client.add_replicas(files=[make_valid_did(f)], rse=args.rse)
                    except RucioException:
                        logger.error("A Rucio exception occurred when adding replicas for %s:%s" % (f['scope'], f['name']))
                        raise
                    logger.info('Replicas successfully added')
                    if not dsscope:
                        # only need to add rules for files if no dataset is given
                        logger.info('Adding replication rule on RSE {0} for the file {1}:{2}'.format(args.rse, f['scope'], f['name']))
                        try:
                            client.add_replication_rule([make_valid_did(f)], copies=1, rse_expression=args.rse, lifetime=args.lifetime)
                        except RucioException:
                            logger.error("A Rucio exception occurred when adding a replication rule for %s:%s" % (f['scope'], f['name']))
                directory = revert_dict[f['scope'], f['name']]
                trace['remoteSite'] = rse_settings['rse']
                trace['protocol'] = rse_settings['protocols'][0]['scheme']
                trace['filesize'] = f['bytes']
                trace['transferStart'] = time.time()
                logger.debug("Uploading source file to RSE")
                f['upstate'] = rsemgr.upload(rse_settings=rse_settings,
                                             lfns=[{'filename': f['filename'],
                                                    'name': f['name'],
                                                    'scope': f['scope'],
                                                    'adler32': f['adler32'],
                                                    'md5': f['md5'],
                                                    'filesize': f['bytes']}],
                                             source_dir=directory,
                                             force_pfn=args.pfn,
                                             transfer_timeout=args.transfer_timeout)
                trace['transferEnd'] = time.time()
                trace['clientState'] = 'DONE'
                logger.info('File {0}:{1} successfully uploaded on the storage'.format(f['scope'], f['name']))
                send_trace(trace, client.host, args.user_agent)
                summary.append(deepcopy(f))
                f.pop('upstate', None)
            except (Duplicate, FileAlreadyExists) as error:
                logger.warning(error)
                return FAILURE
            except ResourceTemporaryUnavailable as error:
                logger.error(error)
                return FAILURE
        except DataIdentifierAlreadyExists as error:
            logger.debug(error)
            logger.error("Some of the files already exist in the catalog. No one will be added.")
        except RucioException:
            if logger.isEnabledFor(logging.DEBUG):
                logger.exception("A Rucio exception occurred when processing a file for upload.")
            else:
                logger.error("A Rucio exception occurred when processing a file for upload.")
            raise
    logger.debug("Finished uploading files to RSE.")

    if dsname:
        # A dataset is provided. Must add the files to the dataset.
        for f in list_files:
            try:
                client.add_files_to_dataset(scope=dsscope, name=dsname, files=[make_valid_did(f)])
            except Exception as error:
                logger.warning('Failed to attach file {0} to the dataset'.format(f))
                logger.warning(error)
                logger.warning("Continuing with the next one")

    replicas = []
    replica_dictionary = {}
    for chunk_files_to_list in chunks(files_to_list, 50):
        for rep in client.list_replicas([make_valid_did(f) for f in chunk_files_to_list]):
            replica_dictionary[rep['scope'], rep['name']] = list(rep['rses'].keys())
    for file in list_files:
        if (file['scope'], file['name']) not in replica_dictionary:
            file['state'] = 'A'
            replicas.append(file)
        elif args.rse not in replica_dictionary[file['scope'], file['name']]:
            file['state'] = 'A'
            replicas.append(file)
    if args.no_register is False and replicas != []:
        logger.info('Will update the file replicas states')
        for chunk_replicas in chunks(replicas, 20):
            try:
                client.update_replicas_states(rse=args.rse, files=[make_valid_did(f) for f in chunk_replicas])
            except AccessDenied as error:
                logger.error(error)
                return FAILURE
        logger.info('File replicas states successfully updated')
    if args.summary:
        final_summary = {}
        for file in summary:
            final_summary['%s:%s' % (file['scope'],
                                     file['name'])] = {'scope': file['scope'],
                                                       'name': file['name'],
                                                       'bytes': file['bytes'],
                                                       'rse': args.rse,
                                                       'pfn': file['upstate']['pfn'],
                                                       'guid': file['meta']['guid'],
                                                       'adler32': file['adler32'],
                                                       'md5': file['md5']}
        with open('rucio_upload.json', 'wb') as summary_file:
            json.dump(final_summary, summary_file, sort_keys=True, indent=1)
    return SUCCESS


def _file_exists(type, scope, name, directory, dsn=None, no_subdir=False):
    file_exists = False
    dest_dir = None
    if no_subdir:
        dest_dir = '%s' % (directory)
    else:
        if type != 'FILE':
            dest_dir = '%s/%s' % (directory, dsn)
            if os.path.isfile('%s/%s' % (dest_dir, name)):
                file_exists = True
        else:
            dest_dir = '%s/%s' % (directory, scope)
            if os.path.isfile('%s/%s' % (dest_dir, name)):
                file_exists = True
    return file_exists, dest_dir


def _replica_mlstr_to_list(mlstr):
    root = ElementTree.fromstring(mlstr)
    files = []

    # metalink namespace mapping
    ns = {'list_replicas_ml': 'urn:ietf:params:xml:ns:metalink'}

    # loop over all <file> tags of the metalink string
    for file_ml in root.findall('list_replicas_ml:file', ns):
        # search for identity-tag
        cur_did = file_ml.find('list_replicas_ml:identity', ns)
        if not ElementTree.iselement(cur_did):
            raise RucioException('Failed to locate identity-tag inside %s' % ElementTree.tostring(file_ml))

        # try extracting scope,name
        try:
            scope, name = extract_scope(cur_did.text)
        except RucioException:
            raise RucioException('Failed extract scope,name from %s' % cur_did.text)

        cur_file = {'scope': scope,
                    'name': name,
                    'bytes': None,
                    'adler32': None,
                    'rses': {}}

        size = file_ml.find('list_replicas_ml:size', ns)
        if ElementTree.iselement(size):
            cur_file['bytes'] = int(size.text)

        adler32hash = file_ml.find('list_replicas_ml:hash', ns)
        if ElementTree.iselement(adler32hash):
            cur_file['adler32'] = adler32hash.text

        for rse_ml in file_ml.findall('list_replicas_ml:url', ns):
            # check if location attrib (rse name) is given
            rse = rse_ml.get('location')
            if rse is None:
                continue
            pfn = rse_ml.text
            # if rse isnt in cur_file yet, create a new pfn list
            cur_file['rses'].setdefault(rse, []).append(pfn)

        files.append(cur_file)

    return files


def _downloader(args, input_queue, output_queue, threadnb, total_threads, trace_endpoint, trace_pattern):
    rse_dict = {}
    thread_prefix = 'Thread %s/%s' % (threadnb, total_threads)
    while True:
        try:
            file = input_queue.get_nowait()
        except Empty:
            return

        dest_dir = file['dest_dir']
        file_scope = file['scope']
        file_name = file['name']
        file_didstr = '%s:%s' % (file_scope, file_name)

        # arguments for rsemgr.download already known
        dlfile = {}
        dlfile['name'] = file_name
        dlfile['scope'] = file_scope
        dlfile['adler32'] = file['adler32']
        dlfile['md5'] = file['md5']
        ignore_checksum = True if args.pfn else False
        if args.pfn:
            dlfile['pfn'] = args.pfn

        logger.info('%s : Starting the download of %s' % (thread_prefix, file_didstr))

        trace = deepcopy(trace_pattern)
        trace.update({'scope': file_scope,
                      'filename': file_name,
                      'datasetScope': file['dataset_scope'],
                      'dataset': file['dataset_name'],
                      'filesize': file['bytes']})

        rses = list(file['rses'].keys())
        if rses == []:
            logger.warning('%s : File %s has no available replicas. Cannot be downloaded.' % (thread_prefix, file_didstr))
            trace['clientState'] = 'FILE_NOT_FOUND'
            send_trace(trace, trace_endpoint, args.user_agent)
            input_queue.task_done()
            continue

        random.shuffle(rses)

        logger.debug('%s : Potential sources : %s' % (thread_prefix, str(rses)))
        success = False
        while not success and len(rses):
            rse_name = rses.pop()
            if rse_name not in rse_dict:
                try:
                    rse_dict[rse_name] = rsemgr.get_rse_info(rse_name)
                except RSENotFound:
                    logger.warning('%s : Could not get info of RSE %s' % (thread_prefix, rse_name))
                    continue

            rse = rse_dict[rse_name]

            if not rse['availability_read']:
                logger.info('%s : %s is blacklisted for reading' % (thread_prefix, rse_name))
                continue

            try:
                if args.pfn:
                    protocols = [rsemgr.select_protocol(rse, operation='read', scheme=args.pfn.split(':')[0])]
                else:
                    protocols = rsemgr.get_protocols_ordered(rse, operation='read', scheme=args.protocol)
                    protocols.reverse()
            except RSEProtocolNotSupported as error:
                logger.info('%s : The protocol specfied (%s) is not supported by %s' % (thread_prefix, args.protocol, rse_name))
                logger.debug(error)
                continue

            logger.debug('%s : %d possible protocol(s) for read' % (thread_prefix, len(protocols)))
            trace['remoteSite'] = rse_name
            trace['clientState'] = 'DOWNLOAD_ATTEMPT'

            while not success and len(protocols):
                protocol = protocols.pop()

                logger.debug('%s : Trying protocol %s at %s' % (thread_prefix, protocol['scheme'], rse_name))
                trace['protocol'] = protocol['scheme']

                out = {}
                out['dataset_scope'] = file['dataset_scope']
                out['dataset_name'] = file['dataset_name']
                out['scope'] = file_scope
                out['name'] = file_name

                attempt = 0
                retries = 2
                while not success and attempt < retries:
                    attempt += 1
                    out['attemptnr'] = attempt

                    logger.info('%s : File %s trying from %s' % (thread_prefix, file_didstr, rse_name))
                    try:
                        trace['transferStart'] = time.time()

                        # If we use https, we need to check if we need to sign URLs for this RSE.
                        # Since we don't have the PFN yet, we can just use list_replicas to do it for us.
                        # TODO: Obsoleted with download API.
                        if protocol['scheme'] == 'https':
                            c = Client()
                            rse_attrs = c.list_rse_attributes(rse_name)
                            if 'sign_url' in rse_attrs:
                                logger.debug('%s : Signature type %s is required for RSE' % (thread_prefix, rse_attrs['sign_url']))
                                dlfile['pfn'] = list(c.list_replicas(dids=[{'scope': dlfile['scope'],
                                                                            'name': dlfile['name']}],
                                                                     rse_expression=rse_name))[0]['rses'][rse_name][0]

                        rsemgr.download(rse,
                                        files=[dlfile],
                                        dest_dir=dest_dir,
                                        force_scheme=protocol['scheme'],
                                        ignore_checksum=ignore_checksum,
                                        transfer_timeout=args.transfer_timeout)

                        trace['transferEnd'] = time.time()
                        trace['clientState'] = 'DONE'
                        out['clientState'] = 'DONE'
                        success = True
                        output_queue.put(out)
                        logger.info('%s : File %s successfully downloaded from %s' % (thread_prefix, file_didstr, rse_name))
                    except KeyboardInterrupt:
                        logger.warning('You pressed Ctrl+C! Exiting gracefully')
                        os.kill(os.getpgid(), signal.SIGINT)
                        return
                    except FileConsistencyMismatch as error:
                        logger.warning(str(error))
                        try:
                            pfns_dict = rsemgr.lfns2pfns(rse,
                                                         lfns=[{'name': file_name, 'scope': file_scope}],
                                                         operation='read',
                                                         scheme=args.protocol)
                            pfn = pfns_dict[file_didstr]

                            out['clientState'] = 'CORRUPTED'
                            out['pfn'] = pfn
                            output_queue.put(out)
                        except Exception as error:
                            logger.debug('%s : %s' % (thread_prefix, str(error)))
                        trace['clientState'] = 'FAIL_VALIDATE'
                        logger.debug('%s : Failed attempt %s/%s' % (thread_prefix, attempt, retries))
                    except Exception as error:
                        logger.warning(str(error))
                        trace['clientState'] = str(type(error).__name__)
                        logger.debug('%s : Failed attempt %s/%s' % (thread_prefix, attempt, retries))

                send_trace(trace, trace_endpoint, args.user_agent, threadnb=threadnb, total_threads=total_threads)

        if success:
            duration = round(trace['transferEnd'] - trace['transferStart'], 2)
            if args.pfn:
                logger.info('%s : File %s successfully downloaded in %s seconds' % (thread_prefix, file_didstr, duration))
            else:
                logger.info('%s : File %s successfully downloaded. %s in %s seconds = %s MBps' % (thread_prefix,
                                                                                                  file_didstr,
                                                                                                  sizefmt(file['bytes'], args.human),
                                                                                                  duration,
                                                                                                  round((file['bytes'] / duration) * 1e-6, 2)))
        else:
            logger.error('%s : Cannot download file %s' % (thread_prefix, file_didstr))

        input_queue.task_done()


def download_rucio(args, input_queue, output_queue, trace_pattern, trace_endpoint):
    total_workers = 1
    if args.ndownloader and not args.pfn:
        total_workers = args.ndownloader
        nlimit = 5
        if total_workers > nlimit:
            logger.warning('Cannot use more than %s parallel downloader.' % nlimit)
            total_workers = nlimit
    total_workers = min(total_workers, input_queue.qsize())

    logger.debug('Starting %d download threads' % total_workers)
    threads = []
    for worker in range(total_workers):
        kwargs = {'args': args,
                  'input_queue': input_queue,
                  'output_queue': output_queue,
                  'threadnb': worker + 1,
                  'total_threads': total_workers,
                  'trace_endpoint': trace_endpoint,
                  'trace_pattern': trace_pattern}
        try:
            thread = Thread(target=_downloader, kwargs=kwargs)
            thread.start()
            threads.append(thread)
        except:
            logger.warning('Failed to start thread %d' % (worker + 1))

    try:
        logger.debug('Waiting for threads to finish')
        for thread in threads:
            thread.join()
    except KeyboardInterrupt:
        logger.warning('You pressed Ctrl+C! Exiting gracefully')
        for thread in threads:
            thread.kill_received = True
    logger.debug('All threads finished')


def _stop_aria_rpc(auth, prox, proc, hard=False):
    try:
        if not hard:
            logger.debug('Shutting down aria rpc')
            prox.aria2.shutdown(auth)
        else:
            logger.debug('Force shutting down aria rpc')
            prox.aria2.forceShutdown(auth)
            proc.terminate()
    except:
        pass


def download_aria(args, input_queue, output_queue, trace_pattern, trace_endpoint):
    logger.info('Using aria2c downloader...')

    # from xmlrpc.client import MultiCall as RPCMultiCall  # py3
    try:
        from xmlrpclib import ServerProxy as RPCServerProxy  # py2
    except ImportError:
        from xmlrpc.client import ServerProxy as RPCServerProxy
    # from xmlrpclib import MultiCall as RPCMultiCall  # py2

    rpc_secret = '%x' % (random.getrandbits(64))
    cmd = 'aria2c '\
          '--enable-rpc '\
          '--certificate=$X509_USER_PROXY '\
          '--private-key=$X509_USER_PROXY '\
          '--ca-certificate=/etc/pki/tls/certs/CERN-bundle.pem '\
          '--quiet=true '\
          '--allow-overwrite=true '\
          '--auto-file-renaming=false '\
          '--stop-with-process=%d '\
          '--rpc-secret=%s '\
          '--rpc-listen-all=false '\
          '--rpc-max-request-size=100M '\
          '--connect-timeout=5 '\
          '--rpc-listen-port=%d'

    logger.info('Starting aria2c rpc server...')

    # trying up to 3 random ports
    for _ in range(3):
        port = random.randint(1024, 65534)
        logger.debug('Trying to start rpc server on port: %d' % port)
        try:
            rpcproc = subprocess.Popen(cmd % (os.getpid(), rpc_secret, port),
                                       shell=True,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
        except:
            raise RucioException('Failed to execute aria2c!')

        # if port is in use aria should fail to start so give it some time
        time.sleep(2)

        # did it fail?
        if rpcproc.poll() is not None:
            (out, err) = rpcproc.communicate()
            logger.debug('Failed to start aria2c with port: %d' % port)
            logger.debug('aria2c output: %s' % out)
        else:
            break

    if rpcproc.poll() is not None:
        raise RucioException('Failed to start aria2c rpc server!')

    try:
        aria_rpc = RPCServerProxy('http://localhost:%d/rpc' % port)
    except:
        rpcproc.kill()
        raise RucioException('Failed to initialise rpc proxy!')

    try:
        rpc_auth = 'token:' + rpc_secret

        gid_to_file = {}
        all_files_queued = False

        # files get removed from gid_to_file when they are complete or failed
        while len(gid_to_file) or not all_files_queued:
            num_queued = 0

            # queue up to 100 files and then check arias status
            while (num_queued < 100) and not all_files_queued:
                try:
                    file = input_queue.get_nowait()
                except Empty:
                    all_files_queued = True
                    break

                file_scope = file['scope']
                file_name = file['name']
                file_didstr = '%s:%s' % (file_scope, file_name)

                # get pfns from all replicas
                file['pfn_to_rse'] = {'': ''}
                pfns = []
                for rse in file['rses']:
                    rse_pfns = file['rses'][rse]
                    pfns.extend(rse_pfns)
                    file['pfn_to_rse'].update(dict.fromkeys(rse_pfns, rse))

                # any replica available?
                if len(pfns) > 0:
                    options = {'dir': file['dest_dir'],
                               'out': file['name'] + '.part'}
                    # 'checksum': 'adler32=' + file['adler32']}
                    gid = aria_rpc.aria2.addUri(rpc_auth, pfns, options)
                    gid_to_file[gid] = file
                    num_queued += 1
                    logger.debug('Queued file: %s' % file_didstr)
                    del file['rses']
                else:
                    trace = trace_pattern  # no need for deep copy
                    trace['scope'] = file_scope
                    trace['filename'] = file_name
                    trace['datasetScope'] = file['dataset_scope']
                    trace['dataset'] = file['dataset_name']
                    trace['filesize'] = file['bytes']
                    trace['clientState'] = 'FILE_NOT_FOUND'
                    send_trace(trace, trace_endpoint, args.user_agent)
                    logger.warning('File %s has no available replicas.' % file_didstr)

            # get some statistics
            aria_stat = aria_rpc.aria2.getGlobalStat(rpc_auth)
            num_active = int(aria_stat['numActive'])
            num_waiting = int(aria_stat['numWaiting'])
            num_stopped = int(aria_stat['numStoppedTotal'])

            complete = []
            faulty = []

            # save start time if one of the active downloads has started
            active = aria_rpc.aria2.tellActive(rpc_auth, ['gid', 'completedLength'])
            for dlinfo in active:
                gid = dlinfo['gid']
                if 'transferStart' not in gid_to_file[gid]:
                    if int(dlinfo['completedLength']) > 0:
                        gid_to_file[gid]['transferStart'] = time.time()

            stopped = aria_rpc.aria2.tellStopped(rpc_auth, -1, num_stopped, ['gid', 'status', 'files'])
            for dlinfo in stopped:
                gid = dlinfo['gid']
                file = gid_to_file[gid]

                # ensure we didnt miss the active state
                if 'transferStart' not in file:
                    logger.debug('Missed active state of DL with GID %s' % gid)
                    gid_to_file[gid]['transferStart'] = time.time()
                file['transferEnd'] = time.time()

                # get used pfn for traces
                file['used_pfn'] = ''
                for uri in dlinfo['files'][0]['uris']:
                    if uri['status'].lower() == 'used':
                        file['used_pfn'] = uri['uri']

                # append file to the completed/error list
                status = dlinfo['status'].lower()
                if status == 'complete':
                    complete.append(gid)
                elif status == 'error':
                    faulty.append(gid)
                else:
                    del gid_to_file[gid]
                    logger.warning('Download with GID %s has unexpected status: %s' % (gid, status))

            for gid in complete:
                file = gid_to_file[gid]
                file_scope = file['scope']
                file_name = file['name']
                file_didstr = '%s:%s' % (file_scope, file_name)
                file_path = '%s/%s' % (file['dest_dir'], file_name)

                # ensure file exists
                if not os.path.isfile(file_path + '.part'):
                    logger.error('Failed to locate downloaded file %s.part' % file_path)
                    del gid_to_file[gid]
                    continue

                # remove .part ending
                os.rename(file_path + '.part', file_path)

                # checksum check
                local_hash = adler32(file_path)
                if local_hash != file['adler32']:
                    logger.debug('File: %s, localHash: %s, serverHash: %s' % (file_didstr,
                                                                              local_hash,
                                                                              file['adler32']))
                    file['validation_failed'] = True
                    faulty.append(gid)
                    continue

                # calculate duration
                duration = round(file['transferEnd'] - file['transferStart'], 2)
                if duration == 0:
                    duration += 0.01
                logger.info('File %s successfully downloaded. %s in %s seconds = %s MBps' % (file_didstr,
                                                                                             sizefmt(file['bytes'], args.human),
                                                                                             duration,
                                                                                             round((file['bytes'] / duration) * 1e-6, 2)))

                # insert to output queue
                out = {}
                out['dataset_scope'] = file['dataset_scope']
                out['dataset_name'] = file['dataset_name']
                out['scope'] = file_scope
                out['name'] = file_name
                out['clientState'] = 'DONE'
                out['attemptnr'] = 1
                output_queue.put(out)

                # send trace
                trace = trace_pattern  # no need for deep copy
                trace['remoteSite'] = ''
                trace['remoteSite'] = file['pfn_to_rse'][file['used_pfn']]
                trace['protocol'] = 'https'
                trace['transferStart'] = file['transferStart']
                trace['transferEnd'] = file['transferEnd']
                trace['scope'] = file_scope
                trace['filename'] = file_name
                trace['datasetScope'] = file['dataset_scope']
                trace['dataset'] = file['dataset_name']
                trace['filesize'] = file['bytes']
                trace['clientState'] = 'DONE'
                send_trace(trace, trace_endpoint, args.user_agent)

                aria_rpc.aria2.removeDownloadResult(rpc_auth, gid)
                del gid_to_file[gid]

            for gid in faulty:
                file = gid_to_file[gid]
                file_scope = file['scope']
                file_name = file['name']
                file_didstr = '%s:%s' % (file['scope'], file['name'])

                trace = trace_pattern  # no need for deep copy
                if 'validation_failed' in file:
                    logger.info('Validation of %s failed.' % file_didstr)

                    # insert to output queue
                    out = {}
                    out['dataset_scope'] = file['dataset_scope']
                    out['dataset_name'] = file['dataset_name']
                    out['scope'] = file_scope
                    out['name'] = file_name
                    out['clientState'] = 'CORRUPTED'
                    out['attemptnr'] = 1
                    out['pfn'] = file['used_pfn']
                    output_queue.put(out)

                    trace['clientState'] = 'FAIL_VALIDATE'
                else:
                    logger.info('Download of %s failed.' % file_didstr)
                    trace['clientState'] = 'DOWNLOAD_ATTEMPT'

                trace['remoteSite'] = ''
                trace['remoteSite'] = file['pfn_to_rse'][file['used_pfn']]
                trace['protocol'] = 'https'
                trace['transferStart'] = file['transferStart']
                trace['transferEnd'] = file['transferEnd']
                trace['scope'] = file_scope
                trace['filename'] = file_name
                trace['datasetScope'] = file['dataset_scope']
                trace['dataset'] = file['dataset_name']
                trace['filesize'] = file['bytes']
                send_trace(trace, trace_endpoint, args.user_agent)

                aria_rpc.aria2.removeDownloadResult(rpc_auth, gid)
                del gid_to_file[gid]
            if len(stopped) > 0:
                logger.info('Active: %d, Waiting: %d, Stopped: %d' % (num_active, num_waiting, num_stopped))
    except (KeyboardInterrupt, SystemExit):
        pass
    except Exception as error:
        _stop_aria_rpc(rpc_auth, aria_rpc, rpcproc, True)
        raise error
    except:
        _stop_aria_rpc(rpc_auth, aria_rpc, rpcproc, True)
        raise RucioException('Caught unknown exception! Killed aria2c rpc!')

    _stop_aria_rpc(rpc_auth, aria_rpc, rpcproc)


@exception_handler
def download(args):
    """
    %(prog)s download [options] <field1=value1 field2=value2 ...>

    Download files from Rucio using new threaded model and RSE expression support
    """
    if args.old:
        logger.warning('--old option has no effect, since it is obsolete and has been removed.')

    client = get_client(args)
    trace_endpoint = client.host
    trace_pattern = {'hostname': socket.getfqdn(),
                     'account': client.account,
                     'uuid': generate_uuid(),
                     'eventType': 'download',
                     'eventVersion': version.RUCIO_VERSION[0]}

    if args.trace_appid:
        trace_pattern['appid'] = args.trace_appid
    if args.trace_dataset:
        trace_pattern['dataset'] = args.trace_dataset
    if args.trace_datasetscope:
        trace_pattern['datasetScope'] = args.trace_datasetscope
    if args.trace_eventtype:
        trace_pattern['eventType'] = args.trace_eventtype
    if args.trace_pq:
        trace_pattern['pq'] = args.trace_pq
    if args.trace_taskid:
        trace_pattern['taskid'] = args.trace_taskid
    if args.trace_usrdn:
        trace_pattern['usrdn'] = args.trace_usrdn

    # is used account an admin account?
    account_attributes = [acc for acc in client.list_account_attributes(client.account)]
    is_admin = False
    for attr in account_attributes[0]:
        if attr['key'] == 'admin' and attr['value'] is True:
            logger.debug('Admin mode enabled')
            is_admin = True
            break

    # extend RSE expression to exclude tape RSEs for non-admin accounts
    rse_expression = args.rse
    if not is_admin:
        rse_expression = 'istape=False'
        if args.rse and len(args.rse.strip()) > 0:
            rse_expression = '(%s)&istape=False' % args.rse
    logger.debug('RSE-Expression: %s' % rse_expression)

    # Extract the scope, name from the did(s)
    dids = []
    for did in args.dids:
        try:
            scope, name = extract_scope(did)
            if name.find('*') > -1:
                for dsn in client.list_dids(scope, filters={'name': name}):
                    dids.append({'scope': scope, 'name': dsn})
            else:
                dids.append({'scope': scope, 'name': name})
        except ValueError as error:
            logger.error('Cannot extract the scope and name from %s : [%s]' % (did, error))
            return FAILURE

    # check if we shall use aria2c
    use_aria = False
    if args.aria and not args.pfn:
        logger.debug('Checking if aria2c is executable...')
        try:
            (exitcode, out, err) = execute('aria2c --version')
            if exitcode == 0:
                # aria2c is executable
                use_aria = True
            else:
                logger.debug('aria2c --version exited with code %d' % exitcode)
                logger.warning('--aria given but failed to execute aria2c!')
        except:
            logger.warning('--aria given but failed to execute aria2c!')
    elif args.aria:
        logger.warning('--pfn with --aria is not supported yet!')

    # if user wants to use aria, check if every file has a https pfn
    if use_aria:
        logger.debug('aria2c is executable! Checking if all files have https PFNs...')
        try:
            # metalink string for replicas of all arg.dids
            mlstr = client.list_replicas(dids, rse_expression=rse_expression, metalink=True, schemes=['https'])
            files = _replica_mlstr_to_list(mlstr)
            for file in files:
                if len(file['rses']) == 0:
                    use_aria = False
                    filename = '%s:%s' % (file['scope'], file['name'])
                    logger.warning('--aria given but didnt find rse with https support for file %s' % filename)
                    break
        except Exception as error:
            use_aria = False
            logger.warning('Cannot use aria2c due to following error: %s' % str(error))

    if use_aria and args.protocol:
        logger.warning('Ignoring --protocol, since aria only supports https!')

    if args.pfn:
        if not args.rse:
            logger.error('--rse option is mandatory in combination with --pfn!')
            return FAILURE

        if len(dids) > 1:
            dids = [dids[0]]
            logger.warning('--pfn option and multiple DIDs given! Only considering first DID...')

    summary = {}
    num_files_to_dl = {}
    input_queue = Queue()
    output_queue = Queue()
    # get replicas for every file of the given dids
    for arg_did in dids:
        arg_didstr = '%s:%s' % (arg_did['scope'], arg_did['name'])
        summary[arg_didstr] = {}

        # get type of given did; save did if its a dataset
        if not args.pfn:
            try:
                did_info = client.get_did(arg_did['scope'], arg_did['name'])
                did_type = did_info['type'].upper()
                dataset_scope = '' if did_type == 'FILE' else arg_did['scope']
                dataset_name = '' if did_type == 'FILE' else arg_did['name']
            except:
                logger.error('Failed to get did info for did %s' % arg_didstr)
                return FAILURE

            metalink = True if use_aria else None
            schemes = ['https'] if use_aria else None
            try:
                files_with_replicas = client.list_replicas([arg_did],
                                                           schemes=schemes,
                                                           rse_expression=rse_expression,
                                                           metalink=metalink)
            except:
                logger.error('Failed to get list of files with their replicas for DID %s' % arg_didstr)
                return FAILURE

            if metalink:
                try:
                    files_with_replicas = _replica_mlstr_to_list(files_with_replicas)
                except Exception as error:
                    logger.error('Failed to parse metalink file for did %s with. Error: %s' % (arg_didstr, str(error)))
                    return FAILURE
            else:
                files_with_replicas = [f for f in files_with_replicas]

            if args.nrandom:
                random.shuffle(files_with_replicas)
                files_with_replicas = files_with_replicas[0:args.nrandom]
        else:
            logger.debug('PFN option overrides replica listing')
            did_type = 'FILE'
            dataset_scope = ''
            dataset_name = ''
            files_with_replicas = [{'bytes': None,
                                    'adler32': None,
                                    'md5': None,
                                    'scope': arg_did['scope'],
                                    'name': arg_did['name'],
                                    'pfns': {args.pfn: {'rse': args.rse}},
                                    'rses': {args.rse: [args.pfn]}}]

        num_files_to_dl[arg_didstr] = len(files_with_replicas)
        for file in files_with_replicas:
            file_scope = file['scope']
            file_name = file['name']
            file_didstr = '%s:%s' % (file_scope, file_name)

            file_exists, dest_dir = _file_exists(did_type,
                                                 file_scope,
                                                 file_name,
                                                 args.dir,
                                                 dsn=dataset_name,
                                                 no_subdir=args.no_subdir)
            dest_dir = os.path.abspath(dest_dir)
            if file_exists:
                logger.info('File %s already exists locally' % file_didstr)

                out = {}
                out['dataset_scope'] = dataset_scope
                out['dataset_name'] = dataset_name
                out['scope'] = file_scope
                out['name'] = file_name
                out['clientState'] = 'ALREADY_DONE'
                output_queue.put(out)

                trace = deepcopy(trace_pattern)

                if 'datasetScope' not in trace:
                    trace['datasetScope'] = dataset_scope
                if 'dataset' not in trace:
                    trace['dataset'] = dataset_name
                trace.update({'scope': file_scope,
                              'filename': file_name,
                              'filesize': file['bytes'],
                              'transferStart': time.time(),
                              'transferEnd': time.time(),
                              'clientState': 'ALREADY_DONE'})
                send_trace(trace, trace_endpoint, args.user_agent)
            else:
                if not os.path.isdir(dest_dir):
                    logger.debug('Destination dir not found: %s' % dest_dir)
                    try:
                        os.makedirs(dest_dir)
                    except:
                        logger.error('Failed to create missing destination directory %s' % dest_dir)
                        return FAILURE
                if args.no_subdir and os.path.isfile('%s/%s' % (dest_dir, file_name)):
                    # Overwrite the files
                    logger.debug('Deleteing existing files: %s' % file_name)
                    os.remove("%s/%s" % (dest_dir, file_name))
                file['dataset_scope'] = dataset_scope
                file['dataset_name'] = dataset_name
                file['dest_dir'] = dest_dir
                input_queue.put(file)
    try:
        if use_aria:
            download_aria(args, input_queue, output_queue, trace_pattern, trace_endpoint)
        else:
            download_rucio(args, input_queue, output_queue, trace_pattern, trace_endpoint)
    except Exception as error:
        logger.error('Exception during download: %s' % str(error))

    while True:
        try:
            item = output_queue.get_nowait()
            output_queue.task_done()
            ds_didstr = '%s:%s' % (item['dataset_scope'], item['dataset_name'])
            file_didstr = '%s:%s' % (item['scope'], item['name'])

            if ds_didstr in summary or file_didstr in summary:
                if item['dataset_scope'] == '':
                    summary[file_didstr][file_didstr] = item['clientState']
                else:
                    summary[ds_didstr][file_didstr] = item['clientState']
                if item['clientState'] == 'CORRUPTED':
                    try:
                        client.declare_suspicious_file_replicas([item['pfn']], reason='Corrupted')
                    except:
                        logger.warning('File replica %s might be corrupted. Failure to declare it bad to Rucio' % item['pfn'])
        except Empty:
            break

    not_downloaded_files = 0
    total_successes = 0
    total_failures = 0
    print('----------------------------------')
    print('Download summary')
    if len(summary) > 0:
        for did in summary:
            print('-' * 40)
            print('DID %s' % (did))
            downloaded_files = 0
            not_downloaded_files = 0
            local_files = 0
            for file in summary[did]:
                if summary[did][file] == 'DONE':
                    downloaded_files += 1
                elif summary[did][file] == 'ALREADY_DONE':
                    local_files += 1
            not_downloaded_files = num_files_to_dl[did] - downloaded_files - local_files
            total_successes += downloaded_files + local_files
            total_failures += not_downloaded_files
            print('{0:40} {1:6d}'.format('Total files : ', num_files_to_dl[did]))
            print('{0:40} {1:6d}'.format('Downloaded files : ', downloaded_files))
            print('{0:40} {1:6d}'.format('Files already found locally : ', local_files))
            print('{0:40} {1:6d}'.format('Files that cannot be downloaded : ', not_downloaded_files))
    else:
        print('-' * 40)
        print('No DID matching the pattern')

    if total_successes == 0 and total_failures > 0:
        raise NoFilesDownloaded
    if total_successes > 0 and total_failures > 0:
        raise NotAllFilesDownloaded
    return SUCCESS


@exception_handler
def get_metadata(args):
    """
    %(prog)s get_metadata [options] <field1=value1 field2=value2 ...>

    Get data identifier metadata
    """
    client = get_client(args)
    for did in args.dids:
        scope, name = extract_scope(did)
        meta = client.get_metadata(scope=scope, name=name)
        for k in meta:
            print('%s: %s' % (k, meta[k]))
    return SUCCESS


@exception_handler
def set_metadata(args):
    """
    %(prog)s set_metadata [options] <field1=value1 field2=value2 ...>

    Set data identifier metadata
    """
    client = get_client(args)
    value = args.value
    if args.key == 'lifetime':
        value = float(args.value)
    scope, name = extract_scope(args.did)
    client.set_metadata(scope=scope, name=name, key=args.key, value=value)
    return SUCCESS


def delete_metadata(args):
    """
    %(prog)s set_metadata [options] <field1=value1 field2=value2 ...>

    Delete data identifier metadata
    """
    # For the moment..
    raise NotImplementedError


@exception_handler
def add_rule(args):
    """
    %(prog)s add-rule <did> <copies> <rse-expression> [options]

    Add a rule to a did.
    """
    client = get_client(args)
    dids = []
    rule_ids = []
    for did in args.dids:
        scope, name = extract_scope(did)
        dids.append({'scope': scope, 'name': name})
    try:
        rule_ids = client.add_replication_rule(dids=dids,
                                               copies=args.copies,
                                               rse_expression=args.rse_expression,
                                               weight=args.weight,
                                               lifetime=args.lifetime,
                                               grouping=args.grouping,
                                               account=args.rule_account,
                                               locked=args.locked,
                                               source_replica_expression=args.source_replica_expression,
                                               notify=args.notify,
                                               activity=args.activity,
                                               comment=args.comment,
                                               ask_approval=args.ask_approval,
                                               asynchronous=args.asynchronous)
    except DuplicateRule as error:
        if args.ignore_duplicate:
            for did in dids:
                try:
                    rule_id = client.add_replication_rule(dids=[did],
                                                          copies=args.copies,
                                                          rse_expression=args.rse_expression,
                                                          weight=args.weight,
                                                          lifetime=args.lifetime,
                                                          grouping=args.grouping,
                                                          account=args.rule_account,
                                                          locked=args.locked,
                                                          source_replica_expression=args.source_replica_expression,
                                                          notify=args.notify,
                                                          activity=args.activity,
                                                          comment=args.comment,
                                                          ask_approval=args.ask_approval,
                                                          asynchronous=args.asynchronous)
                    rule_ids.extend(rule_id)
                except DuplicateRule as error:
                    print('Duplicate rule for %s:%s found; Skipping.' % (did['scope'], did['name']))
        else:
            raise error

    for rule in rule_ids:
        print(rule)
    return SUCCESS


@exception_handler
def delete_rule(args):
    """
    %(prog)s delete-rule [options] <ruleid>

    Delete a rule.
    """
    client = get_client(args)
    try:
        # Test if the rule_id is a real rule_id
        uuid.UUID(args.rule_id)
        client.delete_replication_rule(rule_id=args.rule_id, purge_replicas=args.purge_replicas)
    except ValueError:
        # Otherwise, trying to extract the scope, name from args.rule_id
        if not args.rse_expression:
            logger.error('A RSE expression must be specified if you do not provide a rule_id but a DID')
            return FAILURE
        scope, name = extract_scope(args.rule_id)
        rules = client.list_did_rules(scope=scope, name=name)
        if args.rule_account is None:
            account = client.account
        else:
            account = args.rule_account
        deletion_success = False
        for rule in rules:
            if args.delete_all:
                account_checked = True
            else:
                account_checked = rule['account'] == account
            if rule['rse_expression'] == args.rse_expression and account_checked:
                client.delete_replication_rule(rule_id=rule['id'], purge_replicas=args.purge_replicas)
                deletion_success = True
        if not deletion_success:
            logger.error('No replication rule was deleted from the DID')
            return FAILURE
    return SUCCESS


@exception_handler
def update_rule(args):
    """
    %(prog)s update-rule [options] <ruleid>

    Update a rule.
    """
    client = get_client(args)
    options = {}
    if args.lifetime:
        options['lifetime'] = None if args.lifetime.lower() == "none" else int(args.lifetime)
    if args.locked:
        if args.locked == "True":
            options['locked'] = True
        elif args.locked == "False":
            options['locked'] = False
    if args.rule_account:
        options['account'] = args.rule_account
    if args.state_stuck:
        options['state'] = 'STUCK'
    if args.state_suspended:
        options['state'] = 'SUSPENDED'
    if args.rule_activity:
        options['activity'] = args.rule_activity
    if args.source_replica_expression:
        options['source_replica_expression'] = None if args.source_replica_expression.lower() == 'none' else args.source_replica_expression
    if args.cancel_requests:
        options['cancel_requests'] = True
    if args.priority:
        options['priority'] = int(args.priority)
    if args.child_rule_id:
        options['child_rule_id'] = args.child_rule_id
    client.update_replication_rule(rule_id=args.rule_id, options=options)
    print('Updated Rule')
    return SUCCESS


@exception_handler
def move_rule(args):
    """
    %(prog)s move-rule [options] <ruleid> <rse_expression>

    Update a rule.
    """
    client = get_client(args)
    print(client.move_replication_rule(rule_id=args.rule_id, rse_expression=args.rse_expression))
    return SUCCESS


@exception_handler
def info_rule(args):
    """
    %(prog)s rule-info [options] <ruleid>

    Retrieve information about a rule.
    """
    client = get_client(args)
    if args.examine:
        analysis = client.examine_replication_rule(rule_id=args.rule_id)
        print('Status of the replication rule: %s' % analysis['rule_error'])
        if analysis['transfers']:
            print('STUCK Requests:')
            for transfer in analysis['transfers']:
                print('  %s:%s' % (transfer['scope'], transfer['name']))
                print('    RSE:                  %s' % str(transfer['rse']))
                print('    Attempts:             %s' % str(transfer['attempts']))
                print('    Last Retry:           %s' % str(transfer['last_time']))
                print('    Last error:           %s' % str(transfer['last_error']))
                print('    Last source:          %s' % str(transfer['last_source']))
                print('    Available sources:    %s' % ', '.join([source[0] for source in transfer['sources'] if source[1]]))
                print('    Blacklisted sources:  %s' % ', '.join([source[0] for source in transfer['sources'] if not source[1]]))
    else:
        if args.estimate_ttc:
            rule = client.get_replication_rule(rule_id=args.rule_id, estimate_ttc=True)
        else:
            rule = client.get_replication_rule(rule_id=args.rule_id)
        print("Id:                         %s" % rule['id'])
        print("Account:                    %s" % rule['account'])
        print("Scope:                      %s" % rule['scope'])
        print("Name:                       %s" % rule['name'])
        print("RSE Expression:             %s" % rule['rse_expression'])
        print("Copies:                     %s" % rule['copies'])
        print("State:                      %s" % rule['state'])
        print("Locks OK/REPLICATING/STUCK: %s/%s/%s" % (rule['locks_ok_cnt'], rule['locks_replicating_cnt'], rule['locks_stuck_cnt']))
        print("Grouping:                   %s" % rule['grouping'])
        print("Expires at:                 %s" % rule['expires_at'])
        print("Locked:                     %s" % rule['locked'])
        print("Weight:                     %s" % rule['weight'])
        print("Created at:                 %s" % rule['created_at'])
        print("Updated at:                 %s" % rule['updated_at'])
        print("Error:                      %s" % rule['error'])
        print("Subscription Id:            %s" % rule['subscription_id'])
        print("Source replica expression:  %s" % rule['source_replica_expression'])
        print("Activity:                   %s" % rule['activity'])
        print("Comment:                    %s" % rule['comments'])
        print("Ignore Quota:               %s" % rule['ignore_account_limit'])
        print("Ignore Availability:        %s" % rule['ignore_availability'])
        print("Purge replicas:             %s" % rule['purge_replicas'])
        print("Notification:               %s" % rule['notification'])
        print("End of life:                %s" % rule['eol_at'])
        print("Child Rule Id:              %s" % rule['child_rule_id'])
        if args.estimate_ttc:
            if result[0] > 0:
                print("Expected transfer start in  %0.2f minutes." % (rule['estimated_start_in'] / 60.))
                print("Expected transfer end in    %0.2f minutes." % (rule['estimated_end_in'] / 60.))
            else:
                print('Couldn\'t calculate the TTC for any of the transfers. No sources selected yet?')
    return SUCCESS


@exception_handler
def list_rules(args):
    """
    %(prog)s list-rules ...

    List rules.
    """
    client = get_client(args)
    if args.rule_id:
        rules = [client.get_replication_rule(args.rule_id)]
    elif args.file:
        scope, name = extract_scope(args.file)
        rules = client.list_associated_rules_for_file(scope=scope, name=name)
    elif args.traverse:
        scope, name = extract_scope(args.did)
        locks = client.get_dataset_locks(scope=scope, name=name)
        rules = []
        for rule_id in list(set([lock['rule_id'] for lock in locks])):
            rules.append(client.get_replication_rule(rule_id))
    elif args.did:
        scope, name = extract_scope(args.did)
        meta = client.get_metadata(scope=scope, name=name)
        rules = client.list_did_rules(scope=scope, name=name)
        try:
            next(rules)
            rules = client.list_did_rules(scope=scope, name=name)
        except StopIteration:
            rules = []
            # looking for other rules
            if meta['did_type'] == u'CONTAINER':
                for dsn in client.list_content(scope, name):
                    rules.extend(client.list_did_rules(scope=dsn['scope'], name=dsn['name']))
                if rules:
                    print('No rules found, listing rules for content')
            if meta['did_type'] == u'DATASET':
                for container in client.list_parent_dids(scope, name):
                    rules.extend(client.list_did_rules(scope=container['scope'], name=container['name']))
                if rules:
                    print('No rules found, listing rules for parents')
    elif args.rule_account:
        rules = client.list_account_rules(account=args.rule_account)
    elif args.subscription:
        account = args.subscription[0]
        name = args.subscription[1]
        rules = client.list_subscription_rules(account=account, name=name)
    else:
        print('At least one option has to be given. Use -h to list the options.')
        return FAILURE
    if args.csv:
        for rule in rules:
            print("{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}".format(rule['id'],
                                                                  rule['account'],
                                                                  '%s:%s' % (rule['scope'], rule['name']),
                                                                  '%s[%d/%d/%d]' % (rule['state'], rule['locks_ok_cnt'], rule['locks_replicating_cnt'], rule['locks_stuck_cnt']),
                                                                  rule['rse_expression'],
                                                                  rule['copies'],
                                                                  rule['expires_at'],
                                                                  rule['created_at']))
    else:
        table = []
        for rule in rules:
            table.append([rule['id'],
                          rule['account'],
                          '%s:%s' % (rule['scope'], rule['name']),
                          '%s[%d/%d/%d]' % (rule['state'], rule['locks_ok_cnt'], rule['locks_replicating_cnt'], rule['locks_stuck_cnt']),
                          rule['rse_expression'],
                          rule['copies'],
                          rule['expires_at'],
                          rule['created_at']])
        print(tabulate.tabulate(table, tablefmt='simple', headers=['ID', 'ACCOUNT', 'SCOPE:NAME', 'STATE[OK/REPL/STUCK]', 'RSE_EXPRESSION', 'COPIES', 'EXPIRES (UTC)', 'CREATED (UTC)']))
    return SUCCESS


@exception_handler
def list_rules_history(args):
    """
    %(prog)s list-rules_history ...

    List replication rules history for a DID.
    """
    rule_dict = []
    client = get_client(args)
    scope, name = extract_scope(args.did)
    for rule in client.list_replication_rule_full_history(scope, name):
        if rule['rule_id'] not in rule_dict:
            rule_dict.append(rule['rule_id'])
            print('-' * 40)
            print('Rule insertion')
            print('Account : %s' % rule['account'])
            print('RSE expression : %s' % (rule['rse_expression']))
            print('Time : %s' % (rule['created_at']))
        else:
            rule_dict.remove(rule['rule_id'])
            print('-' * 40)
            print('Rule deletion')
            print('Account : %s' % rule['account'])
            print('RSE expression : %s' % (rule['rse_expression']))
            print('Time : %s' % (rule['updated_at']))
    return SUCCESS


@exception_handler
def list_rses(args):
    """
    %(prog)s list-rses [options] <field1=value1 field2=value2 ...>

    List rses.

    """
    client = get_client(args)
    rse_expression = None
    if args.rse_expression:
        rse_expression = args.rse_expression
    rses = client.list_rses(rse_expression)
    for rse in rses:
        print('%(rse)s' % rse)
    return SUCCESS


@exception_handler
def list_rse_attributes(args):
    """
    %(prog)s list-rse-attributes [options] <field1=value1 field2=value2 ...>

    List rses.

    """
    client = get_client(args)
    attributes = client.list_rse_attributes(rse=args.rse)
    for k in attributes:
        print('  ' + k + ': ' + str(attributes[k]))
    return SUCCESS


@exception_handler
def list_rse_usage(args):
    """
    %(prog)s list-rse-usage [options] <rse>

    Show the space usage of a given rse

    """
    client = get_client(args)
    usages = client.get_rse_usage(rse=args.rse)
    print('USAGE:')
    for usage in usages:
        if usage['source'] not in ['srm', 'gsiftp', 'webdav']:
            print('------')
            print_free = False
            if usage['source'] in ['storage']:
                print_free = True
            for elem in usage:
                if not (elem == 'free' or elem == 'total') or print_free:
                    if elem == 'free' or elem == 'total' or elem == 'used':
                        print('  {0}: {1}'.format(elem, sizefmt(usage[elem], args.human)))
                    else:
                        print('  {0}: {1}'.format(elem, usage[elem]))
    print('------')
    return SUCCESS


@exception_handler
def list_account_limits(args):
    """
    %(prog)s list [options] <field1=value1 field2=value2 ...>

    List account limits.

    """
    client = get_client(args)
    table = []
    if args.rse:
        limits = client.get_account_limit(account=args.limit_account, rse=args.rse)
    else:
        limits = client.get_account_limits(account=args.limit_account)
    for limit in list(limits.items()):
        table.append([limit[0], sizefmt(limit[1], args.human)])
    table.sort()
    print(tabulate.tabulate(table, tablefmt=tablefmt, headers=['RSE', 'LIMIT']))
    return SUCCESS


@exception_handler
def list_account_usage(args):
    """
    %(prog)s list [options] <field1=value1 field2=value2 ...>

    List account usage.

    """
    client = get_client(args)
    table = []
    if args.rse:
        usage = client.get_account_usage(account=args.usage_account, rse=args.rse)
    else:
        usage = client.get_account_usage(account=args.usage_account)
    for item in usage:
        remaining = 0 if float(item['bytes_remaining']) < 0 else float(item['bytes_remaining'])
        table.append([item['rse'], sizefmt(item['bytes'], args.human), sizefmt(item['bytes_limit'], args.human), sizefmt(remaining, args.human)])
    table.sort()
    print(tabulate.tabulate(table, tablefmt=tablefmt, headers=['RSE', 'USAGE', 'LIMIT', 'QUOTA LEFT']))
    return SUCCESS


@exception_handler
def list_datasets_rse(args):
    """
    %(prog)s list [options] <field1=value1 field2=value2 ...>

    List the datasets in a site.

    """
    client = get_client(args)
    if args.long:
        table = []
        for dsn in client.list_datasets_per_rse(args.rse):
            table.append(['%s:%s' % (dsn['scope'], dsn['name']), '%s/%s' % (str(dsn['available_length']), str(dsn['length'])), '%s/%s' % (str(dsn['available_bytes']), str(dsn['bytes']))])
        table.sort()
        print(tabulate.tabulate(table, tablefmt=tablefmt, headers=['DID', 'LOCAL FILES/TOTAL FILES', 'LOCAL BYTES/TOTAL BYTES']))
    else:
        dsns = list(set(['%s:%s' % (dsn['scope'], dsn['name']) for dsn in client.list_datasets_per_rse(args.rse)]))
        dsns.sort()
        print("SCOPE:NAME")
        print('----------')
        for dsn in dsns:
            print(dsn)
    return SUCCESS


def test_server(args):
    """"
    %(prog)s test-server [options] <field1=value1 field2=value2 ...>
    Test the client against a server.
    """
    import nose
    config = nose.config.Config()
    config.verbosity = 2
    nose.run(argv=sys.argv[1:], defaultTest='rucio.tests.test_rucio_server', config=config)
    return SUCCESS


def touch(args):
    """
    %(prog)s touch [options] <did1 did2 ...>
    """

    client = get_client(args)

    for did in args.dids:
        scope, name = extract_scope(did)
        client.touch(scope, name, args.rse)


def rse_completer(prefix, parsed_args, **kwargs):
    """
    Completes the argument with a list of RSEs
    """
    client = get_client(parsed_args)
    return ["%(rse)s" % rse for rse in client.list_rses()]


def get_parser():
    """
    Returns the argparse parser.
    """
    oparser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]), add_help=True)
    subparsers = oparser.add_subparsers()

    # Main arguments
    oparser.add_argument('--version', action='version', version='%(prog)s ' + version.version_string())
    oparser.add_argument('--verbose', '-v', default=False, action='store_true', help="Print more verbose output.")
    oparser.add_argument('-H', '--host', dest="host", metavar="ADDRESS", help="The Rucio API host.")
    oparser.add_argument('--auth-host', dest="auth_host", metavar="ADDRESS", help="The Rucio Authentication host.")
    oparser.add_argument('-a', '--account', dest="account", metavar="ACCOUNT", help="Rucio account to use.")
    oparser.add_argument('-S', '--auth-strategy', dest="auth_strategy", default=None, help="Authentication strategy (userpass, x509...)")
    oparser.add_argument('-T', '--timeout', dest="timeout", type=float, default=None, help="Set all timeout values to seconds.")
    oparser.add_argument('--robot', '-R', dest="human", default=True, action='store_false', help="All output in bytes and without the units. This output format is preferred by parsers and scripts.")
    oparser.add_argument('--user-agent', '-U', dest="user_agent", default='rucio-clients', action='store', help="Rucio User Agent")

    # Options for the userpass auth_strategy
    oparser.add_argument('-u', '--user', dest='username', default=None, help='username')
    oparser.add_argument('-pwd', '--password', dest='password', default=None, help='password')

    # Options for the x509  auth_strategy
    oparser.add_argument('--certificate', dest='certificate', default=None, help='Client certificate file.')
    oparser.add_argument('--ca-certificate', dest='ca_certificate', default=None, help='CA certificate to verify peer against (SSL).')

    # Ping command
    ping_parser = subparsers.add_parser('ping', formatter_class=argparse.RawDescriptionHelpFormatter, help='Ping Rucio server.',
                                        epilog='Usage example\n'
                                               '"""""""""""""\n'
                                               '\n'
                                               'To ping the server::\n'
                                               '\n'
                                               '    $ rucio ping\n'
                                               '    1.14.8\n'
                                               '\n'
                                               'The returned value is the version of Rucio installed on the server.'
                                               '\n')
    ping_parser.set_defaults(which='ping')

    # The whoami command
    whoami_parser = subparsers.add_parser('whoami', help='Get information about account whose token is used.', formatter_class=argparse.RawDescriptionHelpFormatter,
                                          epilog='''Usage example
"""""""""""""
::
    $ rucio whoami
    jdoe
The returned value is the account currently used
                                                 ''')

    whoami_parser.set_defaults(which='whoami_account')

    # The list-file-replicas command
    list_file_replicas_parser = subparsers.add_parser('list-file-replicas', help='List the replicas of a DID and it\'s PFNs.', description='This method allows to list all the replicas of a given Data IDentifier (DID). \
The only mandatory parameter is the DID which can be a container/dataset/files. By default all the files replicas in state available are returned.', formatter_class=argparse.RawDescriptionHelpFormatter,
                                                      epilog='''Usage example
^^^^^^^^^^^^^

To list the file replicas for a given dataset::

    $ rucio list-file-replicas user.jdoe:user.jdoe.test.data.1234.1
    +-----------+---------------------------------+------------+-----------+-----------------------------------------------------------------------------------+
    | SCOPE      | NAME                              | FILESIZE   | ADLER32   | RSE: REPLICA                                                                   |
    |-----------+---------------------------------+------------+-----------+-----------------------------------------------------------------------------------+
    | user.jdoe | user.jdoe.test.data.1234.file.1 | 94.835 MB  | 5d000974  | SITE1_DISK: srm://blahblih/path/to/file/user.jdoe/user.jdoe.test.data.1234.file.1 |
    | user.jdoe | user.jdoe.test.data.1234.file.1 | 94.835 MB  | 5d000974  | SITE2_DISK: file://another/path/to/file/user.jdoe/user.jdoe.test.data.1234.file.1 |
    | user.jdoe | user.jdoe.test.data.1234.file.2 | 82.173 MB  | 01e56f23  | SITE2_DISK: file://another/path/to/file/user.jdoe/user.jdoe.test.data.1234.file.2 |
    +------------+-----------------------------------+------------+-----------+--------------------------------------------------------------------------------+

To list the missing replica of a dataset of a given RSE::

    $ rucio list-file-replicas --rse SITE1_DISK user.jdoe:user.jdoe.test.data.1234.1
    +---------------+------------------------+
    | SCOPE     | NAME                       |
    |---------------+------------------------|
    | user.jdoe | user.jdoe.test.data.1234.2 |
    +-----------+----------------------------+
    ''')
    list_file_replicas_parser.set_defaults(which='list_file_replicas')
    list_file_replicas_parser.add_argument('--protocols', dest='protocols', action='store', help='List of comma separated protocols. (i.e. https, root, srm).', required=False)
    list_file_replicas_parser.add_argument('--all-states', dest='all_states', action='store_true', default=False, help='To select all replicas (including unavailable ones).', required=False)
    list_file_replicas_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')
    list_file_replicas_parser.add_argument('--pfns', default=False, action='store_true', help='Show only the PFNs.', required=False)
    list_file_replicas_parser.add_argument('--domain', default=None, action='store', help='Force the networking domain. Available options: wan, lan, all.', required=False)
    list_file_replicas_parser.add_argument('--link', dest='link', default=None, action='store', help='Symlink PFNs with directory substitution.', required=False)
    list_file_replicas_parser.add_argument('--rse', dest='selected_rse', default=False, action='store', help='Show only results for this RSE.', required=False).completer = rse_completer
    list_file_replicas_parser.add_argument('--missing', dest='missing', default=False, action='store_true', help='To list missing replicas at a RSE. Must be used with --rse option', required=False)
    list_file_replicas_parser.add_argument('--metalink', dest='metalink', default=False, action='store_true', help='Output available replicas as metalink.', required=False)
    list_file_replicas_parser.add_argument('--sort', dest='sort', default=None, action='store', help='Replica sort algorithm. Available options: random (default), geoip', required=False)
    list_file_replicas_parser.add_argument('--expression', dest='rse_expression', default=None, action='store', help='The RSE filter expression. A comprehensive help about RSE expressions\
            can be found in ' + Color.BOLD + 'http://rucio.cern.ch/client_tutorial.html#adding-rules-for-replication' + Color.END)

    # The list-dataset-replicas command
    list_dataset_replicas_parser = subparsers.add_parser('list-dataset-replicas', help='List the dataset replicas.',
                                                         formatter_class=argparse.RawDescriptionHelpFormatter,
                                                         epilog='''Usage example
"""""""""""""
::

    $ rucio list-dataset-replicas user.jdoe:user.jdoe.test.data.1234.1

    DATASET: user.jdoe:user.jdoe.test.data.1234.1
    +------------+---------+---------+
    | RSE        |   FOUND |   TOTAL |
    |------------+---------+---------|
    | SITE1_DISK |       1 |       2 |
    | SITE2_DISK |       2 |       2 |
    +------------+---------+---------+
    ''')
    list_dataset_replicas_parser.set_defaults(which='list_dataset_replicas')
    list_dataset_replicas_parser.add_argument(dest='did', action='store', help='The name of the DID to search.')
    list_dataset_replicas_parser.add_argument('--deep', action='store_true', help='Make a deep check.')
    list_dataset_replicas_parser.add_argument('--csv', dest='csv', action='store_true', default=False, help='Comma Separated Value output.',)

    # The add-dataset command
    add_dataset_parser = subparsers.add_parser('add-dataset', help='Add a dataset to Rucio Catalog.',
                                               formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''Usage example
"""""""""""""
::

    $ rucio add-dataset user.jdoe:user.jdoe.test.data.1234.1
    Added user.jdoe:user.jdoe.test.data.1234.1

    ''')

    add_dataset_parser.set_defaults(which='add_dataset')
    add_dataset_parser.add_argument('--monotonic', action='store_true', help='Monotonic status to True.')
    add_dataset_parser.add_argument(dest='did', action='store', help='The name of the dataset to add.')
    add_dataset_parser.add_argument('--lifetime', dest='lifetime', action='store', type=int, help='Lifetime in seconds.')

    # The add-container command
    add_container_parser = subparsers.add_parser('add-container', help='Add a container to Rucio Catalog.',
                                                 formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''Usage example
"""""""""""""
::

    $ rucio add-container user.jdoe:user.jdoe.test.cont.1234.1
    Added user.jdoe:user.jdoe.test.cont.1234.1

    ''')

    add_container_parser.set_defaults(which='add_container')
    add_container_parser.add_argument('--monotonic', action='store_true', help='Monotonic status to True.')
    add_container_parser.add_argument(dest='did', action='store', help='The name of the container to add.')
    add_container_parser.add_argument('--lifetime', dest='lifetime', action='store', type=int, help='Lifetime in seconds.')

    # The attach command
    attach_parser = subparsers.add_parser('attach', help='Attach a list of DIDs to a parent DID.',
                                          description='Attach a list of Data IDentifiers (file, dataset or container) to an other Data IDentifier (dataset or container).',
                                          formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''Usage example
"""""""""""""
::

    $ rucio attach user.jdoe:user.jdoe.test.cont.1234.1 user.jdoe:user.jdoe.test.data.1234.1
    DIDs successfully attached to user.jdoe:user.jdoe.test.cont.1234.1

    ''')

    attach_parser.set_defaults(which='attach')
    attach_parser.add_argument(dest='todid', action='store', help='Destination Data IDentifier (either dataset or container).')
    attach_parser.add_argument('-f', '--from-file', dest='fromfile', action='store_true', default=False, help='Attach the DIDs contained in a file. The file should contain one did per line.')
    attach_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers (or a file containing one did per line, if -f is present).')

    # The detach command
    detach_parser = subparsers.add_parser('detach', help='Detach a list of DIDs from a parent DID.',
                                          description='Detach a list of Data Identifiers (file, dataset or container) from an other Data Identifier (dataset or container).',
                                          formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''Usage example
"""""""""""""
::

    $ rucio detach user.jdoe:user.jdoe.test.cont.1234.1 user.jdoe:user.jdoe.test.data.1234.1
    DIDs successfully detached from user.jdoe:user.jdoe.test.cont.1234.1

    ''')

    detach_parser.set_defaults(which='detach')
    detach_parser.add_argument(dest='fromdid', action='store', help='Target Data IDentifier (must be a dataset or container).')
    detach_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The list command
    ls_parser = subparsers.add_parser('ls', help='List the data identifiers matching some metadata (synonym for list-dids).', description='List the Data IDentifiers matching certain pattern.\
Only the collections (i.e. dataset or container) are returned by default. With the filter option, you can specify a list of metadata that the Data IDentifier should match.',
                                      formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''Usage example
"""""""""""""
You can query the DIDs matching a certain pattern. It always requires to specify the scope in which you want to search::

    $ rucio ls user.jdoe:*
    +-------------------------------------------+--------------+
    | SCOPE:NAME                                | [DID TYPE]   |
    |-------------------------------------------+--------------|
    | user.jdoe:user.jdoe.test.container.1234.1 | CONTAINER    |
    | user.jdoe:user.jdoe.test.container.1234.2 | CONTAINER    |
    | user.jdoe:user.jdoe.test.cont.1234.2      | CONTAINER    |
    | user.jdoe:user.jdoe.test.dataset.1        | DATASET      |
    | user.jdoe:user.jdoe.test.dataset.2        | DATASET      |
    | user.jdoe:user.jdoe.test.data.1234.1      | DATASET      |
    | user.jdoe:test.file.1                     | FILE         |
    | user.jdoe:test.file.2                     | FILE         |
    | user.jdoe:test.file.3                     | FILE         |
    |-------------------------------------------+--------------|

You can filter by key/value, e.g.::

    $ rucio ls --filter type=CONTAINER
    +-------------------------------------------+--------------+
    | SCOPE:NAME                                | [DID TYPE]   |
    |-------------------------------------------+--------------|
    | user.jdoe:user.jdoe.test.container.1234.1 | CONTAINER    |
    | user.jdoe:user.jdoe.test.container.1234.2 | CONTAINER    |
    | user.jdoe:user.jdoe.test.cont.1234.2      | CONTAINER    |
    |-------------------------------------------+--------------|
    ''')

    ls_parser.set_defaults(which='list_dids')
    ls_parser.add_argument('-r', '--recursive', dest='recursive', action='store_true', default=False, help='List data identifiers recursively.')
    ls_parser.add_argument('--filter', dest='filter', action='store', help='Filter arguments in form `key=value,another_key=next_value`. Valid keys are name, type.')
    ls_parser.add_argument('--short', dest='short', action='store_true', help='Just dump the list of DIDs.')
    ls_parser.add_argument(dest='did', nargs=1, action='store', default=None, help='Data IDentifier pattern.')

    list_parser = subparsers.add_parser('list-dids', help='List the data identifiers matching some metadata (synonym for ls).', description='List the Data IDentifiers matching certain pattern. \
Only the collections (i.e. dataset or container) are returned by default. With the filter option, you can specify a list of metadata that the Data IDentifier should match.',
                                        formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''Usage example
"""""""""""""

You can query the DIDs matching a certain pattern. It always requires to specify the scope in which you want to search::

    $ rucio list-dids user.jdoe:*
    +-------------------------------------------+--------------+
    | SCOPE:NAME                                | [DID TYPE]   |
    |-------------------------------------------+--------------|
    | user.jdoe:user.jdoe.test.container.1234.1 | CONTAINER    |
    | user.jdoe:user.jdoe.test.container.1234.2 | CONTAINER    |
    | user.jdoe:user.jdoe.test.cont.1234.2      | CONTAINER    |
    | user.jdoe:user.jdoe.test.dataset.1        | DATASET      |
    | user.jdoe:user.jdoe.test.dataset.2        | DATASET      |
    | user.jdoe:user.jdoe.test.data.1234.1      | DATASET      |
    | user.jdoe:test.file.1                     | FILE         |
    | user.jdoe:test.file.2                     | FILE         |
    | user.jdoe:test.file.3                     | FILE         |
    |-------------------------------------------+--------------|

You can filter by key/value, e.g.::

    $ rucio list-dids --filter type=CONTAINER
    +-------------------------------------------+--------------+
    | SCOPE:NAME                                | [DID TYPE]   |
    |-------------------------------------------+--------------|
    | user.jdoe:user.jdoe.test.container.1234.1 | CONTAINER    |
    | user.jdoe:user.jdoe.test.container.1234.2 | CONTAINER    |
    | user.jdoe:user.jdoe.test.cont.1234.2      | CONTAINER    |
    |-------------------------------------------+--------------|''')

    list_parser.set_defaults(which='list_dids')
    list_parser.add_argument('--recursive', dest='recursive', action='store_true', default=False, help='List data identifiers recursively.')
    list_parser.add_argument('--filter', dest='filter', action='store', help='Filter arguments in form `key=value,another_key=next_value`. Valid keys are name, type.')
    list_parser.add_argument('--short', dest='short', action='store_true', help='Just dump the list of DIDs.')
    list_parser.add_argument(dest='did', nargs=1, action='store', default=None, help='Data IDentifier pattern')

    # The list parent-dids command
    list_parent_parser = subparsers.add_parser('list-parent-dids', help='List parent DIDs for a given DID', description='List all parents Data IDentifier that contains the target Data IDentifier.',
                                               formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''Usage example
"""""""""""""
::

    $ rucio list-parent-dids user.jdoe:user.jdoe.test.data.1234.1
    +--------------------------------------+--------------+
    | SCOPE:NAME                           | [DID TYPE]   |
    |--------------------------------------+--------------|
    | user.jdoe:user.jdoe.test.cont.1234.2 | CONTAINER    |
    +--------------------------------------+--------------+

    ''')
    list_parent_parser.set_defaults(which='list_parent_dids')
    list_parent_parser.add_argument(dest='did', action='store', nargs='?', default=None, help='Data identifier.')
    list_parent_parser.add_argument('--pfn', dest='pfns', action='store', nargs='+', help='List parent dids for these pfns.')
    list_parent_parser.add_argument('--guid', dest='guids', action='store', nargs='+', help='List parent dids for these guids.')

    # argparse 2.7 does not allow aliases for commands, thus the list-parent-datasets is a copy&paste from list-parent-dids
    list_parent_datasets_parser = subparsers.add_parser('list-parent-datasets', help='List parent DIDs for a given DID', description='List all parents Data IDentifier that contains the target Data IDentifier.',
                                                        formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''Usage example
"""""""""""""
::

    $ rucio list-parent-datasets user.jdoe:user.jdoe.test.data.1234.1
    +--------------------------------------+--------------+
    | SCOPE:NAME                           | [DID TYPE]   |
    |--------------------------------------+--------------|
    | user.jdoe:user.jdoe.test.cont.1234.2 | CONTAINER    |
    +--------------------------------------+--------------+

    ''')

    list_parent_datasets_parser.set_defaults(which='list_parent_datasets')
    list_parent_datasets_parser.add_argument(dest='did', action='store', nargs='?', default=None, help='Data identifier.')
    list_parent_datasets_parser.add_argument('--pfn', dest='pfns', action='store', nargs='+', help='List parent dids for these pfns.')
    list_parent_datasets_parser.add_argument('--guid', dest='guids', action='store', nargs='+', help='List parent dids for these guids.')

    # The list-scopes command
    scope_list_parser = subparsers.add_parser('list-scopes', help='List all available scopes.',
                                              formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''Usage example
"""""""""""""
::

    $ rucio list-scopes
    mc
    data
    user.jdoe
    user.janedoe

    ''')

    scope_list_parser.set_defaults(which='list_scopes')

    # The close command
    close_parser = subparsers.add_parser('close', help='Close a dataset or container.')
    close_parser.set_defaults(which='close')
    close_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The reopen command
    reopen_parser = subparsers.add_parser('reopen', help='Reopen a dataset or container (only for privileged users).')
    reopen_parser.set_defaults(which='reopen')
    reopen_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The stat command
    stat_parser = subparsers.add_parser('stat', help='List attributes and statuses about data identifiers.')
    stat_parser.set_defaults(which='stat')
    stat_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The erase command
    erase_parser = subparsers.add_parser('erase', help='Delete a data identifier.', description='This command sets the lifetime of the DID in order to expire in the next 24 hours.\
            After this time, the dataset is eligible for deletion. The deletion is not reversible after 24 hours grace time period expired.')
    erase_parser.set_defaults(which='erase')
    erase_parser.add_argument('--undo', dest='undo', action='store_true', default=False, help='Undo erase DIDs. Only works if has been less than 24 hours since erase operation.')
    erase_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The list_files command
    list_files_parser = subparsers.add_parser('list-files', help='List DID contents', description='List all the files in a Data IDentifier. The DID can be a container, dataset or a file.\
                                                                  What is returned is a list of files in the DID with : <scope>:<name>\t<filesize>\t<checksum>\t<guid>')
    list_files_parser.set_defaults(which='list_files')
    list_files_parser.add_argument('--csv', dest='csv', action='store_true', default=False, help='Comma Separated Value output. This output format is preferred for easy parsing and scripting.')
    list_files_parser.add_argument('--pfc', dest='LOCALPATH', action='store', default=False, help='Outputs the list of files in the dataset with the LOCALPATH prepended as a PoolFileCatalog')
    list_files_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The list_content command
    list_content_parser = subparsers.add_parser('list-content', help='List the content of a collection.')
    list_content_parser.set_defaults(which='list_content')
    list_content_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')
    list_content_parser.add_argument('--short', dest='short', action='store_true', help='Just dump the list of DIDs.')

    # The list_content_history command
    list_content_history_parser = subparsers.add_parser('list-content-history', help='List the content history of a collection.')
    list_content_history_parser.set_defaults(which='list_content_history')
    list_content_history_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The upload subparser
    upload_parser = subparsers.add_parser('upload', help='Upload method.')
    upload_parser.set_defaults(which='upload')
    upload_parser.add_argument('--rse', dest='rse', action='store', help='Rucio Storage Element (RSE) name.', required=True).completer = rse_completer
    upload_parser.add_argument('--lifetime', type=int, action='store', help='Lifetime of the rule in seconds.')
    upload_parser.add_argument('--scope', dest='scope', action='store', help='Scope name.')
    # The --no-register option is hidden. This is pilot ONLY. Users should not use this. Will lead to unregistered data on storage!
    upload_parser.add_argument('--no-register', dest='no_register', action='store_true', default=False, help=argparse.SUPPRESS)
    upload_parser.add_argument('--summary', dest='summary', action='store_true', default=False, help='Create rucio_upload.json summary file')
    upload_parser.add_argument('--guid', dest='guid', action='store', help='Manually specify the GUID for the file.')
    upload_parser.add_argument('--protocol', action='store', help='Force the protocol to use')
    upload_parser.add_argument('--pfn', dest='pfn', action='store', help='Specify the exact PFN for the upload.')
    upload_parser.add_argument('--name', dest='name', action='store', help='Specify the exact LFN for the upload.')
    upload_parser.add_argument('--transfer-timeout', dest='transfer_timeout', type=float, action='store', default=config_get('upload', 'transfer_timeout', False, 3600), help='Transfer timeout (in seconds).')
    upload_parser.add_argument(dest='args', action='store', nargs='+', help='files and datasets.')

    # The download subparser
    get_parser = subparsers.add_parser('get', help='Download method (synonym for download)')
    get_parser.set_defaults(which='download')
    get_parser.add_argument('--dir', dest='dir', default='.', action='store', help='The directory to store the downloaded file.')
    get_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')
    get_parser.add_argument('--rse', action='store', help='RSE Expression to specify allowed sources.')
    get_parser.add_argument('--protocol', action='store', help='Force the protocol to use.')
    get_parser.add_argument('--nrandom', type=int, action='store', help='Download N random files from the DID.')
    get_parser.add_argument('--ndownloader', type=int, default=3, action='store', help='Choose the number of parallel processes for download.')
    get_parser.add_argument('--no-subdir', action='store_true', default=False, help="Don't create a subdirectory for the scope of the files. Existing files in the directory will be overwritten.")
    get_parser.add_argument('--old', action='store_true', default=False, help=argparse.SUPPRESS)
    get_parser.add_argument('--pfn', dest='pfn', action='store', help="Specify the exact PFN for the download.")
    get_parser.add_argument('--transfer-timeout', dest='transfer_timeout', type=float, action='store', default=config_get('download', 'transfer_timeout', False, 3600), help='Transfer timeout (in seconds).')
    get_parser.add_argument('--aria', action='store_true', default=False, help="Use aria2c utility if possible. (EXPERIMENTAL)")
    get_parser.add_argument('--trace_appid', dest='trace_appid', action='store', default=os.environ.get('RUCIO_TRACE_APPID', None), help=argparse.SUPPRESS)
    get_parser.add_argument('--trace_dataset', dest='trace_dataset', action='store', default=os.environ.get('RUCIO_TRACE_DATASET', None), help=argparse.SUPPRESS)
    get_parser.add_argument('--trace_datasetscope', dest='trace_datasetscope', action='store', default=os.environ.get('RUCIO_TRACE_DATASETSCOPE', None), help=argparse.SUPPRESS)
    get_parser.add_argument('--trace_eventtype', dest='trace_eventtype', action='store', default=os.environ.get('RUCIO_TRACE_EVENTTYPE', None), help=argparse.SUPPRESS)
    get_parser.add_argument('--trace_pq', dest='trace_pq', action='store', default=os.environ.get('RUCIO_TRACE_PQ', None), help=argparse.SUPPRESS)
    get_parser.add_argument('--trace_taskid', dest='trace_taskid', action='store', default=os.environ.get('RUCIO_TRACE_TASKID', None), help=argparse.SUPPRESS)
    get_parser.add_argument('--trace_usrdn', dest='trace_usrdn', action='store', default=os.environ.get('RUCIO_TRACE_USRDN', None), help=argparse.SUPPRESS)

    download_parser = subparsers.add_parser('download', help='Download method (synonym for get)')
    download_parser.set_defaults(which='download')
    download_parser.add_argument('--dir', dest='dir', default='.', action='store', help='The directory to store the downloaded file.')
    download_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')
    download_parser.add_argument('--rse', action='store', help='RSE Expression to specify allowed sources')
    download_parser.add_argument('--protocol', action='store', help='Force the protocol to use.')
    download_parser.add_argument('--nrandom', type=int, action='store', help='Download N random files from the DID.')
    download_parser.add_argument('--ndownloader', type=int, default=3, action='store', help='Choose the number of parallel processes for download.')
    download_parser.add_argument('--no-subdir', action='store_true', default=False, help="Don't create a subdirectory for the scope of the files. Existing files in the directory will be overwritten.")
    download_parser.add_argument('--old', action='store_true', default=False, help=argparse.SUPPRESS)
    download_parser.add_argument('--pfn', dest='pfn', action='store', help="Specify the exact PFN for the download.")
    download_parser.add_argument('--transfer-timeout', dest='transfer_timeout', type=float, action='store', default=config_get('download', 'transfer_timeout', False, 3600), help='Transfer timeout (in seconds).')
    download_parser.add_argument('--aria', action='store_true', default=False, help="Use aria2c utility if possible. (EXPERIMENTAL)")
    download_parser.add_argument('--trace_appid', dest='trace_appid', action='store', default=os.environ.get('RUCIO_TRACE_APPID', None), help=argparse.SUPPRESS)
    download_parser.add_argument('--trace_dataset', dest='trace_dataset', action='store', default=os.environ.get('RUCIO_TRACE_DATASET', None), help=argparse.SUPPRESS)
    download_parser.add_argument('--trace_datasetscope', dest='trace_datasetscope', action='store', default=os.environ.get('RUCIO_TRACE_DATASETSCOPE', None), help=argparse.SUPPRESS)
    download_parser.add_argument('--trace_eventtype', dest='trace_eventtype', action='store', default=os.environ.get('RUCIO_TRACE_EVENTTYPE', None), help=argparse.SUPPRESS)
    download_parser.add_argument('--trace_pq', dest='trace_pq', action='store', default=os.environ.get('RUCIO_TRACE_PQ', None), help=argparse.SUPPRESS)
    download_parser.add_argument('--trace_taskid', dest='trace_taskid', action='store', default=os.environ.get('RUCIO_TRACE_TASKID', None), help=argparse.SUPPRESS)
    download_parser.add_argument('--trace_usrdn', dest='trace_usrdn', action='store', default=os.environ.get('RUCIO_TRACE_USRDN', None), help=argparse.SUPPRESS)

    # The get-metadata subparser
    get_metadata_parser = subparsers.add_parser('get-metadata', help='Get metadata for DIDs.')
    get_metadata_parser.set_defaults(which='get_metadata')
    get_metadata_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The set-metadata subparser
    set_metadata_parser = subparsers.add_parser('set-metadata', help='set-metadata method')
    set_metadata_parser.set_defaults(which='set_metadata')
    set_metadata_parser.add_argument('--did', dest='did', action='store', help='Data identifier to add', required=True)
    set_metadata_parser.add_argument('--key', dest='key', action='store', help='Attribute key', required=True)
    set_metadata_parser.add_argument('--value', dest='value', action='store', help='Attribute value', required=True)

    # The delete-metadata subparser
    # delete_metadata_parser = subparsers.add_parser('delete-metadata', help='Delete metadata')
    # delete_metadata_parser.set_defaults(which='delete_metadata')

    # The list-rse-usage subparser
    list_rse_usage_parser = subparsers.add_parser('list-rse-usage', help='Shows the total/free/used space for a given RSE. This values can differ for different RSE source.')
    list_rse_usage_parser.set_defaults(which='list_rse_usage')
    list_rse_usage_parser.add_argument(dest='rse', action='store', help='Rucio Storage Element (RSE) name.').completer = rse_completer
    list_rse_usage_parser.add_argument('--history', dest='history', default=False, action='store', help='List RSE usage history. [Unimplemented]')

    # The list-account-usage subparser
    list_account_usage_parser = subparsers.add_parser('list-account-usage', help='Shows the space used, the quota limit and the quota left for an account for every RSE where the user have quota.')
    list_account_usage_parser.set_defaults(which='list_account_usage')
    list_account_usage_parser.add_argument(dest='usage_account', action='store', help='Account name.')
    list_account_usage_parser.add_argument('--rse', action='store', help='Show usage for only for this RSE.')

    # The list-account-limits subparser
    list_account_limits_parser = subparsers.add_parser('list-account-limits', help='List quota limits for an account in every RSEs.')
    list_account_limits_parser.set_defaults(which='list_account_limits')
    list_account_limits_parser.add_argument('limit_account', action='store', help='The account name.')
    list_account_limits_parser.add_argument('--rse', dest='rse', action='store', help='If this option is given, the results are restricted to only this RSE.').completer = rse_completer

    # Add replication rule subparser
    add_rule_parser = subparsers.add_parser('add-rule', help='Add replication rule.')
    add_rule_parser.set_defaults(which='add_rule')
    add_rule_parser.add_argument(dest='dids', action='store', nargs='+', help='DID(s) to apply the rule to')
    add_rule_parser.add_argument(dest='copies', action='store', type=int, help='Number of copies')
    add_rule_parser.add_argument(dest='rse_expression', action='store', help='RSE Expression')
    add_rule_parser.add_argument('--weight', dest='weight', action='store', help='RSE Weight')
    add_rule_parser.add_argument('--lifetime', dest='lifetime', action='store', type=int, help='Rule lifetime (in seconds)')
    add_rule_parser.add_argument('--grouping', dest='grouping', action='store', choices=['DATASET', 'ALL', 'NONE'], help='Rule grouping')
    add_rule_parser.add_argument('--locked', dest='locked', action='store_true', help='Rule locking')
    add_rule_parser.add_argument('--source-replica-expression', dest='source_replica_expression', action='store', help='Source Replica Expression')
    add_rule_parser.add_argument('--notify', dest='notify', action='store', help='Notification strategy : Y (Yes), N (No), C (Close)')
    add_rule_parser.add_argument('--activity', dest='activity', action='store', help='Activity to be used (e.g. User, Data Consolidation')
    add_rule_parser.add_argument('--comment', dest='comment', action='store', help='Comment about the replication rule')
    add_rule_parser.add_argument('--ask-approval', dest='ask_approval', action='store_true', help='Ask for rule approval')
    add_rule_parser.add_argument('--asynchronous', dest='asynchronous', action='store_true', help='Create rule asynchronously')
    add_rule_parser.add_argument('--account', dest='rule_account', action='store', help='The account owning the rule')
    add_rule_parser.add_argument('--skip-duplicates', dest='ignore_duplicate', action='store_true', help='Skip duplicate rules')

    # Delete replication rule subparser
    delete_rule_parser = subparsers.add_parser('delete-rule', help='Delete replication rule.')
    delete_rule_parser.set_defaults(which='delete_rule')
    delete_rule_parser.add_argument(dest='rule_id', action='store', help='Rule id or DID. If DID, the RSE expression is mandatory.')
    delete_rule_parser.add_argument('--purge-replicas', dest='purge_replicas', action='store_true', help='Purge rule replicas')
    delete_rule_parser.add_argument('--all', dest='delete_all', action='store_true', default=False, help='Delete all the rules, even the ones that are not owned by the account')
    delete_rule_parser.add_argument('--rse_expression', dest='rse_expression', action='store', help='The RSE expression. Must be specified if a DID is provided.')
    delete_rule_parser.add_argument('--account', dest='rule_account', action='store', help='The account of the rule that must be deleted')

    # Info replication rule subparser
    info_rule_parser = subparsers.add_parser('rule-info', help='Retrieve information about a rule.')
    info_rule_parser.set_defaults(which='info_rule')
    info_rule_parser.add_argument(dest='rule_id', action='store', help='The rule ID')
    info_rule_parser.add_argument('--examine', dest='examine', action='store_true', help='Detailed analysis of transfer errors')
    info_rule_parser.add_argument('--estimate-ttc', dest='estimate_ttc', action='store_true', help='Show Estimated Time To Complete for the rule. Calculation can be time consuming.')

    # The list_rules command
    list_rules_parser = subparsers.add_parser('list-rules', help='List replication rules.', formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''Usage example
"""""""""""""

You can list the rules for a particular DID::

    $ rucio list-rules user.jdoe:user.jdoe.test.container.1234.1
    ID                                ACCOUNT    SCOPE:NAME                                 STATE[OK/REPL/STUCK]    RSE_EXPRESSION        COPIES  EXPIRES (UTC)
    --------------------------------  ---------  -----------------------------------------  ----------------------  ------------------  --------  -------------------
    a12e5664555a4f12b3cc6991db5accf9  jdoe       user.jdoe:user.jdoe.test.container.1234.1  OK[3/0/0]               tier=1&disk=1       1         2018-02-09 03:57:46
    b0fcde2acbdb489b874c3c4537595adc  janedoe    user.jdoe:user.jdoe.test.container.1234.1  REPLICATING[4/1/1]      tier=1&tape=1       2
    4a6bd85c13384bd6836fbc06e8b316d7  mc         user.jdoe:user.jdoe.test.container.1234.1  OK[3/0/0]               tier=1&tape=1       2

You can filter by account::

    $ rucio list-rules --account jdoe
    ID                                ACCOUNT    SCOPE:NAME                                 STATE[OK/REPL/STUCK]    RSE_EXPRESSION        COPIES  EXPIRES (UTC)
    --------------------------------  ---------  -----------------------------------------  ----------------------  ------------------  --------  -------------------
    a12e5664555a4f12b3cc6991db5accf9  jdoe       user.jdoe:user.jdoe.test.container.1234.1  OK[3/0/0]               tier=1&disk=1       1         2018-02-09 03:57:46
    08537b2176843d92e05317938a89d148  jdoe       user.jdoe:user.jdoe.test.data.1234.1       OK[2/0/0]               SITE2_DISK          1

                                              ''')

    list_rules_parser.set_defaults(which='list_rules')
    list_rules_parser.add_argument(dest='did', action='store', nargs='?', default=None, help='List by did')
    list_rules_parser.add_argument('--id', dest='rule_id', action='store', help='List by rule id')
    list_rules_parser.add_argument('--traverse', dest='traverse', action='store_true', help='Traverse the did tree and search for rules affecting this did')
    list_rules_parser.add_argument('--csv', dest='csv', action='store_true', default=False, help='Comma Separated Value output')
    list_rules_parser.add_argument('--file', dest='file', action='store', help='List associated rules of an affected file')
    list_rules_parser.add_argument('--account', dest='rule_account', action='store', help='List by account')
    list_rules_parser.add_argument('--subscription', dest='subscription', action='store', help='List by account and subscription name', metavar=('ACCOUNT', 'SUBSCRIPTION'), nargs=2)

    # The list_rules_history command
    list_rules_history_parser = subparsers.add_parser('list-rules-history', help='List replication rules history for a DID.')
    list_rules_history_parser.set_defaults(which='list_rules_history')
    list_rules_history_parser.add_argument(dest='did', action='store', help='The Data IDentifier.')

    # The update_rule command
    update_rule_parser = subparsers.add_parser('update-rule', help='Update replication rule.')
    update_rule_parser.set_defaults(which='update_rule')
    update_rule_parser.add_argument(dest='rule_id', action='store', help='Rule id')
    update_rule_parser.add_argument('--lifetime', dest='lifetime', action='store', help='Lifetime in seconds.')
    update_rule_parser.add_argument('--locked', dest='locked', action='store', help='Locked (True/False).')
    update_rule_parser.add_argument('--account', dest='rule_account', action='store', help='Account to change.')
    update_rule_parser.add_argument('--stuck', dest='state_stuck', action='store_true', help='Set state to STUCK.')
    update_rule_parser.add_argument('--suspend', dest='state_suspended', action='store_true', help='Set state to SUSPENDED.')
    update_rule_parser.add_argument('--activity', dest='rule_activity', action='store', help='Activity of the rule.')
    update_rule_parser.add_argument('--source-replica-expression', dest='source_replica_expression', action='store', help='Source replica expression of the rule.')
    update_rule_parser.add_argument('--cancel-requests', dest='cancel_requests', action='store_true', help='Cancel requests when setting rules to stuck.')
    update_rule_parser.add_argument('--priority', dest='priority', action='store', help='Priority of the requests of the rule.')
    update_rule_parser.add_argument('--child-rule-id', dest='child_rule_id', action='store', help='Child rule id of the rule.')

    # The move_rule command
    move_rule_parser = subparsers.add_parser('move-rule', help='Move a replication rule to another RSE.')
    move_rule_parser.set_defaults(which='move_rule')
    move_rule_parser.add_argument(dest='rule_id', action='store', help='Rule id')
    move_rule_parser.add_argument(dest='rse_expression', action='store', help='RSE expression of new rule')

    # The list-rses command
    list_rses_parser = subparsers.add_parser('list-rses', help='Show the list of all the registered Rucio Storage Elements (RSEs).')
    list_rses_parser.set_defaults(which='list_rses')
    list_rses_parser.add_argument('--expression', dest='rse_expression', action='store', help='The RSE filter expression. A comprehensive help about RSE expressions \
can be found in ' + Color.BOLD + 'http://rucio.cern.ch/client_tutorial.html#adding-rules-for-replication' + Color.END)

    # The list-rses-attributes command
    list_rse_attributes_parser = subparsers.add_parser('list-rse-attributes', help='List the attributes of an RSE.', description='This command is useful to create RSE filter expressions.')
    list_rse_attributes_parser.set_defaults(which='list_rse-attributes')
    list_rse_attributes_parser.add_argument(dest='rse', action='store', help='The RSE name').completer = rse_completer

    # The list-datasets-rse command
    list_datasets_rse_parser = subparsers.add_parser('list-datasets-rse', help='List all the datasets at a RSE', description='This method allows to list all the datasets on a given Rucio Storage Element.\
        ' + Color.BOLD + 'Warning: ' + Color.END + 'This command can take a long time depending on the number of datasets in the RSE.')
    list_datasets_rse_parser.set_defaults(which='list_datasets_rse')
    list_datasets_rse_parser.add_argument(dest='rse', action='store', default=None, help='The RSE name').completer = rse_completer
    list_datasets_rse_parser.add_argument('--long', dest='long', action='store_true', default=False, help='The long option')

    # The test-server command
    test_server_parser = subparsers.add_parser('test-server', help='Test Server', description='Run a battery of nosetests against the Rucio Servers.')
    test_server_parser.set_defaults(which='test_server')

    # The get-metadata subparser
    touch_parser = subparsers.add_parser('touch', help='Touch one or more DIDs and set the last accessed date to the current date')
    touch_parser.set_defaults(which='touch')
    touch_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')
    touch_parser.add_argument('--rse', dest='rse', action='store', help="The RSE of the DIDs that are touched.").completer = rse_completer

    return oparser


if __name__ == '__main__':
    oparser = get_parser()
    argcomplete.autocomplete(oparser)

    if len(sys.argv) == 1:
        oparser.print_help()
        sys.exit(FAILURE)

    args = oparser.parse_args(sys.argv[1:])

    commands = {'add_container': add_container,
                'add_dataset': add_dataset,
                'add_rule': add_rule,
                'attach': attach,
                'close': close,
                'reopen': reopen,
                'erase': erase,
                'delete_rule': delete_rule,
                'detach': detach,
                'download': download,
                'get': download,
                'get_metadata': get_metadata,
                'info_rule': info_rule,
                'list_account_limits': list_account_limits,
                'list_account_usage': list_account_usage,
                'list_datasets_rse': list_datasets_rse,
                'list_parent_dids': list_parent_dids,
                'list_parent_datasets': list_parent_dids,
                'list_files': list_files,
                'list_content': list_content,
                'list_content_history': list_content_history,
                'list_dids': list_dids,
                'list_dataset_replicas': list_dataset_replicas,
                'list_file_replicas': list_file_replicas,
                'list_rses': list_rses,
                'list_rse-attributes': list_rse_attributes,
                'list_rse_usage': list_rse_usage,
                'list_rules': list_rules,
                'list_rules_history': list_rules_history,
                'list_scopes': list_scopes,
                'ping': ping,
                'set_metadata': set_metadata,
                'stat': stat,
                'test_server': test_server,
                'touch': touch,
                'update_rule': update_rule,
                'move_rule': move_rule,
                'upload': upload,
                'whoami_account': whoami_account}

    try:
        if args.verbose:
            logger.setLevel(logging.DEBUG)
        start_time = time.time()
        command = commands.get(args.which)
        result = command(args)
        end_time = time.time()
        if args.verbose:
            print("Completed in %-0.4f sec." % (end_time - start_time))
        sys.exit(result)
    except Exception as error:
        logger.error("Strange error: {0}".format(error))
        sys.exit(FAILURE)
