#!/usr/bin/env python

# Copyright European Organization for Nuclear Research (CERN)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Mario Lassnig, <mario.lassnig@cern.ch>, 2012-2014, 2016
# - Vincent Garonne, <vincent.garonne@cern.ch>, 2012-2016
# - Thomas Beermann, <thomas.beermann@cern.ch>, 2012
# - Yun-Pin Sun, <yun-pin.sun@cern.ch>, 2013
# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2016
# - Martin Barisits <martin.barisits@cern.ch>, 2013-2015
# - Joaquin Bogado <joaquin.bogado@cern.ch>, 2014, 2015
# - Evangelia Liotiri <evangelia.liotiri@cern.ch>, 2015

"""
    Rucio CLI.
"""

import argcomplete
import argparse
import hashlib
import json
import logging
import os
import random
import requests
import signal
import socket
import subprocess
import sys
import tabulate
import time
import traceback
import uuid

from ConfigParser import NoOptionError, NoSectionError
from copy import deepcopy
from functools import wraps
from multiprocessing import Process
from Queue import Queue, Empty
from threading import Thread, Event

from rucio.client import Client
from rucio import version
from rucio.common.config import config_get
from rucio.common.exception import (DataIdentifierAlreadyExists, Duplicate, FileAlreadyExists, AccessDenied, ResourceTemporaryUnavailable,
                                    DataIdentifierNotFound, InvalidObject, RSENotFound, InvalidRSEExpression, DuplicateContent, RSEProtocolNotSupported,
                                    RuleNotFound, CannotAuthenticate, MissingDependency, UnsupportedOperation, FileConsistencyMismatch,
                                    RucioException)
from rucio.common.utils import adler32, generate_uuid, execute, chunks, sizefmt, Color
from rucio.rse import rsemanager as rsemgr

SUCCESS = 0
FAILURE = 1

DEFAULT_SECURE_PORT = 443
DEFAULT_PORT = 80

STOP_REQUEST = Event()

logger = logging.getLogger("user")
tablefmt = 'psql'


def setup_logger(logger):
    logger.setLevel(logging.INFO)
    hdlr = logging.StreamHandler()

    def emit_decorator(fnc):
        def func(*args):
            if 'RUCIO_LOGGING_FORMAT' not in os.environ:
                levelno = args[0].levelno
                if levelno >= logging.CRITICAL:
                    color = '\033[31;1m'
                elif levelno >= logging.ERROR:
                    color = '\033[31;1m'
                elif levelno >= logging.WARNING:
                    color = '\033[33;1m'
                elif levelno >= logging.INFO:
                    color = '\033[32;1m'
                elif levelno >= logging.DEBUG:
                    color = '\033[36;1m'
                else:
                    color = '\033[0m'
                formatter = logging.Formatter('{0}%(asctime)s %(levelname)s [%(message)s]\033[0m'.format(color))
            else:
                formatter = logging.Formatter(os.environ['RUCIO_LOGGING_FORMAT'])
            hdlr.setFormatter(formatter)
            return fnc(*args)
        return func
    hdlr.emit = emit_decorator(hdlr.emit)
    logger.addHandler(hdlr)

setup_logger(logger)


def signal_handler(sig, frame):
    logger.warning('You pressed Ctrl+C! Exiting gracefully')
    child_processes = subprocess.Popen('ps -o pid --ppid %s --noheaders' % os.getpid(), shell=True, stdout=subprocess.PIPE)
    child_processes = child_processes.stdout.read()
    for pid in child_processes.split("\n")[:-1]:
        try:
            os.kill(int(pid), signal.SIGTERM)
        except Exception:
            print 'Cannot kill child process'
    STOP_REQUEST.set()
    sys.exit(1)

signal.signal(signal.SIGINT, signal_handler)


def extract_scope(did):
    # Try to extract the scope from the DSN
    if did.find(':') > -1:
        if len(did.split(':')) > 2:
            raise RucioException('Too many colons. Cannot extract scope and name')
        scope, name = did.split(':')[0], did.split(':')[1]
        if name.endswith('/'):
            name = name[:-1]
        return scope, name
    else:
        scope = did.split('.')[0]
        if did.startswith('user') or did.startswith('group'):
            scope = ".".join(did.split('.')[0:2])
        if did.endswith('/'):
            did = did[:-1]
        return scope, did


def send_trace(trace, trace_endpoint, retries=5):
    for dummy in xrange(retries):
        try:
            requests.post(trace_endpoint + '/traces/', verify=False, data=json.dumps(trace))
            return 0
        except Exception, error:
            logger.debug(error)
    return 1


def exception_handler(function):
    @wraps(function)
    def new_funct(*args, **kwargs):
        try:
            return function(*args, **kwargs)
        except InvalidObject, error:
            logger.error(error)
            return FAILURE
        except DataIdentifierNotFound, error:
            logger.error(error)
            logger.debug('This means that the Data IDentifier you provided is not known by Rucio.')
            return FAILURE
        except AccessDenied, error:
            logger.error(error)
            logger.debug('This error is a permission issue. You cannot run this command with your account.')
            return FAILURE
        except DataIdentifierAlreadyExists, error:
            logger.error(error)
            logger.debug('This means that the data IDentifier you try to add is already registered in Rucio.')
            return FAILURE
        except RSENotFound, error:
            logger.error(error)
            logger.debug('This means that the Rucio Storage Element you provided is not known by Rucio.')
            return FAILURE
        except InvalidRSEExpression, error:
            logger.error(error)
            logger.debug('This means the RSE expression you provided is not syntactically correct.')
            return FAILURE
        except DuplicateContent, error:
            logger.error(error)
            logger.debug('This means that the DID you want to attach is already in the target DID.')
            return FAILURE
        except TypeError, error:
            logger.error(error)
            logger.debug('This means the parameter you passed has a wrong type.')
            return FAILURE
        except RuleNotFound, error:
            logger.error(error)
            logger.debug('This means the rule you specified does not exist.')
            return FAILURE
        except UnsupportedOperation, error:
            logger.error(error)
            logger.debug('That means you cannot change the status of the DID.')
            return FAILURE
        except MissingDependency, error:
            logger.error(error)
            logger.debug('This means one dependency is missing.')
            return FAILURE
        except KeyError, error:
            if 'x-rucio-auth-token' in error:
                logger.error(error)
                logger.error('This means that your RUCIO_ACCOUNT environment variable not match with a registered identity to your account.')
            else:
                logger.error(error)
                logger.debug(traceback.format_exc())
                logger.error("This means that the object doesn't have the property " + str(error) + ". This should never happen. Please rerun the last command with the '-v' option and provide the traceback to the developers.")
            return FAILURE
        except RucioException, error:
            logger.error(error)
            return FAILURE
        except IOError, error:
            logger.error(error)
            return FAILURE
        except Exception, error:
            logger.error(error)
            logger.error('Rucio exited with an unexpected/unknown error. Please re run the last command with "-v" option and provide the traceback below to the developers.')
            logger.debug(traceback.format_exc())
            return FAILURE
    return new_funct


def get_client(args):
    """
    Returns a new client object.
    """
    if args.auth_strategy == 'userpass':
        creds = {'username': args.username, 'password': args.password}
    else:
        creds = None

    try:
        client = Client(rucio_host=args.host, auth_host=args.auth_host,
                        account=args.account,
                        auth_type=args.auth_strategy, creds=creds,
                        ca_cert=args.ca_certificate, timeout=args.timeout,
                        user_agent=args.user_agent)
    except CannotAuthenticate, error:
        logger.error(error)
        if not args.auth_strategy:
            if 'RUCIO_AUTH_TYPE' in os.environ:
                auth_type = os.environ['RUCIO_AUTH_TYPE']
            else:
                try:
                    auth_type = config_get('client', 'auth_type')
                except (NoOptionError, NoSectionError):
                    logger.error('Cannot get AUTH_TYPE')
                    sys.exit(FAILURE)
        if auth_type == 'x509_proxy':
            logger.error('Please verify that your proxy is still valid and renew it if needed.')
        sys.exit(FAILURE)
    return client


@exception_handler
def ping(args):
    """
    Pings a Rucio server.
    """
    client = get_client(args)
    server_info = client.ping()
    if server_info:
        print server_info['version']
        return SUCCESS
    logger.error('Ping failed')
    return FAILURE


@exception_handler
def whoami_account(args):
    """
    %(prog)s show [options] <field1=value1 field2=value2 ...>

    Show extended information of a given account
    """
    client = get_client(args)
    info = client.whoami()
    for k in info:
        print k.ljust(10) + ' : ' + str(info[k])
    return SUCCESS


@exception_handler
def add_file(args):
    """
    %(prog)s show [options] <field1=value1 field2=value2 ...>

    Add file.
    """
    client = get_client(args)
    scope = args.scope
    lfn = args.lfn
    if scope is None:
        scope, lfn = extract_scope(lfn)
    client.add_file(rse=args.rse, scope=scope, lfn=lfn)
    print 'Added new file replica: %s-%s' % (args.rse, args.lfn)
    return SUCCESS


@exception_handler
def list_dataset_replicas(args):
    """
    %(prog)s list [options] <field1=value1 field2=value2 ...>

    List dataset replicas
    """
    client = get_client(args)
    datasets = {}
    result = {}
    scope, name = extract_scope(args.did)
    meta = client.get_metadata(scope, name)
    if meta['did_type'] != 'DATASET' or args.deep:
        dids = client.scope_list(scope=scope, name=name, recursive=True)
        for did in dids:
            if did['type'] == 'FILE':
                dsn = '%s:%s' % (did['parent']['scope'], did['parent']['name'])
                if dsn not in datasets:
                    datasets[dsn] = 0
                datasets[dsn] += 1
    else:
        datasets['%s:%s' % (scope, name)] = 0
    for dsn in datasets:
        scope, name = extract_scope(dsn)
        result[dsn] = {}
        if args.deep:
            replicas = client.list_replicas([{'scope': scope, 'name': name}])
            for replica in replicas:
                for rse in replica['rses']:
                    if rse not in result[dsn]:
                        result[dsn][rse] = [rse, 0, datasets[dsn]]
                    if replica['rses'][rse] != []:
                        result[dsn][rse][1] += 1
        else:
            for rep in client.list_dataset_replicas(scope, name):
                result[dsn][rep['rse']] = [rep['rse'], rep['available_length'], rep['length']]
    if args.csv:
        for dsn in result:
            for rse in result[dsn].values():
                print "{0}, {1}, {2}".format(rse[0], rse[1], rse[2])
    else:
        for dsn in result:
            print '\nDATASET: %s' % (dsn)
            print tabulate.tabulate(result[dsn].values(), tablefmt=tablefmt, headers=['RSE', 'FOUND', 'TOTAL'])
    return SUCCESS


@exception_handler
def list_file_replicas(args):
    """
    %(prog)s list [options] <field1=value1 field2=value2 ...>

    List file replicas
    """
    client = get_client(args)
    protocols = None
    if args.protocols:
        protocols = args.protocols.split(',')

    table = []
    dids = []
    rse_dict = {}
    if args.missing and not args.selected_rse:
        print 'Cannot use --missing without specifying a RSE'
        return FAILURE
    if args.selected_rse and args.list_collections:
        print 'Cannot use --rse with --list_collections option'
        return FAILURE
    if args.list_collections and len(args.dids) > 1:
        print 'Cannot use --list_collections option with multiple dids'
        return FAILURE

    for did in args.dids:
        scope, name = extract_scope(did)
        dids.append({'scope': scope, 'name': name})
    replicas = client.list_replicas(dids, schemes=protocols,
                                    all_states=args.all_states,
                                    rse_expression=args.rse_expression)
    if args.missing:
        for replica in replicas:
            rses = replica['rses'].keys()
            if args.selected_rse not in rses:
                table.append([replica['scope'], replica['name']])
        print tabulate.tabulate(table, tablefmt=tablefmt, headers=['SCOPE', 'NAME'])
    else:
        for replica in replicas:
            if 'bytes' in replica:
                for rse in replica['rses']:
                    if args.list_collections:
                        if rse not in rse_dict:
                            rse_dict[rse] = 0
                        if replica['rses'][rse] != []:
                            rse_dict[rse] += 1
                    else:
                        for pfn in replica['rses'][rse]:
                            if args.selected_rse:
                                if rse == args.selected_rse:
                                    table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], '{0}: {1}'.format(rse, pfn)])
                            else:
                                table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], '{0}: {1}'.format(rse, pfn)])
            elif not args.list_collections:
                table.append([replica['scope'], replica['name'], '???', '???', 'Unavailable'])
        if args.list_collections:
            nbfiles = len([i for i in client.list_files(dids[0]['scope'], dids[0]['name'])])
            sorted_key = rse_dict.keys()
            sorted_key.sort()
            table = []
            for rse in sorted_key:
                table.append([rse, rse_dict[rse], nbfiles])
        if args.list_collections:
            logger.warning('This option is deprecated. Please use rucio list-dataset-replicas instead.')
            print tabulate.tabulate(table, tablefmt=tablefmt, headers=['RSE', 'Found', 'Total'])
        else:
            print tabulate.tabulate(table, tablefmt=tablefmt, headers=['SCOPE', 'NAME', 'FILESIZE', 'ADLER32', 'RSE: REPLICA'])
    return SUCCESS


@exception_handler
def add_dataset(args):
    """
    %(prog)s add-dataset [options] <dsn>

    Add a dataset identifier.
    """
    client = get_client(args)
    scope, name = extract_scope(args.did)
    client.add_dataset(scope=scope, name=name, statuses={'monotonic': args.monotonic}, lifetime=args.lifetime)
    print 'Added %s:%s' % (scope, name)
    return SUCCESS


@exception_handler
def add_container(args):
    """
    %(prog)s add-container [options] <dsn>

    Add a container identifier.
    """
    client = get_client(args)
    scope, name = extract_scope(args.did)
    client.add_container(scope=scope, name=name, statuses={'monotonic': args.monotonic}, lifetime=args.lifetime)
    print 'Added %s:%s' % (scope, name)
    return SUCCESS


@exception_handler
def attach(args):
    """
    %(prog)s attach [options] <field1=value1 field2=value2 ...>

    Attach a data identifier.
    """
    client = get_client(args)
    scope, name = extract_scope(args.todid)
    dids = []
    if args.fromfile:
        if len(args.dids) > 1:
            logger.error("If --fromfile option is active, only one file is supported. The file should contain a list of dids, one per line.")
            return FAILURE
        try:
            f = open(args.dids[0], 'r')
            for did in f.readlines():
                cscope, cname = extract_scope(did.rstrip())
                dids.append({'scope': cscope, 'name': cname})
        except IOError:
            logger.error("Can't open file '" + args.dids[0] + "'.")
            return FAILURE
    else:
        for did in args.dids:
            cscope, cname = extract_scope(did)
            dids.append({'scope': cscope, 'name': cname})
    client.attach_dids(scope=scope, name=name, dids=dids)
    print 'DIDs successfully attached to %s:%s' % (scope, name)
    return SUCCESS


@exception_handler
def detach(args):
    """
    %(prog)s detach [options] <field1=value1 field2=value2 ...>

    Detach data identifier.
    """
    client = get_client(args)
    scope, name = extract_scope(args.fromdid)
    dids = []
    for did in args.dids:
        cscope, cname = extract_scope(did)
        dids.append({'scope': cscope, 'name': cname})
    client.detach_dids(scope=scope, name=name, dids=dids)
    return SUCCESS


@exception_handler
def list_dids(args):
    """
    %(prog)s list-dids scope[:*|:name] [--filter 'key=value' | --recursive]

    List the data identifiers for a given scope.
    """
    client = get_client(args)
    filters = {}
    type = 'collection'
    table = []
    if args.filter:
        if args.recursive:
            logger.error('Option recursive and filter cannot be used together')
            return FAILURE
        else:
            try:
                for key, value in [(a.split('=')[0], a.split('=')[1]) for a in args.filter.split(',')]:
                    if key == 'type':
                        if value.upper() in ['ALL', 'COLLECTION', 'CONTAINER', 'DATASET', 'FILE']:
                            type = value.lower()
                        else:
                            logger.error('{0} is not a valid type. Valid types are {1}'.format(value, ['ALL', 'COLLECTION', 'CONTAINER', 'DATASET', 'FILE']))
                            return FAILURE
                    else:
                        filters[key] = value
            except Exception:
                logger.error("Invalid Filter. Filter must be 'key=value'")
                return FAILURE
    try:
        scope, name = extract_scope(args.did[0])
        if name == '':
            name = '*'
    except InvalidObject:
        scope = args.did[0]
        name = '*'
    if scope not in client.list_scopes():
        logger.error('Scope not found')
        return FAILURE
    if name.find('*') > -1:
        if args.recursive:
            logger.error('Option recursive cannot be used with wildcards')
            return FAILURE
        elif ('name' in filters) and (name != '*'):
            logger.error('You cannot use a wildcard query and a filter by name')
            return FAILURE
        filters['name'] = name
        for did in client.list_dids(scope, filters=filters, type=type, long=True):
            table.append(['%s:%s' % (did['scope'], did['name']), did['did_type']])
    else:
        dids = client.scope_list(scope=scope, name=name, recursive=args.recursive)
        no_result = True
        for did in dids:
            table.append(['%s:%s' % (did['scope'], did['name']), did['type'].upper()])
        if no_result:
            did_info = client.get_did(scope=scope, name=name)
            table.append(['%s:%s' % (did_info['scope'], did_info['name']), did_info['type'].upper()])

    if args.short:
        for did, dummy in table:
            print did
    else:
        print tabulate.tabulate(table, tablefmt=tablefmt, headers=['SCOPE:NAME', '[DID TYPE]'])
    return SUCCESS


@exception_handler
def list_scopes(args):
    """
    %(prog)s list-scopes <scope>

    List scopes.
    """
    # For the moment..
    client = get_client(args)
    scopes = client.list_scopes()
    for scope in scopes:
        print scope
    return SUCCESS


@exception_handler
def list_files(args):
    """
    %(prog)s list-files [options] <field1=value1 field2=value2 ...>

    List data identifier contents.
    """
    client = get_client(args)
    if args.csv:
        for did in args.dids:
            scope, name = extract_scope(did)
            for f in client.list_files(scope=scope, name=name):
                guid = f['guid']
                guid = '%s-%s-%s-%s-%s' % (guid[0:8], guid[8:12], guid[12:16], guid[16:20], guid[20:32])
                print "{0}:{1},{2},{3},{4},{5}".format(f['scope'], f['name'], guid, f['adler32'], sizefmt(f['bytes'], args.human), f['events'])
        return SUCCESS
    table = []
    for did in args.dids:
        totfiles = 0
        totsize = 0
        totevents = 0
        scope, name = extract_scope(did)
        for file in client.list_files(scope=scope, name=name):
            totfiles += 1
            totsize += int(file['bytes'])
            if file['events']:
                totevents += int(file.get('events', 0))
            guid = file['guid']
            guid = '%s-%s-%s-%s-%s' % (guid[0:8], guid[8:12], guid[12:16], guid[16:20], guid[20:32])
            table.append(['%s:%s' % (file['scope'], file['name']), guid, 'ad:%s' % file['adler32'], sizefmt(file['bytes'], args.human), file['events']])
        print tabulate.tabulate(table, tablefmt=tablefmt, headers=['SCOPE:NAME', 'GUID', 'ADLER32', 'FILESIZE', 'EVENTS'])
        print 'Total files : %s' % totfiles
        print 'Total size : %s' % sizefmt(totsize, args.human)
        if totevents:
            print 'Total events : %s' % totevents
    return SUCCESS


@exception_handler
def list_content(args):
    """
    %(prog)s list-content [options] <field1=value1 field2=value2 ...>

    List data identifier contents.
    """
    client = get_client(args)
    table = []
    for did in args.dids:
        scope, name = extract_scope(did)
        for content in client.list_content(scope=scope, name=name):
            table.append(['%s:%s' % (content['scope'], content['name']), content['type'].upper()])
    print tabulate.tabulate(table, tablefmt=tablefmt, headers=['SCOPE:NAME', '[DID TYPE]'])
    return SUCCESS


@exception_handler
def list_parent_dids(args):
    """
    %(prog)s list-parent-dids

    List parent data identifier.
    """
    client = get_client(args)
    table = []
    scope, name = extract_scope(args.did)
    for dataset in client.list_parent_dids(scope=scope, name=name):
        table.append(['%s:%s' % (dataset['scope'], dataset['name']), dataset['type']])
    print tabulate.tabulate(table, tablefmt=tablefmt, headers=['SCOPE:NAME', '[DID TYPE]'])
    return SUCCESS


@exception_handler
def close(args):
    """
    %(prog)s close [options] <field1=value1 field2=value2 ...>

    Close a dataset or container.
    """
    client = get_client(args)
    for did in args.dids:
        scope, name = extract_scope(did)
        client.set_status(scope=scope, name=name, open=False)
        print '%(scope)s:%(name)s has been closed.' % locals()
    return SUCCESS


@exception_handler
def reopen(args):
    """
    %(prog)s reopen [options] <field1=value1 field2=value2 ...>

    Reopen a dataset or container (only for privileged users).
    """
    client = get_client(args)
    for did in args.dids:
        scope, name = extract_scope(did)
        client.set_status(scope=scope, name=name, open=True)
        print '%(scope)s:%(name)s has been reopened.' % locals()
    return SUCCESS


@exception_handler
def stat(args):
    """
    %(prog)s stat [options] <field1=value1 field2=value2 ...>

    List attributes and statuses about data identifiers..
    """
    client = get_client(args)
    for did in args.dids:
        scope, name = extract_scope(did)
        info = client.get_did(scope=scope, name=name)
        for key, val in info.iteritems():
            print '%(key)s: %(val)s' % locals()
    return SUCCESS


def erase(args):
    """
    %(prog)s erase [options] <field1=value1 field2=value2 ...>

    Delete data identifier.
    """
    client = get_client(args)
    for did in args.dids:
        scope, name = extract_scope(did)
        if args.undo:
            try:
                client.set_metadata(scope=scope, name=name, key='lifetime', value=None)
                logger.info('Erase undo for DID: {0}:{1}'.format(scope, name))
            except Exception:
                logger.warning('Cannot undo erase operation on DID. DID not existent or grace period of 24 hours already expired.')
                logger.warning('    DID: {0}:{1}'.format(scope, name))
        else:
            # set lifetime to expire in 24 hours (value is in seconds).
            client.set_metadata(scope=scope, name=name, key='lifetime', value=86400)
            logger.info('CAUTION! erase operation is irreversible after 24 hours. To cancel this operation you can run the following command:')
            print "rucio erase --undo {0}:{1}".format(scope, name)
    return SUCCESS


def __get_dataset(args):
    '''Parse helper for upload'''
    dsscope = None
    dsname = None
    for item in args:
        if item.count(':') == 1:
            if dsscope:
                raise Exception("Only one dataset should be given")
            else:
                dsscope, dsname = item.split(':')
    return dsscope, dsname


def __get_files(args):
    '''Parse helper for upload'''
    files = []
    for item in args:
        if os.path.isdir(item):
            dname, subdirs, fnames = os.walk(item).next()
            for fname in fnames:
                files.append(os.path.join(dname, fname))
        elif os.path.isfile(item):
            files.append(item)
        else:
            logger.warning('%s does not exist' % item)
    return files


@exception_handler
def upload(args):
    """
    rucio upload [scope:datasetname] [folder/] [files1 file2 file3]
    %(prog)s upload [options] <field1=value1 field2=value2 ...>

    Upload files into Rucio
    """
    client = get_client(args)
    try:
        dsscope, dsname = __get_dataset(args.args)    # None, None if no dataset given
    except Exception:
        logger.error('rucio upload only allows to upload files to one dataset, more than one provided.')
        return FAILURE
    files = __get_files(args.args)               # a list of file names (even if a directory is given)
    if not files:
        return FAILURE
    list_files = []
    files_to_list = []
    lfns = {}
    revert_dict = {}
    if args.scope:
        fscope = args.scope
    else:
        fscope = 'user.' + client.account
    if args.no_register is False and fscope not in client.list_scopes_for_account(client.account):
        logger.error("Cannot guess the scope for the files. You must specify one with --scope option. The scope must be one of: " + ', '.join(client.list_scopes_for_account(client.account)))
        return FAILURE

    trace = {}
    trace['hostname'] = socket.getfqdn()
    trace['scope'] = fscope
    trace['uuid'] = generate_uuid()
    for name in files:
        try:
            size = os.stat(name).st_size
            checksum = adler32(name)
            logger.debug('Extracting filesize (%s) and checksum (%s) for file %s:%s' % (str(size), checksum, fscope, os.path.basename(name)))
            files_to_list.append({'scope': fscope, 'name': os.path.basename(name)})
            if 'pool.root' in name.lower():  # is a root file, getting the GUID
                status, output, err = execute('pool_extractFileIdentifier {0}'.format(name))
                if status != 0:
                    logger.error('Trying to upload ROOT files but pool_extractFileIdentifier tool can not be found.')
                    logger.error('Setup your ATHENA environment and try again.')
                    return FAILURE
                try:
                    logger.debug('Extracting GUID from POOL file: {0}'.format(output.splitlines()[-1].split()[0].replace('-', '').lower()))
                    guid = output.splitlines()[-1].split()[0].replace('-', '').lower()
                except Exception:
                    logger.error('Error during GUID extraction. Failing. None of the files will be uploaded.')
                    return FAILURE
                list_files.append({'scope': fscope, 'name': os.path.basename(name), 'bytes': size, 'adler32': checksum, 'state': 'C', 'meta': {'guid': guid}})
            else:
                list_files.append({'scope': fscope, 'name': os.path.basename(name), 'bytes': size, 'adler32': checksum, 'state': 'C', 'meta': {'guid': generate_uuid()}})
            if not os.path.dirname(name) in lfns:
                lfns[os.path.dirname(name)] = []
            lfns[os.path.dirname(name)].append({'name': os.path.basename(name), 'scope': fscope, 'adler32': checksum, 'filesize': size})
            revert_dict[fscope, os.path.basename(name)] = os.path.dirname(name)

        except OSError, error:
            logger.error(error)
            logger.error("No operation will be performed. Exiting!")
            return FAILURE

    rse_settings = rsemgr.get_rse_info(args.rse)
    if rse_settings['availability_write'] != 1:
        logger.critical('RSE is not available for write now')
        return FAILURE

    if args.account is None:
        account = client.whoami()['account']
    else:
        account = args.account
    logger.debug('Using account %s' % (account))

    trace['account'] = client.account
    trace['dataset'] = ''
    trace['datasetScope'] = ''
    trace['eventType'] = 'upload'
    trace['eventVersion'] = version.RUCIO_VERSION[0]
    if dsscope and dsname:
        if files_to_list.count({'scope': dsscope, 'name': dsname}) > 0:
            # There is a file with the name of the destination dataset.
            logger.error('scope:name for the files must be different from scope:name for the destination dataset. {0}:{1}'.format(dsscope, dsname))
            return FAILURE
        try:
            client.add_dataset(scope=dsscope, name=dsname, rules=[{'account': client.account, 'copies': 1, 'rse_expression': args.rse, 'grouping': 'DATASET', 'lifetime': args.lifetime}])
            logger.info('Dataset successfully created')
            trace['dataset'] = dsname
            trace['datasetScope'] = dsscope
        except DataIdentifierAlreadyExists:
            # TODO: Need to check the rules thing!!
            logger.warning("The dataset name already exist")

    # Adding files to the catalog
    for f in list_files:
        try:  # If the did already exist in the catalog, only should be upload if the checksum is the same
            meta = client.get_metadata(f['scope'], f['name'])
            replicastate = [rep for rep in client.list_replicas([{'scope': f['scope'], 'name': f['name']}], all_states=True)]
            if args.rse not in replicastate[0]['rses']:
                client.add_replicas(files=[f], rse=args.rse)
            # logger.warning("The file {0}:{1} already exist in the catalog and will not be added.".format(f['scope'], f['name']))
            if rsemgr.exists(rse_settings=rse_settings, files={'name': f['name'], 'scope': f['scope']}):
                logger.warning('File {0}:{1} already exists on RSE. Will not try to reupload'.format(f['scope'], f['name']))
            else:
                if meta['adler32'] == f['adler32']:
                    logger.info('Local files and file %s:%s recorded in Rucio have the same checksum. Will try the upload' % (f['scope'], f['name']))
                    directory = revert_dict[f['scope'], f['name']]
                    trace['remoteSite'] = rse_settings['rse']
                    trace['protocol'] = rse_settings['protocols'][0]['scheme']
                    trace['filesize'] = f['bytes']
                    trace['transferStart'] = time.time()
                    rsemgr.upload(rse_settings=rse_settings, lfns=[{'name': f['name'], 'scope': f['scope'], 'adler32': f['adler32'], 'filesize': f['bytes']}], source_dir=directory)
                    trace['transferEnd'] = time.time()
                    trace['clientState'] = 'DONE'
                    logger.info('File %s:%s successfully uploaded on the storage' % (f['scope'], f['name']))
                    retries = 2
                    for dummy in xrange(retries):
                        try:
                            requests.post(client.host + '/traces/', verify=False, data=json.dumps(trace))
                            break
                        except:
                            pass
                else:
                    raise DataIdentifierAlreadyExists
        except NotImplementedError, error:
            for proto in rse_settings['protocols']:
                if proto['domains']['wan']['read'] == 1:
                    prot = proto['scheme']
            logger.error('Protocol {0} for RSE {1} not supported!'.format(prot, args.rse))
            return FAILURE
        except DataIdentifierNotFound:
            try:
                if args.no_register is False:
                    # Skiping registration for pilot
                    logger.info('Adding replicas in Rucio catalog')
                    client.add_replicas(files=[f], rse=args.rse)
                    logger.info('Replicas successfully added')
                    if not dsscope:
                        # only need to add rules for files if no dataset is given
                        logger.info('Adding replication rule on RSE {0} for the file {1}:{2}'.format(args.rse, f['scope'], f['name']))
                        client.add_replication_rule([f], copies=1, rse_expression=args.rse, lifetime=args.lifetime)
                directory = revert_dict[f['scope'], f['name']]
                trace['remoteSite'] = rse_settings['rse']
                trace['protocol'] = rse_settings['protocols'][0]['scheme']
                trace['filesize'] = f['bytes']
                trace['transferStart'] = time.time()
                rsemgr.upload(rse_settings=rse_settings, lfns=[{'name': f['name'], 'scope': f['scope'], 'adler32': f['adler32'], 'filesize': f['bytes']}], source_dir=directory)
                trace['transferEnd'] = time.time()
                trace['clientState'] = 'DONE'
                logger.info('File {0}:{1} successfully uploaded on the storage'.format(f['scope'], f['name']))
                retries = 2
                for dummy in xrange(retries):
                    try:
                        requests.post(client.host + '/traces/', verify=False, data=json.dumps(trace))
                        break
                    except:
                        pass
            except (Duplicate, FileAlreadyExists), error:
                logger.warning(error)
                return FAILURE
            except ResourceTemporaryUnavailable, error:
                logger.error(error)
                return FAILURE
        except DataIdentifierAlreadyExists, error:
            logger.debug(error)
            logger.error("Some of the files already exist in the catalog. No one will be added.")
    if dsname:
        # A dataset is provided. Must add the files to the dataset.
        for f in list_files:
            try:
                client.add_files_to_dataset(scope=dsscope, name=dsname, files=[f])
            except Exception, error:
                logger.warning('Failed to attach file {0} to the dataset'.format(f))
                logger.warning(error)
                logger.warning("Continuing with the next one")

    replicas = []
    replica_dictionary = {}
    for chunk_files_to_list in chunks(files_to_list, 50):
        for rep in client.list_replicas(chunk_files_to_list):
            replica_dictionary[rep['scope'], rep['name']] = rep['rses'].keys()
    for file in list_files:
        if (file['scope'], file['name']) not in replica_dictionary:
            file['state'] = 'A'
            replicas.append(file)
        elif args.rse not in replica_dictionary[file['scope'], file['name']]:
            file['state'] = 'A'
            replicas.append(file)
    if args.no_register is False and replicas != []:
        logger.info('Will update the file replicas states')
        for chunk_replicas in chunks(replicas, 20):
            try:
                client.update_replicas_states(rse=args.rse, files=chunk_replicas)
            except AccessDenied, error:
                logger.error(error)
                return FAILURE
        logger.info('File replicas states successfully updated')
    return SUCCESS


def _downloader2(total_number, scope, name, files, tape_endpoints, is_admin, last_chosen_rse, rse_dict, dest_dir, trace, client, exit=False):
    for file in files:
        logger.info('Starting the download of %s:%s' % (file['scope'], file['name']))
        stime = time.time()
        trace['scope'] = file['scope']
        trace['filename'] = file['name']
        if scope == file['scope'] and name == file['name']:
            trace['dataset'] = ''
            trace['datasetScope'] = ''
        else:
            trace['dataset'] = name
            trace['datasetScope'] = scope
        rses = file['rses'].keys()
        if rses == []:
            logger.warning('File %s:%s has no available replicas. Cannot be downloaded.' % (file['scope'], file['name']))
            trace['clientState'] = 'FILE_NOT_FOUND'
            trace['eventType'] = 'download'
            trace['eventVersion'] = version.RUCIO_VERSION[0]
            requests.post(client.host + '/traces/', verify=False, data=json.dumps(trace))
        else:
            trace['filesize'] = file['bytes']
        trace['eventType'] = 'download'
        trace['eventVersion'] = version.RUCIO_VERSION[0]
        rse_to_remove = []
        for rse in rses:
            if str(rse) in tape_endpoints:
                rse_to_remove.append(rse)
        if len(rses) == len(rse_to_remove):
            if not is_admin:
                logger.warning('File %s:%s has no replicas available on disk endpoints and cannot be downloaded. Please ask for a replication' % (file['scope'], file['name']))
                sys.exit(1)
            else:
                logger.warning('File %s:%s has no replicas available on disk endpoints. Will be downloaded from TAPE.' % (file['scope'], file['name']))
        if last_chosen_rse and last_chosen_rse in rses:
            idx = rses.index(last_chosen_rse)
            rses[idx], rses[0] = rses[0], rses[idx]
        else:
            random.shuffle(rses)
        logger.debug('Choosing RSE')
        index = 0
        download_ok = 0
        if args.rse:
            if args.rse in rses:
                rses = [args.rse]
            else:
                rses = []
        for rse in rses:
            if rse not in rse_dict:
                rse_dict[rse] = rsemgr.get_rse_info(rse)
            if args.protocol:
                try:
                    protocol = rsemgr.select_protocol(rse_dict[rse], operation='read', scheme=args.protocol)
                except RSEProtocolNotSupported, error:
                    logger.error('The protocol specfied (%s) is not supported by the RSE %s' % (args.protocol, rse))
                    logger.debug(error)
                    return FAILURE
                rse_dict[rse]['protocols'] = [protocol, ]
            if rse_dict[rse]['availability_read']:
                logger.debug('Getting file %s:%s from %s' % (file['scope'], file['name'], rse))
                trace['remoteSite'] = rse_dict[rse]['rse']
                trace['protocol'] = rse_dict[rse]['protocols'][0]['scheme']
                trace['transferStart'] = time.time()
                trace['clientState'] = 'DOWNLOAD_ATTEMPT'
                success = False
                retries = 5
                for attempt in xrange(0, retries):
                    try:
                        rsemgr.download(rse_dict[rse], files=[{'name': file['name'], 'scope': file['scope'], 'adler32': file['adler32']}, ], dest_dir=dest_dir, printstatements=True)
                        logger.info('File %s:%s successfully downloaded from %s' % (file['scope'], file['name'], rse))
                        download_ok = 1
                        success = True
                        last_chosen_rse = rse
                        break
                    except FileConsistencyMismatch, error:
                        logger.warning(str(error))
                        try:
                            pfns_dict = rsemgr.lfns2pfns(rse_dict[rse], lfns=[{'name': file['name'], 'scope': file['scope']}, ], operation='read', scheme=args.protocol)
                            pfn = pfns_dict['%s:%s' % (file['scope'], file['name'])]
                            client.declare_suspicious_file_replicas([pfn, ], reason='Corrupted')
                        except Exception, error:
                            logger.debug(str(error))
                        trace['clientState'] = 'FAIL_VALIDATE'
                        logger.debug('Failed attempt %s/%s' % (attempt + 1, retries))
                    except Exception, error:
                        logger.warning(str(error))
                        trace['clientState'] = str(type(error).__name__)
                        logger.debug('Failed attempt %s/%s' % (attempt + 1, retries))
                if success:
                    trace['clientState'] = 'DONE'
                trace['transferEnd'] = time.time()
                for dummy in xrange(retries):
                    try:
                        requests.post(client.host + '/traces/', verify=False, data=json.dumps(trace))
                        break
                    except Exception:
                        pass
                if success:
                    break
                index += 1
                if index != len(rses):
                    logger.debug('Will retry download on an other RSE')
        if not download_ok:
            logger.error('Cannot download file %s:%s' % (file['scope'], file['name']))
        else:
            logger.info('File %s:%s successfully downloaded. %s bytes downloaded in %s seconds' % (file['scope'], file['name'], sizefmt(file['bytes'], args.human), time.time() - stime))
    if exit:
        sys.exit(0)
    return


def _downloader1(input_queue, output_queue, threadnb, total_threads, tape_endpoints, is_admin, trace_endpoint, trace_pattern):
    last_chosen_rse = None
    rse_dict = {}
    try:
        while(True):
            trace = deepcopy(trace_pattern)
            stime = time.time()
            file = input_queue.get_nowait()
            logger.info('Thread %s/%s : Starting the download of %s:%s' % (threadnb, total_threads, file['scope'], file['name']))
            trace.update({'scope': file['scope'], 'name': file['name'], 'datasetScope': file['datasetScope'], 'dataset': file['datasetName'], 'eventType': 'download', 'eventVersion': version.RUCIO_VERSION[0], 'filesize': file['bytes']})
            rses = file['rses'].keys()
            dest_dir = file['dest_dir']
            if rses == []:
                logger.warning('Thread %s/%s : File %s:%s has no available replicas. Cannot be downloaded.' % (threadnb, total_threads, file['scope'], file['name']))
                trace['clientState'] = 'FILE_NOT_FOUND'
                if send_trace(trace, trace_endpoint, retries=5):
                    logger.warning('Thread %s/%s : Problem sending traces' % (threadnb, total_threads))
            rse_to_remove = []
            for rse in rses:
                if str(rse) in tape_endpoints:
                    rse_to_remove.append(rse)
            if len(rses) == len(rse_to_remove):
                if not is_admin:
                    logger.warning('Thread %s/%s : File %s:%s has no replicas available on disk endpoints and cannot be downloaded. Please ask for a replication' % (threadnb, total_threads, file['scope'], file['name']))
                    sys.exit(1)
                else:
                    logger.warning('Thread %s/%s : File %s:%s has no replicas available on disk endpoints. Will be downloaded from TAPE.' % (threadnb, total_threads, file['scope'], file['name']))
            if last_chosen_rse and last_chosen_rse in rses:
                idx = rses.index(last_chosen_rse)
                rses[idx], rses[0] = rses[0], rses[idx]
            else:
                random.shuffle(rses)
            logger.debug('Thread %s/%s : Choosing RSE' % (threadnb, total_threads))
            index = 0
            download_ok = 0
            if args.rse:
                if args.rse in rses:
                    rses = [args.rse]
                else:
                    rses = []
            for rse in rses:
                if rse not in rse_dict:
                    rse_dict[rse] = rsemgr.get_rse_info(rse)
                if args.protocol:
                    try:
                        protocol = rsemgr.select_protocol(rse_dict[rse], operation='read', scheme=args.protocol)
                    except RSEProtocolNotSupported, error:
                        logger.error('Thread %s/%s : The protocol specfied (%s) is not supported by the RSE %s' % (threadnb, total_threads, args.protocol, rse))
                        logger.debug(error)
                        return FAILURE
                    rse_dict[rse]['protocols'] = [protocol, ]
                if rse_dict[rse]['availability_read']:
                    logger.debug('Thread %s/%s : Getting file %s:%s from %s' % (threadnb, total_threads, file['scope'], file['name'], rse))
                    trace.update({'remoteSite': rse_dict[rse]['rse'], 'protocol': rse_dict[rse]['protocols'][0]['scheme'], 'transferStart': time.time(), 'clientState': 'DOWNLOAD_ATTEMPT'})
                    success = False
                    retries = 5
                    for attempt in xrange(0, retries):
                        try:
                            rsemgr.download(rse_dict[rse], files=[{'name': file['name'], 'scope': file['scope'], 'adler32': file['adler32']}, ], dest_dir=dest_dir, printstatements=True)
                            logger.info('Thread %s/%s : File %s:%s successfully downloaded from %s' % (threadnb, total_threads, file['scope'], file['name'], rse))
                            download_ok = 1
                            success = True
                            last_chosen_rse = rse
                            break
                        except KeyboardInterrupt:
                            logger.warning('You pressed Ctrl+C! Exiting gracefully')
                            os.kill(os.getpgid(), signal.SIGINT)
                            break
                        except FileConsistencyMismatch, error:
                            logger.warning(str(error))
                            try:
                                pfns_dict = rsemgr.lfns2pfns(rse_dict[rse], lfns=[{'name': file['name'], 'scope': file['scope']}, ], operation='read', scheme=args.protocol)
                                pfn = pfns_dict['%s:%s' % (file['scope'], file['name'])]
                                output_queue.put({'dataset_scope': file['datasetScope'], 'dataset_name': file['datasetName'], 'scope': file['scope'], 'name': file['name'], 'clientState': 'CORRUPTED', 'attemptnr': attempt + 1, 'pfn': pfn})
                            except Exception, error:
                                logger.debug(str(error))
                            trace['clientState'] = 'FAIL_VALIDATE'
                            logger.debug('Thread %s/%s : Failed attempt %s/%s' % (threadnb, total_threads, attempt + 1, retries))
                        except Exception, error:
                            logger.warning(str(error))
                            trace['clientState'] = str(type(error).__name__)
                            logger.debug('Thread %s/%s : Failed attempt %s/%s' % (threadnb, total_threads, attempt + 1, retries))
                    if success:
                        trace['clientState'] = 'DONE'
                        output_queue.put({'dataset_scope': file['datasetScope'], 'dataset_name': file['datasetName'], 'scope': file['scope'], 'name': file['name'], 'clientState': 'DONE', 'attemptnr': attempt + 1})
                    trace['transferEnd'] = time.time()
                    if send_trace(trace, trace_endpoint, retries=5):
                        logger.warning('You pressed Ctrl+C! Exiting gracefully')
                    if success:
                        break
                    index += 1
                    if index != len(rses):
                        logger.debug('Thread %s/%s : Will retry download on an other RSE' % (threadnb, total_threads))
            if not download_ok:
                logger.error('Thread %s/%s : Cannot download file %s:%s' % (threadnb, total_threads, file['scope'], file['name']))
            else:
                logger.info('Thread %s/%s : File %s:%s successfully downloaded. %s bytes downloaded in %s seconds' % (threadnb, total_threads, file['scope'], file['name'], sizefmt(file['bytes'], args.human), time.time() - stime))
            input_queue.task_done()
    except Empty:
        return


def _file_exists(type, scope, name, directory, dsn=None, no_subdir=False):
    file_exists = False
    dest_dir = None
    if no_subdir:
        dest_dir = '%s' % (directory)
    else:
        if type != 'FILE':
            dest_dir = '%s/%s' % (directory, dsn)
            if os.path.isfile('%s/%s' % (dest_dir, name)):
                file_exists = True
        else:
            dest_dir = '%s/%s' % (directory, scope)
            if os.path.isfile('%s/%s' % (dest_dir, name)):
                file_exists = True
    return file_exists, dest_dir


def download(args):
    """
    %(prog)s download [options] <field1=value1 field2=value2 ...>

    Download files from Rucio
    """
    if args.new:
        return download1(args)
    else:
        return download2(args)


def download1(args):
    """
    %(prog)s download [options] <field1=value1 field2=value2 ...>

    Download files from Rucio
    """
    client = get_client(args)
    trace_endpoint = client.host
    summary = {}
    trace_pattern = {'hostname': socket.getfqdn(), 'account': client.account, 'uuid': generate_uuid(), 'eventType': 'download', 'eventVersion': version.RUCIO_VERSION[0]}
    nbfiles_to_download = {}
    try:
        tape_endpoints = [str(rse['rse']) for rse in client.list_rses('rse_type=TAPE')]
    except InvalidRSEExpression, error:
        logger.warning(error)
        tape_endpoints = []
    account_attributes = [acc for acc in client.list_account_attributes(client.account)]
    is_admin = False
    for attr in account_attributes[0]:
        logger.debug(attr)
        if attr['key'] == 'admin' and attr['value'] is True:
            is_admin = True
            break

    # Extract the scope, name from the did(s)
    dids = []
    for did in args.dids:
        try:
            scope, name = extract_scope(did)
            if name.find('*') > -1:
                for dsn in client.list_dids(scope, filters={'name': name}):
                    dids.append((scope, dsn))
            else:
                dids.append((scope, name))
        except ValueError, error:
            logger.error('ERROR cannot extract the scope and name from %s : [%s]' % (did, error))
            return FAILURE

    total_workers = 1
    if args.ndownloader:
        total_workers = args.ndownloader
        nlimit = 5
        if total_workers > nlimit:
            logger.warning('Cannot use more than %s parallel downloader.' % nlimit)
            total_workers = nlimit

    input_queue = Queue()
    output_queue = Queue()
    for scope, name in dids:
        try:
            summary['%s:%s' % (scope, name)] = {}
            did_info = client.get_did(scope, name)
            did_type = did_info['type']
            dataset_scope = '' if did_type == 'FILE' else scope
            dataset_name = '' if did_type == 'FILE' else name
            logger.debug('Getting the list of replicas')
            replicas = [f for f in client.list_replicas([{'scope': scope, 'name': name}])]
            if args.nrandom:
                files_to_download = replicas
                random.shuffle(files_to_download)
                files_to_download = files_to_download[0:args.nrandom]
            else:
                files_to_download = replicas
            nbfiles_to_download['%s:%s' % (scope, name)] = len(files_to_download)
            logger.info('Starting download for %s:%s with %s files' % (scope, name, len(files_to_download)))
            for file in files_to_download:
                file_download = file
                dest_dir = None
                file_exists, dest_dir = _file_exists(did_type, file['scope'], file['name'], args.dir, dsn=name, no_subdir=args.no_subdir)
                if file_exists:
                    logger.info('File %s:%s already exists locally' % (file['scope'], file['name']))
                    summary['%s:%s' % (scope, name)]['%s:%s' % (file['scope'], file['name'])] = 2
                    trace = deepcopy(trace_pattern)
                    # Filling and sending the trace
                    trace.update({'scope': file['scope'], 'filename': file['name'], 'datasetScope': dataset_scope, 'datasetName': dataset_name,
                                 'filesize': file['bytes'], 'transferStart': time.time(), 'transferEnd': time.time(), 'clientState': 'ALREADY_DONE'})
                    if send_trace(trace, trace_endpoint, retries=5):
                        logger.warning('Problem sending traces')
                    output_queue.put({'dataset_scope': dataset_scope, 'dataset_name': dataset_name, 'scope': file['scope'], 'name': file['name'], 'clientState': 'ALREADY_DONE'})
                else:
                    logger.debug('Will start downloading file %s:%s' % (file['scope'], file['name']))
                    if not os.path.isdir(dest_dir):
                        os.mkdir(dest_dir)
                    if args.no_subdir is True and os.path.isfile('%s/%s' % (dest_dir, file['name'])):
                        # Overwrite the files
                        os.remove("%s/%s" % (dest_dir, file['name']))
                    file_download['datasetScope'] = dataset_scope
                    file_download['datasetName'] = dataset_name
                    file_download['dest_dir'] = dest_dir
                    input_queue.put(file_download)

        except Exception, error:
            logger.error('Failed to download %(scope)s:%(name)s' % locals())
            logger.error(error)

    threads = []
    for worker in range(total_workers):
        thread = Thread(target=_downloader1, kwargs={'input_queue': input_queue, 'output_queue': output_queue, 'threadnb': worker + 1, 'total_threads': total_workers,
                        'tape_endpoints': tape_endpoints, 'is_admin': is_admin, 'trace_endpoint': trace_endpoint, 'trace_pattern': trace_pattern})
        thread.daemon = True
        thread.start()
        threads.append(thread)

    graceful_stop = None
    while not input_queue.empty():
        try:
            pass
        except KeyboardInterrupt:
            graceful_stop = True
            for thread in threads:
                thread.kill_received = True
            break

    if graceful_stop:
        logger.warning('You pressed Ctrl+C! Exiting gracefully')
    else:
        input_queue.join()

    while(True):
        try:
            item = output_queue.get_nowait()
            print item
            output_queue.task_done()
            if '%s:%s' % (item['dataset_scope'], item['dataset_name']) in summary:
                summary['%s:%s' % (item['dataset_scope'], item['dataset_name'])]['%s:%s' % (item['scope'], item['name'])] = item['clientState']
                if item['clientState'] == 'CORRUPTED':
                    client.declare_suspicious_file_replicas([item['pfn'], ], reason='Corrupted')
        except Empty:
            break

    print '----------------------------------'
    print 'Download summary'
    for did in summary:
        print '-' * 40
        print 'DID %s' % (did)
        downloaded_files = 0
        not_downloaded_files = 0
        local_files = 0
        for file in summary[did]:
            if summary[did][file] == 'DONE':
                downloaded_files += 1
            elif summary[did][file] == 'ALREADY_DONE':
                local_files += 1

        not_downloaded_files = nbfiles_to_download[did] - downloaded_files - local_files
        print '{0:40} {1:6d}'.format('Total files : ', nbfiles_to_download[did])
        print '{0:40} {1:6d}'.format('Downloaded files : ', downloaded_files)
        print '{0:40} {1:6d}'.format('Files already found locally : ', local_files)
        print '{0:40} {1:6d}'.format('Files that cannot be downloaded : ', not_downloaded_files)


def download2(args):
    """
    %(prog)s download [options] <field1=value1 field2=value2 ...>

    Download files from Rucio
    """
    client = get_client(args)
    rse_dict = {}
    summary = {}
    trace = {}
    trace_uuid = generate_uuid()
    trace['hostname'] = socket.getfqdn()
    trace['account'] = client.account
    trace['uuid'] = trace_uuid
    nbfiles_to_download = {}
    try:
        tape_endpoints = [str(rse['rse']) for rse in client.list_rses('rse_type=TAPE')]
    except InvalidRSEExpression, error:
        logger.warning(error)
        tape_endpoints = []
    account_attributes = [acc for acc in client.list_account_attributes(client.account)]
    is_admin = False
    for attr in account_attributes[0]:
        logger.debug(attr)
        if attr['key'] == 'admin' and attr['value'] is True:
            is_admin = True
            break

    dids = []
    for did in args.dids:
        try:
            scope, name = extract_scope(did)
            if name.find('*') > -1:
                for dsn in client.list_dids(scope, filters={'name': name}):
                    dids.append((scope, dsn))
            else:
                dids.append((scope, name))
        except ValueError, error:
            logger.error('ERROR cannot extract the scope and name from %s : [%s]' % (did, error))
            return FAILURE

    for scope, name in dids:
        try:
            did_info = client.get_did(scope, name)
            files_to_download = [f for f in client.list_files(scope=scope, name=name)]
            if args.nrandom:
                files_to_download = files_to_download[0:args.nrandom]
            nbfiles_to_download['%s:%s' % (scope, name)] = len(files_to_download)
            logger.info('Starting download for %s:%s with %s files' % (scope, name, len(files_to_download)))
            summary['%s:%s' % (scope, name)] = {}
            logger.debug('Getting the list of replicas')
            files = client.list_replicas([{'scope': scope, 'name': name}])
            if args.nrandom:
                files_to_download = [f for f in files]
                random.shuffle(files_to_download)
                files_to_download = files_to_download[0:args.nrandom]
            else:
                files_to_download = files
            last_chosen_rse = None
            dict_files_to_download = {}
            dict_files_to_download[0] = []
            total_workers = 1
            if args.ndownloader:
                total_workers = args.ndownloader
                nlimit = 5
                if total_workers > nlimit:
                    logger.warning('Cannot use more than %s parallel downloader.' % nlimit)
                    total_workers = nlimit
                for i in xrange(total_workers):
                    dict_files_to_download[i] = []
            for file in files_to_download:
                dest_dir = None
                file_exists, dest_dir = _file_exists(did_info['type'], file['scope'], file['name'], args.dir, dsn=name, no_subdir=args.no_subdir)
                if file_exists:
                    logger.info('File %s:%s already exists locally' % (file['scope'], file['name']))
                    summary['%s:%s' % (scope, name)]['%s:%s' % (file['scope'], file['name'])] = 2
                    # Filling and sending the trace
                    trace['scope'] = file['scope']
                    trace['filename'] = file['name']
                    if scope == file['scope'] and name == file['name']:
                        trace['dataset'] = ''
                        trace['datasetScope'] = ''
                    else:
                        trace['dataset'] = name
                        trace['datasetScope'] = scope
                    trace['filesize'] = file['bytes']
                    trace['eventType'] = 'download'
                    trace['eventVersion'] = version.RUCIO_VERSION[0]
                    trace['transferStart'] = time.time()
                    trace['transferEnd'] = time.time()
                    trace['clientState'] = 'ALREADY_DONE'
                    for dummy in xrange(5):
                        try:
                            requests.post(client.host + '/traces/', verify=False, data=json.dumps(trace))
                            break
                        except Exception:
                            pass
                else:
                    logger.debug('Will start downloading file %s:%s' % (file['scope'], file['name']))
                    if not os.path.isdir(dest_dir):
                        os.mkdir(dest_dir)
                    if args.no_subdir is True and os.path.isfile('%s/%s' % (dest_dir, file['name'])):
                        # Overwrite the files
                        os.remove("%s/%s" % (dest_dir, file['name']))
                    hash_md5 = hashlib.md5()
                    hash_md5.update('%s:%s' % (file['scope'], file['name']))
                    worker_number = int(hash_md5.hexdigest(), 16) % total_workers
                    dict_files_to_download[worker_number].append(file)

            if total_workers == 1 and dict_files_to_download:
                _downloader2(total_workers, scope, name, dict_files_to_download[0],
                             tape_endpoints, is_admin, last_chosen_rse,
                             rse_dict, dest_dir, trace, client, exit=False)
            else:
                pool_list = []
                for worker_number in dict_files_to_download:
                    proc = Process(target=_downloader2, args=(total_workers, scope, name,
                                   dict_files_to_download[worker_number], tape_endpoints,
                                   is_admin, last_chosen_rse, rse_dict, dest_dir, trace,
                                   client))
                    proc.start()
                    pool_list.append(proc)
                active_workers = pool_list

                while active_workers:
                    time.sleep(2)
                    active_workers = [worker for worker in pool_list if worker.is_alive()]

            for worker in dict_files_to_download:
                for file in dict_files_to_download[worker]:
                    file_exists, dest_dir = _file_exists(did_info['type'], file['scope'], file['name'], args.dir, dsn=name, no_subdir=args.no_subdir)
                    # If no_subdir is set, "Files already found locally" should be 0, truly existing files will be overwriten...
                    if file_exists:
                        summary['%s:%s' % (scope, name)]['%s:%s' % (file['scope'], file['name'])] = 1
                    elif os.path.isfile('%s/%s' % (dest_dir, file['name'])):
                        summary['%s:%s' % (scope, name)]['%s:%s' % (file['scope'], file['name'])] = 1

        except DataIdentifierAlreadyExists, error:
            logger.error(error)
            return FAILURE
        except Exception, error:
            logger.error('Failed to download %(scope)s:%(name)s' % locals())
            logger.error(error)

    logger.info('Download operation for %s:%s done' % (scope, name))
    print '----------------------------------'
    print 'Download summary'
    for did in summary:
        print '-' * 40
        print 'DID %s' % (did)
        downloaded_files = 0
        not_downloaded_files = 0
        local_files = 0
        for file in summary[did]:
            if summary[did][file] == 1:
                downloaded_files += 1
            elif summary[did][file] == 2:
                local_files += 1
        not_downloaded_files = nbfiles_to_download[did] - downloaded_files - local_files
        print '{0:40} {1:6d}'.format('Total files : ', nbfiles_to_download[did])
        print '{0:40} {1:6d}'.format('Downloaded files : ', downloaded_files)
        print '{0:40} {1:6d}'.format('Files already found locally : ', local_files)
        print '{0:40} {1:6d}'.format('Files that cannot be downloaded : ', not_downloaded_files)
    return SUCCESS


@exception_handler
def get_metadata(args):
    """
    %(prog)s get_metadata [options] <field1=value1 field2=value2 ...>

    Get data identifier metadata
    """
    client = get_client(args)
    for did in args.dids:
        scope, name = extract_scope(did)
        meta = client.get_metadata(scope=scope, name=name)
        for k in meta:
            print '%s: %s' % (k, meta[k])
    return SUCCESS


@exception_handler
def set_metadata(args):
    """
    %(prog)s set_metadata [options] <field1=value1 field2=value2 ...>

    Set data identifier metadata
    """
    client = get_client(args)
    scope, name = extract_scope(args.did)
    client.set_metadata(scope=scope, name=name, key=args.key, value=args.value)
    return SUCCESS


def delete_metadata(args):
    """
    %(prog)s set_metadata [options] <field1=value1 field2=value2 ...>

    Delete data identifier metadata
    """
    # For the moment..
    raise NotImplementedError


@exception_handler
def add_rule(args):
    """
    %(prog)s add-rule <did> <copies> <rse-expression> [options]

    Add a rule to a did.
    """
    client = get_client(args)
    dids = []
    for did in args.dids:
        scope, name = extract_scope(did)
        dids.append({'scope': scope, 'name': name})
    rule_ids = client.add_replication_rule(dids=dids,
                                           copies=args.copies,
                                           rse_expression=args.rse_expression,
                                           weight=args.weight,
                                           lifetime=args.lifetime,
                                           grouping=args.grouping,
                                           account=args.rule_account,
                                           locked=args.locked,
                                           source_replica_expression=args.source_replica_expression,
                                           notify=args.notify,
                                           activity=args.activity,
                                           comment=args.comment,
                                           ask_approval=args.ask_approval,
                                           asynchronous=args.asynchronous)
    for rule in rule_ids:
        print rule
    return SUCCESS


@exception_handler
def delete_rule(args):
    """
    %(prog)s delete-rule [options] <ruleid>

    Delete a rule.
    """
    client = get_client(args)
    try:
        # Test if the rule_id is a real rule_id
        uuid.UUID(args.rule_id)
        client.delete_replication_rule(rule_id=args.rule_id, purge_replicas=args.purge_replicas)
    except ValueError:
        # Otherwise, trying to extract the scope, name from args.rule_id
        if not args.rse_expression:
            logger.error('A RSE expression must be specified if you do not provide a rule_id but a DID')
            return FAILURE
        scope, name = extract_scope(args.rule_id)
        rules = client.list_did_rules(scope=scope, name=name)
        if args.rule_account is None:
            account = client.account
        else:
            account = args.rule_account
        for rule in rules:
            if args.delete_all:
                account_checked = True
            else:
                account_checked = rule['account'] == account
            if rule['rse_expression'] == args.rse_expression and account_checked:
                client.delete_replication_rule(rule_id=rule['id'], purge_replicas=args.purge_replicas)
    return SUCCESS


@exception_handler
def update_rule(args):
    """
    %(prog)s update-rule [options] <ruleid>

    Update a rule.
    """
    client = get_client(args)
    options = {}
    if args.lifetime:
        options['lifetime'] = None if args.lifetime.lower() == "none" else int(args.lifetime)
    if args.locked:
        if args.locked == "True":
            options['locked'] = True
        elif args.locked == "False":
            options['locked'] = False
    if args.rule_account:
        options['account'] = args.rule_account
    if args.state_stuck:
        options['state'] = 'STUCK'
    if args.state_suspended:
        options['state'] = 'SUSPENDED'
    if args.rule_activity:
        options['activity'] = args.rule_activity
    if args.source_replica_expression:
        options['source_replica_expression'] = None if args.source_replica_expression.lower() == 'none' else args.source_replica_expression
    if args.cancel_requests:
        options['cancel_requests'] = True
    if args.priority:
        options['priority'] = int(args.priority)
    client.update_replication_rule(rule_id=args.rule_id, options=options)
    print 'Updated Rule'
    return SUCCESS


@exception_handler
def info_rule(args):
    """
    %(prog)s rule-info [options] <ruleid>

    Retrieve information about a rule.
    """
    client = get_client(args)
    if args.examine:
        analysis = client.examine_replication_rule(rule_id=args.rule_id)
        print 'Status of the replication rule: %s' % analysis['rule_error']
        if analysis['transfers']:
            print 'STUCK Requests:'
            for transfer in analysis['transfers']:
                print '  %s:%s' % (transfer['scope'], transfer['name'])
                print '    RSE:                  %s' % str(transfer['rse'])
                print '    Attempts:             %s' % str(transfer['attempts'])
                print '    Last Retry:           %s' % str(transfer['last_time'])
                print '    Last error:           %s' % str(transfer['last_error'])
                print '    Last source:          %s' % str(transfer['last_source'])
                print '    Available sources:    %s' % ', '.join([source for source in transfer['sources'] if source[1]])
                print '    Blacklisted sources:  %s' % ', '.join([source for source in transfer['sources'] if not source[1]])
    else:
        rule = client.get_replication_rule(rule_id=args.rule_id)
        print "Id:                         %s" % rule['id']
        print "Account:                    %s" % rule['account']
        print "Scope:                      %s" % rule['scope']
        print "Name:                       %s" % rule['name']
        print "RSE Expression:             %s" % rule['rse_expression']
        print "Copies:                     %s" % rule['copies']
        print "State:                      %s" % rule['state']
        print "Locks OK/REPLICATING/STUCK: %s/%s/%s" % (rule['locks_ok_cnt'], rule['locks_replicating_cnt'], rule['locks_stuck_cnt'])
        print "Grouping:                   %s" % rule['grouping']
        print "Expires at:                 %s" % rule['expires_at']
        print "Locked:                     %s" % rule['locked']
        print "Weight:                     %s" % rule['weight']
        print "Created at:                 %s" % rule['created_at']
        print "Updated at:                 %s" % rule['updated_at']
        print "Error:                      %s" % rule['error']
        print "Subscription Id:            %s" % rule['subscription_id']
        print "Source replica expression:  %s" % rule['source_replica_expression']
        print "Activity:                   %s" % rule['activity']
        print "Comment:                    %s" % rule['comments']
        print "Ignore Quota:               %s" % rule['ignore_account_limit']
        print "Ignore Availability:        %s" % rule['ignore_availability']
        print "Purge replicas:             %s" % rule['purge_replicas']
        print "Notification:               %s" % rule['notification']
    return SUCCESS


@exception_handler
def list_rules(args):
    """
    %(prog)s list-rules ...

    List rules.
    """
    client = get_client(args)
    if args.rule_id:
        rules = [client.get_replication_rule(args.rule_id)]
    elif args.file:
        scope, name = extract_scope(args.file)
        rules = client.list_associated_rules_for_file(scope=scope, name=name)
    elif args.traverse:
        scope, name = extract_scope(args.did)
        locks = client.get_dataset_locks(scope=scope, name=name)
        rules = []
        for rule_id in list(set([lock['rule_id'] for lock in locks])):
            rules.append(client.get_replication_rule(rule_id))
    elif args.did:
        scope, name = extract_scope(args.did)
        meta = client.get_metadata(scope=scope, name=name)
        rules = client.list_did_rules(scope=scope, name=name)
        try:
            rules.next()
            rules = client.list_did_rules(scope=scope, name=name)
        except StopIteration:
            rules = []
            # looking for other rules
            if meta['did_type'] == u'CONTAINER':
                for dsn in client.list_content(scope, name):
                    rules.extend(client.list_did_rules(scope=dsn['scope'], name=dsn['name']))
            if meta['did_type'] == u'DATASET':
                for container in client.list_parent_dids(scope, name):
                    rules.extend(client.list_did_rules(scope=container['scope'], name=container['name']))
    elif args.rule_account:
        rules = client.list_account_rules(account=args.rule_account)
    elif args.subscription:
        account = args.subscription[0]
        name = args.subscription[1]
        rules = client.list_subscription_rules(account=account, name=name)
    else:
        print 'At least one option has to be given. Use -h to list the options.'
        return FAILURE
    if args.csv:
        for rule in rules:
            print "{0}, {1}, {2}, {3}, {4}, {5}, {6}".format(rule['id'],
                                                             rule['account'],
                                                             '%s:%s' % (rule['scope'], rule['name']),
                                                             '%s[%d/%d/%d]' % (rule['state'], rule['locks_ok_cnt'], rule['locks_replicating_cnt'], rule['locks_stuck_cnt']),
                                                             rule['rse_expression'],
                                                             rule['copies'],
                                                             rule['expires_at'])
    else:
        table = []
        for rule in rules:
            table.append([rule['id'],
                          rule['account'],
                          '%s:%s' % (rule['scope'], rule['name']),
                          '%s[%d/%d/%d]' % (rule['state'], rule['locks_ok_cnt'], rule['locks_replicating_cnt'], rule['locks_stuck_cnt']),
                          rule['rse_expression'],
                          rule['copies'],
                          rule['expires_at']])
        print tabulate.tabulate(table, tablefmt='simple', headers=['ID', 'ACCOUNT', 'SCOPE:NAME', 'STATE[OK/REPL/STUCK]', 'RSE_EXPRESSION', 'COPIES', 'EXPIRES (UTC)'])
    return SUCCESS


@exception_handler
def list_rules_history(args):
    """
    %(prog)s list-rules_history ...

    List replication rules history for a DID.
    """
    rule_dict = []
    client = get_client(args)
    scope, name = extract_scope(args.did)
    for rule in client.list_replication_rule_full_history(scope, name):
        if rule['rule_id'] not in rule_dict:
            rule_dict.append(rule['rule_id'])
            print '-' * 40
            print 'Rule insertion'
            print 'Account : %s' % rule['account']
            print 'RSE expression : %s' % (rule['rse_expression'])
            print 'Time : %s' % (rule['created_at'])
        else:
            rule_dict.remove(rule['rule_id'])
            print '-' * 40
            print 'Rule deletion'
            print 'Account : %s' % rule['account']
            print 'RSE expression : %s' % (rule['rse_expression'])
            print 'Time : %s' % (rule['updated_at'])
    return SUCCESS


@exception_handler
def list_rses(args):
    """
    %(prog)s list-rses [options] <field1=value1 field2=value2 ...>

    List rses.

    """
    client = get_client(args)
    rse_expression = None
    if args.rse_expression:
        rse_expression = args.rse_expression
    rses = client.list_rses(rse_expression)
    for rse in rses:
        print '%(rse)s' % rse
    return SUCCESS


@exception_handler
def list_rse_attributes(args):
    """
    %(prog)s list-rse-attributes [options] <field1=value1 field2=value2 ...>

    List rses.

    """
    client = get_client(args)
    attributes = client.list_rse_attributes(rse=args.rse)
    for k in attributes:
        print '  ' + k + ': ' + str(attributes[k])
    return SUCCESS


@exception_handler
def list_rse_usage(args):
    """
    %(prog)s list-rse-usage [options] <rse>

    Show the space usage of a given rse

    """
    client = get_client(args)
    usages = client.get_rse_usage(rse=args.rse)
    print 'USAGE:'
    for usage in usages:
        print '------'
        print_free = False
        if usage['source'] in ['srm']:
            print_free = True
        for elem in usage:
            if not (elem == 'free' or elem == 'total') or print_free:
                if elem == 'free' or elem == 'total' or elem == 'used':
                    print '  {0}: {1}'.format(elem, sizefmt(usage[elem], args.human))
                else:
                    print '  {0}: {1}'.format(elem, usage[elem])
    print '------'
    return SUCCESS


@exception_handler
def list_account_limits(args):
    """
    %(prog)s list [options] <field1=value1 field2=value2 ...>

    List account limits.

    """
    client = get_client(args)
    table = []
    if args.rse:
        limits = client.get_account_limit(account=args.limit_account, rse=args.rse)
    else:
        limits = client.get_account_limits(account=args.limit_account)
    for limit in limits.items():
        table.append([limit[0], sizefmt(limit[1], args.human)])
    table.sort()
    print tabulate.tabulate(table, tablefmt=tablefmt, headers=['RSE', 'LIMIT'])
    return SUCCESS


@exception_handler
def list_account_usage(args):
    """
    %(prog)s list [options] <field1=value1 field2=value2 ...>

    List account usage.

    """
    client = get_client(args)
    table = []
    if args.rse:
        usage = client.get_account_usage(account=args.usage_account, rse=args.rse)
    else:
        usage = client.get_account_usage(account=args.usage_account)
    for item in usage:
        remaining = 0 if float(item['bytes_remaining']) < 0 else float(item['bytes_remaining'])
        table.append([item['rse'], sizefmt(item['bytes'], args.human), sizefmt(item['bytes_limit'], args.human), sizefmt(remaining, args.human)])
    table.sort()
    print tabulate.tabulate(table, tablefmt=tablefmt, headers=['RSE', 'USAGE', 'LIMIT', 'QUOTA LEFT'])
    return SUCCESS


@exception_handler
def list_datasets_rse(args):
    """
    %(prog)s list [options] <field1=value1 field2=value2 ...>

    List the datasets in a site.

    """
    client = get_client(args)
    if args.long:
        table = []
        for dsn in client.list_datasets_per_rse(args.rse):
            print dsn
            table.append(['%s:%s' % (dsn['scope'], dsn['name']), '%s/%s' % (str(dsn['available_length']), str(dsn['length'])), '%s/%s' % (str(dsn['available_bytes']), str(dsn['bytes']))])
        table.sort()
        print tabulate.tabulate(table, tablefmt=tablefmt, headers=['DID', 'LOCAL FILES/TOTAL FILES', 'LOCAL BYTES/TOTAL BYTES'])
    else:
        dsns = list(set(['%s:%s' % (dsn['scope'], dsn['name']) for dsn in client.list_datasets_per_rse(args.rse)]))
        dsns.sort()
        print "SCOPE:NAME"
        print '----------'
        for dsn in dsns:
            print dsn
    return SUCCESS


@exception_handler
def list_pfns(args):
    """
    %(prog)s list [options] <field1=value1 field2=value2 ...>

    List the possible PFN for a file at a site.

    """
    print ''
    print 'The functionality of this command has been moved to ' + Color.BOLD + 'rucio-admin' + Color.END + ' tool. In order to get the pfns of a replica, '\
          + Color.BOLD + 'rucio list-file-replicas' + Color.END + ' command must be used.'
    print ''


@exception_handler
def list_parent_datasets(args):
    """
    %(prog)s list [options] <field1=value1 field2=value2 ...>

    List the possible PFN for a file at a site.

    """
    client = get_client(args)
    inputs = args.inputs.split(',')
    guids = []
    pfns = []
    for input in inputs:
        try:
            uuid.UUID(input)
            guids.append(input)
        except ValueError:
            pfns.append(input)

    print pfns, guids
    dict_datasets = {}
    if pfns:
        for res in client.get_did_from_pfns(pfns):
            for key in res:
                if key not in dict_datasets:
                    dict_datasets[key] = []
                for rule in client.list_associated_rules_for_file(res[key]['scope'], res[key]['name']):
                    if '%s:%s' % (rule['scope'], rule['name']) not in dict_datasets[key]:
                        dict_datasets[key].append('%s:%s' % (rule['scope'], rule['name']))
    for guid in guids:
        for did in client.get_dataset_by_guid(guid):
            if guid not in dict_datasets:
                dict_datasets[guid] = []
            for rule in client.list_associated_rules_for_file(did['scope'], did['name']):
                if '%s:%s' % (rule['scope'], rule['name']) not in dict_datasets[guid]:
                    dict_datasets[guid].append('%s:%s' % (rule['scope'], rule['name']))

    for pfn in dict_datasets:
        print 'PFN or GUID : ', pfn
        print 'Parents : ', ','.join(dict_datasets[pfn])
    return SUCCESS


def test_server(args):
    """"
    %(prog)s test-server [options] <field1=value1 field2=value2 ...>
    Test the client against a server.
    """
    import nose
    config = nose.config.Config()
    config.verbosity = 2
    nose.run(argv=sys.argv[1:], defaultTest='rucio.tests.test_rucio_server', config=config)
    return SUCCESS


if __name__ == '__main__':
    usage = """
usage: %(prog)s <command> [options] [args]

Commands:

    help <command>  Output help for one of the commands below


"""
    oparser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]), add_help=True)
    subparsers = oparser.add_subparsers()

    # Main arguments
    oparser.add_argument('--version', action='version', version='%(prog)s ' + version.version_string())
    oparser.add_argument('--verbose', '-v', default=False, action='store_true', help="Print more verbose output.")
    oparser.add_argument('-H', '--host', dest="host", metavar="ADDRESS", help="The Rucio API host.")
    oparser.add_argument('--auth-host', dest="auth_host", metavar="ADDRESS", help="The Rucio Authentication host.")
    oparser.add_argument('-a', '--account', dest="account", metavar="ACCOUNT", help="Rucio account to use.")
    oparser.add_argument('-S', '--auth-strategy', dest="auth_strategy", default=None, help="Authentication strategy (userpass or x509)")
    oparser.add_argument('-T', '--timeout', dest="timeout", type=float, default=None, help="Set all timeout values to seconds.")
    oparser.add_argument('--robot', '-R', dest="human", default=True, action='store_false', help="All output in bytes and without the units. This output format is preferred by parsers and scripts.")
    oparser.add_argument('--user-agent', '-U', dest="user_agent", default='rucio-clients', action='store', help="Rucio User Agent")

    # Options for the userpass auth_strategy
    oparser.add_argument('-u', '--user', dest='username', default=None, help='username')
    oparser.add_argument('-pwd', '--password', dest='password', default=None, help='password')

    # Options for the x509  auth_strategy
    oparser.add_argument('--certificate', dest='certificate', default=None, help='Client certificate file.')
    oparser.add_argument('--ca-certificate', dest='ca_certificate', default=None, help='CA certificate to verify peer against (SSL).')

    # Ping command
    ping_parser = subparsers.add_parser('ping', help='Ping Rucio server.')
    ping_parser.set_defaults(which='ping')

    # The whoami command
    whoami_parser = subparsers.add_parser('whoami', help='Get information about account whose token is used.')
    whoami_parser.set_defaults(which='whoami_account')

    # The list-file-replicas command
    list_file_replicas_parser = subparsers.add_parser('list-file-replicas', help='List the replicas of a DID and it\'s PFNs.', description='This method allows to list all the replicas of a given Data IDentifier (DID).\
                                                 The only mandatory parameter is the DID which can be a container/dataset/files. By default all the files replicas in state available are returned.')
    list_file_replicas_parser.set_defaults(which='list_file_replicas')
    list_file_replicas_parser.add_argument('--protocols', dest='protocols', action='store', help='List of comma separated protocols. (i.e. https, root, srm).', required=False)
    list_file_replicas_parser.add_argument('--all-states', dest='all_states', action='store_true', default=False, help='To select all replicas (including unavailable ones).', required=False)
    list_file_replicas_parser.add_argument('--list-collections', dest='list_collections', action='store_true', default=False, help='To have the number of files of the dataset by site\
        (' + Color.BOLD + 'DEPRECATED: ' + Color.END + 'please use list-dataset-replicas instead.)', required=False)
    list_file_replicas_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')
    list_file_replicas_parser.add_argument('--rse', dest='selected_rse', default=False, action='store', help='Show only results for this RSE', required=False)
    list_file_replicas_parser.add_argument('--missing', dest='missing', default=False, action='store_true', help='To list missing replicas at a RSE.', required=False)
    list_file_replicas_parser.add_argument('--expression', dest='rse_expression', default=None, action='store', help='The RSE filter expression. A comprehensive help about RSE expressions\
            can be found in ' + Color.BOLD + 'http://rucio.cern.ch/client_tutorial.html#adding-rules-for-replication' + Color.END)

    # The list_parent_datasets command
    list_parent_datasets_parser = subparsers.add_parser('list-parent-datasets', help='List the datasets associated to a PFN.')
    list_parent_datasets_parser.set_defaults(which='list_parent_datasets')
    list_parent_datasets_parser.add_argument(dest='inputs', action='store', help='The PFN of the DID is required. Must be a SURL or GUID.')

    # The list-dataset-replicas command
    list_dataset_replicas_parser = subparsers.add_parser('list-dataset-replicas', help='List the dataset replicas.')
    list_dataset_replicas_parser.set_defaults(which='list_dataset_replicas')
    list_dataset_replicas_parser.add_argument(dest='did', action='store', help='The name of the DID to search.')
    list_dataset_replicas_parser.add_argument('--deep', action='store_true', help='Make a deep check.')
    list_dataset_replicas_parser.add_argument('--csv', dest='csv', action='store_true', default=False, help='Coma Separated Value output.')

    # The add-dataset command
    add_dataset_parser = subparsers.add_parser('add-dataset', help='Add a dataset to Rucio Catalog. ')
    add_dataset_parser.set_defaults(which='add_dataset')
    add_dataset_parser.add_argument('--monotonic', action='store_true', help='Monotonic status to True.')
    add_dataset_parser.add_argument(dest='did', action='store', help='The name of the dataset to add.')
    add_dataset_parser.add_argument('--lifetime', dest='lifetime', action='store', help='Lifetime in seconds.')

    # The add-container command
    add_container_parser = subparsers.add_parser('add-container', help='Add a container to Rucio Catalog.')
    add_container_parser.set_defaults(which='add_container')
    add_container_parser.add_argument('--monotonic', action='store_true', help='Monotonic status to True.')
    add_container_parser.add_argument(dest='did', action='store', help='The name of the container to add.')
    add_container_parser.add_argument('--lifetime', dest='lifetime', action='store', help='Lifetime in seconds.')

    # The attach command
    attach_parser = subparsers.add_parser('attach', help='Attach a list of DIDs to a parent DID.',
                                          description='Attach a list of Data IDentifiers (file, dataset or container) to an other Data IDentifier (dataset or container).')
    attach_parser.set_defaults(which='attach')
    attach_parser.add_argument(dest='todid', action='store', help='Destination Data IDentifier (either dataset or container).')
    attach_parser.add_argument('-f', '--from-file', dest='fromfile', action='store_true', default=False, help='Attach the DIDs contained in a file. The file should contain one did per line.')
    attach_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers (or a file containing one did per line, if -f is present).')

    # The detach command
    detach_parser = subparsers.add_parser('detach', help='Detach a list of DIDs from a parent DID.',
                                          description='Detach a list of Data Identifiers (file, dataset or container) from an other Data Identifier (dataset or container).')
    detach_parser.set_defaults(which='detach')
    detach_parser.add_argument(dest='fromdid', action='store', help='Target Data IDentifier (must be a dataset or container).')
    detach_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The list command
    ls_parser = subparsers.add_parser('ls', help='List the data identifiers matching some metadata (synonym for list-dids).', description='List the Data IDentifiers matching certain pattern.\
                                         Only the collections (i.e. dataset or container) are returned by default.\
                                         With the filter option, you can specify a list of metadata that the Data IDentifier should match.')
    ls_parser.set_defaults(which='list_dids')
    ls_parser.add_argument('-r', '--recursive', dest='recursive', action='store_true', default=False, help='List data identifiers recursively.')
    ls_parser.add_argument('--filter', dest='filter', action='store', help='Filter arguments in form `key=value,another_key=next_value`. Valid keys are name, type.')
    ls_parser.add_argument('--short', dest='short', action='store_true', help='Just dump the list of DIDs.')
    ls_parser.add_argument(dest='did', nargs=1, action='store', default=None, help='Data IDentifier pattern.')

    list_parser = subparsers.add_parser('list-dids', help='List the data identifiers matching some metadata (synonym for ls).', description='List the Data IDentifiers matching certain pattern.\
                                         Only the collections (i.e. dataset or container) are returned by default.\
                                         With the filter option, you can specify a list of metadata that the Data IDentifier should match.')
    list_parser.set_defaults(which='list_dids')
    list_parser.add_argument('--recursive', dest='recursive', action='store_true', default=False, help='List data identifiers recursively.')
    list_parser.add_argument('--filter', dest='filter', action='store', help='Filter arguments in form `key=value,another_key=next_value`. Valid keys are name, type.')
    list_parser.add_argument('--short', dest='short', action='store_true', help='Just dump the list of DIDs.')
    list_parser.add_argument(dest='did', nargs=1, action='store', default=None, help='Data IDentifier pattern')

    # The list parent-dids command
    list_parent_parser = subparsers.add_parser('list-parent-dids', help='List parent DIDs for a given DID', description='List all parents Data IDentifier that contains the target Data IDentifier.')
    list_parent_parser.set_defaults(which='list_parent_dids')
    list_parent_parser.add_argument(dest='did', action='store', default=None, help='Data identifier.')

    # The list-scopes command
    scope_list_parser = subparsers.add_parser('list-scopes', help='List all available scopes.')
    scope_list_parser.set_defaults(which='list_scopes')

    # The close command
    close_parser = subparsers.add_parser('close', help='Close a dataset or container.')
    close_parser.set_defaults(which='close')
    close_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The reopen command
    reopen_parser = subparsers.add_parser('reopen', help='Reopen a dataset or container (only for privileged users).')
    reopen_parser.set_defaults(which='reopen')
    reopen_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The stat command
    stat_parser = subparsers.add_parser('stat', help='List attributes and statuses about data identifiers.')
    stat_parser.set_defaults(which='stat')
    stat_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The erase command
    erase_parser = subparsers.add_parser('erase', help='Delete a data identifier.', description='This command sets the lifetime of the DID in order to expire in the next 24 hours.\
            After this time, the dataset is eligible for deletion. The deletion is not reversible after 24 hours grace time period expired.')
    erase_parser.set_defaults(which='erase')
    erase_parser.add_argument('--undo', dest='undo', action='store_true', default=False, help='Undo erase DIDs. Only works if has been less than 24 hours since erase operation.')
    erase_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The list_files command
    list_files_parser = subparsers.add_parser('list-files', help='List DID contents', description='List all the files in a Data IDentifier. The DID can be a container, dataset or a file.\
                                                                  What is returned is a list of files in the DID with : <scope>:<name>\t<filesize>\t<checksum>\t<guid>')
    list_files_parser.set_defaults(which='list_files')
    list_files_parser.add_argument('--csv', dest='csv', action='store_true', default=False, help='Coma Separated Value output. This output format is preferred for easy parsing and scripting.')
    list_files_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The list_content command
    list_content_parser = subparsers.add_parser('list-content', help='List the content of a collection.')
    list_content_parser.set_defaults(which='list_content')
    list_content_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The upload subparser
    upload_parser = subparsers.add_parser('upload', help='Upload method.')
    upload_parser.set_defaults(which='upload')
    upload_parser.add_argument('--rse', dest='rse', action='store', help='Rucio Storage Element (RSE) name.', required=True)
    upload_parser.add_argument('--lifetime', type=int, action='store', help='Lifetime of the rule in seconds.')
    upload_parser.add_argument('--scope', dest='scope', action='store', help='Scope name.')
    # The --no-register option is hidden. This is pilot ONLY. Users should not used. Can lead to dark data increase and data loss
    # No replicas are registered for files, nor rules created.
    upload_parser.add_argument('--no-register', dest='no_register', action='store_true', default=False, help=argparse.SUPPRESS)
    upload_parser.add_argument(dest='args', action='store', nargs='+', help='files and datasets.')

    # The download subparser
    get_parser = subparsers.add_parser('get', help='Download method (synonym for download)')
    get_parser.set_defaults(which='download')
    get_parser.add_argument('--dir', dest='dir', default='.', action='store', help='The directory to store the downloaded file.')
    get_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers')
    get_parser.add_argument('--rse', action='store', help='Force the RSE from where the DID is downloaded')
    get_parser.add_argument('--protocol', action='store', help='Force the protocol to use')
    get_parser.add_argument('--nrandom', type=int, action='store', help='Download N random files from the DID')
    get_parser.add_argument('--ndownloader', type=int, default=3, action='store', help='Choose the number of parallel processes for download')
    get_parser.add_argument('--no-subdir', action='store_true', default=False, help="Don't create a subdirectory for the scope of the files. Existing files in the directory will be overwritten.")
    get_parser.add_argument('--new', action='store_true', default=False, help="Choose the new download thread model.")

    download_parser = subparsers.add_parser('download', help='Download method (synonym for get)')
    download_parser.set_defaults(which='download')
    download_parser.add_argument('--dir', dest='dir', default='.', action='store', help='The directory to store the downloaded file.')
    download_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')
    download_parser.add_argument('--rse', action='store', help='Force the RSE from where the DID is downloaded.')
    download_parser.add_argument('--protocol', action='store', help='Force the protocol to use.')
    download_parser.add_argument('--nrandom', type=int, action='store', help='Download N random files from the DID.')
    download_parser.add_argument('--ndownloader', type=int, default=3, action='store', help='Choose the number of parallel processes for download.')
    download_parser.add_argument('--no-subdir', action='store_true', default=False, help="Don't create a subdirectory for the scope of the files. Existing files in the directory will be overwritten.")
    download_parser.add_argument('--new', action='store_true', default=False, help="Choose the new download thread model.")

    # The get-metadata subparser
    get_metadata_parser = subparsers.add_parser('get-metadata', help='Get metadata for DIDs.')
    get_metadata_parser.set_defaults(which='get_metadata')
    get_metadata_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')

    # The list-pfns subparser
    list_pfns_parser = subparsers.add_parser('list-pfns', help=Color.BOLD + 'DEPRECATED: ' + Color.END + 'Please use rucio list-file-replicas instead.',
                                             description='The functionality of this command has been moved to rucio-admin. If you want the actual PFN for a replica, you should use ' + Color.BOLD + 'rucio list-file-replicas' + Color.END + ' instead.')
    list_pfns_parser.set_defaults(which='list-pfns')

    # The set-metadata subparser
    set_metadata_parser = subparsers.add_parser('set-metadata', help='set-metadata method')
    set_metadata_parser.set_defaults(which='set_metadata')
    set_metadata_parser.add_argument('--did', dest='did', action='store', help='Data identifier to add', required=True)
    set_metadata_parser.add_argument('--key', dest='key', action='store', help='Attribute key', required=True)
    set_metadata_parser.add_argument('--value', dest='value', action='store', help='Attribute value', required=True)

    # The delete-metadata subparser
    # delete_metadata_parser = subparsers.add_parser('delete-metadata', help='Delete metadata')
    # delete_metadata_parser.set_defaults(which='delete_metadata')

    # The list-rse-usage subparser
    list_rse_usage_parser = subparsers.add_parser('list-rse-usage', help='Shows the total/free/used space for a given RSE. This values can differ for different RSE source.')
    list_rse_usage_parser.set_defaults(which='list_rse_usage')
    list_rse_usage_parser.add_argument(dest='rse', action='store', help='Rucio Storage Element (RSE) name.')
    list_rse_usage_parser.add_argument('--history', dest='history', default=False, action='store', help='List RSE usage history. [Unimplemented]')

    # The list-account-usage subparser
    list_account_usage_parser = subparsers.add_parser('list-account-usage', help='Shows the space used, the quota limit and the quota left for an account for every RSE where the user have quota.')
    list_account_usage_parser.set_defaults(which='list_account_usage')
    list_account_usage_parser.add_argument(dest='usage_account', action='store', help='Account name.')
    list_account_usage_parser.add_argument('--rse', action='store', help='Show usage for only for this RSE.')

    # The list-account-limits subparser
    list_account_limits_parser = subparsers.add_parser('list-account-limits', help='List quota limits for an account in every RSEs.')
    list_account_limits_parser.set_defaults(which='list_account_limits')
    list_account_limits_parser.add_argument('limit_account', action='store', help='The account name.')
    list_account_limits_parser.add_argument('--rse', dest='rse', action='store', help='If this option is given, the results are restricted to only this RSE.')

    # Add replication rule subparser
    add_rule_parser = subparsers.add_parser('add-rule', help='Add replication rule.')
    add_rule_parser.set_defaults(which='add_rule')
    add_rule_parser.add_argument(dest='dids', action='store', nargs='+', help='DID(s) to apply the rule to')
    add_rule_parser.add_argument(dest='copies', action='store', type=int, help='Number of copies')
    add_rule_parser.add_argument(dest='rse_expression', action='store', help='RSE Expression')
    add_rule_parser.add_argument('--weight', dest='weight', action='store', help='RSE Weight')
    add_rule_parser.add_argument('--lifetime', dest='lifetime', action='store', type=int, help='Rule lifetime (in seconds)')
    add_rule_parser.add_argument('--grouping', dest='grouping', action='store', choices=['DATASET', 'ALL', 'NONE'], help='Rule grouping')
    add_rule_parser.add_argument('--locked', dest='locked', action='store_true', help='Rule locking')
    add_rule_parser.add_argument('--source-replica-expression', dest='source_replica_expression', action='store', help='Source Replica Expression')
    add_rule_parser.add_argument('--notify', dest='notify', action='store', help='Notification strategy : Y (Yes), N (No), C (Close)')
    add_rule_parser.add_argument('--activity', dest='activity', action='store', help='Activity to be used (e.g. User, Data Consolidation')
    add_rule_parser.add_argument('--comment', dest='comment', action='store', help='Comment about the replication rule')
    add_rule_parser.add_argument('--ask-approval', dest='ask_approval', action='store_true', help='Ask for rule approval')
    add_rule_parser.add_argument('--asynchronous', dest='asynchronous', action='store_true', help='Create rule asynchronously')
    add_rule_parser.add_argument('--account', dest='rule_account', action='store', help='The account owning the rule')

    # Delete replication rule subparser
    delete_rule_parser = subparsers.add_parser('delete-rule', help='Delete replication rule.')
    delete_rule_parser.set_defaults(which='delete_rule')
    delete_rule_parser.add_argument(dest='rule_id', action='store', help='Rule id or DID. If DID, the RSE expression is mandatory.')
    delete_rule_parser.add_argument('--purge-replicas', dest='purge_replicas', action='store_true', help='Purge rule replicas')
    delete_rule_parser.add_argument('--all', dest='delete_all', action='store_true', default=False, help='Delete all the rules, even the ones that are not owned by the account')
    delete_rule_parser.add_argument('--rse_expression', dest='rse_expression', action='store', help='The RSE expression. Must be specified if a DID is provided.')
    delete_rule_parser.add_argument('--account', dest='rule_account', action='store', help='The account of the rule that must be deleted')

    # Info replication rule subparser
    info_rule_parser = subparsers.add_parser('rule-info', help='Retrieve information about a rule.')
    info_rule_parser.set_defaults(which='info_rule')
    info_rule_parser.add_argument(dest='rule_id', action='store', help='The rule ID')
    info_rule_parser.add_argument('--examine', dest='examine', action='store_true', help='Detailed analysis of transfer errors')

    # The list_rules command
    list_rules_parser = subparsers.add_parser('list-rules', help='List replication rules.')
    list_rules_parser.set_defaults(which='list_rules')
    list_rules_parser.add_argument(dest='did', action='store', nargs='?', default=None, help='List by did')
    list_rules_parser.add_argument('--id', dest='rule_id', action='store', help='List by rule id')
    list_rules_parser.add_argument('--traverse', dest='traverse', action='store_true', help='Traverse the did tree and search for rules affecting this did')
    list_rules_parser.add_argument('--csv', dest='csv', action='store_true', default=False, help='Coma Separated Value output')
    list_rules_parser.add_argument('--file', dest='file', action='store', help='List associated rules of an affected file')
    list_rules_parser.add_argument('--account', dest='rule_account', action='store', help='List by account')
    list_rules_parser.add_argument('--subscription', dest='subscription', action='store', help='List by account and subscription name', metavar=('ACCOUNT', 'SUBSCRIPTION'), nargs=2)

    # The list_rules_history command
    list_rules_history_parser = subparsers.add_parser('list-rules-history', help='List replication rules history for a DID.')
    list_rules_history_parser.set_defaults(which='list_rules_history')
    list_rules_history_parser.add_argument(dest='did', action='store', help='The Data IDentifier.')

    # The update_rule command
    update_rule_parser = subparsers.add_parser('update-rule', help='Update replication rule.')
    update_rule_parser.set_defaults(which='update_rule')
    update_rule_parser.add_argument(dest='rule_id', action='store', help='Rule id')
    update_rule_parser.add_argument('--lifetime', dest='lifetime', action='store', help='Lifetime in seconds.')
    update_rule_parser.add_argument('--locked', dest='locked', action='store', help='Locked (True/False).')
    update_rule_parser.add_argument('--account', dest='rule_account', action='store', help='Account to change.')
    update_rule_parser.add_argument('--stuck', dest='state_stuck', action='store_true', help='Set state to STUCK.')
    update_rule_parser.add_argument('--suspend', dest='state_suspended', action='store_true', help='Set state to SUSPENDED.')
    update_rule_parser.add_argument('--activity', dest='rule_activity', action='store', help='Activity of the rule.')
    update_rule_parser.add_argument('--source-replica-expression', dest='source_replica_expression', action='store', help='Source replica expression of the rule.')
    update_rule_parser.add_argument('--cancel-requests', dest='cancel_requests', action='store_true', help='Cancel requests when setting rules to stuck.')
    update_rule_parser.add_argument('--priority', dest='priority', action='store', help='Priority of the requests of the rule')

    # The list-rses command
    list_rses_parser = subparsers.add_parser('list-rses', help='Show the list of all the registered Rucio Storage Elements (RSEs).')
    list_rses_parser.set_defaults(which='list_rses')
    list_rses_parser.add_argument('--expression', dest='rse_expression', action='store', help='The RSE filter expression. A comprehensive help about RSE expressions\
            can be found in ' + Color.BOLD + 'http://rucio.cern.ch/client_tutorial.html#adding-rules-for-replication' + Color.END)

    # The list-rses-attributes command
    list_rse_attributes_parser = subparsers.add_parser('list-rse-attributes', help='List the attributes of an RSE.', description='This command is useful to create RSE filter expressions.')
    list_rse_attributes_parser.set_defaults(which='list_rse-attributes')
    list_rse_attributes_parser.add_argument(dest='rse', action='store', help='The RSE name')

    # The list-datasets-rse command
    list_datasets_rse_parser = subparsers.add_parser('list-datasets-rse', help='List all the datasets at a RSE', description='This method allows to list all the datasets on a given Rucio Storage Element.\
        ' + Color.BOLD + 'Warning: ' + Color.END + 'This command can take a long time depending on the number of datasets in the RSE.')
    list_datasets_rse_parser.set_defaults(which='list_datasets_rse')
    list_datasets_rse_parser.add_argument(dest='rse', action='store', default=None, help='The RSE name')
    list_datasets_rse_parser.add_argument('--long', dest='long', action='store_true', default=False, help='The long option')

    # The test-server command
    test_server_parser = subparsers.add_parser('test-server', help='Test Server', description='Run a battery of nosetests against the Rucio Servers.')
    test_server_parser.set_defaults(which='test_server')

    argcomplete.autocomplete(oparser)

    if len(sys.argv) == 1:
        oparser.print_help()
        sys.exit(FAILURE)

    args = oparser.parse_args(sys.argv[1:])

    commands = {'add_container': add_container,
                'add_dataset': add_dataset,
                'add_rule': add_rule,
                'attach': attach,
                'close': close,
                'reopen': reopen,
                'erase': erase,
                # 'delete_metadata': delete_metadata,
                'delete_rule': delete_rule,
                'detach': detach,
                'download': download,
                'get_metadata': get_metadata,
                'list-pfns': list_pfns,
                'info_rule': info_rule,
                'list_account_limits': list_account_limits,
                'list_account_usage': list_account_usage,
                'list_datasets_rse': list_datasets_rse,
                'list_parent_dids': list_parent_dids,
                'list_files': list_files,
                'list_content': list_content,
                'list_dids': list_dids,
                'list_dataset_replicas': list_dataset_replicas,
                'list_file_replicas': list_file_replicas,
                'list_parent_datasets': list_parent_datasets,
                'list_rses': list_rses,
                'list_rse-attributes': list_rse_attributes,
                'list_rse_usage': list_rse_usage,
                'list_rules': list_rules,
                'list_rules_history': list_rules_history,
                'list_scopes': list_scopes,
                'ping': ping,
                'set_metadata': set_metadata,
                'stat': stat,
                'test_server': test_server,
                'update_rule': update_rule,
                'upload': upload,
                'whoami_account': whoami_account}

    try:
        if args.verbose:
            logger.setLevel(logging.DEBUG)
        start_time = time.time()
        command = commands.get(args.which)
        result = command(args)
        end_time = time.time()
        if args.verbose:
            print "Completed in %-0.4f sec." % (end_time - start_time)
        sys.exit(result)
    except Exception, error:
        logger.error("Strange error {0}".format(error))
        sys.exit(FAILURE)
