# -*- coding: utf-8 -*-
#!/usr/bin/env python
#
# Copyright 2012-2104 BigML
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Resources management functions

"""
from __future__ import absolute_import

import sys
import time

try:
    import simplejson as json
except ImportError:
    import json

import bigml.api

from bigmler.utils import (dated, get_url, log_message, plural, check_resource,
                           check_resource_error, log_created_resources,
                           is_shared)
from bigmler.labels import label_model_args, get_all_labels
from bigmler.reports import report
from bigml.util import bigml_locale


EVALUATE_SAMPLE_RATE = 0.8
SEED = "BigML, Machine Learning made easy"
LOCALE_DEFAULT = "en_US"
FIELDS_QS = 'only_model=true'
ALL_FIELDS_QS = "limit=-1"
ADD_PREFIX = '+'
REMOVE_PREFIX = '-'
ADD_REMOVE_PREFIX = [ADD_PREFIX, REMOVE_PREFIX]
BRIEF_FORMAT = 'brief'
NORMAL_FORMAT = 'normal'
FULL_FORMAT = 'full'


def get_basic_seed(order):
    """ Builds a standard seed from a text adding the order

    """
    return "%s - %s" % (SEED, order)


def shared_changed(shared, resource):
    """Returns True if the shared status of the resource differs from the user
       given value

    """
    return is_shared(resource) != shared


def configure_input_fields(fields, user_given_fields):
    """ Returns the input fields used in the new resource creation as given

        The user can choose to write all the fields that will be used in the
        new resource or modify the set of fields retrieved from the
        resource that will be used to create the new one.
    """
    # case of adding and removing fields to the dataset preferred field set
    if all([name[0] in ADD_REMOVE_PREFIX for name in user_given_fields]):
        preferred_fields = fields.preferred_fields()
        input_fields = preferred_fields.keys()
        for name in user_given_fields:
            field_id = fields.field_id(name[1:])
            if name[0] == ADD_PREFIX:
                if not field_id in input_fields:
                    input_fields.append(field_id)
            elif field_id in input_fields:
                input_fields.remove(field_id)
    # case of user given entire list of fields
    else:
        input_fields = []
        for name in user_given_fields:
            input_fields.append(fields.field_id(name))
    return input_fields


def relative_input_fields(fields, user_given_fields):
    """Returns the user given input fields using relative syntax

    """

    input_fields = []
    if all([(name[0] in ADD_REMOVE_PREFIX) for name in user_given_fields]):
        return user_given_fields

    preferred_fields = fields.preferred_fields()
    for field_id in preferred_fields.keys():
        name = fields.fields[field_id]['name']
        if not name in user_given_fields:
            input_fields.append("%s%s" % (REMOVE_PREFIX, name))
    for name in user_given_fields:
        field_id = fields.field_id(name)
        input_fields.append("%s%s" % (ADD_PREFIX, name))

    return input_fields


def wait_for_available_tasks(inprogress, max_parallel, get_function,
                             resource_type, query_string=None, wait_step=2):
    """According to the max_parallel number of parallel resources to be
       created, when the number of in progress resources reaches the limit,
       it checks the ones in inprogress to see if there's a
       FINISHED or FAULTY resource. If found, it is removed from the
       inprogress list and returns to allow another one to be created.

    """

    check_kwargs = {"retries": 0}
    if query_string:
        check_kwargs.update(query_string=query_string)
    while len(inprogress) == max_parallel:
        for j in range(0, len(inprogress)):
            try:
                ready = check_resource(inprogress[j], get_function,
                                       **check_kwargs)
                status = bigml.api.get_status(ready)
                if (status['code'] == bigml.api.FINISHED):
                    del inprogress[j]
                    return
                elif (status['code'] == bigml.api.FAULTY):
                    raise ValueError(status['message'])
            except ValueError, exception:
                sys.exit("Failed to get a finished %s: %s" %
                         (resource_type, str(exception)))
        time.sleep(max_parallel * wait_step)


def set_source_args(data_set_header, name, description, args,
                    multi_label_data=None):
    """Returns a source arguments dict

    """
    source_args = {
        "name": name,
        "description": description,
        "category": args.category,
        "tags": args.tag,
        "source_parser": {"header": data_set_header}}
    # If user has given an OS locale, try to add the locale used in bigml.com
    if args.user_locale is not None:
        source_locale = bigml_locale(args.user_locale)
        if source_locale is None:
            log_message("WARNING: %s locale equivalence not found."
                        " Using %s instead.\n" % (args.user_locale,
                        LOCALE_DEFAULT), log_file=None, console=True)
            source_locale = LOCALE_DEFAULT
        source_args["source_parser"].update({'locale': source_locale})
    # If user has set a training separator, use it.
    if args.training_separator is not None:
        training_separator = args.training_separator.decode("string_escape")
        source_args["source_parser"].update({'separator': training_separator})
    # If uploading a multi-label file, add the user_metadata info needed to
    # manage the multi-label fields
    if args.multi_label and multi_label_data is not None:
        source_args.update(
            {"user_metadata":
                {"multi_label_data": multi_label_data}})
    if args.json_args['source']:
        source_args.update(args.json_args['source'])
    return source_args


def create_source(data_set, source_args, args, api=None, path=None,
                  session_file=None, log=None, source_type=None):
    """Creates remote source

    """
    if api is None:
        api = bigml.api.BigML()
    suffix = "" if source_type is None else "%s " % source_type
    message = dated("Creating %ssource.\n" % suffix)
    log_message(message, log_file=session_file, console=args.verbosity)

    source = api.create_source(data_set, source_args,
                               progress_bar=args.progress_bar)
    if path is not None:
        try:
            suffix = "_" + source_type if source_type else ""
            with open("%s/source%s" % (path, suffix), 'w', 0) as source_file:
                source_file.write("%s\n" % source['resource'])
                source_file.write("%s\n" % source['object']['name'])
        except IOError, exc:
            sys.exit("%s: Failed to write %s/source" % (str(exc), path))
    source_id = check_resource_error(source, "Failed to create source: ")
    try:
        source = check_resource(source, api.get_source,
                                query_string=ALL_FIELDS_QS)
    except ValueError, exception:
        sys.exit("Failed to get a finished source: %s" % str(exception))
    message = dated("Source created: %s\n" % get_url(source))
    log_message(message, log_file=session_file, console=args.verbosity)
    log_message("%s\n" % source_id, log_file=log)
    if args.reports:
        report(args.reports, path, source)
    return source


def data_to_source(training_set, test_set,
                   training_set_header, test_set_header, args):
    """Extracts the flags info to create a source object

    """
    data_set = None
    data_set_header = None
    if (training_set and not args.source and not args.dataset and
            not args.model and not args.models):
        data_set = training_set
        data_set_header = training_set_header
    elif (args.evaluate and test_set and not args.source):
        data_set = test_set
        data_set_header = test_set_header

    return data_set, data_set_header


def get_source(source, api=None, verbosity=True,
               session_file=None):
    """Retrieves the source in its actual state and its field info

    """
    if api is None:
        api = bigml.api.BigML()
    if (isinstance(source, basestring) or
            bigml.api.get_status(source)['code'] != bigml.api.FINISHED):
        message = dated("Retrieving source. %s\n" %
                        get_url(source))
        log_message(message, log_file=session_file,
                    console=verbosity)
        try:
            source = check_resource(source, api.get_source,
                                    query_string=ALL_FIELDS_QS)
        except ValueError, exception:
            sys.exit("Failed to get a finished source: %s" % str(exception))
    return source


def update_source_fields(source, updated_values, fields, api=None,
                         verbosity=True, session_file=None):
    """Update remote source with new fields values

    """
    if api is None:
        api = bigml.api.BigML()
    update_fields = {}
    for (column, value) in updated_values.iteritems():
        update_fields.update({
            fields.field_id(column): value})
    message = dated("Updating source. %s\n" %
                    get_url(source))
    log_message(message, log_file=session_file,
                console=verbosity)
    source = api.update_source(source, {"fields": update_fields})
    check_resource_error(source, "Failed to update source: ")
    return source


def set_basic_dataset_args(name, description, args):
    """Return dataset basic arguments dict

    """
    dataset_args = {
        "name": name,
        "description": description,
        "category": args.category,
        "tags": args.tag
    }

    return dataset_args


def set_dataset_args(name, description, args, fields, dataset_fields,
                     objective_field=None, multi_label_data=None):
    """Return dataset arguments dict

    """
    dataset_args = set_basic_dataset_args(name, description, args)
    if multi_label_data is not None and objective_field is None:
        objective_field = multi_label_data['objective_name']
    if objective_field is not None and fields is not None:
        objective_id = fields.field_id(objective_field)
        dataset_args.update(objective_field={'id': objective_id})

    if args.json_filter:
        dataset_args.update(json_filter=args.json_filter)
    elif args.lisp_filter:
        dataset_args.update(lisp_filter=args.lisp_filter)

    if dataset_fields and fields is not None:
        input_fields = configure_input_fields(fields, dataset_fields)
        dataset_args.update(input_fields=input_fields)
    if args.multi_label and multi_label_data is not None:
        dataset_args.update(
            user_metadata={'multi_label_data': multi_label_data})
    if args.json_args['dataset']:
        dataset_args.update(args.json_args['dataset'])

    return dataset_args


def set_dataset_split_args(name, description, args, sample_rate,
                           out_of_bag=False, multi_label_data=None):
    """Return dataset arguments dict to split a dataset

    """
    dataset_args = {
        "name": name,
        "description": description,
        "category": args.category,
        "tags": args.tag,
        "seed": SEED if args.seed is None else args.seed,
        "sample_rate": sample_rate,
        "out_of_bag": out_of_bag
    }
    if args.multi_label and multi_label_data is not None:
        dataset_args.update(
            user_metadata={'multi_label_data': multi_label_data})
    return dataset_args


def create_dataset(source_or_dataset, dataset_args, args, api=None,
                   path=None, session_file=None, log=None, dataset_type=None):
    """Creates remote dataset from source, dataset or datasets list

    """
    if api is None:
        api = bigml.api.BigML()
    message = dated("Creating dataset.\n")
    log_message(message, log_file=session_file, console=args.verbosity)
    dataset = api.create_dataset(source_or_dataset, dataset_args)
    suffix = "_" + dataset_type if dataset_type else ""
    log_created_resources("dataset%s" % suffix, path,
                          bigml.api.get_dataset_id(dataset), open_mode='a')
    dataset_id = check_resource_error(dataset, "Failed to create dataset: ")
    try:
        dataset = check_resource(dataset, api.get_dataset,
                                 query_string=ALL_FIELDS_QS)
    except ValueError, exception:
        sys.exit("Failed to get a finished dataset: %s" % str(exception))
    message = dated("Dataset created: %s\n" % get_url(dataset))
    log_message(message, log_file=session_file, console=args.verbosity)
    log_message("%s\n" % dataset_id, log_file=log)
    if args.reports:
        report(args.reports, path, dataset)
    return dataset


def get_dataset(dataset, api=None, verbosity=True, session_file=None):
    """Retrieves the dataset in its actual state

    """
    if api is None:
        api = bigml.api.BigML()
    if (isinstance(dataset, basestring) or
            bigml.api.get_status(dataset)['code'] != bigml.api.FINISHED):
        message = dated("Retrieving dataset. %s\n" %
                        get_url(dataset))
        log_message(message, log_file=session_file,
                    console=verbosity)
        dataset = check_resource(dataset, api.get_dataset,
                                 query_string=ALL_FIELDS_QS)
        check_resource_error(dataset, "Failed to get dataset: ")
    return dataset


def publish_dataset(dataset, args, api=None, session_file=None):
    """Publishes dataset and sets its price (if any)

    """
    if api is None:
        api = bigml.api.BigML()
    public_dataset = {"private": False}
    if args.dataset_price:
        public_dataset.update(price=args.dataset_price)
    return update_dataset(dataset, public_dataset, args.verbosity, api=api,
                          session_file=session_file)


def update_dataset(dataset, dataset_args, args,
                   api=None, path=None, session_file=None):
    """Updates dataset properties

    """
    if api is None:
        api = bigml.api.BigML()
    message = dated("Updating dataset. %s\n" %
                    get_url(dataset))
    log_message(message, log_file=session_file,
                console=args.verbosity)
    dataset = api.update_dataset(dataset, dataset_args)
    if is_shared(dataset):
        message = dated("Shared dataset link. %s\n" %
                        get_url(dataset, shared=True))
        log_message(message, log_file=session_file,
                    console=args.verbosity)
        if args.reports:
            report(args.reports, path, dataset)
    check_resource_error(dataset, "Failed to update dataset: ")
    return dataset


def set_model_args(name, description,
                   args, objective_field=None, fields=None,
                   model_fields=None, other_label=None):
    """Return model arguments dict

    """
    model_args = {
        "name": name,
        "description": description,
        "category": args.category,
        "tags": args.tag
    }
    if objective_field is not None and fields is not None:
        model_args.update({"objective_field":
                           fields.field_id(objective_field)})
    # If evaluate flag is on and no test_split flag is provided,
    # we choose a deterministic sampling with
    # args.sample_rate (80% by default) of the data to create the model
    # If cross_validation_rate = n/100, then we choose to run 2 * n evaluations
    # by holding out a n% of randomly sampled data.

    if ((args.evaluate and args.test_split == 0 and args.test_datasets is None)
            or args.cross_validation_rate > 0):
        model_args.update(seed=SEED)
        if args.cross_validation_rate > 0:
            args.sample_rate = 1 - args.cross_validation_rate
            args.replacement = False
        elif args.sample_rate == 1 and args.test_datasets is None:
            args.sample_rate = EVALUATE_SAMPLE_RATE
    if model_fields and fields is not None:
        input_fields = configure_input_fields(fields, model_fields)
        model_args.update(input_fields=input_fields)

    if args.pruning and args.pruning != 'smart':
        model_args.update(stat_pruning=(args.pruning == 'statistical'))

    if args.node_threshold > 0:
        model_args.update(node_threshold=args.node_threshold)

    if args.balance:
        model_args.update(balance_objective=True)

    if args.weight_field:
        weight_field = fields.field_id(args.weight_field)
        model_args.update(weight_field=weight_field)

    if args.objective_weights:
        model_args.update(objective_weights=args.objective_weights_json)

    if args.max_categories > 0:
        model_args.update(
            user_metadata={'other_label': other_label,
                           'max_categories': args.max_categories})

    if args.json_args['model']:
        model_args.update(args.json_args['model'])

    model_args.update(sample_rate=args.sample_rate,
                      replacement=args.replacement,
                      randomize=args.randomize)
    return model_args


def set_label_model_args(name, description, args, labels, multi_label_data,
                         fields, model_fields, objective_field):
    """Set of args needed to build a model per label

    """

    if model_fields is None:
        model_fields = []
    else:
        model_fields = relative_input_fields(fields, model_fields)
    if objective_field is None:
        objective_field = fields.objective_field
    objective_id = fields.field_id(objective_field)
    objective_field = fields.field_name(objective_id)
    all_labels = get_all_labels(multi_label_data)
    model_args_list = []

    for index in range(args.number_of_models - 1, -1, -1):
        label = labels[index]
        (new_name, label_field, single_label_fields) = label_model_args(
            name, label, all_labels, model_fields, objective_field)
        model_args = set_model_args(new_name, description, args,
                                    label_field, fields,
                                    single_label_fields)
        if multi_label_data is not None:
            model_args.update(
                user_metadata={'multi_label_data': multi_label_data})
        model_args_list.append(model_args)
    return model_args_list


def create_models(datasets, model_ids, model_args,
                  args, api=None, path=None,
                  session_file=None, log=None):
    """Create remote models

    """
    if api is None:
        api = bigml.api.BigML()

    models = model_ids[:]
    existing_models = len(models)
    model_args_list = []
    if args.dataset_off and args.evaluate:
        args.test_dataset_ids = datasets[:]
    if not args.multi_label:
        datasets = datasets[existing_models:]
    # if resuming and all models were created, there will be no datasets left
    if datasets:
        dataset = datasets[0]
        if isinstance(model_args, list):
            model_args_list = model_args
        if args.number_of_models > 0:
            message = dated("Creating %s.\n" %
                            plural("model", args.number_of_models))
            log_message(message, log_file=session_file,
                        console=args.verbosity)

            single_model = args.number_of_models == 1 and existing_models == 0
            # if there's more than one model the first one must contain
            # the entire field structure to be used as reference.
            query_string = (FIELDS_QS if single_model
                            else ALL_FIELDS_QS)
            inprogress = []
            for i in range(0, args.number_of_models):
                wait_for_available_tasks(inprogress, args.max_parallel_models,
                                         api.get_model, "model",
                                         query_string=query_string)
                if model_args_list:
                    model_args = model_args_list[i]
                if args.cross_validation_rate > 0:
                    new_seed = get_basic_seed(i + existing_models)
                    model_args.update(seed=new_seed)
                # one model per dataset (--max-categories or single model)
                if (args.max_categories or
                        (args.test_datasets and args.evaluate)) > 0:
                    dataset = datasets[i]
                    model = api.create_model(dataset, model_args)
                elif args.dataset_off and args.evaluate:
                    multi_dataset = args.test_dataset_ids[:]
                    del multi_dataset[i + existing_models]
                    model = api.create_model(multi_dataset, model_args)
                else:
                    model = api.create_model(datasets, model_args)
                model_id = check_resource_error(model,
                                                "Failed to create model: ")
                log_message("%s\n" % model_id, log_file=log)
                model_ids.append(model_id)
                inprogress.append(model_id)
                models.append(model)
                log_created_resources("models", path, model_id, open_mode='a')

            if args.number_of_models < 2 and args.verbosity:
                if bigml.api.get_status(model)['code'] != bigml.api.FINISHED:
                    try:
                        model = check_resource(model, api.get_model,
                                               query_string=query_string)
                    except ValueError, exception:
                        sys.exit("Failed to get a finished model: %s" %
                                 str(exception))
                    models[0] = model
                message = dated("Model created: %s.\n" %
                                get_url(model))
                log_message(message, log_file=session_file,
                            console=args.verbosity)
                if args.reports:
                    report(args.reports, path, model)

    return models, model_ids


def update_model(model, model_args, args,
                 api=None, path=None, session_file=None):
    """Updates model properties

    """
    if api is None:
        api = bigml.api.BigML()
    message = dated("Updating model. %s\n" %
                    get_url(model))
    log_message(message, log_file=session_file,
                console=args.verbosity)
    model = api.update_model(model, model_args)
    check_resource_error(model, "Failed to update model: %s"
                         % model['resource'])
    if is_shared(model):
        message = dated("Shared model link. %s\n" %
                        get_url(model, shared=True))
        log_message(message, log_file=session_file, console=args.verbosity)
        if args.reports:
            report(args.reports, path, model)

    return model


def get_models(model_ids, args, api=None, session_file=None):
    """Retrieves remote models in its actual status

    """
    if api is None:
        api = bigml.api.BigML()
    model_id = ""
    models = model_ids
    single_model = len(model_ids) == 1
    if single_model:
        model_id = model_ids[0]
    message = dated("Retrieving %s. %s\n" %
                    (plural("model", len(model_ids)),
                     get_url(model_id)))
    log_message(message, log_file=session_file, console=args.verbosity)
    if len(model_ids) < args.max_batch_models:
        models = []
        for model in model_ids:
            try:
                # if there's more than one model the first one must contain
                # the entire field structure to be used as reference.
                query_string = (ALL_FIELDS_QS if not single_model
                                and (len(models) == 0 or args.multi_label)
                                else FIELDS_QS)
                model = check_resource(model, api.get_model,
                                       query_string=query_string)
            except ValueError, exception:
                sys.exit("Failed to get a finished model: %s" %
                         str(exception))
            models.append(model)
        model = models[0]
    else:
        try:
            query_string = (ALL_FIELDS_QS if not single_model
                            else FIELDS_QS)
            model = check_resource(model_ids[0], api.get_model,
                                   query_string=query_string)
        except ValueError, exception:
            sys.exit("Failed to get a finished model: %s" % str(exception))
        models[0] = model
    return models, model_ids


def set_label_ensemble_args(name, description, args, labels, multi_label_data,
                            number_of_ensembles, fields, model_fields,
                            objective_field):
    """Set of args needed to build an ensemble per label

    """
    if model_fields is None:
        model_fields = []
    else:
        model_fields = relative_input_fields(fields, model_fields)
    if objective_field is None:
        objective_field = fields.objective_field
    objective_id = fields.field_id(objective_field)
    objective_field = fields.fields[objective_id]['name']
    ensemble_args_list = []

    for index in range(number_of_ensembles - 1, -1, -1):
        label = labels[index]
        all_labels = get_all_labels(multi_label_data)
        (new_name, label_field, single_label_fields) = label_model_args(
            name, label, all_labels, model_fields, objective_field)
        ensemble_args = set_ensemble_args(new_name, description, args,
                                          single_label_fields,
                                          label_field, fields)
        if multi_label_data is not None:
            ensemble_args.update(
                user_metadata={'multi_label_data': multi_label_data})
        ensemble_args_list.append(ensemble_args)
    return ensemble_args_list


def set_ensemble_args(name, description, args, model_fields,
                      objective_field=None, fields=None):
    """Return ensemble arguments dict

    """
    ensemble_args = {
        "name": name,
        "description": description,
        "number_of_models": args.number_of_models,
        "category": args.category,
        "tags": args.tag
    }
    if objective_field is not None and fields is not None:
        ensemble_args.update({"objective_field":
                              fields.field_id(objective_field)})
    # If evaluate flag is on and no test_split flag is provided,
    # we choose a deterministic sampling with
    # args.sample_rate (80% by default) of the data to create the model

    if (args.evaluate and args.test_split == 0 and
            args.test_datasets is None and not args.dataset_off):
        ensemble_args.update(seed=SEED)
        if args.sample_rate == 1:
            args.sample_rate = EVALUATE_SAMPLE_RATE

    if model_fields and fields is not None:
        input_fields = configure_input_fields(fields, model_fields)
        ensemble_args.update(input_fields=input_fields)

    if args.pruning and args.pruning != 'smart':
        ensemble_args.update(stat_pruning=(args.pruning == 'statistical'))
    if args.node_threshold > 0:
        ensemble_args.update(node_threshold=args.node_threshold)
    if args.balance:
        ensemble_args.update(balance_objective=True)
    if args.weight_field:
        weight_field = fields.field_id(args.weight_field)
        ensemble_args.update(weight_field=weight_field)
    if args.objective_weights:
        ensemble_args.update(objective_weights=args.objective_weights_json)
    if args.json_args['model']:
        ensemble_args.update(args.json_args['model'])
    ensemble_args.update(sample_rate=args.sample_rate,
                         replacement=args.replacement,
                         randomize=args.randomize,
                         tlp=args.tlp)
    if args.json_args['ensemble']:
        ensemble_args.update(args.json_args['ensemble'])
    return ensemble_args


def create_ensembles(datasets, ensemble_ids, ensemble_args, args,
                     number_of_ensembles=1,
                     api=None, path=None, session_file=None, log=None):
    """Create ensembles from input data

    """
    if api is None:
        api = bigml.api.BigML()
    ensembles = ensemble_ids[:]
    model_ids = []
    ensemble_args_list = []
    if isinstance(ensemble_args, list):
        ensemble_args_list = ensemble_args
    if number_of_ensembles > 0:
        message = dated("Creating %s.\n" %
                        plural("ensemble", number_of_ensembles))
        log_message(message, log_file=session_file,
                    console=args.verbosity)
        query_string = ALL_FIELDS_QS
        inprogress = []
        for i in range(0, number_of_ensembles):
            wait_for_available_tasks(inprogress, args.max_parallel_ensembles,
                                     api.get_ensemble, "ensemble",
                                     query_string=query_string,
                                     wait_step=args.number_of_models)

            if ensemble_args_list:
                ensemble_args = ensemble_args_list[i]
            ensemble = api.create_ensemble(datasets, ensemble_args)
            ensemble_id = check_resource_error(ensemble,
                                               "Failed to create ensemble: ")
            log_message("%s\n" % ensemble_id, log_file=log)
            ensemble_ids.append(ensemble_id)
            inprogress.append(ensemble_id)
            ensembles.append(ensemble)
            log_created_resources("ensembles", path, ensemble_id,
                                  open_mode='a')
        models, model_ids = retrieve_ensembles_models(ensembles, api, path)
        if number_of_ensembles < 2 and args.verbosity:
            message = dated("Ensemble created: %s.\n" %
                            get_url(ensemble))
            log_message(message, log_file=session_file,
                        console=args.verbosity)
            if args.reports:
                report(args.reports, path, ensemble)

    return ensembles, ensemble_ids, models, model_ids


def retrieve_ensembles_models(ensembles, api, path=None):
    """Retrieves the models associated to a list of ensembles

    """
    models = []
    model_ids = []
    for index in range(0, len(ensembles)):
        ensemble = ensembles[index]
        if (isinstance(ensemble, basestring) or
                bigml.api.get_status(ensemble)['code'] != bigml.api.FINISHED):
            try:
                ensemble = check_resource(ensemble, api.get_ensemble)
                ensembles[index] = ensemble
            except ValueError, exception:
                sys.exit("Failed to get a finished ensemble: %s" %
                         str(exception))
        model_ids.extend(ensemble['object']['models'])
    if path is not None:
        for model_id in model_ids:
            log_created_resources("models", path, model_id, open_mode='a')
    models = model_ids[:]
    models[0] = check_resource(models[0], api.get_model,
                               query_string=ALL_FIELDS_QS)
    return models, model_ids


def get_ensemble(ensemble, api=None, verbosity=True, session_file=None):
    """Retrieves remote ensemble in its actual status

    """
    if api is None:
        api = bigml.api.BigML()
    if (isinstance(ensemble, basestring) or
            bigml.api.get_status(ensemble)['code'] != bigml.api.FINISHED):
        message = dated("Retrieving ensemble. %s\n" %
                        get_url(ensemble))
        log_message(message, log_file=session_file,
                    console=verbosity)
        ensemble = check_resource(ensemble, api.get_ensemble)
        check_resource_error(ensemble, "Failed to get ensemble: ")
    return ensemble


def set_publish_model_args(args):
    """Set args to publish model

    """
    public_model = {}
    if args.black_box:
        public_model = {"private": False}
    if args.white_box:
        public_model = {"private": False, "white_box": True}
        if args.model_price:
            public_model.update(price=args.model_price)
        if args.cpp:
            public_model.update(credits_per_prediction=args.cpp)
    return public_model


def map_fields(fields_map, model_fields, dataset_fields):
    """Build a dict to map model to dataset fields

    """
    update_map = {}
    for (model_column, dataset_column) in fields_map.iteritems():
        update_map.update({
            model_fields.field_id(model_column):
            dataset_fields.field_id(dataset_column)})

    return update_map


def set_evaluation_args(name, description, args, fields=None,
                        dataset_fields=None, fields_map=None):
    """Return evaluation args dict

    """
    evaluation_args = {
        "name": name,
        "description": description,
        "tags": args.tag
    }

    if (args.number_of_models > 1 or args.ensemble):
        evaluation_args.update(combiner=args.method)
    if fields_map is not None and fields is not None:
        if dataset_fields is None:
            dataset_fields = fields
        evaluation_args.update({"fields_map": map_fields(fields_map,
                                                         fields,
                                                         dataset_fields)})
    if args.missing_strategy:
        evaluation_args.update(missing_strategy=args.missing_strategy)
    if args.json_args['evaluation']:
        evaluation_args.update(args.json_args['evaluation'])
    # Two cases to use out_of_bag and sample_rate: standard evaluations where
    # only the training set is provided, and cross_validation
    # [--dataset|--test] [--model|--models|--model-tag|--ensemble] --evaluate
    if ((args.dataset or args.test_set)
            and (args.model or args.models or args.model_tag or
                 (args.ensemble and args.number_of_models == 1))):
        return evaluation_args
    # [--train|--dataset] --test-split --evaluate
    if (args.test_split > 0 and (args.training_set or args.dataset)):
        return evaluation_args
    # --datasets --test-datasets
    if (args.datasets and (args.test_datasets or args.dataset_off)):
        return evaluation_args
    if args.sample_rate == 1:
        args.sample_rate = EVALUATE_SAMPLE_RATE
    evaluation_args.update(out_of_bag=True, seed=SEED,
                           sample_rate=args.sample_rate)
    return evaluation_args


def set_label_evaluation_args(name, description, args, labels, all_labels,
                              number_of_evaluations, fields, dataset_fields,
                              fields_map, objective_field):
    """Set of args needed to build an evaluation per label

    """
    if objective_field is None:
        objective_id = fields.field_id(fields.objective_field)
        objective_field = fields.fields[objective_id]['name']
    evaluation_args_list = []

    for index in range(number_of_evaluations - 1, -1, -1):
        label = labels[index]
        new_name = label_model_args(
            name, label, all_labels, [], objective_field)[0]
        evaluation_args = set_evaluation_args(new_name, description, args,
                                              fields, dataset_fields,
                                              fields_map)
        evaluation_args_list.append(evaluation_args)
    return evaluation_args_list


def create_evaluations(model_ids, datasets, evaluation_args, args, api=None,
                       path=None, session_file=None, log=None,
                       existing_evaluations=0):
    """Create evaluations for a list of models

       ``model_ids``: list of model ids to create an evaluation of
       ``datasets``: dataset objects or ids to evaluate with
       ``evaluation_args``: arguments for the ``create_evaluation`` call
       ``args``: input values for bigmler flags
       ``api``: api to remote objects in BigML
       ``path``: directory to store the BigMLer generated files in
       ``session_file``: file to store the messages of that session
       ``log``: user provided log file
       ``existing_evaluations``: evaluations found when attempting resume
    """
    evaluations = []
    dataset = datasets[0]
    evaluation_args_list = []
    if isinstance(evaluation_args, list):
        evaluation_args_list = evaluation_args
    if api is None:
        api = bigml.api.BigML()
    remaining_ids = model_ids[existing_evaluations:]
    if args.test_dataset_ids or args.dataset_off:
        remaining_datasets = datasets[existing_evaluations:]
    number_of_evaluations = len(remaining_ids)
    message = dated("Creating evaluations.\n")
    log_message(message, log_file=session_file,
                console=args.verbosity)

    inprogress = []
    for i in range(0, number_of_evaluations):
        model = remaining_ids[i]
        if args.test_dataset_ids or args.dataset_off:
            dataset = remaining_datasets[i]
        wait_for_available_tasks(inprogress, args.max_parallel_evaluations,
                                 api.get_evaluation, "evaluation")

        if evaluation_args_list != []:
            evaluation_args = evaluation_args_list[i]
        if args.cross_validation_rate > 0:
            new_seed = get_basic_seed(i + existing_evaluations)
            evaluation_args.update(seed=new_seed)

        evaluation = api.create_evaluation(model, dataset, evaluation_args)
        evaluation_id = check_resource_error(evaluation,
                                             "Failed to create evaluation: ")
        inprogress.append(evaluation_id)
        log_created_resources("evaluations", path, evaluation_id,
                              open_mode='a')
        evaluations.append(evaluation)
        log_message("%s\n" % evaluation['resource'], log_file=log)

    if (args.number_of_evaluations < 2 and len(evaluations) == 1
            and args.verbosity):
        evaluation = evaluations[0]
        if bigml.api.get_status(evaluation)['code'] != bigml.api.FINISHED:
            try:
                evaluation = check_resource(evaluation, api.get_evaluation)
            except ValueError, exception:
                sys.exit("Failed to get a finished evaluation: %s" %
                         str(exception))
            evaluations[0] = evaluation
        message = dated("Evaluation created: %s.\n" %
                        get_url(evaluation))
        log_message(message, log_file=session_file,
                    console=args.verbosity)
        if args.reports:
            report(args.reports, path, evaluation)

    return evaluations


def get_evaluation(evaluation, api=None, verbosity=True, session_file=None):
    """Retrieves evaluation in its actual state

    """
    if api is None:
        api = bigml.api.BigML()
    message = dated("Retrieving evaluation. %s\n" %
                    get_url(evaluation))
    log_message(message, log_file=session_file, console=verbosity)
    try:
        evaluation = check_resource(evaluation, api.get_evaluation)
    except ValueError, exception:
        sys.exit("Failed to get a finished evaluation: %s" % str(exception))
    return evaluation


def update_evaluation(evaluation, evaluation_args, args,
                      api=None, path=None, session_file=None):
    """Updates evaluation properties

    """
    if api is None:
        api = bigml.api.BigML()
    message = dated("Updating evaluation. %s\n" %
                    get_url(evaluation))
    log_message(message, log_file=session_file,
                console=args.verbosity)
    evaluation = api.update_evaluation(evaluation, evaluation_args)
    check_resource_error(evaluation, "Failed to update evaluation: %s"
                         % evaluation['resource'])
    if is_shared(evaluation):
        message = dated("Shared evaluation link. %s\n" %
                        get_url(evaluation, shared=True))
        log_message(message, log_file=session_file, console=args.verbosity)
        if args.reports:
            report(args.reports, path, evaluation)

    return evaluation


def save_evaluation(evaluation, output, api=None):
    """Creates the evaluation .txt and .json files

    """
    if api is None:
        api = bigml.api.BigML()
    evaluation_json = open(output + '.json', 'w', 0)
    evaluation = evaluation.get('object', evaluation).get('result', evaluation)
    evaluation_json.write(json.dumps(evaluation))
    evaluation_json.flush()
    evaluation_json.close()
    evaluation_txt = open(output + '.txt', 'w', 0)
    api.pprint(evaluation, evaluation_txt)
    evaluation_txt.flush()
    evaluation_txt.close()


def set_batch_prediction_args(name, description, args, fields=None,
                              dataset_fields=None, fields_map=None):
    """Return batch prediction args dict

    """
    batch_prediction_args = {
        "name": name,
        "description": description,
        "tags": args.tag,
        "header": args.prediction_header,
        "combiner": args.method
    }

    if fields_map is not None and fields is not None:
        if dataset_fields is None:
            dataset_fields = fields
        batch_prediction_args.update({
            "fields_map": map_fields(fields_map, fields, dataset_fields)})

    if args.prediction_info == NORMAL_FORMAT:
        batch_prediction_args.update(confidence=True)

    if args.prediction_info == FULL_FORMAT:
        batch_prediction_args.update(all_fields=True)
    if args.missing_strategy:
        batch_prediction_args.update(missing_strategy=args.missing_strategy)
    if args.json_args['batch_prediction']:
        batch_prediction_args.update(args.json_args['batch_prediction'])

    return batch_prediction_args


def create_batch_prediction(model_or_ensemble, test_dataset,
                            batch_prediction_args, args,
                            api=None, session_file=None,
                            path=None, log=None):
    """Creates remote batch_prediction

    """
    if api is None:
        api = bigml.api.BigML()
    message = dated("Creating batch prediction.\n")
    log_message(message, log_file=session_file, console=args.verbosity)
    batch_prediction = api.create_batch_prediction(model_or_ensemble,
                                                   test_dataset,
                                                   batch_prediction_args)
    log_created_resources("batch_prediction", path,
                          bigml.api.get_batch_prediction_id(batch_prediction),
                          open_mode='a')
    batch_prediction_id = check_resource_error(
        batch_prediction, "Failed to create batch prediction: ")
    try:
        batch_prediction = check_resource(batch_prediction,
                                          api.get_batch_prediction)
    except ValueError, exception:
        sys.exit("Failed to get a finished batch prediction: %s"
                 % str(exception))
    message = dated("Batch prediction created: %s\n"
                    % get_url(batch_prediction))
    log_message(message, log_file=session_file, console=args.verbosity)
    log_message("%s\n" % batch_prediction_id, log_file=log)
    if args.reports:
        report(args.reports, path, batch_prediction)
    return batch_prediction
