#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Find files and extract its contents into a `Data` object.

If files contain the ``.gz`` suffix, the system assumes they are
gzip-compressed.

"""

from __future__ import print_function
from __future__ import absolute_import

__author__ = "Lluís Vilanova"
__copyright__ = "Copyright 2009-2014, Lluís Vilanova"
__license__ = "GPL version 3 or later"

__maintainer__ = "Lluís Vilanova"
__email__ = "vilanova@ac.upc.edu"


import re
import os
import collections
import functools
import numpy
from six.moves import zip
import six
import sys
import warnings

from sciexp2.common import progress
from sciexp2.common import utils
from sciexp2.common import varref
import sciexp2.common.filter
import sciexp2.common.instance
import sciexp2.common.pp
import sciexp2.data
import sciexp2.launcher
import sciexp2.system
from . import _lazy
from ._errors import *
from . import _source


###########################################################################
## find_files

def _find_files(expression, *filters):
    res = sciexp2.common.instance.InstanceGroup()

    # normalize `filters` format
    new_filters = []
    for f in filters:
        if isinstance(f, collections.Iterable) and \
           not isinstance(f, six.string_types):
            new_filters += f
        else:
            new_filters.append(f)
    filters = sciexp2.common.filter.Filter(*new_filters)

    files = None
    if isinstance(expression, six.string_types) and os.path.isfile(expression):
        try:
            launcher = sciexp2.launcher.load(expression)
            jd = True
            state = sciexp2.system.Job.DONE
            files = launcher._system.build(set([state]), filters)
            with progress.progressable_simple(
                    files, None, msg="finding files...") as pfiles:
                for instance in pfiles:
                    instance = sciexp2.common.instance.Instance(instance)
                    instance["FILE"] = os.sep.join([launcher._system._base,
                                                    instance["DONE"]])
                    for key in ["_STATE", "LAUNCHER", "DONE", "FAIL"]:
                        del instance[key]
                    if filters.match(instance):
                        res += instance
        except sciexp2.launcher.LauncherLoadError:
            pass

    if files is None:
        jd = False
        variables = set(varref.expr_get_vars("@", expression, True)) |\
                    set(["FILE"])
        filters.validate(variables)
        files = utils.find_files("@", expression, "FILE")

        with progress.progressable_simple(
                files, None, msg="finding files...") as pfiles:
            for instance in pfiles:
                if filters.match(instance):
                    res += instance

    return res, jd


def find_files(expression, *filters):
    """Find paths matching a regular expression.

    For each path matching `expression` and `filters`, extract it into an
    `~sciexp2.common.instance.Instance` and add to the result.

    The expression can contain arbitrarily complex `regular expressions
    <http://docs.python.org/library/re.html#regular-expression-syntax>`_,
    and variables are extracted from:

        - variable references: ``@varname@``
        - named regular expression groups: ``(?P<varname>regexp)``


    Parameters
    ----------
    expression : expression
        An expression for selecting paths.
    filters : list of filters, optional
        Filters that paths must match to be considered. All provided filters
        are AND'ed.


    Returns
    -------
    InstanceGroup
        An `~sciexp2.common.instance.InstanceGroup` with the found files.


    Notes
    -----
    Implicitly added variables:
        ======== ======================
        Name     Contents
        ======== ======================
        ``FILE`` the complete path name
        ======== ======================

    If argument *expression* points to a job descriptor file generated by
    `~sciexp2.launchgen.Launchgen.launcher`, the resulting *FILE* values will
    be the path to the *DONE* files for these jobs that have been successfully
    executed.

    For convenience, the elements in *filters* can also be lists of filters.

    The results of multiple calls to `find_files` can be aggregated by
    extending the contents of an `~sciexp2.common.instance.InstanceGroup`
    object.

    You can also set the ``FILE`` variable to a `file`, `StringIO` or any other
    `io.IOBase` subclass instance by either modifying the result of
    `find_files` or by creating an `~sciexp2.common.instance.InstanceGroup` by
    hand.

    """
    return _find_files(expression, *filters)[0]


def _as_find_files(arg):
    """Treat argument as the result of `find_files` or arguments to call it.

    """
    if isinstance(arg, sciexp2.common.instance.InstanceGroup):
        return arg, False
    elif isinstance(arg, str):
        return _find_files(arg)
    elif isinstance(arg, tuple):
        return _find_files(arg[0], *arg[1:])
    else:
        raise TypeError("Invalid 'source' value: %s" % arg)


###########################################################################
## extract_*

#: Data type for string-like elements
STRING_DTYPE = "|S300"


class _FieldInfo (sciexp2.common.pp.Pretty):
    """Descriptor of a field in a source file.

    Parameters
    ----------
    name : str
        Field name.
    index : int
        Field number.
    type
        Field type.
    converter : optional
        Field converter (from string). Default is `type`.
    default : optional
        Default value when field is not present. Default is the default
        constructor of `type`.
    selected : bool, optional
        Whether the field has been selected for extraction. Default is `False`.

    """
    def __init__(self, name, index, type,
                 converter=None, default=None, selected=False):
        self.name = name
        self.index = index
        self.field_name = "f%d" % index
        self.type = type
        if converter is None:
            self.converter = type
        else:
            self.converter = converter
        if default is None:
            self.default = self.type()
        else:
            self.default = default
        self.selected = selected

    def copy(self):
        return _FieldInfo(name=self.name, index=self.index, type=self.type,
                          converter=self.converter, default=self.default,
                          selected=self.selected)

    def _repr_pretty_(self, p, cycle):
        names = ["name", "index", "field_name", "type", "converter", "default",
                 "selected"]
        with self.pformat(p, cycle):
            for name in names[:-1]:
                p.text("%s=%r," % (name, getattr(self, name)))
                p.breakable()
            name = names[-1]
            p.text("%s=%r" % (name, getattr(self, name)))

    def __repr__(self):
        return sciexp2.common.pp.Pretty.__repr__(self)


class _SourceInfo (sciexp2.common.pp.Pretty):
    """Descriptor of the fields in a source file.

    Parameters
    ----------
    fields : list or _SourceInfo
        List of fields (instances of `_FieldInfo`).

    """
    def __init__(self, fields):
        if isinstance(fields, _SourceInfo):
            self._fields = [field.copy() for field in fields]
        else:
            for field in fields:
                if not isinstance(field, _FieldInfo):
                    raise TypeError("Not a `_FieldInfo`: %r" % field)
            self._fields = fields

    def where(self, attribute, value):
        return [field
                for field in self
                if getattr(field, attribute) == value]

    def where_one(self, attribute, value):
        fields = self.where(attribute, value)
        assert len(fields) == 1
        return fields[0]

    def __len__(self):
        return len(self._fields)

    def __iter__(self):
        return iter(self._fields)

    def __getitem__(self, key):
        return self._fields[key]

    def __setitem__(self, key, value):
        self._fields[key] = value

    def _repr_pretty_(self, p, cycle):
        with self.pformat(p, cycle):
            p.pretty(self._fields)

    def __repr__(self):
        return sciexp2.common.pp.Pretty.__repr__(self)


def _probe_fsv_file(fobj, header, delimiter, genfromtxt_args, info=None):
    """Read FSV-formatted `fobj` and return its configuration.

    If `info` is given, returns new information metadata with updated `type`
    and `default` fields according to the `converter` field.
    """
    if info is not None:
        info = _SourceInfo(info)
        info_usecols = [i.index
                        for i in info if i.selected]
        info_converters = dict([
            (i.index, i.converter)
            for i in info
            if i.selected and i.converter is not None])
        if header is None:
            header = [i.name if i.name is not None else i.field_name
                      for i in info]
    else:
        info_usecols = None
        info_converters = None

    with _source.current_position(fobj):
        data = numpy.genfromtxt(fobj, delimiter=delimiter,
                                dtype=None,
                                names=header,
                                usecols=info_usecols,
                                converters=info_converters,
                                **genfromtxt_args)

    if len(data.dtype) > 0:
        names = data.dtype.names
        types = [data.dtype.fields[n][0].type for n in names]
    else:
        if info_usecols is None:
            names = ["f0"]
        else:
            names = [info[i].name for i in info_usecols]
        types = [data.dtype.type]

    # Get options affecting data extraction
    genfromtxt_data_args = {}
    for name in ["missing_values", "filling_values", "invalid_raise"]:
        if name in genfromtxt_args:
            genfromtxt_data_args[name] = genfromtxt_args[name]

    default_contents = six.StringIO(("," * len(names)) + "1")
    dtype = [(n, t) for n, t in zip(names, types)]
    default_data = numpy.genfromtxt(default_contents,
                                    usecols=list(range(len(names))) + [-1],
                                    delimiter=",",
                                    dtype=dtype + [("last", int)],
                                    **genfromtxt_data_args)
    defaults = default_data.tolist()[:-1]

    if info is None:
        info = _SourceInfo([
            _FieldInfo(name=n,
                       index=i,
                       type=t,
                       converter=t,
                       default=d,
                       selected=False)
            for i, (n, t, d) in enumerate(zip(names, types, defaults))
        ])
    else:
        for i in info:
            if not i.selected:
                continue
            didx = info_usecols.index(i.index)
            j = info.where_one("index", didx)
            j.type = types[didx]
            j.default = defaults[didx]

    return info


###########################################################################
### extract_func

def _extract_parse_source(source):
    source_, jd = _as_find_files(source)
    if len(source_) == 0:
        raise ValueError("Cannot accept 'source' with no contents")

    if not isinstance(source, six.string_types) or jd:
        source_vars = list(source_.variables())
        del source_vars[source_vars.index("FILE")]
    else:
        source_vars = varref.expr_get_vars("@", source, True)
    source = source_
    return source, source_vars


def _extract_parse_count(source, kwargs):
    count = kwargs.pop("count", None)
    if count is not None:
        if not isinstance(count, six.string_types):
            raise TypeError("Argument 'count' must be a string")
        if count in source:
            raise ValueError("Variable in 'count' already present in 'source'")
    return count


def _extract_parse_fields_to_vars(kwargs):
    res = kwargs.pop("fields_to_vars", [])
    if not isinstance(res, collections.Iterable):
        raise TypeError("Argument `fields_to_vars` must be a list of "
                        "field names or regular expressions")
    if isinstance(res, six.string_types):
        res = [res]
    res = list(res)
    if not all([isinstance(elem, six.string_types) for elem in res]):
        raise TypeError("Argument `fields_to_vars` must be a list of "
                        "field names or regular expressions")
    return res


def _extract_parse_vars_to_fields(kwargs, source, source_vars):
    names = kwargs.pop("vars_to_fields", [])
    if not isinstance(names, collections.Iterable):
        raise TypeError("Argument `vars_to_fields` must be a list of "
                        "variable names or regular expressions")
    if isinstance(names, six.string_types):
        names = [names]
    names = list(names)
    if not all([isinstance(elem, six.string_types) for elem in names]):
        raise TypeError("Argument `vars_to_fields` must be a list of "
                        "variable names or regular expressions")
    v2f = []
    for v2f_re in names:
        match = False
        v2f_cre = re.compile(v2f_re)
        for source_var in source.variables():
            if v2f_cre.match(source_var) is not None:
                match = True
                if source_var not in v2f:
                    v2f.append(source_var)
        if not match:
            raise ValueError("Value in `vars_to_fields` does not match "
                             "any variable: %s" % v2f_re)
    if len(v2f) == 0:
        return [], source_vars
    source_vars = [var for var in source_vars if var not in v2f]
    if len(source_vars) == 0:
        raise ValueError("Cannot specify all variables in `vars_to_fields`")
    return v2f, source_vars


_EXTRACT_FUNC_ARGS = ["probe_file", "count", "fields_to_vars",
                      "vars_to_fields", "allow_empty"]


def extract_func(source, func, *args, **kwargs):
    r"""Extract data using the provided function.

    Returns a 1-dimensional structured `~sciexp2.data.Data` array, with the
    results of extracting the data of each file in `source`. The result will
    have as many fields as the data produced by `func` (must be the same for
    each source), and as many elements as produced by `func` on each source.

    The variables in `source` (and their values) will be used as the dimension
    variables for the resulting array.


    Parameters
    ----------
    source : str or tuple or result of `find_files`
        `InstanceGroup` as returned by `find_files` with sources to extract
        data from. Can also be a tuple with arguments to `find_files`, or a
        single string with the first argument to `find_files`.
    func : callable
        Function to extract data from a single source file.
    probe_file : optional
        First file to extract data from. Accepts any value valid in the
        ``FILE`` key of `source`. If not in `source`, will only be used to
        probe for data types, but the actual data is ignored. Defaults to the
        first element of `source`.
    count : str, optional
        Dimension variable added to uniquely identify each per-source element
        by its index.
    fields_to_vars : list of str, optional
        Names of fields in the array returned by `func` that will be instead
        used as dimension variables (field values will be used as dimension
        tick values). You can also use regular expressions to select the
        matching fields.
    vars_to_fields : list of str, optional
        Names of variables in `source` that will be instead used as fields in
        the result (variable values will be used as field values). You can also
        use regular expressions to select the matching variables.
    allow_empty : bool, optional
        Whether to allow empty source files, which are ignored (default is
        `True`).
    args, kwargs
        Arguments passed to `func`.


    Raises
    ------
    ProbeError
        An error happened while probing the first source.
    ExtractError
        An error happened while extracting data.
    EmptyFileError
        The first source was empty, or some other was while `allow_empty` was
        `False`.


    Notes
    -----
    If `func` returns an array with one single element for each source, `count`
    and `fields_to_vars` are not necessary. Otherwise, at least one of them is
    required to disambiguate the elements of the resulting array (i.e., cannot
    have two elements with the same dimension ticks).

    Using `fields_to_vars` will drop the selected data fields and instead use
    their values in the dimension metadata.

    Before the actual data extraction, a single source will be selected (the
    first in `source`, or `probe_file` if provided) and used for probing the
    resulting data format (common to all sources). This probed information is
    discarded when later performing the actual data extraction. Thus, probing
    with the first element of `source` will actually extract it twice.

    The provided function `func` must accept the following arguments (plus
    those passed by the user through `args` and `kwargs`):

    `source`
        The file to extract data from (assumes a `file`-like interface).
    `source_first` : bool
        Whether `source` is being used for probing.

    The result of `func` must be a structured `numpy.ndarray` with at least one
    field and one or two dimensions (depending on whether `count` is being
    used).

    """
    ##################################################
    # parse arguments

    source, source_vars = _extract_parse_source(source)
    count = _extract_parse_count(source, kwargs)
    f2v = _extract_parse_fields_to_vars(kwargs)
    v2f, source_vars = _extract_parse_vars_to_fields(
        kwargs, source, source_vars)
    allow_empty = kwargs.pop("allow_empty", True)
    if not callable(func):
        raise TypeError("Argument 'func' is not callable")

    source_expr = "@" + "@-@".join(source_vars) + "@" \
                  if len(source_vars) > 0 else ""

    probe_file = kwargs.pop("probe_file", None)
    if probe_file is None:
        probe_file = source.get_index(0)["FILE"]
        probe_id = _source.get_name(probe_file, 0)
    else:
        probe_id = _source.get_name(probe_file, None)

    ##################################################
    # extract first file and post-process arguments

    def _extract_func_file(source_file, source_id, error_cls, probe,
                           *args, **kwargs):
        if _source.is_empty(source_file):
            if allow_empty:
                return []
            else:
                raise EmptyFileError(source_id)
        try:
            data = func(source=source_file, source_first=probe,
                        *args, **kwargs)
            # sanity checks
            if not isinstance(data, numpy.ndarray):
                raise TypeError("Extraction function returned non-numpy data")
            if len(data.dtype) == 0 or len(data.dtype.names) == 0:
                raise TypeError("Extraction function returned data without fields")
            if data.shape == tuple():
                raise TypeError("Extraction function returned data without shape")

            if len(data) > 1 and count is None and len(f2v) == 0:
                raise ValueError("Source has multiple results, you must use "
                                 "'count' and/or 'fields_to_vars'")
        except EmptyFileError:
            raise
        except Exception as e:
            _, _, exc_traceback = sys.exc_info()
            new_e = error_cls(source_id, e)
            six.reraise(new_e.__class__, new_e, exc_traceback)

        if len(data) == 0 and not allow_empty:
            raise EmptyFileError(source_id)
        return data

    probe_file = _source.get_file(probe_file)
    if _source.is_empty(probe_file):
        raise EmptyFileError(probe_id)
    data = _extract_func_file(probe_file, probe_id, ProbeError, True,
                              *args, **kwargs)

    # find fields in `fields_to_vars`
    if len(f2v) > 0:
        f2v_ = []
        for pidx, p in enumerate(f2v):
            pfound = False
            pre = re.compile(p + "$")
            for idx, name in enumerate(data.dtype.names):
                field_name = "f%d" % idx
                if (pre.match(name) or pre.match(field_name)):
                    f2v_.append(name)
                    pfound = True
            if not pfound:
                raise ValueError(
                    ("Entry number %d (%s) in " % (pidx, p)) +
                    "`fields_to_vars` does not match any extracted variable")
        f2v = f2v_

    # get index of `fields_to_vars` elements
    if len(f2v) > 0:
        def _find_f2v_idx(name):
            for idx, n in enumerate(data.dtype.names):
                if name == n:
                    return idx
        f2v_dict = dict([(p, _find_f2v_idx(p)) for p in f2v])
        f2v_indexes = [f2v_dict[name] for name in f2v]
        f2v_indexes.sort()

    # final sanity checks for arguments
    if count in f2v:
        raise ValueError("Variable in 'count' already present in "
                         "`fields_to_vars`")
    if all(name in f2v for name in data.dtype.names):
        raise ValueError("Cannot move all fields into variables "
                         "(`fields_to_vars`)")

    ##################################################
    # mass-extraction

    def _extract_func_postprocess(data, source, v2f):
        data_dim = []
        data_values = []

        for i, ldata in enumerate(data):
            variables = []
            values = list(ldata.tolist())

            if count is not None:
                variables.append((count, i))
            if len(f2v) > 0:
                for name in f2v:
                    variables.append((name, values[f2v_dict[name]]))
                for pidx in reversed(f2v_indexes):
                    del values[pidx]

            data_dim.append(dict(
                [(key, val) for key, val in source.items()
                 if key not in v2f] + variables))
            data_values.append(tuple(values + [source[var] for var in v2f]))

        return data, data_values, data_dim

    dim = []
    dtype = [(data.dtype.names[i], data.dtype[i])
             for i in range(len(data.dtype))
             if data.dtype.names[i] not in f2v]
    for var in v2f:
        tmp = numpy.array(list(source[var].keys()))
        if numpy.issubdtype(tmp.dtype, str):
            vdtype = STRING_DTYPE
        else:
            vdtype = tmp.dtype
        dtype.append((var, vdtype))
    arr, growable = _arr_growable_init(
        len(source), data, dtype)

    with progress.progressable_simple(
            source, None, msg="extracting files...") as psource:
        for source_idx, source_name in enumerate(psource):
            source_file = _source.get_file(source_name["FILE"])
            source_id = _source.get_name(source_file, source_idx)
            data = _extract_func_file(source_file, source_id, ExtractError,
                                      False, *args, **kwargs)
            if len(data) > 0:
                data, data_values, data_dim = _extract_func_postprocess(
                    data, source_name, v2f)
                arr, growable = _arr_growable_fill(arr, growable,
                                                   len(source), dim,
                                                   data_values, data_dim)

    arr = _arr_growable_fini(arr, dim)

    if len(f2v) > 0:
        if source_expr != "":
            source_expr += "-"
        source_expr += "@" + "@-@".join(f2v) + "@"
    if count:
        if source_expr != "":
            source_expr += "-"
        source_expr += "@%s@" % count
    return sciexp2.data.data_array(arr,
                                   dims=[(source_expr, dim)],
                                   dtype=arr.dtype)


def _arr_growable_init(sources, data, dtype):
    avg_lines = float(len(data))
    avg_count = 1
    arr = numpy.ndarray(sources * int(avg_lines), dtype=dtype)
    return arr, (avg_lines, avg_count)


def _arr_growable_fill(arr, info, sources, dim, arr_data, dim_data):
    """Fill-in contents and grow if necessary."""
    assert len(arr_data) == len(dim_data)
    avg_lines, avg_count = info
    avg_lines = (len(arr_data[0]) * len(arr_data)) + (avg_count * avg_lines)
    avg_lines = avg_lines / (avg_count + 1)
    avg_count += len(arr_data)

    # grow array, accounting for averages
    if len(dim) + len(dim_data) > len(arr):
        size = max(sources * int(avg_lines),
                   len(dim) + len(dim_data))
        arr_ = numpy.ndarray(size, dtype=arr.dtype)
        arr_[:len(arr)] = arr
        arr = arr_

    arr[len(dim):len(dim)+len(dim_data)] = arr_data
    dim.extend(dim_data)

    return arr, (avg_lines, avg_count)


def _arr_growable_fini(arr, dim):
    """Trims excess elements."""
    if len(dim) < len(arr):
        arr = arr[:len(dim)]
    return arr


###########################################################################
### extract_txt

def _txt_normalize_descriptor(index, descriptor):
    error = False
    fields = {"regex": None, "name": None, "converter": None}

    if isinstance(descriptor, six.string_types):
        fields["regex"] = descriptor
    elif callable(descriptor):
        fields["converter"] = descriptor
    elif isinstance(descriptor, collections.Iterable):
        if len(descriptor) == 2:
            if isinstance(descriptor[0], six.string_types) and \
               isinstance(descriptor[1], six.string_types):
                fields["regex"] = descriptor[0]
                fields["name"] = descriptor[1]
            elif isinstance(descriptor[0], six.string_types) and \
                 callable(descriptor[1]):
                fields["regex"] = descriptor[0]
                fields["converter"] = descriptor[1]
            else:
                error = True
        elif len(descriptor) == 3:
            if isinstance(descriptor[0], six.string_types) and \
               isinstance(descriptor[1], six.string_types) and \
               callable(descriptor[2]):
                fields["regex"] = descriptor[0]
                fields["name"] = descriptor[1]
                fields["converter"] = descriptor[2]
            else:
                error = True
        else:
            error = True
    else:
        error = True

    if error:
        msg = "Invalid element in position %d " % index
        msg += "of `fields`: %s" % repr(descriptor)
        raise ValueError(msg)

    if fields["regex"] is not None:
        fields["regex"] = re.compile(fields["regex"] + "$")

    return fields


def extract_txt(source, *fields, **kwargs):
    r"""Extract data from structured text files.

    Uses `extract_func` to extract data from each file in `source` and returns a
    1-dimensional structured `~sciexp2.data.Data` array. Sources must be in a
    field-separated-value format (e.g., CSV with one column per field). For each
    source, the array has as many fields as selected by `fields`, and as many
    elements as lines.

    If any data element is missing, its default value is set according to
    `numpy.genfromtxt`.


    Parameters
    ----------
    source
        Argument to `extract_func`.
    fields : optional
        List describing which and how to extract fields from the source files
        (by default extracts all fields and auto-detects their format).
    delimiter : str, optional
        The string used to separate values (default is ``,``, which corresponds
        to CSV format).
    header : bool, optional
        Whether the source files contain a header describing the fields.
        The header is used to provide a field name, which otherwise defaults
        to ``f<num>`` (default is `True`).
    probe_file, count, fields_to_vars, allow_empty : optional
        Arguments to `extract_func`.
    comments, skip_header, skip_footer, missing_values, filling_values,
    excludelist, deletechars, defaultfmt, autostrip, replace_space,
    case_sensitive, invalid_raise : optional
        Arguments passed to `numpy.genfromtxt`.


    Returns
    -------
    Data
        A 1-dimensional `~sciexp2.data.Data` array with the extracted data.


    Notes
    -----
    Elements in argument `fields` have the base format ``(regexp, name, type or
    func)``, where each is:

    * Regular expression to match on each field name.

      Can be applied to both the source header (if available) and the default
      field name (see `defaultfmt`).
      If present, only fields matched by any regexp will be extracted.

    * New field name.

      Used to rename a field, can reference groups in the previous regular
      expression.

    * Field value type or function to convert the string in the source file to
      its actual value.

      If the converter raises an exception, a default value will be assigned
      according to the field's type.

    A few shortcuts are also available:

        ==========================  ===========================================
        Shortcut                    Description
        ==========================  ===========================================
        ``regexp``                  Select matching fields.
        ``type or func``            Use specified converter on selected field
                                    (selects all fields if none was).
        ``(regexp, name)``          Select matching fields and rename.
        ``(regexp, type or func)``  Select matching fields and convert.
        ==========================  ===========================================

    Since `fields` is a list, the last matching element on each field takes
    precedence on the selected action. Thus the order in `fields` can be used
    to refine the actions.

    When used, `fields_to_vars` is applied after renaming fields.


    Examples
    --------
    >>> cat data/extract_txt/foo.csv    # doctest: +SKIP
    time,iterations,test
    12.3,5,fast
    53.8,9,slow
    ,,failed
    >>> extract_txt("data/extract_txt/@configuration@.csv",
    ...             count="line")
    data_array([(8.6, 2, 'fast'),
                (53.36, 1, 'slow'),
                (nan, -1, 'failed'),
                (12.3, 5, 'fast'),
                (53.8, 9, 'slow'),
                (nan, -1, 'failed')],
               dtype=[('time', '<f8'), ('iterations', '<i8'), ('test', '|S300')],
               dims=[('@configuration@-@line@',
                      ['bar-0', 'bar-1', 'bar-2', 'foo-0', 'foo-1', 'foo-2'])])

    You can limit which files are extracted without explicitly using
    `find_files`:

    >>> extract_txt(("data/extract_txt/@configuration@.csv",
    ...              "configuration == 'bar'"),
    ...             count="line")
    data_array([(8.6, 2, 'fast'), (53.36, 1, 'slow'), (nan, -1, 'failed')],
               dtype=[('time', '<f8'), ('iterations', '<i8'), ('test', '|S300')],
               dims=[('@configuration@-@line@', ['bar-0', 'bar-1', 'bar-2'])])

    Using `fields_to_vars` is usually more convenient than `count` when
    discriminating between multiple elements:

    >>> extract_txt("data/extract_txt/@configuration@.csv",
    ...             fields_to_vars=["test"])
    data_array([(8.6, 2), (53.36, 1), (nan, -1), (12.3, 5), (53.8, 9), (nan, -1)],
               dtype=[('time', '<f8'), ('iterations', '<i8')],
               dims=[('@configuration@-@test@',
                      ['bar-fast',
                       'bar-slow',
                       'bar-failed',
                       'foo-fast',
                       'foo-slow',
                       'foo-failed'])])

    You can use Python's support for regular expressions to rename fields, as
    well as use your own functions to convert field values:

    >>> def str_float2int(n): return int(float(n))
    >>> extract_txt("data/extract_txt/@configuration@.csv",
    ...             ("(.*)", r"Col_\1"),
    ...             ("time", str_float2int),
    ...             count="line")
    data_array([(8, 2, 'fast'),
                (53, 1, 'slow'),
                (-1, -1, 'failed'),
                (12, 5, 'fast'),
                (53, 9, 'slow'),
                (-1, -1, 'failed')],
               dtype=[('Col_time', '<i8'),
                      ('Col_iterations', '<i8'),
                      ('Col_test', '|S300')],
               dims=[('@configuration@-@line@',
                      ['bar-0', 'bar-1', 'bar-2', 'foo-0', 'foo-1', 'foo-2'])])

    See also
    --------
    extract_func
    re

    """
    delimiter = kwargs.pop("delimiter", ",")

    header = kwargs.pop("header", True)
    header = True if header else None

    # Arguments specific to genfromtxt
    genfromtxt_args = {}
    for name in [
            "comments", "skip_header", "skip_footer", "missing_values",
            "filling_values", "excludelist", "deletechars", "defaultfmt",
            "autostrip", "replace_space", "case_sensitive", "invalid_raise"
    ]:
        if name in kwargs:
            genfromtxt_args[name] = kwargs.pop(name)

    kwargs_unhandled = set(kwargs) - set(_EXTRACT_FUNC_ARGS)
    if len(kwargs_unhandled) > 0:
        raise TypeError("Unexpected arguments %s" %
                        ", ".join(kwargs_unhandled))

    state = {}
    return extract_func(source, _extract_txt_func,
                        fields=fields,
                        delimiter=delimiter,
                        header=header,
                        genfromtxt_args=genfromtxt_args,
                        state=state,
                        **kwargs)


def _extract_txt_func(source, source_first, header, delimiter, fields,
                      genfromtxt_args, state):
    if source_first:
        return _extract_txt_probe(source,
                                  fields=fields,
                                  delimiter=delimiter,
                                  header=header,
                                  genfromtxt_args=genfromtxt_args,
                                  state=state)
    else:
        return _extract_txt_file(source,
                                 delimiter=delimiter,
                                 genfromtxt_args=genfromtxt_args,
                                 **state)


def _extract_txt_probe(source, fields, delimiter, header, genfromtxt_args,
                       state):
    ##################################################
    # parse field information and transform according to `fields`

    info_orig = _probe_fsv_file(source,
                                header=header,
                                delimiter=delimiter,
                                genfromtxt_args=genfromtxt_args)
    info = _SourceInfo(info_orig)

    for index, descriptor in enumerate(fields):
        # normalize descriptors
        fields = _txt_normalize_descriptor(index, descriptor)

        # modify fields in info_orig with descriptors
        if fields["regex"] is None:
            assert fields["name"] is None
            if index == 0:
                # select all
                for i in info:
                    i.selected = True
            # apply on selected
            for i in info:
                if i.selected and fields["converter"] is not None:
                    i.converter = fields["converter"]
        else:
            # apply on regexp
            for i in info_orig:
                match = fields["regex"].match(i.name)
                if match is None:
                    match = fields["regex"].match(i.field_name)

                if match is not None:
                    j = info.where_one("field_name", i.field_name)
                    j.selected = True
                    if fields["name"] is not None:
                        j.name = match.expand(fields["name"])
                    if j.selected and fields["converter"] is not None:
                        j.converter = fields["converter"]

    if len(fields) == 0:
        for i in info_orig:
            info.where_one("field_name", i.field_name).selected = True

    ##################################################
    # recalculate types by applying converters

    info = _probe_fsv_file(source, header=header, delimiter=delimiter,
                           genfromtxt_args=genfromtxt_args, info=info)

    ##################################################
    # adjust arguments

    # skip header when extracting actual data
    if header:
        if "skip_header" not in genfromtxt_args:
            genfromtxt_args["skip_header"] = 0
        genfromtxt_args["skip_header"] += 1

    # make strings long enough
    for i in info:
        if issubclass(i.type, six.string_types):
            i.type = STRING_DTYPE

    # arguments for extraction using `genfromtxt`
    state["x_dtype"] = [
        (i.name, i.type)
        for i in info if i.selected]
    state["x_usecols"] = [
        i.index
        for i in info if i.selected]
    state["x_converters"] = dict([
        (i.index, i.converter)
        for i in info
        if i.selected and i.converter is not None])
    if len(state["x_converters"]) == 0:
        state["x_converters"] = None

    ##################################################
    # extract probed file

    return _extract_txt_file(source,
                             delimiter=delimiter,
                             genfromtxt_args=genfromtxt_args,
                             **state)


def _extract_txt_file(source, delimiter, x_usecols, x_converters, x_dtype,
                      genfromtxt_args):
    data = numpy.genfromtxt(source, delimiter=delimiter,
                            usecols=x_usecols,
                            converters=x_converters,
                            dtype=x_dtype,
                            names=None,
                            **genfromtxt_args)

    # single-field source
    if len(data.dtype) == 0:
        if data.shape == tuple():
            data = numpy.array([(data,)], dtype=x_dtype)
        else:
            data = numpy.array([(i,) for i in data], dtype=x_dtype)
    # single-row source
    if data.shape == tuple():
        data = numpy.array([data], dtype=data.dtype)

    return data


###########################################################################
### extract_regex

def _regex_normalize(expression, index):
    """Replace variable references with `re` named groups or backreferences."""
    if isinstance(expression, tuple):
        if len(expression) != 2 or \
           not isinstance(expression[0], six.string_types):
            raise TypeError("Invalid regex entry %d: %s" % (index, expression))
        converter = expression[1]
        expression = expression[0]
        if not isinstance(converter, dict):
            converter = {None: converter}
        converter_default = converter.pop(None, None)
        for var in varref.expr_get_vars("@", expression, accept_regexp=True):
            converter[var] = converter.get(var, converter_default)
        if not all(callable(c) or c is None for c in converter.values()):
            raise TypeError("Invalid regex entry %d: %s" % (index, expression))
    elif not isinstance(expression, six.string_types):
        raise TypeError("Invalid regex entry %d: %s" % (index, expression))
    else:
        converter = {}
        for var in varref.expr_get_vars("@", expression, accept_regexp=True):
            converter[var] = None

    return varref.expr_to_regexp("@", expression), converter


def _regex_extract(source, cre):
    if isinstance(cre, list):           # multi_line
        cres = cre
        contents = "".join(source.readlines())
        values = []
        value = {}
        for cre in cres:
            for i, m in enumerate(cre.finditer(contents)):
                values = values + [{}] * (i - len(values) + 1)
                if m is None:
                    continue
                for k, v in six.iteritems(m.groupdict()):
                    if v is None:
                        continue
                    values[i][k] = v
        return values
    else:
        values = []
        value = {}
        for line in source:
            m = cre.search(line)
            if m is None:
                continue
            if any(v is not None and k in value and value[k] is not None
                   for k, v in six.iteritems(m.groupdict())):
                # new set of results
                values.append(value)
                value = {}
            for k, v in six.iteritems(m.groupdict()):
                if v is None:
                    continue
                value[k] = v
        values.append(value)
        return values


def extract_regex(source, *fields, **kwargs):
    r"""Extract data using regular expressions.

    Uses `extract_func` to extract data from each file in `source` and returns
    a 1-dimensional structured `~sciexp2.data.Data` array. Data in sources is
    extracted using the regular expressions in `fields`. For each source, the
    array has as many fields as regular expressions in `fields`, and as many
    elements as different matches in `fields` (e.g., if two lines match the
    same regular expression, the resulting data will have two elements).

    If any data element is missing, its default value is set according to
    `numpy.genfromtxt`.


    Parameters
    ----------
    source
        Argument to `extract_func`.
    fields
        List describing how to extract data from the source files.
    multi_line : bool, optional
        Whether to allow regexps in `fields` to match multiple lines (default
        is no, which is faster).
    probe_file, count, fields_to_vars, allow_empty : optional
        Arguments to `extract_func`.


    Returns
    -------
    Data
        A 1-dimensional `~sciexp2.data.Data` array with the extracted data.


    Notes
    -----
    Elements in argument `fields` have the base format ``(regexp, dict)``,
    where each is:

    * Regular expression to match on the source contents.

      Variable references in the regular expression will be used to extract a
      value into a field named as the variable.

      In the case of complex patterns, you can use named regular expression
      groups instead of variable references; i.e., ``@var@`` is a shorthand for
      ``(?P<var>.+)``.

    * Dictionary mapping variables (in the regexp) to value types or functions
      used to convert the string in the source file to its actual value.

      Using ``None`` as a key will apply that type or function to convert the
      values of all variables not explicitly listed as keys in the dictionary.

      If no value was extracted (i.e., an optional regexp group), the converter
      will receive `None`.

      If the converter raises an exception, a default value will be assigned
      according to the field's type.

    Every element in `fields` can also be expressed in any of the following
    shortcuts:

    * ``regexp``

      The field type will be inferred automatically from the probing file.

    * ``(regexp, type or func)``

      A shortcut for ``(regexp, {None: type or func})``.


    Examples
    --------
    As an expression can contain multiple variable references, you can extract
    more than one value from every line:

    >>> cat data/extract_regex/foo.txt  # doctest: +SKIP
    test: fast
      time = 12.3 (5 iterations)
      test: complete
    test: slow
      time = 53.8 (9 iterations)
      test: complete
    test: failed
      test: failed
    >>> def str_float2int(n): return int(float(n))
    >>> extract_regex("data/extract_regex/@configuration@.txt",
    ...               "^test: @test@",
    ...               ("time = @time@ \\(@iterations@ ", str_float2int),
    ...               count = "num")
    data_array([('fast', 53, 1),
                ('fast', 12, 5),
                ('slow', 53, 9),
                ('failed', -1, -1)],
               dtype=[('test', '|S300'), ('time', '<i8'), ('iterations', '<i8')],
               dims=[('@configuration@-@num@',
                      ['bar-0', 'foo-0', 'foo-1', 'foo-2'])])

    You can limit which files are extracted without explicitly using
    `find_files`:

    >>> extract_regex(("data/extract_regex/@configuration@.txt",
    ...                "configuration=='bar'"),
    ...               "^test: @test@",
    ...               ("time = @time@ \\(@iterations@ ", str_float2int),
    ...               count = "num")
    data_array([('fast', 53, 1)],
               dtype=[('test', '|S300'), ('time', '<i8'), ('iterations', '<i8')],
               dims=[('@configuration@-@num@', ['bar-0'])])

    Using `fields_to_vars` is usually more convenient than `count` when
    discriminating between multiple elements:

    >>> extract_regex("data/extract_regex/@configuration@.txt",
    ...               "^test: @test@",
    ...               ("time = @time@ \\(@iterations@ ", str_float2int),
    ...               fields_to_vars=["test"])
    data_array([(53, 1), (12, 5), (53, 9), (-1, -1)],
               dtype=[('time', '<i8'), ('iterations', '<i8')],
               dims=[('@configuration@-@test@',
                      ['bar-fast', 'foo-fast', 'foo-slow', 'foo-failed'])])


    See also
    --------
    extract_func
    re

    """
    # `fields`
    if len(fields) == 0:
        raise ValueError("Argument `fields` cannot be empty")
    fields = [_regex_normalize(r, i) for i, r in enumerate(fields)]

    multi_line = kwargs.pop("multi_line", False)

    kwargs_unhandled = set(kwargs) - set(_EXTRACT_FUNC_ARGS)
    if len(kwargs_unhandled) > 0:
        raise TypeError("Unexpected arguments: %s" %
                        ", ".join(kwargs_unhandled))

    converters = {}
    for field in fields:
        convs = field[1]
        for key in convs:
            if key in converters:
                raise ValueError("Duplicate field name: %s" % key)
            converters[key] = convs[key]
    regexes = [i[0] for i in fields]

    state = {}
    return extract_func(source, _extract_regex_func,
                        regexes=regexes,
                        converters=converters,
                        multi_line=multi_line,
                        state=state,
                        **kwargs)


def _extract_regex_func(source, source_first, regexes, converters, multi_line,
                        state):
    if source_first:
        return _extract_regex_probe(source, regexes, converters, multi_line,
                                    state)
    else:
        return _extract_regex_file(source, **state)


def _extract_regex_probe(source, regexes, converters, multi_line, state):
    ##################################################
    # establish data types from inital source

    variables = []
    for regex in regexes:
        regex_vars = varref.regexp_get_names(regex)
        variables.extend(regex_vars)

    if multi_line:
        state["x_cre"] = [re.compile(regex, flags=re.MULTILINE)
                          for regex in regexes]
    else:
        state["x_cre"] = re.compile("(" + ")|(".join(regexes) + ")")

    with _source.current_position(source):
        data = _regex_extract(source, state["x_cre"])
        if len(data) == 0:
            raise EmptyFileError(_source.get_name(source, None))
        data = data[0]
        if len(data) == 0:
            raise EmptyFileError(_source.get_name(source, None))

    data_contents = []
    for var in variables:
        conv = converters[var]
        val = data.get(var, None)
        if conv is not None:
            # if invalid v, conversion can throw an exception (e.g., float(""))
            try:
                val = conv(val)
            except Exception:
                val = ""
            val = str(val)
        elif val is None:
            val = ""
        else:
            # XXX: avoid spurious complaints
            val = val.replace("\n", "")
        data_contents.append(val)
    data_contents = ",".join(data_contents)
    contents = ",".join(variables) + "\n" + data_contents
    # TODO: do not probe non-missing entries with explicit converter
    assert len(contents) > 0
    info = _probe_fsv_file(six.StringIO(contents),
                           header=True,
                           delimiter=",",
                           genfromtxt_args={})

    # apply user-defined converters
    for regex in regexes:
        for var in varref.regexp_get_names(regex):
            conv = converters[var]
            if conv is not None:
                info.where_one("name", var).converter = conv
    for i in info:
        i.selected = True

    ##################################################
    # recalculate types by applying converters

    if len(data_contents) == 0:
        data_contents = ","
    info = _probe_fsv_file(six.StringIO(data_contents),
                           header=None,
                           delimiter=",",
                           genfromtxt_args={},
                           info=info)

    ##################################################
    # adjust arguments

    # make strings long enough
    for i in info:
        if issubclass(i.type, six.string_types):
            i.type = STRING_DTYPE

    # arguments for extraction
    state["x_dtype"] = [(i.name, i.type) for i in info]
    state["x_defaults"] = [i.default for i in info]
    state["x_converters"] = [i.converter for i in info]
    state["x_map"] = dict([(i.name, i.index) for i in info])

    ##################################################
    # extract probed file

    return _extract_regex_file(source, **state)


def _extract_regex_file(source,
                        x_cre, x_dtype, x_defaults, x_converters, x_map):
    data = _regex_extract(source, x_cre)

    variables = [var for var, _ in sorted(x_map.items(),
                                          key=lambda (var, idx): idx)]

    # apply converters, use defaults otherwise
    data_values = []
    for ldata in data:
        values = []
        for idx, (conv, var) in enumerate(zip(x_converters, variables)):
            val = ldata.get(var, None)
            # conversion can throw an exception (e.g., float(""))
            try:
                val = conv(val)
            except Exception:
                val = x_defaults[idx]
            values.append(val)

        data_values.append(tuple(values))

    data = numpy.ndarray(len(data_values), dtype=x_dtype)
    data[:] = data_values

    return data


###########################################################################
### lazy

def lazy(func, *args, **kwargs):
    """Lazily evaluate function with given arguments.

    Returns an object for lazy evaluation, which can be used as a parameter to
    other lazily evaluated functions, or can be used to lazily invoke methods
    on it.

    The `func` argument can be any arbitrary function, including those defined
    by the user.

    The actual result is computed by calling the `realize(path)` method on the
    resulting object. If `path` does not exist, the result will be computed and
    saved to disk. If `path` exists but was generated from older results or
    different arguments, the result is also computed and saved to disk. In the
    common case where it is supposed to compute the exact same result, it
    simply loads the result from disk.

    The `path` argument to `realize` is optional. If it is not provided, it
    will compute the result, but will not save it to disk.

    Similarly, the `checkpoint(path)` method returns a new lazily evaluated
    result that will be saved to disk (just like `realize`).


    Parameters
    ----------
    func
        Function to lazily evaluate.
    args, kwargs
        Arguments to `func`.


    See also
    --------
    lazy_wrap, lazy_wrap_realize, lazy_wrap_checkpoint


    Examples
    --------
    The use of `lazy` has some knowledge about data extraction, and will force
    it only if newer data is available, otherwise loading the previous result
    from "/tmp/extracted":

    >>> data = lazy(extract_txt, "./data/maybe/@configuration@.csv",
    ...             fields_to_vars=["test"])
    >>> data = data.realize("/tmp/extracted")
    >>> data
    data_array([(8.6, 2), (53.36, 1), (nan, -1), (12.3, 5), (53.8, 9), (nan, -1)],
               dtype=[('time', '<f8'), ('iterations', '<i8')],
               dims=[('@configuration@-@test@',
                      ['bar-fast',
                       'bar-slow',
                       'bar-failed',
                       'foo-fast',
                       'foo-slow',
                       'foo-failed'])])

    For convenience, invoking a method on a lazily evaluated result returns a
    new lazily evaluated result:

    >>> data = lazy(extract_txt, "./data/maybe/@configuration@.csv",
    ...             fields_to_vars=["test"])
    >>> data = data.reshape(["configuration"])
    >>> data = data.realize("/tmp/reshaped")

    This is equivalent to:

    >>> data = lazy(extract_txt, "./data/maybe/@configuration@.csv",
    ...             fields_to_vars=["test"])
    >>> data = lazy(sciexp2.data.Data.reshape, data, ["configuration"])
    >>> data = data.realize("/tmp/reshaped")
    >>> data
    data_array([[(8.6, 2), (53.36, 1), (nan, -1)],
                [(12.3, 5), (53.8, 9), (nan, -1)]],
               dtype=[('time', '<f8'), ('iterations', '<i8')],
               dims=[('@configuration@', ['bar', 'foo']),
                     ('@test@', ['fast', 'slow', 'failed'])])

    If an intermediate result is used by many other functions, you can save it
    to disk with `checkpoint(path)`. In this example, only the calculations for
    `reshape` are reexecuted if "/tmp/extracted" already exists and is up to
    date:

    >>> data = lazy(extract_txt, "./data/maybe/@configuration@.csv",
    ...             fields_to_vars=["test"])
    >>> data = data.checkpoint("/tmp/extracted")
    >>> data1 = data.reshape(["configuration"])
    >>> data1 = data1.realize("/tmp/reshaped1")
    >>> data1
    data_array([[(8.6, 2), (53.36, 1), (nan, -1)],
                [(12.3, 5), (53.8, 9), (nan, -1)]],
               dtype=[('time', '<f8'), ('iterations', '<i8')],
               dims=[('@configuration@', ['bar', 'foo']),
                     ('@test@', ['fast', 'slow', 'failed'])])
    >>> data2 = data.reshape(["test"])
    >>> data2 = data2.realize("/tmp/reshaped2")
    >>> data2
    data_array([[(8.6, 2), (12.3, 5)],
                [(53.36, 1), (53.8, 9)],
                [(nan, -1), (nan, -1)]],
               dtype=[('time', '<f8'), ('iterations', '<i8')],
               dims=[('@test@', ['fast', 'slow', 'failed']),
                     ('@configuration@', ['bar', 'foo'])])

    """
    return _lazy.Result(func, args, kwargs)


def lazy_wrap(func):
    """Convenience decorator/wrapper for lazily-evaluated user functions.

    Invoking ``func(args)`` when `func` has been wrapped/decorated with
    `lazy_wrap` is equivalent to calling ``lazy(func, args)``.


    See also
    --------
    lazy, lazy_wrap_realize, lazy_wrap_checkpoint


    Examples
    --------
    The slightly shorter and more readable version:

    >>> @lazy_wrap
    ... def foo (data):
    ...     "Return first element"
    ...     return data[:3]
    >>> data = lazy(extract_txt, "./data/maybe/@configuration@.csv",
    ...             fields_to_vars=["test"])
    >>> data = foo(data)
    >>> data.realize()
    data_array([(8.6, 2), (53.36, 1), (nan, -1)],
               dtype=[('time', '<f8'), ('iterations', '<i8')],
               dims=[('@configuration@-@test@',
                      ['bar-fast', 'bar-slow', 'bar-failed'])])

    is equivalent to an explicit call to `lazy` on a user function:

    >>> def foo (data):
    ...     "Return first element"
    ...     return data[:3]
    >>> data = lazy(extract_txt, "./data/maybe/@configuration@.csv",
    ...             fields_to_vars=["test"])
    >>> data = lazy(foo, data)
    >>> data.realize()
    data_array([(8.6, 2), (53.36, 1), (nan, -1)],
               dtype=[('time', '<f8'), ('iterations', '<i8')],
               dims=[('@configuration@-@test@',
                      ['bar-fast', 'bar-slow', 'bar-failed'])])

    """
    @functools.wraps(func)
    def do_func(*args, **kwargs):
        return lazy(func, *args, **kwargs)
    return do_func


def lazy_wrap_realize(path=None):
    """Convenience decorator/wrapper for lazily-evaluated user functions.

    Returns the result of a call to `realize(path)` on the lazyly evaluated
    result.


    See also
    --------
    lazy_wrap


    Examples
    --------
    >>> @lazy_wrap_realize("/tmp/foo")
    ... def foo (data):
    ...     "Return first element"
    ...     return data[:3]
    >>> data = lazy(extract_txt, "./data/maybe/@configuration@.csv",
    ...             fields_to_vars=["test"])
    >>> foo(data)
    data_array([(8.6, 2), (53.36, 1), (nan, -1)],
               dtype=[('time', '<f8'), ('iterations', '<i8')],
               dims=[('@configuration@-@test@',
                      ['bar-fast', 'bar-slow', 'bar-failed'])])

    """
    def do_func1(func):
        @functools.wraps(func)
        def do_func2(*args, **kwargs):
            return lazy(func, *args, **kwargs).realize(path)
        return do_func2
    return do_func1


def lazy_wrap_checkpoint(path):
    """Convenience decorator/wrapper for lazily-evaluated user functions.

    Returns the result of a call to `checkpoint(path)` on the lazyly evaluated
    result.


    See also
    --------
    lazy_wrap


    Examples
    --------
    >>> @lazy_wrap_checkpoint("/tmp/foo")
    ... def foo (data):
    ...     "Return first element"
    ...     return data[:3]
    >>> data = lazy(extract_txt, "./data/maybe/@configuration@.csv",
    ...             fields_to_vars=["test"])
    >>> data = foo(data)

    """
    def do_func1(func):
        @functools.wraps(func)
        def do_func2(*args, **kwargs):
            return lazy(func, *args, **kwargs).checkpoint(path)
        return do_func2
    return do_func1


__all__ = [
    "find_files",
    "extract_func", "extract_txt", "extract_regex",
    "lazy", "lazy_wrap", "lazy_wrap_realize", "lazy_wrap_checkpoint",
]

__all__ += _errors.__all__
