#!/usr/bin/env python
# -*- coding: UTF-8 -*-
#
# Copyright 2014 European Commission (JRC);
# Licensed under the EUPL (the 'Licence');
# You may not use this work except in compliance with the Licence.
# You may obtain a copy of the Licence at: http://ec.europa.eu/idabc/eupl
"""
A mini-language for "throwing the rope" around rectangular areas of Excel-sheets.

.. default-role:: term
.. currentmodule:: pandalone.xleash

About
=====
Any *decent* dataset is stored in **csv**.
Consequently, many datasets are still trapped in excel-sheets.

*XLeash* defines a url-fragment notation (`xl-ref`) that renders
the `capturing` of tables from sheets as practical as reading a **csv**,
even when the exact position of those tables are not known beforehand.

An additional goal is to apply the same `lassoing` operation recursively,
to  build *data-trees*. For that end, the syntax supports `filter`
transformations such as:

    - setting the dimensionality of the result tables,
    - creating higher-level objects from 2D `capture-rect`
      (dictionaries, *numpy-arrays* & *dataframes*).

It is based on `xlrd <http://www.python-excel.org/>`_ library but also
checked for compatibility with `xlwings <http://xlwings.org/quickstart/>`_
*COM-client* library.
It requires *numpy* and (optionally) *pandas*.
It is developed on python-3 but also tested on python-2 for compatibility.


Overview
========
The `xl-ref` notation extends ordinary *A1* and *RC* excel `coordinates` with
conditional `traversing` operations, based on the cell's empty/full `state`.
For instance, to extract a contigious table near the ``A1`` cell,
and make a ``pandas.DataFrame`` out of it use this::

    from pandalone import xleash, SheetsFactory

    shfac = SheetsFactory()
    shfac.list_sheetnames(''path/to/workbook.xlsx')
    [Sheet1', ...]

    ## Search and capture the first contiguous table from the 1st sheet
    #  as a pandas-DataFrame:
    df = xleash.lasso('path/to/workbook.xlsx#0!A1(DR):..(DR):RLDU:["df"]',
                      sheets_factory=shfac)

    ## Assuming the sheet contain a single table, a lone `:` fetches
    #  the same contents.  Additionally, it is possible
    #  to skip the sheetname/sheet-index (1st 1st sheet implied).
    df = xleash.lasso('#:["df"]',
                      url_file=path/to/workbook.xlsx,
                      sheets_factory=shfac)

Xl-ref Syntax
-------------
::

    [<url>]#[<sheet>!][<1st-edge>][:[<2nd-edge>][:<expansions>]][:<filters>]

- See `edge`, `expansion-moves`, `filters` for details.
- Missing *edges* are implicitly replaced by ``^^:__`` (top-left/bottom-right).
- Spaces are allowed only in `filters`.


Annotated Example
-----------------
::

  target-moves─────┐
  landing-cell──┐  │
               ┌┤ ┌┤
              #C3(UL):..(RD):RULD:["pipe": ["odict", "recursive"]]
               └─┬──┘ └─┬──┘ └┬─┘ └──────────────┬───────────────┘
  1st-edge───────┘      │     │                  │
  2nd-edge──────────────┘     │                  │
  expansions──────────────────┘                  │
  filters────────────────────────────────────────┘

Which means:

    1. `Target` the `1st` `edge` of the `capture-rect` by starting from ``C3``
       `landing-cell`. If it is a `full-cell`, stop, otherwise start moving
       above and to the left of ``C3`` and stop on the first `full-cell`;
    2. continue from the last `target` and travel the `exterior` row and column
       right and down, stopping on their last `full-cell`;
    3. `capture` all the cells between the 2 targets.
    4. try `expansions` to all directions if any neighbouring `full-cell`;
    5. finally `filter` the values of the `capture-rect` to wrap them up
       in an ordered- dictionary, and dive into its values searching for
       `xl-ref`, and replace them.


Basic Usage
-----------
The simplest way to `lasso` a `xl-ref` is through :func:`lasso()`.
A common task is to  capture all non-empty cells of the 1st workbook-sheet but
without any bordering nulls::

    >>> from pandalone import xleash

    >>> values = xleash.lasso('path/to/workbook.xlsx#:')  # doctest: +SKIP

Assuming that the `full-cell` of the 1st sheet of the workbook on disk are
those marked with ``'X'``, then the result  `capture-rect` of the above call
would be a 2D *list-of-lists* with the values contained in ``C2:E4``::

      A B C D E
    1    ┌─────┐
    2    │    X│
    3    │X    │
    4    │  X  │
    5    └─────┘


If another sheet is desired, add its name or 0-based ordinal immediately after ``#``
separated by a ``!`` with the rest of the `xl-ref` - which inthat case
might be empty::

    >>> lasso = xleash.lasso
    >>> lasso('Book.xlsx#Sheet1!') == lasso('Book.xlsx#0!') == lasso('Book.xlsx#:') # doctest: +SKIP
    True


If you do not wish to let the library read your workbooks, you can
invoke the function with a pre-loaded sheet.
Here we will use the utility :class:`ArraySheet` with a more complicated
`xl-ref` expression::

    >>> sheet = xleash.ArraySheet([[None, None,  'A',   None],
    ...                          [None, 2.2,   'foo', None],
    ...                          [None, None,   2,    None],
    ...                          [None, None,   None, 3.14],
    ... ])
    >>> xleash.lasso('#A1(DR):..(DR):RULD', sheet=sheet)
    [[None, 'A'],
     [2.2, 'foo'],
     [None, 2]]

This `capture-rect` in this case was *B1* and *C3* as can be seen by inspecting
the ``st`` and ``nd`` fields of the full :class:`Xlref` results returned::

    >>> xleash.lasso('#A1(DR):..(DR):RULD', sheet=sheet, return_lasso=True)
    Lasso(xl_ref='#A1(DR):..(DR):RULD',
          url_file=None,
          sh_name=None,
          st_edge=Edge(land=Cell(row='1', col='A'), mov='DR', mod=None),
          nd_edge=Edge(land=Cell(row='.', col='.'), mov='DR', mod=None),
          exp_moves='RULD',
          call_spec=None,
          sheet=ArraySheet(SheetId(book='wb', ids=['sh', 0]),
                                 [[None None 'A' None]
                                  [None 2.2 'foo' None]
                                  [None None 2 None]
                                  [None None None 3.14]]),
          st=Coords(row=0, col=1),
          nd=Coords(row=2, col=2),
          values=[[None, 'A'],
                  [2.2, 'foo'],
                  [None, 2]],
          base_coords=None,
          ...


For controlling explicitly the configuration parameters and the opening of
workbooks, use separate instances of :class:`Ranger` and :class:`SheetsFactory`,
that are the workhorses of this library::

    >>> with xleash.SheetsFactory() as sf:
    ...     sf.add_sheet(sheet, wb_ids='foo_wb', sh_ids='Sheet1')
    ...     ranger = xleash.Ranger(sf, base_opts={'verbose': True})
    ...     ranger.do_lasso('foo_wb#Sheet1!__').values
    3.14

Notice that it returned a scalar value since we specified only the `1st` `edge`
as ``'__'``, which points to the bottom row and most-left column of the sheet.


Alternatively you can call the :func:`make_default_Ranger` for extending
library's defaults.



More Syntax Examples
--------------------
Another typical but more advanced case is when a sheet contains a single table
with a "header"-row and a "index"-column.
There are (at least) 3 ways to do it, beyond specifying
the exact `coordinates`::


      A B C D E
    1  ┌───────┐     Β2:E4          ## Exact referencing.
    2  │  X X X│     ^^.__  or :    ## From top-left full-cell to bottom-right.
    3  │X X X X│     A1(DR):__:U1   ## Start from A1 and move down and right
    3  │X X X X│                    #    until B3; capture till bottom-left;
    4  │X X X X│                    #    expand once upwards (to header row).
       └───────┘     A1(RD):__:L1   ## Start from A1 and move down by row
                                    #    until C1; capture till bottom-left;
                                    #    expand once left (to index column).


Note that if ``B1`` were full, the results would still be the same, because
``?`` expands only if any full-cell found in row/column.

In case where the sheet contains more than one *disjoint* tables, the
bottom-left cell of the sheet would not coincide with table-end, so the handy
last two `xl-ref` above would not work.

For that we may resort to `dependent` referencing for the `2nd` `edge`, and
define its position in relation to the `1st` `target`::

      A B C D E
    1  ┌─────┐    _^:..(LD+):L1     ## Start from top-right(E2) and target left
    2  │  X X│                      #    left(D2); from there capture left-down
    3  │X X X│                      #    till 1st empty-cell(C4, regardless of
    4  │X X X│                      #    col/row order); expand left once.
       └─────┘    ^_(U):..(UR):U1   ## Start from B5 and target 1st cell up;
    5         Χ                     #    capture from there till D3; expand up.


In the presence of `empty-cell` breaking the `exterior` row/column of
the `1st` `landing-cell`, the capturing becomes more intricate::

      A B C D E
    1  ┌─────┐      Β2:D_
    2  │  X X│      A1(RD):..(RD):L1D
    3  │X X  │      D_:^^
    3  │X    │      A^(DR):D_:U
    4  │  X  │X
       └─────┘


      A B C D E
    1    ┌───┐      ^^(RD):..(RD)
    2    │X X│      _^(R):^.(DR)
    3   X│X  │
         └───┘
    3   X
    4     X   X


      A B C D E
    1  ┌───┐        Β2:C4
    2  │  X│X       A1(RD):^_
    3  │X X│        C_:^^
    3  │X  │        A^(DR):C_:U
    4  │  X│  X     ^^(RD):..(D):D
       └───┘        D2(L+):^_


.. seealso:: Example spreadsheet: :download:`xleash.xlsx`


Definitions
===========

.. glossary::

    lasso
    lassoing
        It may denote 3 things:

        - the whole procedure of `parsing` the `xl-ref` syntax,
          `capturing` values from spreadsheet rect-regions and sending them
          through any `filters` specified in the xl-ref;
        - the :func:`lasso()` and/or :meth:`Ranger.do_lasso()` functions
          performing the above job;
        - the :class:`Lasso` storing intermediate and final results of the
          above algorithm.

    xl-ref
        Any url with its fragment abiding to the syntax defined herein.

        - The *fragment* describes how to `capture` rects from excel-sheets, and
          it is composed of 2 `edge` references followed by `expansions` and
          `filters`.
        - The *file-part* should resolve to an excel-file.

    parse
    parsing
        The stage where the input string gets splitted and checked for validity
        against the `xl-ref` syntax.

    edge
        An *edge* might signify:

        - the syntactic construct of the `xl-ref`, composed of a pair
          of row/column `coordinates`, optionally followed by parenthesized
          `target-moves`, like ``A1(LU)``;
        - the bounding cells of the `target-rect`;
        - the bounding cells of the `capture-rect`.

        In all cases above there are 2 instances; the `1st` and `2nd`.

    1st
    2nd
        It may refer to the *1st*/*2nd*:

        - `edge` of some `xl-ref`;
        - `landing-cell` of an `edge`;
        - `target-cell` of an `edge`;
        - `capture-cell` of a `capture-rect`.

        The *1st-edge` supports `absolute` `coordinates` only, while the
        *2nd-edge* supports also `dependent` ones from the *1st* `target-cell`.

    landing-cell
        The cell identified by the `coordinates` of the `edge` alone.

    target-cell
    target-rect
        The bounding *cell* identified after applying `target-moves` on the
        `landing-cell`.


    target
    targeting
        The process of identifying any `target-cell` bounding the
        `target-rect`.

        - The search for the `target-cell` starts from the `landing-cell`,
          follows the specified `target-moves`, and ends when a `state-change`
          is detected on an `exterior` column or row, according to the enacted
          `termination-rule`.
        - Failure to identify any target-cell raises a :class:`EmptyCaptureException`
          which is subsequently translated as *empty* `capture-rect` by :class:`Ranger`
          when `opts` contain ``{"no_empty": false}`` (default).
        - The process is followed by `expansions` to identify the
          `capture-rect`.

        Note that in the case of a `dependent` `2nd` `edge`, the `target-rect`
        would always be the same, irrespective of whether `target-moves`
        denoted a row-by-row or column-by-column traversal.

    capture
    capturing
        It is the overall procedure of:

            1. `targeting` both `edge` refs to come up with the `target-rect`;
            2. performing `expansions` to identify the `capture-rect`;
            3. extracting the values and feed them to `filters`.

    capture-rect
    capture-cell
        The *rectangular-area* of the sheet denoted by the two *capture-cells*
        identified by `capturing`, that is, after applying `expansions` on
        `target-rect`.

    directions
        The 4 primitive *directions* that are denoted with one of the letters
        ``LURD``.
        Thee are used to express both `target-moves` and `expansions`.

    coordinate
    coordinates
        Any pair of a cell/column *coordinates* specifying cell positions,
        (i.e. `landing-cell`, `target-cell`, bounds of the `capture-rect`)
        written as the first part of the `edge` syntax, or implicitely resolved.
        They can be expressed in ``A1`` or ``RC`` format or as a zero-based
        ``(row, col)`` tuple (*num*).
        Each *coordinate* might be `absolute` or `dependent`, independently.

    traversing
    traversal-operations
        Either the `target-moves` or the `expansion-moves` that comprise the
        `capturing`.

    target-moves
        Specify the cell traversing order while `targeting` using
        primitive `directions` pairs.
        The pairs ``UD`` and ``LR`` (and their inverse) are invalid.
        I.e. ``DR`` means:

            *"Start going right, column-by-column, traversing each column
            from top to bottom."*

    move-modifier
        One of ``+`` and ``-`` chars that might trail the `target-moves`
        and define which the `termination-rule` to follow if `landing-cell`
        is `full-cell`, i.e. ``A1(RD+)``

    expansions
    expansion-moves
        Due to `state-change` on the 'exterior' cells the `capture-rect`
        might be smaller that a wider contigious but "convex" rectangular area.

        The *expansions* attempt to remedy this by providing for expanding on
        arbitrary `directions` accompanied by a multiplicity for each one.
        If multiplicity is unspecified, infinite assumed, so it expands
        until an empty/full row/column is met.

    absolute
        Any cell row/col identified with column-characters, row-numbers, or
        the following special-characters:

        - ``^``          The top/Left full-cell `coordinate`.
        - ``_``          The bottom/right full-cell `coordinate`.

    dependent
    base-cell
        A `landing-cell` whose any `coordinate` is identified with a dot(``.``),
        which resolves to the *base-coordinate* depending on which `edge`
        it is referring to:

        - `1st` edge: The coordinates of the `base-cell` field of the
          `Lasso` given to the :meth:`Ranger.do_lasso()`; must not be ``None``.
        - `2nd` edge: the `target-cell` coordinates of the `1st` edge.

        An `edge` might contain a "mix" of `absolute` and *dependent*
        coordinates.

    state
    full-cell
    empty-cell
        A cell is *full* when it is not *empty* / *blank* (in Excel's parlance).

    states-matrix
        A boolean matrix denoting the `state` of the cells, having the same
        size as a sheet it was derived from.

    state-change
        Whether we are traversing from an `empty-cell` to a `full-cell`, and
        vice-versa, while `targeting`.

    termination-rule
        The condition to stop `targeting` while traversing from `landing-cell`.
        The are 2 rules: `search-same` and `search-opposite`.

        .. seealso::
            Check `Target-termination enactment`_ for the enactment of the rules.

    search-opposite
        The `target-cell` is the FIRST `full-cell` found while traveling
        from the `landing-cell` according to the `target-moves`.

    search-same
        The coordinates of the `target-cell` are given by the LAST `full-cell`
        on the `exterior` column/row according to the `target-moves`;
        the order of the moves is insignificant in that case.

    exterior
        The *column* and the *row* of the `landing-cell`; the `search-same`
        `termination-rule` gets to be triggered by 'full-cells' only on them.

    filter
    filters
        The last part of the `xl-ref` specifying predefined functions to
        apply for transforming the cell-values of `capture-rect`,
        abiding to the  **json** syntax.
        They may be `bulk` or `element-wise`.

    bulk
    bulk-filter
        A `filter` treating `capture-rect` values as a whole , i.e.
        transposing arrays, is_empty

    element-wise
    element-wise-filter
        A `filter` diving into `capture-rect` values, i.e. for python-eval.

    call-specifier
    call-spec
        The structure to specify some function call in the `filter` part;
        it can either be a json *string*, *list* or *object* like that:

        - string: ``"func_name"``
        - list:   ``["func_name", ["arg1", "arg2"], {"k1": "v1"}]``
          where the last 2 parts are optional and can be given in any order;
        - object: ``{"func": "func_name", "args": ["arg1"], "kwds": {"k":"v"}}``
          where the ``args`` and ``kwds`` are optional.

        If the outer-most filter is a dictionary, a ``'pop'`` kwd is popped-out
        as the `opts`.

    opts
        Key-value pairs affecting the `lassoing` (i.e. opening xlrd-workbooks).
        Read the code to be sure what are the available choices :-(
        They are a combination of options specified in code (i.e. in the
        :func:`lasso()` and those extracted from `filters` by the 'opts' key,
        and they are stored in the :class:`Lasso`.

    backend
    backends
        *IO* level object providing the actual spreadsheet cells for `capturing`.
        Each *backend* may provide for its workbooks and sheets corresponding to:
        - different implementations (e.g.``xlrd`` or ``xlwings`` library), or
        - different origins (e.g. file-based, network-based per url ).

        The decision which *backend* to use is taken by the `sheet-factory`
        following a `bidding` process.

    sheets-factory
        *IO* level object acting as the caching manager for `spreadsheets`
        fetched from different `backends`.  The caching happens per
        *spreadsheet*.

    bid
    backend-bidding
        All *backends* are asked to provide their willingness to handle
        some `xl-ref` (see :meth:`SimpleSheetFactory.decide_backend`)).
        For a *sibling* sheet, always the parent *backend* is used.

    sheet
    spreadsheet
        *IO* level object that acts as the container of cells.

Details
=======

Target-moves
---------------

There are 12 `target-moves` named with a *single* or a *pair* of
letters denoting the 4 primitive `directions`, ``LURD``::

            U
     UL◄───┐▲┌───►UR
    LU     │││     RU
     ▲     │││     ▲
     │     │││     │
     └─────┼│┼─────┘
    L◄──────X──────►R
     ┌─────┼│┼─────┐
     │     │││     │
     ▼     │││     ▼
    LD     │││     RD
     DL◄───┘▼└───►DR
            D

    - The 'X' at the center points the starting cell.


So a ``RD`` move means *"traverse cells first by rows then by columns"*,
or more lengthy description would be:

    *"Start moving *right* till 1st state change, and then
    move *down* to the next row, and start traversing right again."*


Target-cells
------------

Using these moves we can identify a `target-cell` in relation to
the `landing-cell`. For instance, given this xl-sheet below, there are
multiple ways to identify (or target) the non-empty values ``X``, below::

      A B C D E F
    1
    2
    3     X        ──────► C3    A1(RD)   _^(L)      F3(L)
    4         X    ──────► E4    A4(R)    _4(L)      D1(DR)
    5   X          ──────► B5    A1(DR)   A_(UR)     _5(L)
    6           X  ──────► F6    __       _^(D)      A_(R)

    - The 'X' signifies non-empty cells.


So we can target cells with "absolute coordinates", the usual ``A1`` notation,
augmented with the following special characters:

  - undesrcore(``_``) for bottom/right, and
  - accent(``^``) for top/left

columns/rows of the sheet with non-empty values.

When no ``LURD`` moves are specified, the target-cell coinceds with the starting one.

.. Seealso:: `Target-termination enactment`_ section


Capturing
---------

To specify a complete `capture-rect` we need to identify a 2nd cell.
The 2nd target-cell may be specified:

  - either with `absolute` coordinates, as above, or
  - with `dependent` coords, using the dot(``.``) to refer to the 1st cell.


In the above example-sheet, here are some ways to specify refs::

      A  B C D E  F
    1

    2
          ┌─────┐
       ┌──┼─┐   │
    3  │  │X│   │
       │┌─┼─┼───┼┐
    4  ││ │ │  X││
       ││ └─┼───┴┼───► C3:E4   A1(RD):..(RD)   _^(L):..(DR)   _4(L):A1(RD)
    5  ││X  │    │
       │└───┼────┴───► B4:E5   A_(UR):..(RU)   _5(L):1_(UR)    E1(D):A.(DR)
    6  │    │     X
       └────┴────────► Β3:C6   A1(RD):^_       ^^:C_           C_:^^


.. Warning::
   Of course, the above rects WILL FAIL since the `target-moves`
   will stop immediately due to ``X`` values being surrounded by empty-cells.

   But the above diagram was to just convey the general idea.
   To make it work, all the in-between cells of the peripheral row and columns
   should have been also non-empty.

.. Note::
    The `capturing` moves from `1st` `target-cell` to `2nd` `target-cell` are
    independent from the implied `target-moves` in the case of `dependent`
    coords.

    More specifically, the `capturing` will always fetch the same values
    regardless of "row-first" or "column-first" order; this is not the case
    with `targeting` (``LURD``) moves.

    For instance, to capture ``B4:E5`` in the above sheet we may use
    ``_5(L):E.(U)``.
    In that case the target cells are ``B5`` and ``E4`` and the `target-moves`
    to reach the 2nd one are ``UR`` which are different from the ``U``
    specified on the 2nd cell.



Target-termination enactment
----------------------------

The guiding principle for when to enact each rule is to always `capture`
a matrix of `full-cell`.

- If the `landing-cell` is `empty-cell`, always `search-opposite`, that is,
  stop on the first `full-cell`.
- When the `landing-cell` is `full-cell`, it depends on the 'move-modifier':

  - If ``+`` exists, apply `search-same`.
  - If ``-`` exists, stop on `landing-cell`.
  - If no modifier, behave like ```-` (stop on `landing-cell`) except when
    on a `2nd` edge with both its coordinates `dependent` (``..``),
    where the `search-same` is applied

So, both `move-modifier` apply only when `landing-cell` is `full-cell`
, and ``-`` actually makes sense only when `2nd` edge is `dependent`.

If the termination conditions is not met, an :class:`EmptyCaptureException`
is raised, which is translated as *empty* `capture-rect` by :class:`Ranger`
when `opts` contain ``{"no_empty": false}`` (default).



Expansions
----------

Captured-rects ("values") may be limited due to `empty-cell` in the 1st
row/column traversed.  To overcome this, the xl-ref may specify `expansions`
directions using a 3rd ``:``-section like that::

    _5(L):1_(UR):RDL1U1

This particular case means:

     *"Try expanding Right and Down repeatedly and then try once Left and Up."*

Expansion happens on a row-by-row or column-by-column basis, and terminates
when a full empty(or non-empty) line is met.

Example-refs are given below for capturing the 2 marked tables::

      A  B C D E F  G
    1
       ┌───────────┐
       │┌─────────┐│
    2  ││  1 X X  ││
       ││         ││
    3  ││X X   X X││
       ││         ││
    4  ││X X X 2 X││
       ││         ││
    5  ││X   X X X││
       └┼─────────┼┴──► A1(RD):..(RD):DRL1
    6   │X        │
        └─────────┴───► A1(RD):..(RD):L1DR       A_(UR):^^(RD)
    7               X

    - The 'X' signify non-empty cells.
    - The '1' and '2' signify the identified target-cells.


Plugin Extensions
=================
The *xleash* library already uses `setuptools entry-points
<https://setuptools.readthedocs.io/en/latest/setuptools.html#dynamic-discovery-of-services-and-plugins>`_
to attach `backends` and pandas `filters`.
Read :func:`init_plugins` to learn how to implement other plugins.

.. default-role:: obj


API
===

- User-facing higher-level functionality:

  .. currentmodule:: pandalone.xleash._lasso
  .. autosummary::

      Lasso
      lasso
      Ranger
      Ranger.do_lasso
      make_default_Ranger
      get_default_opts

- Related to :term:`capturing` algorithm:

  .. currentmodule:: pandalone.xleash._capture
  .. autosummary::

      resolve_capture_rect
      coords2Cell
      EmptyCaptureException


  .. currentmodule:: pandalone.xleash._filter
  .. autosummary::

      installed_filters
      xlwings_dims_call_spec


- Related to parsing and basic structure used throughout:
  .. currentmodule:: pandalone.xleash._parse
  .. autosummary::

      parse_xlref
      parse_expansion_moves
      parse_call_spec
      Cell
      Coords
      Edge

- **IO** back-end functionality:

  .. currentmodule:: pandalone.xleash.io
  .. autosummary::

      io_backends
      _sheets.SheetsFactory
      _sheets.ABCSheet.read_rect
      _sheets.ArraySheet
      _sheets.ABCSheet
      _xlrd.XlrdSheet
      _xlrd.open_sheet

- Plugin related
  .. autosummary::
      _init_plugins
     _plugins_installed
     _PLUGIN_GROUP_NAME

.. currentmodule:: pandalone.xleash
"""


from __future__ import unicode_literals

from collections import namedtuple, OrderedDict
import logging
import os
import pkg_resources

from .. import utils as pndlutils

log = logging.getLogger(__name__)

Lasso = namedtuple('Lasso',
                   ('xl_ref', 'url_file', 'sh_name',
                    'st_edge', 'nd_edge', 'exp_moves',
                    'call_spec',
                    'sheet', 'st', 'nd', 'values', 'base_coords',
                    'opts'))
"""
All the fields used by the algorithm, populated stage-by-stage by :class:`Ranger`.

:param str xl_ref:
        The full url, populated on parsing.
:param str sh_name:
        Parsed sheet name (or index, but still as string), populated on parsing.

        .. Note::
            If you need the name of the *captured* sheet, use::

                lasso.sheet.get_sheet_ids().ids[0]

:param Edge st_edge:
        The 1st edge, populated on parsing.
:param Edge nd_edge:
        The 2nd edge, populated on parsing.
:param Coords st:
        The top-left targeted coords of the :term:`capture-rect`,
        populated on :term:`capturing`.`
:param Coords nd:
        The bottom-right targeted coords of the :term:`capture-rect`,
        populated on :term:`capturing`
:param ABCSheet sheet:
        The fetched from factory or ranger's current sheet, populated
        after :term:`capturing` before reading.
:param values:
        The excel's table-values captured by the :term:`lasso`,
        populated after reading updated while applying :term:`filters`.
:param call_spec:
        The :term:`call-spec` derrived from the parsed filters, to be fed
        into :meth:`Ranger.make_call()`.
:param Coords base_coords:
        On recursive calls it becomes the :term:`base-cell` for the :term:`1st`
        :term:`edge`.
:param dict or ChainMap opts:
        - Before `parsing`, they are just any 'opts' dict found in the
          :term:`filters`.
        - After *parsing, a 2-map ChainMap with :attr:`Ranger.base_opts` and
          options extracted from *filters* on top.
"""

Lasso.__new__.__defaults__ = (None,) * len(Lasso._fields)
"""Make :class:`Lasso` construct with all missing fields as `None`."""

Coords = namedtuple('Coords', ['row', 'col'])
"""
A pair of 0-based integers denoting the "num" coordinates of a cell.

The "A1" coords (1-based coordinates) are specified using :class:`Cell`.
"""


from ._parse import (
    Cell, Edge, CallSpec,
    parse_xlref,
)

io_backends = []
"""Hook for plugins to append :class:`ABCBackend` instances."""
from pandalone.xleash.io.backend import (
    ABCSheet, ArraySheet, margin_coords_from_states_matrix,
    SheetsFactory,
)

from ._capture import (
    resolve_capture_rect, coords2Cell,
    EmptyCaptureException,
)

installed_filters = {}
"""Hook for plugins to append :term:`filters`."""
from ._filter import (
    XLocation,
    xlwings_dims_call_spec,
    install_default_filters
)
install_default_filters(installed_filters)

from ._lasso import (
    lasso, Ranger,
    make_default_Ranger, get_default_opts,
)


_PLUGIN_GROUP_NAME = 'pandalone.xleash.plugins'
"""Used to discover *setuptools* extension-points."""


def _init_plugins(plugin_group_name=_PLUGIN_GROUP_NAME):
    """
    Discover and load plugins.

    The *xleash* library already uses `setuptools entry-points
    <https://setuptools.readthedocs.io/en/latest/setuptools.html#dynamic-discovery-of-services-and-plugins>`_
    to attach backend :class:`Sheet` and pandas `filters`.

    You may re-invoke after some ``pip install <some-xleash-plugin>``.

    `setup.py` configurations
    -------------------------
    To implement a new plugin, you have to package your code as a regular
    python distribution and add the following declaration inside its
    :file:`setup.py`::

        setup(
            # ...
            entry_points = {
                'pandalone.xleash.plugins': [
                    'plugin_1 = <foo.plugin.module>:<plugin-install-func> ## Load & install.
                    'plugin_2 = <bar.plugin.module>                       ## Load only.
                ]
            }
        )


    Implementing a plugin
    ---------------------
    The plugins are initialized during *import time* in a 2-stage procedure
    by :func:`init_plugins()`.
    A plugin is *loaded* and optionally *installed* if the *setup-configuration*
    above specifies a no-args ``<plugin-install-func>`` callable.
    Any collected ``<plugin-install-func>`` callables are invoked AFTER all
    plugin-modules have finished loading.

    .. Tip::
       For example, study this project how it sets backend and filters.

    .. Warning::
       When appending into "hook" lists during installation, remember to avoid
       re-inserting duplicate items.  In general try to well-behave even when
       **plugins are initialized multiple times**!

    """
    global _plugins_installed

    def stringify_EntryPoint(ep):
        return "%r@%s" % (ep, ep.dist)

    plugin_loaders = []
    entry_points = sorted(
        pkg_resources.working_set.iter_entry_points(plugin_group_name),
        key=lambda ep: ep.name)
    if not entry_points:
        raise ValueError("No xleash-plugins found!"
                         "\n  You have to install AT LEAST on backend plugin."
                         '\n  Try `xlrd` "extras" with this command?\n'
                         "\n      pip install pandalone[xlrd]")
    for ep in entry_points:
        try:
            _plugins_installed[stringify_EntryPoint(ep)] = 0
            plugin_loader = ep.load()
            _plugins_installed[stringify_EntryPoint(ep)] = 1
            if callable(plugin_loader):
                plugin_loaders.append((ep, plugin_loader))
        except Exception as ex:
            log.error('Failed LOADING plugin(%r@%s) due to: %s',
                      ep, ep.dist, ex, exc_info=1)

    for ep, plugin_loader in plugin_loaders:
        try:
            plugin_loader()
            _plugins_installed[stringify_EntryPoint(ep)] = 2
        except Exception as ex:
            log.error('Failed INSTALLING plugin(%r@%s) due to: %s',
                      ep, ep.dist, ex, exc_info=1)

_plugins_installed = OrderedDict()
"""
A list of 2-tuples for each plugin installed of :class:`pkg_resources.EntryPoint`
and the number of completed stages (integer).

The *EntryPoint* gets stringified to avoid memory-leaks.
"""

_init_plugins()

__all__ = [
    '_init_plugins', '_PLUGIN_GROUP_NAME',

    'resolve_capture_rect', 'ABCSheet', 'ArraySheet', 'coords2Cell',
    'EmptyCaptureException', 'margin_coords_from_states_matrix',

    'lasso', 'Ranger', 'SheetsFactory', 'io_backends',
    'make_default_Ranger',
    'XLocation', 'get_default_opts',
    'installed_filters',
    'Lasso',
    'xlwings_dims_call_spec',

    'Cell', 'Coords', 'Edge', 'CallSpec',
    'parse_xlref',
]
