# -*- coding: utf-8
"""
pdfreactor.parsecfg._tokensgroup: TokensGroup class

(helper class for the .parse module)
"""

# Python compatibility:
from __future__ import absolute_import, print_function

from six import string_types as six_string_types
from six.moves import range, zip
from six.moves import StringIO

__author__ = "Tobias Herp <tobias.herp@visaplan.com>"

# Standard library:
from collections import deque
# from tokenize import untokenize  (customize, perhaps?)
from tokenize import (
    COMMENT,
    ENDMARKER,
    INDENT,
    NAME,
    NEWLINE,
    NL,
    OP,
    STRING,
    TokenError,
    generate_tokens,
    tok_name,
    )

# visaplan:
from visaplan.tools.minifuncs import makeBool
from visaplan.tools.minifuncs import translate_dummy as _
from visaplan.tools.sequences import sequence_slide

# Local imports:
from pdfreactor.parsecfg._args import extract_input_specs
from pdfreactor.parsecfg._tokeninfo import TokenInfo


class TokensGroup(list):
    """
    A class to contain a group of tokens, as generated by the
    generate_token_groups function.

    >>> txt = 'config.disableLinks = true;'
    >>> groups = [TokensGroup(grp)
    ...           for grp in generate_token_groups(txt)]
    >>> dn = groups[0]  # a dotted name
    >>> dn
    <NAME config.disableLinks>
    >>> dn.is_terminator
    False
    >>> dn.is_dotted_name
    True
    >>> dn.names_list
    ['config', 'disableLinks']

    >>> op = groups[1]
    >>> op
    <OP '='>
    >>> op.is_terminator
    False
    >>> op.is_dotted_name
    False

    >>> sy = groups[2]  # another dotted name we'll consider a symbol
    >>> sy
    <NAME true>
    >>> sy.is_terminator
    False
    >>> sy.is_dotted_name
    True
    >>> sy.names_list
    ['true']

    >>> tg = groups[3]
    >>> tg
    <OP ';'>
    >>> tg.is_terminator
    True
    >>> tg.is_dotted_name
    False
    >>> tg.names_list
    """

    @property
    def text(self):
        return ''.join([tup[1] for tup in self])

    @property
    def ttype(self):
        """
        The token type for our group is the type of the first token
        """
        return self[0][0]

    @property
    def tname(self):
        """
        The name of our "token type", as given by the tok_name dict
        """
        return tok_name[self.ttype]

    @property
    def opens_brace(self):
        return self[0].opens_brace

    @property
    def closes_brace(self):
        return self[0].closes_brace

    @property
    def expects_brace(self):
        """
        This will yield a KeyError if not .opens_brace!
        """
        return self[0].expects_brace

    def __repr__(self):
        ttype = self.ttype
        if self.is_terminator:
            txt = self.text.strip()
            if txt:
                return '<%s %r>' % (self.tname, txt)
            else:
                return '<%s>' % (self.tname,)
        elif ttype == OP:
            mask = '<%s %r>'
        else:
            mask = '<%s %s>'
        return mask % (self.tname, self.text)

    @property
    def is_terminator(self):
        """
        Would this group of tokens terminate a statement?
        """
        ttype = self.ttype
        if ttype == OP:
            length = len(self)
            if length > 1:
                raise ValueError('Invalid operator %(self)r: '
                     'too long (%(length)d tokens)!' % locals())
            return self.text == ';'
        elif ttype == NAME:
            return False
        else:
            text = self.text.strip()
            return not text

    @property
    def is_dotted_name(self):
        """
        We use dotted names ...
        - as assignment targets, e.g. config.disableLinks
        - as PDFreactor symbols, e.g. Doctype.AUTODETECT
        """
        if self.ttype == NAME:
            for i, tok in zip(list(range(1, len(self))), self[1:]):
                odd = i % 2
                if odd:
                    if tok[1] != '.':
                        raise ValueError('Invalid dotted name %r: '
                                'expected odd tokens to be dots; '
                                'found %r!' % (self, tok[1]))
                elif tok[0] != NAME:
                    tname = tok_name[tok[0]]
                    raise ValueError('Invalid dotted name %r: '
                            'expected even tokens to be names; '
                            'found a %s %r!'
                            % (self, tname, tok[1]))
            return True
        return False

    @property
    def names_list(self):
        """
        The list of names; useful to compile a symbol name
        or the target variable and key
        """
        if self.ttype != NAME:
            return None
        res = []
        for i, tok in zip(list(range(len(self))), self):
            odd = i % 2
            if odd:
                if tok[1] != '.':
                    raise ValueError('Invalid dotted name %r: '
                            'expected odd tokens to be dots; '
                            'found %r!' % (self, tok[1]))
            elif tok[0] != NAME:
                tname = tok_name[tok[0]]
                raise ValueError('Invalid dotted name %r: '
                        'expected even tokens to be names; '
                        'found a %s %r!'
                        % (self, tname, tok[1]))
            else:
                res.append(tok[1])
        return res

    @property
    def dotted_name(self):
        the_list = self.names_list
        if the_list is None:
            return None
        return '.'.join(the_list)


def generate_token_groups(*args, **kwargs):
    r"""
    Generate token groups which we are willing to interpret

    We use the Python parser to parse to parse the given text
    and group tokens together which form some interesting unit.

    We'll use two little helper classes which are implemented above:

    - TokenInfo(tuple), to take a tuple as generated by the
      tokenize.generate_tokens function
    - TokensGroup(list), to take a list of adjacent tuples
      which toether form some useful entity, e.g. a dotted name.

    Both classes should be usable just like their parent classes (tuple and
    list); but they provide some handy properties which should ease our task.

    >>> def lotg(txt):  # list of token groups
    ...     return list(generate_token_groups(txt))

    Let's suppose we have the following configuration text:

    >>> txt = 'config.disableLinks = true'
    >>> lotg(txt)
    ...                                       # doctest: +NORMALIZE_WHITESPACE
    [<NAME config.disableLinks>, <OP '='>, <NAME true>, <ENDMARKER>]

    >>> def lotg(txt):
    ...     return [tg for tg in generate_token_groups(txt)]

    >>> lotg(txt)
    ...                                       # doctest: +NORMALIZE_WHITESPACE
    [<NAME config.disableLinks>, <OP '='>, <NAME true>, <ENDMARKER>]

    At this stage, we are not picky about what we accept;
    we are prepared to accept old (PDFreactor v7-) API method names
    and proprietary control statements:

    >>> txt = '''# some comment
    ... strict on
    ... setAddLinks(False); # another comment
    ... with_images off
    ... '''
    >>> lotg(txt)                             # doctest: +NORMALIZE_WHITESPACE
    [<NAME strict>, <NAME on>,
     <NEWLINE>,
     <NAME setAddLinks>, <OP '('>, <NAME False>, <OP ')'>, <OP ';'>,
     <NAME with_images>, <NAME off>,
     <NEWLINE>]

    >>> txt2 = '''
    ... setAddLinks(False)
    ... # Enable bookmarks in the PDF document
    ... setAddBookmarks(True)
    ... setCleanupTool(CLEANUP_NONE)
    ... setEncoding('UTF-8')
    ... setJavaScriptMode(JAVASCRIPT_MODE_ENABLED_NO_LAYOUT)
    ... '''
    >>> lotg(txt2)
    ...                                       # doctest: +NORMALIZE_WHITESPACE
    [<NAME setAddLinks>,
         <OP '('>, <NAME False>, <OP ')'>,
         <NEWLINE>,
     <NAME setAddBookmarks>,
         <OP '('>, <NAME True>, <OP ')'>,
         <NEWLINE>,
     <NAME setCleanupTool>,
         <OP '('>, <NAME CLEANUP_NONE>, <OP ')'>,
         <NEWLINE>,
     <NAME setEncoding>,
         <OP '('>, <STRING 'UTF-8'>, <OP ')'>,
         <NEWLINE>,
     <NAME setJavaScriptMode>,
         <OP '('>, <NAME JAVASCRIPT_MODE_ENABLED_NO_LAYOUT>, <OP ')'>,
         <NEWLINE>]

    Of course, the conversion to the respective new API config assignments is
    not done here.

    We may put multiple statements in one line, separated by semicolons:

    >>> txt3 = '; ;setA();setB()'
    >>> lotg(txt3)
    ...                                       # doctest: +NORMALIZE_WHITESPACE
    [<NAME setA>,
         <OP '('>,
         <OP ')'>,
         <OP ';'>,
     <NAME setB>,
         <OP '('>,
         <OP ')'>,
         <ENDMARKER>]

    Comments work as expected, of course:

    >>> txt3c = 'setA();#setB()'
    >>> lotg(txt3c)
    ...                                       # doctest: +NORMALIZE_WHITESPACE
    [<NAME setA>,
         <OP '('>,
         <OP ')'>,
         <OP ';'>]

    We mainly expect to see assignments for the current PDFreactor API
    which we support in convenient fashion.

    Instead of something like:

      config['outputFormat'] = {
          'type': PDFreactor.OutputType.JPEG,
          'width': 640,
          }

    (which is what you'll get in the end,
    and which might be supported by a future release),
    you may specify:

    >>> txt4 = '''
    ... config.outputFormat = {
    ...     type: OutputType.JPEG,
    ...     width: 640,
    ... }
    ... '''
    >>> lotg(txt4)                            # doctest: +NORMALIZE_WHITESPACE
    [<NAME config.outputFormat>,
     <OP '='>,
     <OP '{'>,
         <NL>,
         <NAME type>,  <OP ':'>, <NAME OutputType.JPEG>,
         <OP ','>,
         <NL>,
         <NAME width>, <OP ':'>, <NUMBER 640>,
         <OP ','>,
         <NL>,
     <OP '}'>,
     <NEWLINE>]

    We are not strictly limited to using the core PDFreactor API.
    Here is an example from our integration which provides the server-side
    creation of appendixes and tables of contents.

    The Python parser yields a newline after every comment:
    >>> txt5 = '''appendix (
    ... #  glossary,
    ...   images,
    ... )'''
    >>> [TokenInfo(tup) for tup in (generate_tokens(StringIO(txt5).readline))]
    ...                                       # doctest: +NORMALIZE_WHITESPACE
    [(NAME appendix), (OP '('), (NL),
     (COMMENT #  glossary,), (NL),
     (NAME images), (OP ','), (NL), (OP ')'), (ENDMARKER)]

    For proper processing, we need to avoid generating two newlines in
    sequence; thus, we drop those newlines after comments:
    >>> lotg(txt5)                            # doctest: +NORMALIZE_WHITESPACE
    [<NAME appendix>, <OP '('>, <NL>,
     <NAME images>, <OP ','>, <NL>, <OP ')'>, <ENDMARKER>]

    This is how our server-side table-of-contents creation would be
    configured:
    >>> lotg(
    ... '''toc (".headline-level-2", ".headline-level-3",
    ... ".headline-level-4")
    ... ''')                                  # doctest: +NORMALIZE_WHITESPACE
    [<NAME toc>, <OP '('>,
     <STRING ".headline-level-2">, <OP ','>,
     <STRING ".headline-level-3">, <OP ','>, <NL>,
     <STRING ".headline-level-4">, <OP ')'>, <NEWLINE>]

    (See the generate_statements doctest for the resulting statement for better
    readability.)
    Remember, the actuell processing is not part of this package;
    but you get an idea of what you could do.

    >>> lotg(                                 # doctest: +NORMALIZE_WHITESPACE
    ... "integrationStyleSheets = ['++resource++pdfreactor.plone/export.css']")
    [<NAME integrationStyleSheets>, <OP '='>, <OP '['>,
     <STRING '++resource++pdfreactor.plone/export.css'>, <OP ']'>, <ENDMARKER>]

    """
    input_specs = extract_input_specs(args, kwargs)
    text = input_specs.get('text')
    if text is None:
        fo = input_specs['file']
        alltokens = generate_tokens(fo.readline)
    elif not text:
        return
    else:
        alltokens = generate_tokens(StringIO(text).readline)
    verbose = kwargs.pop('verbose', False)
    firstn = 5 if verbose else 2
    braces_stack = deque()
    buf = []
    prev_ttype = None
    yield_singleton = False
    yield_sofar = False
    drop_this = False
    follows_terminator = True
    for toktup in alltokens:
        ti = TokenInfo(toktup)
        ttype = ti.ttype
        if ti.is_terminator:
            drop_this = follows_terminator or prev_ttype == COMMENT
            follows_terminator = True
            yield_singleton = not drop_this
            yield_sofar = True
            if 0 and  ti.text == ';' and braces_stack:
                open_braces = ''.join(braces_stack)
                raise ValueError("We don't expect %(ti)r in braces!"
                                 ' (%(open_braces)r)'
                                 % locals())
        else:
            follows_terminator = False
            if ttype == OP:
                token = ti.text
                if ti.opens_brace:
                    braces_stack.append(ti)
                elif ti.closes_brace:
                    if not braces_stack:
                        raise ValueError('Found closing %(ti)r, '
                            'but the braces stack is empty!'
                            % locals())

                    last_brace = braces_stack.pop()
                    expected = last_brace.expects_brace
                    if ti.text != expected:
                        raise ValueError('Unexpected closing brace %(ti)r; '
                                'expected %(expected)r!'
                                % locals())
                assert token != ';', ('Semicolons should have been treated '
                    'as terminators (above)!')
                if token != '.':
                    yield_sofar = True
                    yield_singleton = True
                elif ti.opens_brace or ti.closes_brace:
                    pass
                elif not buf:
                    raise ValueError("Can't start with a %(ti)r!"
                                     % locals())
                elif token == '.':
                    if prev_ttype != NAME:
                        so_far = ''.join([tup[1] for tup in buf])
                        raise ValueError("Won't accept a %(ti)r "
                            'after %(so_far)r!'
                            % locals())
            elif ttype == NAME and prev_ttype == NAME:
                # for control statements, e.g. 'strict off';
                # not implemented here:
                yield_sofar = True
            elif ttype in (NL, COMMENT):  # insignificant newline, or comment
                drop_this = True
            elif not buf and ttype in (INDENT,
                    ):
                drop_this = True

        if yield_sofar:
            if buf:
                yield TokensGroup(buf)
                buf = []
            yield_sofar = False

        if drop_this:
            assert not yield_singleton
            drop_this = False
        else:
            buf.append(TokenInfo(toktup[:firstn]))

        if yield_singleton:
            yield TokensGroup(buf)
            buf = []
            yield_singleton = False

        prev_ttype = ttype

    if braces_stack:
        cnt = len(braces_stack)
        braces = ' '.join([ti.text for ti in braces_stack])
        raise ValueError('Found %(cnt)d unclosed brace(s)! (%(braces)r)'
            % locals())
    if buf:
        yield TokensGroup(buf)


if __name__ == '__main__':
    # Standard library:
    import doctest
    doctest.testmod()
