# -*- test-case-name: twisted.web.test.test_xml -*-
#
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.


"""
*S*mall, *U*ncomplicated *X*ML.

This is a very simple implementation of XML/HTML as a network
protocol.  It is not at all clever.  Its main features are that it
does not:

  - support namespaces
  - mung mnemonic entity references
  - validate
  - perform *any* external actions (such as fetching URLs or writing files)
    under *any* circumstances
  - has lots and lots of horrible hacks for supporting broken HTML (as an
    option, they're not on by default).
"""

from __future__ import (
    absolute_import, division, print_function, unicode_literals)

import pygogo as gogo

from builtins import *

from chardet import detect
from meza._compat import decode

try:
    from twisted.internet.protocol import Protocol
except ImportError:
    Protocol = object
else:
    from twisted.python.reflect import prefixedMethodNames as find_method_names

logger = gogo.Gogo(__name__, monolog=True).logger

# Elements of the three-tuples in the state table.
BEGIN_HANDLER = 0
DO_HANDLER = 1
END_HANDLER = 2
IDENTCHARS = '.-_:'
LENIENT_IDENTCHARS = IDENTCHARS + ';+#/%~'

nop = lambda *args, **kwargs: None


def zipfndict(*args):
    for fndict in args:
        for key in fndict:
            yield (key, tuple(x.get(key, nop) for x in args))


def get_method_obj_dict(obj, prefix):
    names = find_method_names(obj.__class__, prefix)
    return {name: getattr(obj, prefix + name) for name in names}


class ParseError(Exception):
    def __init__(self, filename, line, col, message):
        self.filename = filename
        self.line = line
        self.col = col
        self.message = message

    def __str__(self):
        return "%s:%s:%s: %s" % (
            self.filename, self.line, self.col, self.message)


class XMLParser(Protocol):
    state = None
    encoding = None
    bom = None
    attrname = ''
    attrval = ''

    # _leadingBodyData will sometimes be set before switching to the
    # 'bodydata' state, when we "accidentally" read a byte of bodydata
    # in a different state.
    _leadingBodyData = None

    def __init__(self, filename='unnamed', **kwargs):
        self.filename = filename
        self.lenient = kwargs.get('lenient')
        self.strict = not self.lenient

    # protocol methods
    def connectionMade(self):
        self.lineno = 1
        self.colno = 0

    def saveMark(self):
        '''Get the line number and column of the last character parsed'''
        # This gets replaced during dataReceived, restored afterwards
        return (self.lineno, self.colno)

    def _raise_parse_error(self, message):
        raise ParseError(*((self.filename,) + self.saveMark() + (message,)))

    def _build_state_table(self):
        '''Return a dictionary of begin, do, end state function tuples'''
        # _build_state_table leaves something to be desired but it does what it
        # does.. probably slowly, so I'm doing some evil caching so it doesn't
        # get called more than once per class.
        stateTable = getattr(self.__class__, '__stateTable', None)

        if stateTable is None:
            prefixes = ('begin_', 'do_', 'end_')
            fndicts = (get_method_obj_dict(self, p) for p in prefixes)
            stateTable = dict(zipfndict(*fndicts))
            self.__class__.__stateTable = stateTable

        return stateTable

    def check_encoding(self, data):
        if self.encoding.startswith('UTF-16'):
            data = data[2:]

        if 'UTF-16' in self.encoding or 'UCS-2' in self.encoding:
            assert not len(data) & 1, 'UTF-16 must come in pairs for now'

    def maybeBodyData(self):
        if self.endtag:
            return 'bodydata'

        # Get ready for fun! We're going to allow
        # <script>if (foo < bar)</script> to work!
        # We do this by making everything between <script> and
        # </script> a Text
        # BUT <script src="foo"> will be special-cased to do regular,
        # lenient behavior, because those may not have </script>
        # -radix
        if (self.tagName == 'script' and 'src' not in self.tagAttributes):
            # we do this ourselves rather than having begin_waitforendscript
            # because that can get called multiple times and we don't want
            # bodydata to get reset other than the first time.
            self.begin_bodydata(None)
            return 'waitforendscript'

        return 'bodydata'

    def dataReceived(self, data):
        stateTable = self._build_state_table()
        self.encoding = self.encoding or detect(data)['encoding']
        self.check_encoding(data)
        self.state = self.state or 'begin'
        content = decode(data, self.encoding)

        # bring state, lineno, colno into local scope
        lineno, colno = self.lineno, self.colno
        curState = self.state

        # replace saveMark with a nested scope function
        saveMark = lambda: (lineno, colno)
        self.saveMark, _saveMark = saveMark, self.saveMark

        # fetch functions from the stateTable
        beginFn, doFn, endFn = stateTable[curState]

        try:
            for char in content:
                # do newline stuff
                if char == '\n':
                    lineno += 1
                    colno = 0
                else:
                    colno += 1

                newState = doFn(char)

                if newState and newState != curState:
                    # this is the endFn from the previous state
                    endFn()
                    curState = newState
                    beginFn, doFn, endFn = stateTable[curState]
                    beginFn(char)
        finally:
            self.saveMark = _saveMark
            self.lineno, self.colno = lineno, colno

        # state doesn't make sense if there's an exception..
        self.state = curState

    def connectionLost(self, reason):
        """
        End the last state we were in.
        """
        stateTable = self._build_state_table()
        stateTable[self.state][END_HANDLER]()

    # state methods

    def do_begin(self, byte):
        if byte.isspace():
            return

        if byte != '<' and self.lenient:
            self._leadingBodyData = byte
            return 'bodydata'
        elif byte != '<':
            msg = "First char of document [%r] wasn't <" % (byte,)
            self._raise_parse_error(msg)

        return 'tagstart'

    def begin_comment(self, byte):
        self.commentbuf = ''

    def do_comment(self, byte):
        self.commentbuf += byte

        if self.commentbuf.endswith('-->'):
            self.gotComment(self.commentbuf[:-3])
            return 'bodydata'

    def begin_tagstart(self, byte):
        self.tagName = ''               # name of the tag
        self.tagAttributes = {}         # attributes of the tag
        self.termtag = 0                # is the tag self-terminating
        self.endtag = 0

    def _get_val(self, byte):
        val = None
        alnum_or_ident = byte.isalnum() or byte in IDENTCHARS
        is_good = alnum_or_ident or byte in '/!?[' or byte.isspace()

        if byte == '-' and self.tagName == '!-':
            val = 'comment'
        elif byte.isspace() and self.tagName:
            # properly strict thing to do here is probably to only
            # accept whitespace
            val = 'waitforgt' if self.endtag else 'attrs'
        elif byte in '>/[':
            def_gt = self.strict and 'bodydata' or self.maybeBodyData()

            switch = {
                '>': 'bodydata' if self.endtag else def_gt,
                '/': 'afterslash'if self.tagName else None,
                '[': 'expectcdata' if self.tagName == '!' else None}

            val = switch[byte]

        if not (self.lenient or val or is_good):
            self._raise_parse_error('Invalid tag character: %r' % byte)

        return val

    def _update_tags(self, byte):
        alnum_or_ident = byte.isalnum() or byte in IDENTCHARS

        if (byte in '!?') or alnum_or_ident:
            self.tagName += byte
        elif byte == '>' and self.endtag:
            self.gotTagEnd(self.tagName)
        elif byte == '>':
            self.gotTagStart(self.tagName, {})
        elif byte == '/' and not self.tagName:
            self.endtag = 1
        elif byte in '!?' and not self.tagName:
            self.tagName += byte
            self.termtag = 1

    def do_tagstart(self, byte):
        if byte.isspace() and not self.tagName:
            self._raise_parse_error("Whitespace before tag-name")
        elif byte in '!?' and self.tagName and self.strict:
            self._raise_parse_error("Invalid character in tag-name")
        elif byte == '[' and not self.tagName == '!':
            self._raise_parse_error("Invalid '[' in tag-name")

        val = self._get_val(byte)
        self._update_tags(byte)
        return val

    def begin_unentity(self, byte):
        self.bodydata += byte

    def do_unentity(self, byte):
        self.bodydata += byte
        return 'bodydata'

    def end_unentity(self):
        self.gotText(self.bodydata)

    def begin_expectcdata(self, byte):
        self.cdatabuf = byte

    def do_expectcdata(self, byte):
        self.cdatabuf += byte
        cdb = self.cdatabuf
        cd = '[CDATA['

        if len(cd) > len(cdb):
            if cd.startswith(cdb):
                return
            elif self.lenient:
                # WHAT THE CRAP!?  MSWord9 generates HTML that includes these
                # bizarre <![if !foo]> <![endif]> chunks, so I've gotta ignore
                # 'em as best I can.  this should really be a separate parse
                # state but I don't even have any idea what these _are_.
                return 'waitforgt'
            else:
                self._raise_parse_error("Mal-formed CDATA header")
        if cd == cdb:
            self.cdatabuf = ''
            return 'cdata'

        self._raise_parse_error("Mal-formed CDATA header")

    def do_cdata(self, byte):
        self.cdatabuf += byte
        if self.cdatabuf.endswith("]]>"):
            self.cdatabuf = self.cdatabuf[:-3]
            return 'bodydata'

    def end_cdata(self):
        self.gotCData(self.cdatabuf)
        self.cdatabuf = ''

    def do_attrs(self, byte):
        if byte.isalnum() or byte in IDENTCHARS:
            # XXX FIXME really handle !DOCTYPE at some point
            if self.tagName == '!DOCTYPE':
                return 'doctype'

            if self.tagName[0] in '!?':
                return 'waitforgt'

            return 'attrname'
        elif byte.isspace():
            return
        elif byte == '>':
            self.gotTagStart(self.tagName, self.tagAttributes)
            return self.strict and 'bodydata' or self.maybeBodyData()
        elif byte == '/':
            return 'afterslash'
        elif self.lenient:
            # discard and move on?  Only case I've seen of this so far was:
            # <foo bar="baz"">
            return

        self._raise_parse_error("Unexpected character: %r" % byte)

    def begin_doctype(self, byte):
        self.doctype = byte

    def do_doctype(self, byte):
        if byte == '>':
            return 'bodydata'

        self.doctype += byte

    def end_doctype(self):
        self.gotDoctype(self.doctype)
        self.doctype = None

    def do_waitforgt(self, byte):
        if byte == '>':
            if self.endtag or self.lenient:
                return 'bodydata'

            return self.maybeBodyData()

    def begin_attrname(self, byte):
        self.attrname = byte
        self._attrname_termtag = 0

    def _get_attrname(self, byte):
        if byte == '=':
            val = 'beforeattrval'
        elif byte.isspace():
            val = 'beforeeq'
        elif self.lenient and byte in '"\'':
            val = 'attrval'
        elif self.lenient and byte == '>':
            val = 'bodydata' if self._attrname_termtag else None
        else:
            # something is really broken. let's leave this attribute where it
            # is and move on to the next thing
            val = None

        return val

    def do_attrname(self, byte):
        if byte.isalnum() or byte in IDENTCHARS:
            self.attrname += byte
        elif self.strict and not (byte.isspace() or byte == '='):
            msg = "Invalid attribute name: %r %r" % (self.attrname, byte)
            self._raise_parse_error(msg)
        elif byte in LENIENT_IDENTCHARS or byte.isalnum():
            self.attrname += byte
        elif byte == '/':
            self._attrname_termtag = 1
        elif byte == '>':
            self.attrval = 'True'
            self.tagAttributes[self.attrname] = self.attrval
            self.gotTagStart(self.tagName, self.tagAttributes)
            self.gotTagEnd(self.tagName) if self._attrname_termtag else None

        return self._get_attrname(byte)

    def do_beforeattrval(self, byte):
        chars = LENIENT_IDENTCHARS
        val = None

        if byte in '"\'':
            val = 'attrval'
        elif byte.isspace():
            pass
        elif self.lenient and (byte in chars or byte.isalnum()):
            val = 'messyattr'
        elif self.lenient and byte == '>':
            self.attrval = 'True'
            self.tagAttributes[self.attrname] = self.attrval
            self.gotTagStart(self.tagName, self.tagAttributes)
            val = self.maybeBodyData()
        elif self.lenient and byte == '\\':
            # I saw this in actual HTML once:
            # <font size=\"3\"><sup>SM</sup></font>
            pass
        else:
            msg = 'Invalid initial attribute value: %r; ' % byte
            msg += 'Attribute values must be quoted.'
            self._raise_parse_error(msg)

        return val

    def begin_beforeeq(self, byte):
        self._beforeeq_termtag = 0

    def do_beforeeq(self, byte):
        if byte == '=':
            return 'beforeattrval'
        elif byte.isspace():
            return
        elif self.lenient:
            if byte.isalnum() or byte in IDENTCHARS:
                self.attrval = 'True'
                self.tagAttributes[self.attrname] = self.attrval
                return 'attrname'
            elif byte == '>':
                self.attrval = 'True'
                self.tagAttributes[self.attrname] = self.attrval
                self.gotTagStart(self.tagName, self.tagAttributes)

                if self._beforeeq_termtag:
                    self.gotTagEnd(self.tagName)
                    return 'bodydata'

                return self.maybeBodyData()
            elif byte == '/':
                self._beforeeq_termtag = 1
                return

        self._raise_parse_error("Invalid attribute")

    def begin_attrval(self, byte):
        self.quotetype = byte
        self.attrval = ''

    def do_attrval(self, byte):
        if byte == self.quotetype:
            return 'attrs'
        self.attrval += byte

    def end_attrval(self):
        self.tagAttributes[self.attrname] = self.attrval
        self.attrname = self.attrval = ''

    def begin_messyattr(self, byte):
        self.attrval = byte

    def do_messyattr(self, byte):
        if byte.isspace():
            return 'attrs'
        elif byte == '>':
            endTag = 0

            if self.attrval.endswith('/'):
                endTag = 1
                self.attrval = self.attrval[:-1]

            self.tagAttributes[self.attrname] = self.attrval
            self.gotTagStart(self.tagName, self.tagAttributes)

            if endTag:
                self.gotTagEnd(self.tagName)
                return 'bodydata'

            return self.maybeBodyData()
        else:
            self.attrval += byte

    def end_messyattr(self):
        if self.attrval:
            self.tagAttributes[self.attrname] = self.attrval

    def begin_afterslash(self, byte):
        self._after_slash_closed = 0

    def do_afterslash(self, byte):
        # this state is only after a self-terminating slash, e.g. <foo/>
        if self._after_slash_closed:
            self._raise_parse_error("Mal-formed")  # XXX When does this happen??

        if byte != '>' and self.lenient:
            return
        elif byte != '>':
            self._raise_parse_error("No data allowed after '/'")

        self._after_slash_closed = 1
        self.gotTagStart(self.tagName, self.tagAttributes)
        self.gotTagEnd(self.tagName)

        # don't need maybeBodyData here because there better not be
        # any javascript code after a <script/>... we'll see :(
        return 'bodydata'

    def begin_bodydata(self, byte):
        if self._leadingBodyData:
            self.bodydata = self._leadingBodyData
            del self._leadingBodyData
        else:
            self.bodydata = ''

    def do_bodydata(self, byte):
        if byte == '<':
            return 'tagstart'
        if byte == '&':
            return 'entityref'
        self.bodydata += byte

    def end_bodydata(self):
        self.gotText(self.bodydata)
        self.bodydata = ''

    def do_waitforendscript(self, byte):
        if byte == '<':
            return 'waitscriptendtag'
        self.bodydata += byte

    def begin_waitscriptendtag(self, byte):
        self.temptagdata = ''
        self.tagName = ''
        self.endtag = 0

    def do_waitscriptendtag(self, byte):
        # 1 enforce / as first byte read
        # 2 enforce following bytes to be subset of "script" until
        #   tagName == "script"
        #   2a when that happens, gotText(self.bodydata) and
        #      gotTagEnd(self.tagName)
        # 3 spaces can happen anywhere, they're ignored
        #   e.g. < / script >
        # 4 anything else causes all data I've read to be moved to the
        #   bodydata, and switch back to waitforendscript state

        # If it turns out this _isn't_ a </script>, we need to
        # remember all the data we've been through so we can append it
        # to bodydata
        self.temptagdata += byte

        # 1
        if byte == '/':
            self.endtag = True
        elif not self.endtag:
            self.bodydata += "<" + self.temptagdata
            return 'waitforendscript'
        # 2
        elif byte.isalnum() or byte in IDENTCHARS:
            self.tagName += byte
            if not 'script'.startswith(self.tagName):
                self.bodydata += "<" + self.temptagdata
                return 'waitforendscript'
            elif self.tagName == 'script':
                self.gotText(self.bodydata)
                self.gotTagEnd(self.tagName)
                return 'waitforgt'
        # 3
        elif byte.isspace():
            return 'waitscriptendtag'
        # 4
        else:
            self.bodydata += "<" + self.temptagdata
            return 'waitforendscript'

    def begin_entityref(self, byte):
        self.erefbuf = ''
        self.erefextra = ''  # extra bit for lenient mode

    def do_entityref(self, byte):
        if byte.isspace() or byte == "<":
            if self.lenient:
                # '&foo' probably was '&amp;foo'
                if self.erefbuf and self.erefbuf != "amp":
                    self.erefextra = self.erefbuf

                self.erefbuf = "amp"

                if byte == "<":
                    return "tagstart"
                else:
                    self.erefextra += byte
                    return 'spacebodydata'

            self._raise_parse_error("Bad entity reference")
        elif byte != ';':
            self.erefbuf += byte
        else:
            return 'bodydata'

    def end_entityref(self):
        self.gotEntityReference(self.erefbuf)

    # hacky support for space after & in entityref in lenient
    # state should only happen in that case
    def begin_spacebodydata(self, byte):
        self.bodydata = self.erefextra
        self.erefextra = None

    do_spacebodydata = do_bodydata
    end_spacebodydata = end_bodydata

    # Sorta SAX-ish API

    def gotTagStart(self, name, attributes):
        '''Encountered an opening tag.

        Default behaviour is to print.'''
        print('begin', name, attributes)

    def gotText(self, data):
        '''Encountered text

        Default behaviour is to print.'''
        print('text:', repr(data))

    def gotEntityReference(self, entityRef):
        '''Encountered mnemonic entity reference

        Default behaviour is to print.'''
        print('entityRef: &%s;' % entityRef)

    def gotComment(self, comment):
        '''Encountered comment.

        Default behaviour is to ignore.'''
        pass

    def gotCData(self, cdata):
        '''Encountered CDATA

        Default behaviour is to call the gotText method'''
        self.gotText(cdata)

    def gotDoctype(self, doctype):
        """Encountered DOCTYPE

        This is really grotty: it basically just gives you everything between
        '<!DOCTYPE' and '>' as an argument.
        """
        print('!DOCTYPE', repr(doctype))

    def gotTagEnd(self, name):
        '''Encountered closing tag

        Default behaviour is to print.'''
        print('end', name)
