# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (c) 2019 Digi International Inc. All Rights Reserved.
import struct

import six
from suitcase.exceptions import SuitcaseChecksumException, SuitcaseProgrammingError, \
    SuitcaseParseError, SuitcaseException, SuitcasePackStructException
from six import BytesIO, StringIO


class FieldPlaceholder(object):
    """Internally used object that holds information about a field schema

    A FieldPlaceholder is what is actually instantiated and stored with
    the message schema class when a message schema is declared.  The
    placeholder stores all instantiation information needed to create
    the fields when the message object is instantiated

    """

    # record the number of instantiations of fields.  This is how
    # we can track the order of fields within a message
    _global_seqno = 0

    def __init__(self, cls, args, kwargs):
        self._field_seqno = FieldPlaceholder._global_seqno
        FieldPlaceholder._global_seqno += 1
        self.cls = cls
        self.args = args
        self.kwargs = kwargs
        self.__accessors = {}

    def create_instance(self, parent):
        """Create an instance based off this placeholder with some parent"""
        self.kwargs['instantiate'] = True
        self.kwargs['parent'] = parent
        instance = self.cls(*self.args, **self.kwargs)
        instance._field_seqno = self._field_seqno
        return instance

    def __getattr__(self, name):
        # In case the same field is being referenced more than once.
        if name in self.__accessors:
            return self.__accessors[name]
        accessor = FieldAccessor(self, name)
        self.__accessors[name] = accessor
        return accessor


class BaseField(object):
    """Base class for all Field instances

    Some magic is used whenever a field is created normally in order to
    allow for a declarative syntax without having to create explicit
    factory methods/classes everywhere.  By default (for a normal call)
    when a field is instantiated, it actually ends up returning a
    FieldPlaceholder object containing information about the field
    that has been declared.

    To get an actual instance of the field, the call to the construction
    must include a keyword argument ``instantiate`` that is set to some
    truthy value.  Typically, this instantiation logic is done by
    calling ``FieldPlaceholder.create_instance(parent)``.  This is the
    recommended way to construct a field as it ensure all invariants
    are in place (having a parent).

    :param instantiate: Create an actual instance instead of a placeholder
    :param parent: Specify the parent of this field (typically a Structure
        instance).

    """

    def __new__(cls, *args, **kwargs):
        instantiate = kwargs.pop('instantiate', False)
        if instantiate:
            return super(BaseField, cls).__new__(cls)
        else:
            return FieldPlaceholder(cls, args, kwargs)

    def __init__(self, *args, **kwargs):
        self._value = None
        self._parent = kwargs.get('parent')

    def _ph2f(self, placeholder):
        """Lookup a field given a field placeholder"""
        if issubclass(placeholder.cls, FieldAccessor):
            return placeholder.cls.access(self._parent, placeholder)
        return self._parent.lookup_field_by_placeholder(placeholder)

    def __repr__(self):
        return repr(self.getval())

    def is_substructure(self):
        """Return True if this field is (effectively) a SubstructureField"""
        return False

    def getval(self):
        return self._value

    def setval(self, value):
        self._value = value

    def unpack(self, data, **kwargs):  # type: (bytes, **bool) -> bytes
        """
        Given the raw data associated with a field, consume the data and return anything left over.
        :param data: Raw data to consume
        :returns: Remaining data
        """
        raise NotImplementedError("Field class must implement unpack.")

    @property
    def is_greedy(self):  # type: () -> bool
        """
        Should the parser consider this field to be greedy, i.e. that it wants to consume
        all remaining data in the input?

        This property currently is only meaningful on ``DispatchTarget``,
        where passing greedy=False allows for nested DispatchTarget structures.
        """
        return True


class CRCField(BaseField):
    r"""Field representing CRC (Cyclical Redundancy Check) in a message

    CRC checks and calculation frequently work quite differently
    from other fields in a protocol and as such are treated differently
    by the message container.  In particular, a CRCField requires
    special steps at either the beginning or end of the message
    pack/unpack process.  For each CRCField, we specify the following:

    :param field: The underlying field defining how the checksum will
        be stored.  This should match the size of the checksum in
        whatever algorithm is being used (16 bits for CRC16).
    :param algo: The algorithm to be used for performing the checksum.
       This is basically a function that takes a chunk of data and
       gives the checksum.  Several of these are provided out of the
       box in ``suitcase.crc``.
    :param start: The offset in the overall message at which we should
        start the checksum.  This may be positive (from the start) or
        negative (from the end of the message).
    :param end: The offset in the overall message at which we should
        end the checksum algo.  This may be positive (from the start)
        or negative (from the end of the message).

    A quick example of a message with a checksum is in order::

        class MyChecksummedMessage(Structure):
            soh = Magic('\x1f\x1f')
            message_id = UBInt16()
            sequence_number = UBInt8()
            payload_length = LengthField(UBInt16())
            payload = VariableRawPayload(payload_length)

            # crc starts after soh and ends before crc
            crc = CRCField(UBInt16(), crc16_ccitt, 2, -3)
            eof = Magic('~')

    """

    def __init__(self, field, algo, start, end, **kwargs):
        BaseField.__init__(self, **kwargs)
        self.field = field.create_instance(self._parent)
        self.field.setval(0)
        self.algo = algo
        self.start = start
        self.end = end
        self._value = None

    @property
    def bytes_required(self):
        return self.field.bytes_required

    def validate(self, data, offset):
        """Raises :class:`SuitcaseChecksumException` if not valid"""
        recorded_checksum = self.field.getval()

        # convert negative offset to positive
        if offset < 0:
            offset += len(data)

        # replace checksum region with zero
        data = b''.join((data[:offset],
                         b"\x00" * self.bytes_required,
                         data[offset + self.bytes_required:]))
        actual_checksum = self.algo(data[self.start:self.end])
        if recorded_checksum != actual_checksum:
            raise SuitcaseChecksumException(
                "recorded checksum %r did not match actual %r.  full data: %r",
                recorded_checksum, actual_checksum, data)

    def packed_checksum(self, data):
        """Given the data of the entire packet return the checksum bytes"""
        self.field.setval(self.algo(data[self.start:self.end]))
        sio = BytesIO()
        self.field.pack(sio)
        return sio.getvalue()

    def getval(self):
        return self.field.getval()

    def setval(self, *args):
        raise SuitcaseProgrammingError("CRC will be set automatically")

    def pack(self, stream):
        # write placeholder during the first pass
        stream.write(b'\x00' * self.field.bytes_required)

    def unpack(self, data, **kwargs):
        return self.field.unpack(data, **kwargs)


class Magic(BaseField):
    """Represent Byte Magic (fixed, expected sequence of bytes)"""

    def __init__(self, expected_sequence, **kwargs):
        BaseField.__init__(self, **kwargs)
        self.expected_sequence = expected_sequence
        self.bytes_required = len(self.expected_sequence)

    def getval(self):
        return self.expected_sequence

    def setval(self, *args):
        raise SuitcaseProgrammingError("One does not simply modify Magic")

    def pack(self, stream):
        stream.write(self.expected_sequence)

    def unpack(self, data, **kwargs):
        if not data == self.expected_sequence:
            raise SuitcaseParseError(
                "Expected sequence %r for magic field but got %r on "
                "message %r" % (self.expected_sequence, data, self._parent))
        return b''

    def __repr__(self):
        return "Magic(%r)" % (self.expected_sequence,)


class FieldProperty(BaseField):
    """Provide the ability to define "Property" getter/setters for other fields

    This is useful and preferable and preferable to inheritance if you want
    to provide a different interface for getting/setting one some field
    within a message.  Take the test case as an example::

        # define the message
        class MyMessage(Structure):
            _version = UBByteSequence(2)
            version = FieldProperty(_version,
                                    onget=lambda v: "%d.%02d" % (v[0], v[1]),
                                    onset=lambda v: tuple(int(x) for x
                                                          in v.split(".", 1)))

        msg = MyMessage()
        msg.unpack('\x10\x03')
        self.assertEqual(msg._version,(16, 3))
        self.assertEqual(msg.version, "16.03")

        msg.version = "22.7"
        self.assertEqual(msg._version, (22, 7))
        self.assertEqual(msg.version, "22.07")

    :param field: The field that is wrapped by this FieldProperty.  The value
        of this field is passed into or set by the associated get/set fns.
    :param onget: This is a function pointer to a function called to mutate
        the value returned on field access.  The function receives a single
        argument containing the value of the wrapped field.
    :oaram onset: This is a function pointer to a function called to map
        between the property and the underlying field.  The function takes
        a single parameter which is the value the property was set to.  It
        should return the value that the underlying field expects (or raise
        an exception if appropriate).

    """

    def __init__(self, field, onget=None, onset=None, **kwargs):
        BaseField.__init__(self, **kwargs)
        self.onget = onget
        self.onset = onset
        self.field = self._ph2f(field)
        self.bytes_required = 0

    def _default_onget(self, value):
        return value

    def _default_onset(self, value):
        return value

    def getval(self):
        onget = self.onget
        if onget is None:
            onget = self._default_onget
        value = self.field.getval()
        return onget(value)

    def setval(self, value):
        onset = self.onset
        if onset is None:
            onset = self._default_onset

        self.field.setval(onset(value))

    def unpack(self, data, **kwargs):
        return data

    def pack(self, stream):
        pass


class DispatchField(BaseField):
    """Decorate a field as a dispatch byte (used as conditional)

    A DispatchField is always used with a DispatchTarget within the same
    message (at some level).

    Example::

        class MyMessage(Structure):
            type = DispatchField(UBInt8())
            body = DispatchTarget(dispatch_field=type, dispatch_mapping={
                0x00: MessageType0,
                0x01: MessageType1,
            })

    :param field: The field containing the dispatch parameter.  This is
        typically an integer but could be any hashable object.

    """

    def __init__(self, field, **kwargs):
        BaseField.__init__(self, **kwargs)
        self.field = field.create_instance(self._parent)

    @property
    def bytes_required(self):
        return self.field.bytes_required

    def getval(self):
        return self.field.getval()

    def setval(self, value):
        return self.field.setval(value)

    def __repr__(self):
        return repr(self.field)

    def pack(self, stream):
        return self.field.pack(stream)

    def unpack(self, data, **kwargs):
        assert len(data) == self.bytes_required
        return self.field.unpack(data)


class DispatchTarget(BaseField):
    """Represent a conditional branch on some DispatchField

    Example::

        class MyMessage(Structure):
            type = DispatchField(UBInt8())
            body = DispatchTarget(dispatch_field=type, dispatch_mapping={
                0x00: MessageType0,
                0x01: MessageType1,
            })

    Observe that it is also possible to create a DispatchTarget based on
    a segment of a BitField::

        class MyMessage(Structure):
            version_and_type = BitField(8,
                version=BitNum(4),
                type=BitNum(4)
            )
            body = DispatchTarget(None, version_and_type.type, {
                0x0: MessageType0,
                0x1: MessageType1,
            })

    :param length_provider: The field providing a length value binding this
        message (if any).  Set this field to None to leave unconstrained.
    :param dispatch_field: The field being target for dispatch.  In most
        protocols this is an integer type byte or something similar.  The
        possible values for this field act as the keys for the dispatch.
    :param dispatch_mapping: This is a dictionary mapping dispatch_field
        values to associated message types to handle the remaining processing.
    :param greedy: If True (the default), the parser can assume that this field
        consumes all remaining data in the input. If False, the parser will only
        consume as much data as necessary to create the structure.
        This comes in handy when a DispatchTarget is inside of a substructure or another
        DispatchTarget, and is not the last field of the message.

    """

    def __init__(self, length_provider, dispatch_field,
                 dispatch_mapping, greedy=True, **kwargs):
        BaseField.__init__(self, **kwargs)
        if length_provider is None:
            self.length_provider = None
        else:
            self.length_provider = self._ph2f(length_provider)
            self.length_provider.associate_length_consumer(self)
        self.dispatch_field = self._ph2f(dispatch_field)
        self.dispatch_mapping = dispatch_mapping
        self.inverse_dispatch_mapping = dict((v, k) for (k, v)
                                             in dispatch_mapping.items())
        self.__greedy = greedy

    def _lookup_msg_type(self):
        target_key = self.dispatch_field.getval()
        target = self.dispatch_mapping.get(target_key, None)
        if target is None:
            target = self.dispatch_mapping.get(None, None)

        return target

    @property
    def is_greedy(self):
        return self.__greedy

    @property
    def bytes_required(self):
        if self.length_provider is None:
            return None
        else:
            return self.length_provider.get_adjusted_length()

    def getval(self):
        return self._value

    def setval(self, value):
        try:
            vtype = type(value)
            key = self.inverse_dispatch_mapping[vtype]
        except KeyError:
            raise SuitcaseProgrammingError("The type specified is not in the "
                                           "dispatch table")

        # OK, things check out.  Set both the value here and the
        # type byte value
        self._value = value
        value._parent = self._parent
        self.dispatch_field.setval(key)

    def pack(self, stream):
        return self._value._packer.write(stream)

    def unpack(self, data, **kwargs):
        target_msg_type = self._lookup_msg_type()
        if target_msg_type is None:
            target_msg_type = self.dispatch_mapping.get(None)
        if target_msg_type is None:  # still none
            raise SuitcaseParseError("Input data contains type byte not"
                                     " contained in mapping")
        message_instance = target_msg_type()
        self.setval(message_instance)
        return self._value.unpack(data, trailing=not self.is_greedy).read()


class LengthField(BaseField):
    """Wraps an existing field marking it as a LengthField

    This field wraps another field which is assumed to return an
    integer value.  A LengthField can be pointed to by a variable
    length field and the two will be coupled appropriately.

    :param length_field: The field providing the actual length value to
        be used.  This field should return an integer value representing
        the length (or a multiple of the length) as a return value
    :param get_length: If specified, this is a function which takes a reference
        to the encapsulated field and is responsible for reading the length
        out from that field.  By default, this just reads the value of the
        field (field is expected to be an integral value).
    :param set_length: If specified, this is a function which takes a reference
        to the encapsulated field and a length value.  Its responsibility
        is to move the provided length value into the field.  By default,
        the length value is just assigned to the field via `setval()`.
    :param multiplier: If specified, this multiplier is applied to the
        length.  If I specify a multiplier of 8, I am saying that each bit
        in the length field represents 8 bytes of actual payload length. By
        default the multiplier is 1 (1 bit/byte).

    """

    def __init__(self, length_field, get_length=None, set_length=None, multiplier=1, **kwargs):
        BaseField.__init__(self, **kwargs)
        self.multiplier = multiplier
        self.get_length = self._default_get_length if get_length is None else get_length
        self.set_length = self._default_set_length if set_length is None else set_length
        self.length_field = length_field.create_instance(self._parent)
        self.length_value_provider = None

    def __repr__(self):
        return repr(self.length_field)

    def _default_get_length(self, field):
        return field.getval()

    def _default_set_length(self, field, length):
        field._value = length  # TODO: use setval() [problem with DependentField tests]?

    def is_substructure(self):
        return self.length_field.is_substructure()

    @property
    def bytes_required(self):
        return self.length_field.bytes_required

    def getval(self):
        return self.length_field.getval()

    def setval(self, value):
        raise SuitcaseProgrammingError("Cannot set the value of a LengthField")

    def associate_length_consumer(self, target_field):
        def _length_value_provider():
            sio = BytesIO()
            target_field.pack(sio)
            target_field_length = len(sio.getvalue())
            if not target_field_length % self.multiplier == 0:
                raise SuitcaseProgrammingError("Payload length not divisible "
                                               "by %s" % self.multiplier)
            return (target_field_length // self.multiplier)

        self.length_value_provider = _length_value_provider

    def pack(self, stream):
        if self.length_value_provider is None:
            raise SuitcaseException("No length_provider added to this LengthField")
        self.set_length(self.length_field, self.length_value_provider())
        self.length_field.pack(stream)

    def unpack(self, data, **kwargs):
        assert len(data) == self.bytes_required
        return self.length_field.unpack(data, **kwargs)

    def get_adjusted_length(self):
        return self.get_length(self.length_field) * self.multiplier


class TypeField(BaseField):
    """Wraps an existing field marking it as a TypeField

    This field wraps another field which is assumed to return an
    integer value.  A TypeField can be pointed to by a variable
    length field and will fix that field's length.

    :param type_field: The field providing the type id that will
        be used to lookup a length value
    :param length_mapping: This is a dictionary mapping type_field
        values to associated length values.

    For example, a TypeField could be used as a replacement for a
    DispatchField so that the associated DispatchTarget is non-greedy,
    without the existence of a seperate LengthField. This enables
    a greedy field to follow the dispatch field, without the usage of
    complex ConditionalField setups::

        class Structure8Bit(Structure):
            value = UBInt8()

        class Structure16Bit(Structure):
            value = UBInt16()

        class DispatchStructure(Structure):
            type = TypeField(UBInt8(), {0x40: 1,
                                        0x80: 2})
            # Here the TypeField is providing both a size and type id
            dispatch = DispatchTarget(type, type, {0x40: Structure8Bit,
                                                   0x80: Structure16Bit})
            greedy = Payload()
    """

    def __init__(self, type_field, length_mapping, **kwargs):
        BaseField.__init__(self, **kwargs)
        self.type_field = type_field.create_instance(self._parent)
        self.length_mapping = length_mapping
        self.length_value_provider = None

    def _lookup_msg_length(self):
        target_key = self.type_field.getval()
        target_length = self.length_mapping.get(target_key, None)
        if target_length is None:
            target_length = self.length_mapping.get(None, None)

        return target_length

    def __repr__(self):
        return repr(self.type_field)

    def is_substructure(self):
        return self.type_field.is_substructure()

    @property
    def bytes_required(self):
        return self.type_field.bytes_required

    def getval(self):
        return self.type_field.getval()

    def setval(self, value):
        self.type_field.setval(value)

    def associate_length_consumer(self, target_field):
        def _length_value_provider():
            sio = BytesIO()
            target_field.pack(sio)
            target_field_length = len(sio.getvalue())
            if target_field_length != self.get_adjusted_length():
                raise SuitcaseProgrammingError("Payload length %i does not"
                                               " match length %i specified by type"
                                               % (target_field_length, self.get_adjusted_length()))
            return target_field_length

        self.length_value_provider = _length_value_provider

    def pack(self, stream):
        if self.length_value_provider is None:
            raise SuitcaseException("No length_provider added to this TypeField")
        # This will throw a SuitcasePackException if the length is not correct
        self.length_value_provider()

        self.type_field.pack(stream)

    def unpack(self, data, **kwargs):
        assert len(data) == self.bytes_required
        return self.type_field.unpack(data, **kwargs)

    def get_adjusted_length(self):
        return self._lookup_msg_length()


class ConditionalField(BaseField):
    """Field which may or may not be included depending on some condition

    In some protocols, there exist fields which may or may not be present
    depending on the values of other fields that would have already been
    parsed at this point in time.  Wrapping such fields in  a ConditionalField
    allows us to define a function to examine that state and only have the
    field capture the bytes if the conditions are right.

    :param field: The field which should handle the parsing if the condition
        evaluates to true.

    :param condition: This is a function which is given access to the parent
        message that is expected to return a boolean value.  If the value
        is true, then ``field`` will handle the bytes.  If not, then the
        field will be skipped and left with its default value (None).

    """

    def __init__(self, field, condition, **kwargs):
        BaseField.__init__(self, **kwargs)
        self.field = field.create_instance(self._parent)
        self.condition = condition

    def __repr__(self):
        if self.condition(self._parent):
            return repr(self.field)
        else:
            return "<ConditionalField: not included>"

    def is_substructure(self):
        return self.field.is_substructure()

    @property
    def bytes_required(self):
        if self.condition(self._parent):
            return self.field.bytes_required
        else:
            return 0

    def pack(self, stream):
        if self.condition(self._parent):
            self.field.pack(stream)

    def unpack(self, data, **kwargs):
        if self.condition(self._parent):
            return self.field.unpack(data, **kwargs)
        return data

    def getval(self):
        if not self.condition(self._parent):
            return None

        return self.field.getval()

    def setval(self, value):
        return self.field.setval(value)

    def associate_length_consumer(self, target_field):
        self.field.associate_length_consumer(target_field)

    def get_adjusted_length(self):
        return self.field.get_adjusted_length()


class Payload(BaseField):
    """Variable length raw (byte string) field

    This field is expected to be used with a LengthField.  The variable
    length field provides a value to be used by the LengthField on
    message pack and vice-versa for unpack.

    :param length_provider: The LengthField with which this variable
        length payload is associated.  If not included, it is assumed that
        the length_provider should consume the remainder of the bytes
        available in the string.  This is only valid in cases where the
        developer knows that they will be dealing with a fixed sequence
        of bytes (already boxed).

    """

    def __init__(self, length_provider=None, **kwargs):
        BaseField.__init__(self, **kwargs)
        if isinstance(length_provider, FieldPlaceholder):
            self.length_provider = self._ph2f(length_provider)
            self.length_provider.associate_length_consumer(self)
        else:
            self.length_provider = None

    @property
    def bytes_required(self):
        if self.length_provider is None:
            return None
        else:
            return self.length_provider.get_adjusted_length()

    def pack(self, stream):
        stream.write(self._value)

    def unpack(self, data, **kwargs):
        self._value = data
        assert self.bytes_required is None or len(data) == self.bytes_required
        return b''


# keep for backwards compatibility
VariableRawPayload = Payload


class BaseVariableByteSequence(BaseField):
    def __init__(self, make_format, length_provider, **kwargs):
        BaseField.__init__(self, **kwargs)
        self.make_format = make_format
        if length_provider is not None:
            self.length_provider = self._ph2f(length_provider)
            self.length_provider.associate_length_consumer(self)

    @property
    def bytes_required(self):
        return self.length_provider.get_adjusted_length()

    def pack(self, stream):
        sfmt = self.make_format(len(self._value))
        try:
            stream.write(struct.pack(sfmt, *self._value))
        except struct.error as e:
            raise SuitcasePackStructException(e)

    def unpack(self, data, **kwargs):
        assert len(data) == self.bytes_required

        length = self.bytes_required
        sfmt = self.make_format(length)
        try:
            self._value = struct.unpack(sfmt, data)
        except struct.error as e:
            raise SuitcasePackStructException(e)


class DependentField(BaseField):
    """Field populated by container packet at lower level

    It is sometimes the case that information from another layer of a
    messaging protocol be needed at another higher-level of the protocol.
    The DependentField is a way of declaring that a message at some layer
    is dependent on a field with some name from the parent layer.

    For instance, let's suppose that my protocol had an option byte that
    I wanted to include in some logic handling packets at some higher
    layer.  I could include that byte in my message as follows::

        class MyDependentMessage(Structure):
            ll_options = DependentField('proto_options')
            data = UBInt8Sequence(16)

        class LowerLevelProtocol(Structure):
            type = DispatchField(UBInt16())
            proto_options = UBInt8()
            body = DispatchTarget(None, type, {
                0x00: MyDependentMessage,
            })

    :param name: The name of the field from the parent message that we
        would like brought into our message namespace.

    """

    def __init__(self, name, **kwargs):
        BaseField.__init__(self, **kwargs)
        self.bytes_required = 0
        self.parent_field_name = name
        self.parent_field = None

    def _get_parent_field(self):
        if self.parent_field is None:
            message_parent = self._parent._parent
            target_field = message_parent.lookup_field_by_name(
                self.parent_field_name)
            self.parent_field = target_field
        return self.parent_field

    def __getattr__(self, attr):
        return getattr(self._get_parent_field(), attr)

    def pack(self, stream):
        pass

    def unpack(self, data, **kwargs):
        return data

    def getval(self):
        return self._get_parent_field().getval()

    def setval(self, value):
        return self._get_parent_field().setval(value)


class SubstructureField(BaseField):
    """Field which contains another non-greedy Structure.

    Often data-types are needed which cannot be easily
    described by a single field, but are representable as
    Structures. For example, Pascal style strings are prefixed
    with their length. It is often desirable to embed these
    data-types within another Structure, to avoid reimplementing
    them in every usage.


    A Pascal style string could be described as follows::

        from suitcase.structure import Structure
        from suitcase.fields import Payload, UBInt16, LengthField, SubstructureField

        class PascalString16(Structure):
            length = LengthField(UBInt16())
            value = Payload(length)

    A structure describing a name of a person might consist of two
    Pascal style strings. Instead of describing the ugly way::

        class NameUgly(Structure):
            first_length = LengthField(UBInt16())
            first_value = Payload(first_length)
            last_length = LengthField(UBInt16())
            last_value = Payload(last_length)

    it could be defined using a SubstructureField::

        class Name(Structure):
            first = SubstructureField(PascalString16)
            last = SubstructureField(PascalString16)
    """

    def __init__(self, substructure, **kwargs):
        BaseField.__init__(self, **kwargs)
        self.substructure = substructure
        self._value = substructure()
        self._value._parent = self._parent

    def is_substructure(self):
        return True

    @property
    def bytes_required(self):
        # We return None but do not count as a greedy field to the packer
        return None

    def pack(self, stream):
        stream.write(self._value.pack())

    def unpack(self, data, **kwargs):
        self._value = self.substructure()
        self._value._parent = self._parent
        return self._value.unpack(data, **kwargs).read()


class FieldArray(BaseField):
    """Field which contains a list of some other field.

    In some protocols, there exist repeated substructures which are present in a
    variable number. The variable nature of these fields make a DispatchField
    and DispatchTarget combination unsuitable; instead a FieldArray may be
    used to pack/unpack these fields to/from a list.

    :param substructure: The type of the array element. Must not be a greedy
        field, or else the array will only ever have one element containing
        the entire contents of the array.
    :param length_provider: The field providing a length value binding this
        message (if any).  Set this field to None to leave unconstrained.
    :param num_elements_provider: The field providing the number of elements
        in the array. Set this field to None to leave unconstrained.

    For example, one can imagine a message listing the zipcodes covered by
    a telephone area code. Depending on the population density, the number
    of zipcodes per area code could vary greatly. One implementation of this
    data structure could be::

        from suitcase.structure import Structure
        from suitcase.fields import UBInt16, FieldArray

        class ZipcodeStructure(Structure):
            zipcode = UBInt16()

        class TelephoneZipcodes(Structure):
            areacode = UBInt16()
            zipcodes = FieldArray(ZipcodeStructure)  # variable number of zipcodes

    """

    def __init__(self, substructure, length_provider=None, num_elements_provider=None, **kwargs):
        BaseField.__init__(self, **kwargs)
        self.substructure = substructure
        self._value = list()
        if isinstance(length_provider, FieldPlaceholder):
            self.length_provider = self._ph2f(length_provider)
            self.length_provider.associate_length_consumer(self)
        else:
            self.length_provider = None
        if isinstance(num_elements_provider, FieldPlaceholder):
            self.num_elements_provider = self._ph2f(num_elements_provider)
            self.num_elements_provider.length_value_provider = lambda: len(self._value)
        else:
            self.num_elements_provider = None


    @property
    def bytes_required(self):
        if self.length_provider is None:
            return None
        else:
            return self.length_provider.get_adjusted_length()

    @property
    def num_elements(self):
        if self.num_elements_provider is None:
            return None
        else:
            return self.num_elements_provider.get_adjusted_length()

    def pack(self, stream):
        for structure in self._value:
            stream.write(structure.pack())

    def unpack(self, data, **kwargs):
        length = self.bytes_required
        if length == 0 or (data == b"" and length is None) or self.num_elements == 0:
            # Array is empty.
            return data

        kwargs['trailing'] = True
        while True:
            structure = self.substructure()
            structure._parent = self._parent
            data = structure.unpack(data, **kwargs).read()
            self._value.append(structure)
            if data == b"":
                break
            if len(self._value) == self.num_elements:
                break

        if self.num_elements is not None and len(self._value) != self.num_elements:
            raise SuitcaseParseError("Expected %s elements but received %d." %
                                     (self.num_elements, len(self._value)))

        return data


class BaseFixedByteSequence(BaseField):
    """Base fixed-length byte sequence field"""

    def __init__(self, make_format, size, **kwargs):
        BaseField.__init__(self, **kwargs)
        self.bytes_required = size
        self.format = make_format(size)

    def pack(self, stream):
        try:
            stream.write(struct.pack(self.format, *self._value))
        except struct.error as e:
            raise SuitcasePackStructException(e)

    def unpack(self, data, **kwargs):
        try:
            self._value = struct.unpack(self.format, data)
            assert len(data) == self.bytes_required
            return b''
        except struct.error as e:
            raise SuitcasePackStructException(e)


def byte_sequence_factory_factory(make_format):
    def byte_sequence_factory(length_or_provider):
        if isinstance(length_or_provider, int):
            return BaseFixedByteSequence(make_format, length_or_provider)
        else:
            return BaseVariableByteSequence(make_format, length_or_provider)

    return byte_sequence_factory


UBInt8Sequence = byte_sequence_factory_factory(lambda l: ">" + "B" * l)
ULInt8Sequence = byte_sequence_factory_factory(lambda l: "<" + "B" * l)
SBInt8Sequence = byte_sequence_factory_factory(lambda l: ">" + "b" * l)
SLInt8Sequence = byte_sequence_factory_factory(lambda l: "<" + "b" * l)


class BaseStructField(BaseField):
    """Base for fields based very directly on python struct module formats

    It is expected that this class will be subclassed and customized by
    defining ``FORMAT`` at the class level.  ``FORMAT`` is expected to
    be a format string that could be used with struct.pack/unpack.  It
    should include endianness information.  If the ``FORMAT`` includes
    multiple elements, the default ``_unpack`` logic assumes that each
    element is a single byte and will OR these together.  To specialize
    this, _unpack should be overridden.

    """

    def __init__(self, **kwargs):
        BaseField.__init__(self, **kwargs)
        self._value = None
        self._keep_bytes = getattr(self, "KEEP_BYTES", None)
        if self._keep_bytes is not None:
            self.bytes_required = self._keep_bytes
        else:
            self.bytes_required = struct.calcsize(self.PACK_FORMAT)

    def pack(self, stream):
        try:
            keep_bytes = getattr(self, 'KEEP_BYTES', None)
            if keep_bytes is not None:
                if self.PACK_FORMAT[0] == b">"[0]:  # The element access makes this compatible with Python 2 and 3
                    to_write = struct.pack(self.PACK_FORMAT, self._value)[-keep_bytes:]
                else:
                    to_write = struct.pack(self.PACK_FORMAT, self._value)[:keep_bytes]
            else:
                to_write = struct.pack(self.PACK_FORMAT, self._value)
        except struct.error as e:
            raise SuitcasePackStructException(e)
        stream.write(to_write)

    def unpack(self, data, **kwargs):
        value = 0
        if self.UNPACK_FORMAT[0] == b">"[0]:  # The element access makes this compatible with Python 2 and 3
            for i, byte in enumerate(reversed(struct.unpack(self.UNPACK_FORMAT, data))):
                value |= (byte << (i * 8))
        else:
            for i, byte in enumerate(struct.unpack(self.UNPACK_FORMAT, data)):
                value |= (byte << (i * 8))
        self._value = value
        assert len(data) == self.bytes_required
        return b''


# ==============================================================================
# Unsigned Big Endian
# ==============================================================================
class UBInt8(BaseStructField):
    """Unsigned Big Endian 8-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b">B"


class UBInt16(BaseStructField):
    """Unsigned Big Endian 16-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b">H"


class UBInt24(BaseStructField):
    """Unsigned Big Endian 24-bit integer field"""
    KEEP_BYTES = 3
    PACK_FORMAT = b">I"
    UNPACK_FORMAT = b">BBB"


class UBInt32(BaseStructField):
    """Unsigned Big Endian 32-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b">I"


class UBInt40(BaseStructField):
    """Unsigned Big Endian 40-bit integer field"""
    KEEP_BYTES = 5
    PACK_FORMAT = b">Q"
    UNPACK_FORMAT = b">BBBBB"


class UBInt48(BaseStructField):
    """Unsigned Big Endian 48-bit integer field"""
    KEEP_BYTES = 6
    PACK_FORMAT = b">Q"
    UNPACK_FORMAT = b">BBBBBB"


class UBInt56(BaseStructField):
    """Unsigned Big Endian 56-bit integer field"""
    KEEP_BYTES = 7
    PACK_FORMAT = b">Q"
    UNPACK_FORMAT = b">BBBBBBB"


class UBInt64(BaseStructField):
    """Unsigned Big Endian 64-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b">Q"


# ==============================================================================
# Signed Big Endian
# ==============================================================================
class SBInt8(BaseStructField):
    """Signed Big Endian 8-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b">b"


class SBInt16(BaseStructField):
    """Signed Big Endian 16-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b">h"


class SBInt24(BaseStructField):
    """Signed Big Endian 24-bit integer field"""
    KEEP_BYTES = 3
    PACK_FORMAT = b">i"
    UNPACK_FORMAT = b">bBB"


class SBInt32(BaseStructField):
    """Signed Big Endian 32-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b">i"


class SBInt40(BaseStructField):
    """Signed Big Endian 40-bit integer field"""
    KEEP_BYTES = 5
    PACK_FORMAT = b">q"
    UNPACK_FORMAT = b">bBBBB"


class SBInt48(BaseStructField):
    """Signed Big Endian 48-bit integer field"""
    KEEP_BYTES = 6
    PACK_FORMAT = b">q"
    UNPACK_FORMAT = b">bBBBBB"


class SBInt56(BaseStructField):
    """Signed Big Endian 56-bit integer field"""
    KEEP_BYTES = 7
    PACK_FORMAT = b">q"
    UNPACK_FORMAT = b">bBBBBBB"


class SBInt64(BaseStructField):
    """Signed Big Endian 64-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b">q"


class SBFloat32(BaseStructField):
    """Unsigned Big Endian 32-bit float field"""
    PACK_FORMAT = UNPACK_FORMAT = b">f"


class SBFloat64(BaseStructField):
    """Unsigned Big Endian 64-bit float field"""
    PACK_FORMAT = UNPACK_FORMAT = b">d"


# ==============================================================================
# Unsigned Little Endian
# ==============================================================================
class ULInt8(BaseStructField):
    """Unsigned Little Endian 8-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b"<B"


class ULInt16(BaseStructField):
    """Unsigned Little Endian 16-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b"<H"


class ULInt24(BaseStructField):
    """Unsigned Little Endian 24-bit integer field"""
    KEEP_BYTES = 3
    PACK_FORMAT = b"<I"
    UNPACK_FORMAT = b"<BBB"


class ULInt32(BaseStructField):
    """Unsigned Little Endian 32-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b"<I"


class ULInt40(BaseStructField):
    """Unsigned Little Endian 40-bit integer field"""
    KEEP_BYTES = 5
    PACK_FORMAT = b"<Q"
    UNPACK_FORMAT = b"<BBBBB"


class ULInt48(BaseStructField):
    """Unsigned Little Endian 48-bit integer field"""
    KEEP_BYTES = 6
    PACK_FORMAT = b"<Q"
    UNPACK_FORMAT = b"<BBBBBB"


class ULInt56(BaseStructField):
    """Unsigned Little Endian 56-bit integer field"""
    KEEP_BYTES = 7
    PACK_FORMAT = b"<Q"
    UNPACK_FORMAT = b"<BBBBBBB"


class ULInt64(BaseStructField):
    """Unsigned Little Endian 64-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b"<Q"


# ==============================================================================
# Signed Little Endian
# ==============================================================================
class SLInt8(BaseStructField):
    """Signed Little Endian 8-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b"<b"


class SLInt16(BaseStructField):
    """Signed Little Endian 16-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b"<h"


class SLInt24(BaseStructField):
    """Signed Little Endian 24-bit integer field"""
    KEEP_BYTES = 3
    PACK_FORMAT = b"<i"
    UNPACK_FORMAT = b"<BBb"


class SLInt32(BaseStructField):
    """Signed Little Endian 32-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b"<i"


class SLInt40(BaseStructField):
    """Signed Little Endian 40-bit integer field"""
    KEEP_BYTES = 5
    PACK_FORMAT = b"<q"
    UNPACK_FORMAT = b"<BBBBb"


class SLInt48(BaseStructField):
    """Signed Little Endian 48-bit integer field"""
    KEEP_BYTES = 6
    PACK_FORMAT = b"<q"
    UNPACK_FORMAT = b"<BBBBBb"


class SLInt56(BaseStructField):
    """Signed Little Endian 56-bit integer field"""
    KEEP_BYTES = 7
    PACK_FORMAT = b"<q"
    UNPACK_FORMAT = b"<BBBBBBb"


class SLInt64(BaseStructField):
    """Signed Little Endian 64-bit integer field"""
    PACK_FORMAT = UNPACK_FORMAT = b"<q"

    
class SLFloat32(BaseStructField):
    """Unsigned Little Endian 32-bit float field"""
    PACK_FORMAT = UNPACK_FORMAT = b"<f"


class SLFloat64(BaseStructField):
    """Unsigned Little Endian 64-bit float field"""
    PACK_FORMAT = UNPACK_FORMAT = b"<d"


# ==============================================================================
# BitField and Bits
# ==============================================================================
def bitfield_placeholder_factory_factory(cls):
    def _factory_fn(*args, **kwargs):
        return cls(*args, **kwargs)

    return _factory_fn


class _BitFieldFieldPlaceholder(object):
    _global_seqno = 0

    def __init__(self, cls, args, kwargs):
        self.cls = cls
        self.args = args
        self.kwargs = kwargs
        self.sequence_number = _BitFieldFieldPlaceholder._global_seqno
        _BitFieldFieldPlaceholder._global_seqno += 1

    def create_instance(self):
        kwargs = self.kwargs
        kwargs['instantiate'] = True
        return self.cls(*self.args, **self.kwargs)


class _BitFieldField(object):
    def __init__(self, *args, **kwargs):
        pass

    def __repr__(self):
        return repr(self.viewget())

    def __new__(cls, *args, **kwargs):
        if 'instantiate' in kwargs:
            return super(_BitFieldField, cls).__new__(cls)
        else:
            return _BitFieldFieldPlaceholder(cls, args, kwargs)


class _BitBool(_BitFieldField):
    def __init__(self, **kwargs):
        _BitFieldField.__init__(self, **kwargs)
        self.size = 1
        self._value = 0

    def getval(self):
        return self._value

    def setval(self, value):
        self._value = value

    def viewget(self):
        return (self._value == 1)

    def viewset(self, value):
        if value:
            self._value = 1
        else:
            self._value = 0


class _BitNum(_BitFieldField):
    def __init__(self, size, **kwargs):
        _BitFieldField.__init__(self, **kwargs)
        self.size = size
        self._value = 0

    def getval(self):
        return self._value

    viewget = getval

    def setval(self, value):
        self._value = value

    viewset = setval


BitNum = bitfield_placeholder_factory_factory(_BitNum)
BitBool = bitfield_placeholder_factory_factory(_BitBool)


class BitField(BaseField):
    """Represent a sequence of bytes broken down into bit segments

    Bit segments may be any BitFieldField instance, with the two most
    commonly used fields being:

    * BitBool(): A single bit flag that is either True or False
    * BitNum(bits): A multi-bit field treated like a big-endian integral
           value.

    Example Usage::

        class TCPFrameHeader(Structure):
            source_address = UBInt16()
            destination_address = UBInt16()
            sequence_number = UBInt32()
            acknowledgement_number = UBInt32()
            options = BitField(16,
                data_offset=BitNum(4),
                reserved=BitNum(3),
                NS=BitBool(),
                CWR=BitBool(),
                ECE=BitBool(),
                URG=BitBool(),
                ACK=BitBool(),
                PSH=BitBool(),
                RST=BitBool(),
                SYN=BitBool(),
                FIN=BitBool()
            )
            window_size = UBInt16()
            checksum = UBInt16()
            urgent_pointer = UBInt16()

        tcp_packet = TCPFrameHeader()
        o = tcp_packet.options
        o.data_offset = 3
        o.NS = True
        o.CWR = False
        o.
        # ... so on, so forth

    """

    def __init__(self, number_bits, field=None, **kwargs):
        BaseField.__init__(self, **kwargs)
        self._ordered_bitfields = []
        self._bitfield_map = {}
        if number_bits % 8 != 0:
            raise SuitcaseProgrammingError("Number of bits must be a factor of "
                                           "8, was %d" % number_bits)

        self.number_bits = number_bits
        self.number_bytes = number_bits // 8
        self.bytes_required = self.number_bytes
        if field is None:
            field = {
                1: UBInt8,
                2: UBInt16,
                3: UBInt24,
                4: UBInt32,
                5: UBInt40,
                6: UBInt48,
                7: UBInt56,
                8: UBInt64,
            }[self.number_bytes]()
        self._field = field.create_instance(self._parent)

        placeholders = []
        for key, value in six.iteritems(kwargs):
            if isinstance(value, _BitFieldFieldPlaceholder):
                placeholders.append((key, value))

        for key, placeholder in sorted(placeholders, key=lambda kv: kv[1].sequence_number):
            value = placeholder.create_instance()
            self._bitfield_map[key] = value
            self._ordered_bitfields.append((key, value))

    def __getattr__(self, key):
        if key in self.__dict__.get('_bitfield_map', {}):
            return self._bitfield_map[key].viewget()
        return object.__getattribute__(self, key)

    def __setattr__(self, key, value):
        if key in self.__dict__.get('_bitfield_map', {}):
            self._bitfield_map[key].viewset(value)
        else:
            self.__dict__[key] = value

    def __repr__(self):
        sio = StringIO()
        sio.write("BitField(\n")
        for key, field in self._ordered_bitfields:
            sio.write("    %s=%r,\n" % (key, field))
        sio.write("  )")
        return sio.getvalue()

    def getval(self):
        return self

    def setval(self, value):
        raise SuitcaseProgrammingError("Setting the value of a bitfield "
                                       "directly is prohibited")

    def pack(self, stream):
        value = 0
        shift = self.number_bits
        for _key, field in self._ordered_bitfields:
            shift -= field.size
            mask = (2 ** field.size - 1)  # mask off size bits
            value |= ((field.getval() & mask) << shift)

        self._field.setval(value)
        sio = BytesIO()
        self._field.pack(sio)
        out = sio.getvalue()[-self.number_bytes:]
        stream.write(out)

    def unpack(self, data, **kwargs):
        self._field.unpack(data, **kwargs)
        value = self._field.getval()
        shift = self.number_bits
        for _key, field in self._ordered_bitfields:
            shift -= field.size
            mask = (2 ** field.size - 1)
            fval = (value >> shift) & mask
            field.setval(fval)


class FieldAccessor(BaseField):
    """Represent attribute access to another field in the structure

    Not meant to be created directly by the user code.
    Instead, this is created automatically when a pattern like the following
    is used::

        class S(Structure):
            bits = BitField(8, top=BitNum(4), bottom=BitNum(4))
            body = DispatchTarget(None, bits.top, { ... })

    `bits.top` is a FieldAccessor, and is responsible for providing the
    value of the `top` field of `bits` when asked for, and updating that
    field when set.

    """

    def __init__(self, field, name, **kwargs):
        BaseField.__init__(self, field, name, **kwargs)
        self._field = field
        self._name = name
        self.bytes_required = 0

    @classmethod
    def access(cls, parent, placeholder):
        """Resolve the deferred field attribute access.

        :param cls: the FieldAccessor class
        :param parent: owning structure of the field being accessed
        :param placeholder: FieldPlaceholder object which holds our info
        :returns: FieldAccessor instance for that field's attribute
        """
        try:
            return parent.lookup_field_by_placeholder(placeholder)
        except KeyError:
            field_placeholder, name = placeholder.args
            # Find the field whose attribute is being accessed.
            field = parent.lookup_field_by_placeholder(field_placeholder)
            # Instantiate the real FieldAccessor that wraps the attribute.
            accessor = cls(field, name, parent=parent, instantiate=True)
            # Keep this instance alive, and reuse it for future references.
            parent._placeholder_to_field[placeholder] = accessor
            return accessor

    def getval(self):
        return getattr(self._field, self._name)

    def setval(self, value):
        setattr(self._field, self._name, value)

    def pack(self, stream):  # pragma: no cover
        pass

    def unpack(self, data, **kwargs):  # pragma: no cover
        pass
