"""
This file is an adaptation of python pickle (not cPickle) to use Cython.

---

Create portable serialized representations of Python objects.

See module copyreg for a mechanism for registering custom picklers.
See module pickletools source for extensive comments.

Classes:

    Pickler
    Unpickler

Functions:

    dump(object, file)
    dumps(object) -> string
    load(file) -> object
    loads(string) -> object

Misc variables:

    __version__
    format_version
    compatible_formats

"""

from copyreg import _extension_registry, _inverted_registry, _extension_cache
from copyreg import dispatch_table
from itertools import islice
from struct import pack, unpack
from sys import maxsize
from types import FunctionType
import _compat_pickle
import codecs
import io
import pickle
import re
import sys


__all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler",
           "Unpickler", "dump", "dumps", "load", "loads"]

# Shortcut for use in isinstance testing
cdef tuple bytes_types = (bytes, bytearray)

# These are purely informational; no code uses these.
cdef str format_version = "4.0"  # File format version we write
cdef list compatible_formats = ["1.0",  # Original protocol 0
                                "1.1",  # Protocol 0 with INST added
                                "1.2",  # Original protocol 1
                                "1.3",  # Protocol 1 with BINFLOAT added
                                "2.0",  # Protocol 2
                                "3.0",  # Protocol 3
                                "4.0",  # Protocol 4
                               ]  # Old format versions we can read

# This is the highest protocol number we know how to read.
cdef int HIGHEST_PROTOCOL = 4

# The protocol we write by default.  May be less than HIGHEST_PROTOCOL.
# We intentionally write a protocol that Python 2.x cannot read;
# there are too many issues with that.
cdef int DEFAULT_PROTOCOL = 3

# Pickle opcodes.  See pickletools.py for extensive docs.  The listing
# here is in kind-of alphabetical order of 1-character pickle code.
# pickletools groups them by purpose.

cdef bytes MARK = b'('  # push special markobject on stack
cdef bytes STOP = b'.'  # every pickle ends with STOP
cdef bytes POP = b'0'  # discard topmost stack item
cdef bytes POP_MARK = b'1'  # discard stack top through topmost markobject
cdef bytes DUP = b'2'  # duplicate top stack item
cdef bytes FLOAT = b'F'  # push float object; decimal string argument
cdef bytes INT = b'I'  # push integer or bool; decimal string argument
cdef bytes BININT = b'J'  # push four-byte signed int
cdef bytes BININT1 = b'K'  # push 1-byte unsigned int
cdef bytes LONG = b'L'  # push long; decimal string argument
cdef bytes BININT2 = b'M'  # push 2-byte unsigned int
cdef bytes NONE = b'N'  # push None
cdef bytes PERSID = b'P'  # push persistent object; id is taken from string arg
cdef bytes BINPERSID = b'Q'  #  "       "         "  ;  "  "   "     "  stack
cdef bytes REDUCE = b'R'  # apply callable to argtuple, both on stack
cdef bytes STRING = b'S'  # push string; NL-terminated string argument
cdef bytes BINSTRING = b'T'  # push string; counted binary string argument
cdef bytes SHORT_BINSTRING = b'U'  #  "     "   ;    "      "       "      " < 256 bytes
cdef bytes UNICODE = b'V'  # push Unicode string; raw-unicode-escaped'd argument
cdef bytes BINUNICODE = b'X'  #   "     "       "  ; counted UTF-8 string argument
cdef bytes APPEND = b'a'  # append stack top to list below it
cdef bytes BUILD = b'b'  # call __setstate__ or __dict__.update()
cdef bytes GLOBAL = b'c'  # push self.find_class(modname, name); 2 string args
cdef bytes DICT = b'd'  # build a dict from stack items
cdef bytes EMPTY_DICT = b'}'  # push empty dict
cdef bytes APPENDS = b'e'  # extend list on stack by topmost stack slice
cdef bytes GET = b'g'  # push item from memo on stack; index is string arg
cdef bytes BINGET = b'h'  #   "    "    "    "   "   "  ;   "    " 1-byte arg
cdef bytes INST = b'i'  # build & push class instance
cdef bytes LONG_BINGET = b'j'  # push item from memo on stack; index is 4-byte arg
cdef bytes LIST = b'l'  # build list from topmost stack items
cdef bytes EMPTY_LIST = b']'  # push empty list
cdef bytes OBJ = b'o'  # build & push class instance
cdef bytes PUT = b'p'  # store stack top in memo; index is string arg
cdef bytes BINPUT = b'q'  #   "     "    "   "   " ;   "    " 1-byte arg
cdef bytes LONG_BINPUT = b'r'  #   "     "    "   "   " ;   "    " 4-byte arg
cdef bytes SETITEM = b's'  # add key+value pair to dict
cdef bytes TUPLE = b't'  # build tuple from topmost stack items
cdef bytes EMPTY_TUPLE = b')'  # push empty tuple
cdef bytes SETITEMS = b'u'  # modify dict by adding topmost key+value pairs
cdef bytes BINFLOAT = b'G'  # push float; arg is 8-byte float encoding

cdef bytes TRUE = b'I01\n'  # not an opcode; see INT docs in pickletools.py
cdef bytes FALSE = b'I00\n'  # not an opcode; see INT docs in pickletools.py

# Protocol 2

cdef bytes PROTO = b'\x80'  # identify pickle protocol
cdef bytes NEWOBJ = b'\x81'  # build object by applying cls.__new__ to argtuple
cdef bytes EXT1 = b'\x82'  # push object from extension registry; 1-byte index
cdef bytes EXT2 = b'\x83'  # ditto, but 2-byte index
cdef bytes EXT4 = b'\x84'  # ditto, but 4-byte index
cdef bytes TUPLE1 = b'\x85'  # build 1-tuple from stack top
cdef bytes TUPLE2 = b'\x86'  # build 2-tuple from two topmost stack items
cdef bytes TUPLE3 = b'\x87'  # build 3-tuple from three topmost stack items
cdef bytes NEWTRUE = b'\x88'  # push True
cdef bytes NEWFALSE = b'\x89'  # push False
cdef bytes LONG1 = b'\x8a'  # push long from < 256 bytes
cdef bytes LONG4 = b'\x8b'  # push really big long

cdef tuple _tuplesize2code = (EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3)

# Protocol 3 (Python 3.x)

cdef bytes BINBYTES = b'B'  # push bytes; counted binary string argument
cdef bytes SHORT_BINBYTES = b'C'  #  "     "   ;    "      "       "      " < 256 bytes

# Protocol 4
cdef bytes SHORT_BINUNICODE = b'\x8c'  # push short string; UTF-8 length < 256 bytes
cdef bytes BINUNICODE8 = b'\x8d'  # push very long string
cdef bytes BINBYTES8 = b'\x8e'  # push very long bytes string
cdef bytes EMPTY_SET = b'\x8f'  # push empty set on the stack
cdef bytes ADDITEMS = b'\x90'  # modify set by adding topmost stack items
cdef bytes FROZENSET = b'\x91'  # build frozenset from topmost stack items
cdef bytes NEWOBJ_EX = b'\x92'  # like NEWOBJ but work with keyword only arguments
cdef bytes STACK_GLOBAL = b'\x93'  # same as GLOBAL but using names on the stacks
cdef bytes MEMOIZE = b'\x94'  # store top of the stack in memo
cdef bytes FRAME = b'\x95'  # indicate the beginning of a new frame

__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)])


cdef class _Framer:
    _FRAME_SIZE_TARGET = 64 * 1024

    def __init__(self, file_write):
        self.file_write = file_write
        self.current_frame = None

    def start_framing(self):
        self.current_frame = io.BytesIO()

    def end_framing(self):
        if self.current_frame and self.current_frame.tell() > 0:
            self.commit_frame(force=True)
            self.current_frame = None

    def commit_frame(self, bint force=False):
        cdef int n
        if self.current_frame:
            f = self.current_frame
            if f.tell() >= self._FRAME_SIZE_TARGET or force:
                with f.getbuffer() as data:
                    n = len(data)
                    write = self.file_write
                    write(FRAME)
                    write(pack("<Q", n))
                    write(data)
                f.seek(0)
                f.truncate()

    def write(self, data):
        if self.current_frame:
            return self.current_frame.write(data)
        else:
            return self.file_write(data)



# Tools used for pickling.

cdef _getattribute(object obj, str name, bint allow_qualname=False):
    cdef list dotted_path = name.split(".")
    cdef str subpath
    if not allow_qualname and len(dotted_path) > 1:
        raise AttributeError("Can't get qualified attribute {!r} on {!r}; " +
                             "use protocols >= 4 to enable support"
                             .format(name, obj))
    for subpath in dotted_path:
        if subpath == '<locals>':
            raise AttributeError("Can't get local attribute {!r} on {!r}"
                                 .format(name, obj))
        try:
            obj = getattr(obj, subpath)
        except AttributeError:
            raise AttributeError("Can't get attribute {!r} on {!r}"
                                 .format(name, obj))
    return obj

cdef whichmodule(object obj, str name, bint allow_qualname=False):
    """Find the module an object belong to."""
    cdef str module_name = getattr(obj, '__module__', None)
    if module_name is not None:
        return module_name
    for module_name, module in sys.modules.items():
        if module_name == '__main__' or module is None:
            continue
        try:
            if _getattribute(module, name, allow_qualname) is obj:
                return module_name
        except AttributeError:
            pass
    return '__main__'

cdef encode_long(x):
    r"""Encode a long to a two's complement little-endian binary string.
    Note that 0 is a special case, returning an empty string, to save a
    byte in the LONG1 pickling context.

    >>> encode_long(0)
    b''
    >>> encode_long(255)
    b'\xff\x00'
    >>> encode_long(32767)
    b'\xff\x7f'
    >>> encode_long(-256)
    b'\x00\xff'
    >>> encode_long(-32768)
    b'\x00\x80'
    >>> encode_long(-128)
    b'\x80'
    >>> encode_long(127)
    b'\x7f'
    >>>
    """
    if x == 0:
        return b''
    cdef int nbytes = (x.bit_length() >> 3) + 1
    cdef bytes result = x.to_bytes(nbytes, byteorder='little', signed=True)
    if x < 0 and nbytes > 1:
        if result[-1] == 0xff and (result[-2] & 0x80) != 0:
            result = result[:-1]
    return result



# Pickling machinery

cdef class Pickler:
    def __init__(self, file, protocol=None, *, fix_imports=True):
        """This takes a binary file for writing a pickle data stream.

        The optional *protocol* argument tells the pickler to use the
        given protocol; supported protocols are 0, 1, 2, 3 and 4.  The
        default protocol is 3; a backward-incompatible protocol designed
        for Python 3.

        Specifying a negative protocol version selects the highest
        protocol version supported.  The higher the protocol used, the
        more recent the version of Python needed to read the pickle
        produced.

        The *file* argument must have a write() method that accepts a
        single bytes argument. It can thus be a file object opened for
        binary writing, a io.BytesIO instance, or any other custom
        object that meets this interface.

        If *fix_imports* is True and *protocol* is less than 3, pickle
        will try to map the new Python 3 names to the old module names
        used in Python 2, so that the pickle data stream is readable
        with Python 2.
        """
        if protocol is None:
            protocol = DEFAULT_PROTOCOL
        if protocol < 0:
            protocol = HIGHEST_PROTOCOL
        elif not 0 <= protocol <= HIGHEST_PROTOCOL:
            raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL)
        try:
            self._file_write = file.write
        except AttributeError:
            raise TypeError("file must have a 'write' attribute")
        self.framer = _Framer(self._file_write)
        self.write = self.framer.write
        self.memo = {}
        self.proto = int(protocol)
        self.bin = protocol >= 1
        self.fast = 0
        self.fix_imports = fix_imports and protocol < 3

    def clear_memo(self):
        """Clears the pickler's "memo".

        The memo is the data structure that remembers which objects the
        pickler has already seen, so that shared or recursive objects
        are pickled by reference and not by value.  This method is
        useful when re-using picklers.
        """
        self.memo.clear()

    def dump(self, obj):
        """Write a pickled representation of obj to the open file."""
        if self.proto >= 2:
            self.write(PROTO + pack("<B", self.proto))
        if self.proto >= 4:
            self.framer.start_framing()
        self.save(obj)
        self.write(STOP)
        self.framer.end_framing()

    def memoize(self, obj):
        """Store an object in the memo."""

        # The Pickler memo is a dictionary mapping object ids to 2-tuples
        # that contain the Unpickler memo key and the object being memoized.
        # The memo key is written to the pickle and will become
        # the key in the Unpickler's memo.  The object is stored in the
        # Pickler memo so that transient objects are kept alive during
        # pickling.

        # The use of the Unpickler memo length as the memo key is just a
        # convention.  The only requirement is that the memo values be unique.
        # But there appears no advantage to any other scheme, and this
        # scheme allows the Unpickler memo to be implemented as a plain (but
        # growable) array, indexed by memo key.
        if self.fast:
            return
        assert id(obj) not in self.memo
        cdef int idx = len(self.memo)
        self.write(self.put(idx))
        self.memo[id(obj)] = idx, obj

    # Return a PUT (BINPUT, LONG_BINPUT) opcode string, with argument i.
    def put(self, int idx):
        if self.proto >= 4:
            return MEMOIZE
        elif self.bin:
            if idx < 256:
                return BINPUT + pack("<B", idx)
            else:
                return LONG_BINPUT + pack("<I", idx)
        else:
            return PUT + repr(idx).encode("ascii") + b'\n'

    # Return a GET (BINGET, LONG_BINGET) opcode string, with argument i.
    def get(self, int i):
        if self.bin:
            if i < 256:
                return BINGET + pack("<B", i)
            else:
                return LONG_BINGET + pack("<I", i)

        return GET + repr(i).encode("ascii") + b'\n'

    def save(self, object obj, bint save_persistent_id=True):
        self.framer.commit_frame()

        # Check for persistent id (defined by a subclass)
        pid = self.persistent_id(obj)
        if pid is not None and save_persistent_id:
            self.save_pers(pid)
            return

        # Check the memo
        x = self.memo.get(id(obj))
        if x is not None:
            self.write(self.get(x[0]))
            return

        # Check the type dispatch table
        cdef type t = type(obj)
        f = self.dispatch.get(t)
        if f is not None:
            f(self, obj)  # Call unbound method with explicit self
            return

        # Check private dispatch table if any, or else copyreg.dispatch_table
        reduce = getattr(self, 'dispatch_table', dispatch_table).get(t)
        if reduce is not None:
            rv = reduce(obj)
        else:
            # Check for a class with a custom metaclass; treat as regular class
            try:
                issc = issubclass(t, type)
            except TypeError:  # t is not a class (old Boost; see SF #502085)
                issc = False
            if issc:
                self.save_global(obj)
                return

            # Check for a __reduce_ex__ method, fall back to __reduce__
            reduce = getattr(obj, "__reduce_ex__", None)
            if reduce is not None:
                rv = reduce(self.proto)
            else:
                reduce = getattr(obj, "__reduce__", None)
                if reduce is not None:
                    rv = reduce()
                else:
                    raise pickle.PicklingError("Can't pickle %r object: %r" %
                                        (t.__name__, obj))

        # Check for string returned by reduce(), meaning "save as global"
        if isinstance(rv, str):
            self.save_global(obj, rv)
            return

        # Assert that reduce() returned a tuple
        if not isinstance(rv, tuple):
            raise pickle.PicklingError("%s must return string or tuple" % reduce)

        # Assert that it returned an appropriately sized tuple
        l = len(rv)
        if not (2 <= l <= 5):
            raise pickle.PicklingError("Tuple returned by %s must have "
                                "two to five elements" % reduce)

        # Save the reduce() output and finally memoize the object
        self.save_reduce(obj=obj, *rv)

    def persistent_id(self, obj):
        # This exists so a subclass can override it
        return None

    def save_pers(self, pid):
        # Save a persistent id reference
        if self.bin:
            self.save(pid, save_persistent_id=False)
            self.write(BINPERSID)
        else:
            self.write(PERSID + str(pid).encode("ascii") + b'\n')

    def save_reduce(self, func, args, state=None, listitems=None,
                    dictitems=None, obj=None):
        # This API is called by some subclasses

        if not isinstance(args, tuple):
            raise pickle.PicklingError("args from save_reduce() must be a tuple")
        if not callable(func):
            raise pickle.PicklingError("func from save_reduce() must be callable")

        save = self.save
        write = self.write

        cdef str func_name = getattr(func, "__name__", "")
        if self.proto >= 4 and func_name == "__newobj_ex__":
            cls, args, kwargs = args
            if not hasattr(cls, "__new__"):
                raise pickle.PicklingError("args[0] from {} args has no __new__"
                                    .format(func_name))
            if obj is not None and cls is not obj.__class__:
                raise pickle.PicklingError("args[0] from {} args has the wrong class"
                                    .format(func_name))
            save(cls)
            save(args)
            save(kwargs)
            write(NEWOBJ_EX)
        elif self.proto >= 2 and func_name == "__newobj__":
            # A __reduce__ implementation can direct protocol 2 or newer to
            # use the more efficient NEWOBJ opcode, while still
            # allowing protocol 0 and 1 to work normally.  For this to
            # work, the function returned by __reduce__ should be
            # called __newobj__, and its first argument should be a
            # class.  The implementation for __newobj__
            # should be as follows, although pickle has no way to
            # verify this:
            #
            # def __newobj__(cls, *args):
            #     return cls.__new__(cls, *args)
            #
            # Protocols 0 and 1 will pickle a reference to __newobj__,
            # while protocol 2 (and above) will pickle a reference to
            # cls, the remaining args tuple, and the NEWOBJ code,
            # which calls cls.__new__(cls, *args) at unpickling time
            # (see load_newobj below).  If __reduce__ returns a
            # three-tuple, the state from the third tuple item will be
            # pickled regardless of the protocol, calling __setstate__
            # at unpickling time (see load_build below).
            #
            # Note that no standard __newobj__ implementation exists;
            # you have to provide your own.  This is to enforce
            # compatibility with Python 2.2 (pickles written using
            # protocol 0 or 1 in Python 2.3 should be unpicklable by
            # Python 2.2).
            cls = args[0]
            if not hasattr(cls, "__new__"):
                raise pickle.PicklingError(
                    "args[0] from __newobj__ args has no __new__")
            if obj is not None and cls is not obj.__class__:
                raise pickle.PicklingError(
                    "args[0] from __newobj__ args has the wrong class")
            args = args[1:]
            save(cls)
            save(args)
            write(NEWOBJ)
        else:
            save(func)
            save(args)
            write(REDUCE)

        if obj is not None:
            # If the object is already in the memo, this means it is
            # recursive. In this case, throw away everything we put on the
            # stack, and fetch the object back from the memo.
            if id(obj) in self.memo:
                write(POP + self.get(self.memo[id(obj)][0]))
            else:
                self.memoize(obj)

        # More new special cases (that work with older protocols as
        # well): when __reduce__ returns a tuple with 4 or 5 items,
        # the 4th and 5th item should be iterators that provide list
        # items and dict items (as (key, value) tuples), or None.

        if listitems is not None:
            self._batch_appends(listitems)

        if dictitems is not None:
            self._batch_setitems(dictitems)

        if state is not None:
            save(state)
            write(BUILD)

    # Methods below this point are dispatched through the dispatch table

    dispatch = {}

    def save_none(self, obj):
        self.write(NONE)
    dispatch[type(None)] = save_none

    def save_bool(self, obj):
        if self.proto >= 2:
            self.write(NEWTRUE if obj else NEWFALSE)
        else:
            self.write(TRUE if obj else FALSE)
    dispatch[bool] = save_bool

    def save_long(self, obj):
        if self.bin:
            # If the int is small enough to fit in a signed 4-byte 2's-comp
            # format, we can store it more efficiently than the general
            # case.
            # First one- and two-byte unsigned ints:
            if obj >= 0:
                if obj <= 0xff:
                    self.write(BININT1 + pack("<B", obj))
                    return
                if obj <= 0xffff:
                    self.write(BININT2 + pack("<H", obj))
                    return
            # Next check for 4-byte signed ints:
            if -0x80000000 <= obj <= 0x7fffffff:
                self.write(BININT + pack("<i", obj))
                return
        if self.proto >= 2:
            encoded = encode_long(obj)
            n = len(encoded)
            if n < 256:
                self.write(LONG1 + pack("<B", n) + encoded)
            else:
                self.write(LONG4 + pack("<i", n) + encoded)
            return
        self.write(LONG + repr(obj).encode("ascii") + b'L\n')
    dispatch[int] = save_long

    def save_float(self, obj):
        if self.bin:
            self.write(BINFLOAT + pack('>d', obj))
        else:
            self.write(FLOAT + repr(obj).encode("ascii") + b'\n')
    dispatch[float] = save_float

    def save_bytes(self, obj):
        if self.proto < 3:
            if not obj:  # bytes object is empty
                self.save_reduce(bytes, (), obj=obj)
            else:
                self.save_reduce(codecs.encode,
                                 (str(obj, 'latin1'), 'latin1'), obj=obj)
            return
        n = len(obj)
        if n <= 0xff:
            self.write(SHORT_BINBYTES + pack("<B", n) + obj)
        elif n > 0xffffffff and self.proto >= 4:
            self.write(BINBYTES8 + pack("<Q", n) + obj)
        else:
            self.write(BINBYTES + pack("<I", n) + obj)
        self.memoize(obj)
    dispatch[bytes] = save_bytes

    def save_str(self, obj):
        if self.bin:
            encoded = obj.encode('utf-8', 'surrogatepass')
            n = len(encoded)
            if n <= 0xff and self.proto >= 4:
                self.write(SHORT_BINUNICODE + pack("<B", n) + encoded)
            elif n > 0xffffffff and self.proto >= 4:
                self.write(BINUNICODE8 + pack("<Q", n) + encoded)
            else:
                self.write(BINUNICODE + pack("<I", n) + encoded)
        else:
            obj = obj.replace("\\", "\\u005c")
            obj = obj.replace("\n", "\\u000a")
            self.write(UNICODE + obj.encode('raw-unicode-escape') +
                       b'\n')
        self.memoize(obj)
    dispatch[str] = save_str

    def save_tuple(self, obj):
        if not obj:  # tuple is empty
            if self.bin:
                self.write(EMPTY_TUPLE)
            else:
                self.write(MARK + TUPLE)
            return

        cdef int n = len(obj)
        save = self.save
        memo = self.memo
        if n <= 3 and self.proto >= 2:
            for element in obj:
                save(element)
            # Subtle.  Same as in the big comment below.
            if id(obj) in memo:
                get = self.get(memo[id(obj)][0])
                self.write(POP * n + get)
            else:
                self.write(_tuplesize2code[n])
                self.memoize(obj)
            return

        # proto 0 or proto 1 and tuple isn't empty, or proto > 1 and tuple
        # has more than 3 elements.
        write = self.write
        write(MARK)
        for element in obj:
            save(element)

        if id(obj) in memo:
            # Subtle.  d was not in memo when we entered save_tuple(), so
            # the process of saving the tuple's elements must have saved
            # the tuple itself:  the tuple is recursive.  The proper action
            # now is to throw away everything we put on the stack, and
            # simply GET the tuple (it's already constructed).  This check
            # could have been done in the "for element" loop instead, but
            # recursive tuples are a rare thing.
            get = self.get(memo[id(obj)][0])
            if self.bin:
                write(POP_MARK + get)
            else:  # proto 0 -- POP_MARK not available
                write(POP * (n + 1) + get)
            return

        # No recursion.
        write(TUPLE)
        self.memoize(obj)

    dispatch[tuple] = save_tuple

    def save_list(self, obj):
        if self.bin:
            self.write(EMPTY_LIST)
        else:  # proto 0 -- can't use EMPTY_LIST
            self.write(MARK + LIST)

        self.memoize(obj)
        self._batch_appends(obj)

    dispatch[list] = save_list

    _BATCHSIZE = 1000

    def _batch_appends(self, items):
        # Helper to batch up APPENDS sequences
        save = self.save
        write = self.write

        if not self.bin:
            for x in items:
                save(x)
                write(APPEND)
            return

        it = iter(items)
        cdef int n
        cdef list tmp
        while True:
            tmp = list(islice(it, self._BATCHSIZE))
            n = len(tmp)
            if n > 1:
                write(MARK)
                for x in tmp:
                    save(x)
                write(APPENDS)
            elif n:
                save(tmp[0])
                write(APPEND)
            # else tmp is empty, and we're done
            if n < self._BATCHSIZE:
                return

    def save_dict(self, obj):
        if self.bin:
            self.write(EMPTY_DICT)
        else:  # proto 0 -- can't use EMPTY_DICT
            self.write(MARK + DICT)

        self.memoize(obj)
        self._batch_setitems(obj.items())
    dispatch[dict] = save_dict

    def _batch_setitems(self, items):
        # Helper to batch up SETITEMS sequences; proto >= 1 only
        save = self.save
        write = self.write

        if not self.bin:
            for k, v in items:
                save(k)
                save(v)
                write(SETITEM)
            return

        it = iter(items)
        while True:
            tmp = list(islice(it, self._BATCHSIZE))
            n = len(tmp)
            if n > 1:
                write(MARK)
                for k, v in tmp:
                    save(k)
                    save(v)
                write(SETITEMS)
            elif n:
                k, v = tmp[0]
                save(k)
                save(v)
                write(SETITEM)
            # else tmp is empty, and we're done
            if n < self._BATCHSIZE:
                return

    def save_set(self, obj):
        save = self.save
        write = self.write

        if self.proto < 4:
            self.save_reduce(set, (list(obj),), obj=obj)
            return

        write(EMPTY_SET)
        self.memoize(obj)

        it = iter(obj)
        while True:
            batch = list(islice(it, self._BATCHSIZE))
            n = len(batch)
            if n > 0:
                write(MARK)
                for item in batch:
                    save(item)
                write(ADDITEMS)
            if n < self._BATCHSIZE:
                return
    dispatch[set] = save_set

    def save_frozenset(self, obj):
        save = self.save
        write = self.write

        if self.proto < 4:
            self.save_reduce(frozenset, (list(obj),), obj=obj)
            return

        write(MARK)
        for item in obj:
            save(item)

        if id(obj) in self.memo:
            # If the object is already in the memo, this means it is
            # recursive. In this case, throw away everything we put on the
            # stack, and fetch the object back from the memo.
            write(POP_MARK + self.get(self.memo[id(obj)][0]))
            return

        write(FROZENSET)
        self.memoize(obj)
    dispatch[frozenset] = save_frozenset

    def save_global(self, obj, name=None):
        write = self.write
        memo = self.memo

        if name is None and self.proto >= 4:
            name = getattr(obj, '__qualname__', None)
        if name is None:
            name = obj.__name__

        module_name = whichmodule(obj, name, allow_qualname=self.proto >= 4)
        try:
            __import__(module_name, level=0)
            module = sys.modules[module_name]
            obj2 = _getattribute(module, name, allow_qualname=self.proto >= 4)
        except (ImportError, KeyError, AttributeError):
            raise pickle.PicklingError(
                "Can't pickle %r: it's not found as %s.%s" %
                (obj, module_name, name))
        else:
            if obj2 is not obj:
                raise pickle.PicklingError(
                    "Can't pickle %r: it's not the same object as %s.%s" %
                    (obj, module_name, name))

        if self.proto >= 2:
            code = _extension_registry.get((module_name, name))
            if code:
                assert code > 0
                if code <= 0xff:
                    write(EXT1 + pack("<B", code))
                elif code <= 0xffff:
                    write(EXT2 + pack("<H", code))
                else:
                    write(EXT4 + pack("<i", code))
                return
        # Non-ASCII identifiers are supported only with protocols >= 3.
        if self.proto >= 4:
            self.save(module_name)
            self.save(name)
            write(STACK_GLOBAL)
        elif self.proto >= 3:
            write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
                  bytes(name, "utf-8") + b'\n')
        else:
            if self.fix_imports:
                r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
                r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
                if (module_name, name) in r_name_mapping:
                    module_name, name = r_name_mapping[(module_name, name)]
                if module_name in r_import_mapping:
                    module_name = r_import_mapping[module_name]
            try:
                write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
                      bytes(name, "ascii") + b'\n')
            except UnicodeEncodeError:
                raise pickle.PicklingError(
                    "can't pickle global identifier '%s.%s' using "
                    "pickle protocol %i" % (module, name, self.proto))

        self.memoize(obj)

    def save_type(self, obj):
        if obj is type(None):
            return self.save_reduce(type, (None,), obj=obj)
        elif obj is type(NotImplemented):
            return self.save_reduce(type, (NotImplemented,), obj=obj)
        elif obj is type(...):
            return self.save_reduce(type, (...,), obj=obj)
        return self.save_global(obj)

    dispatch[FunctionType] = save_global
    dispatch[type] = save_type
