/**
 * Copyright 2016 Confluent Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * ⚠️  WARNING: UPDATE TYPE STUBS WHEN MODIFYING INTERFACES ⚠️
 *
 * This file defines core classes: KafkaError, Message, TopicPartition, Uuid.
 * When changing method signatures, parameters, or defaults, you MUST
 * also update the corresponding type definitions in:
 *   src/confluent_kafka/cimpl.pyi
 *
 * Failure to keep both in sync will result in incorrect type hints.
 */

#include "confluent_kafka.h"

#include <stdarg.h>

#ifdef _WIN32
#include <windows.h>
#endif

/**
 * @brief KNOWN ISSUES
 *
 *  - Partitioners will cause a dead-lock with librdkafka, because:
 *     GIL + topic lock in topic_new  is different lock order than
 *     topic lock in msg_partitioner + GIL.
 *     This needs to be sorted out in librdkafka, preferably making the
 *     partitioner run without any locks taken.
 *     Until this is fixed the partitioner is ignored and librdkafka's
 *     default will be used.
 *  - KafkaError type .tp_doc allocation is lost on exit.
 *
 */


PyObject *KafkaException;


/****************************************************************************
 *
 *
 * KafkaError
 *
 *
 * FIXME: Pre-create simple instances for each error code, only instantiate
 *        a new object if a rich error string is provided.
 *
 ****************************************************************************/
typedef struct {
#ifdef PY3
        PyException_HEAD
#else
        PyObject_HEAD
            /* Standard fields of PyBaseExceptionObject which we inherit from.
             */
            PyObject *dict;
        PyObject *args;
        PyObject *message;
#endif

            rd_kafka_resp_err_t code; /* Error code */
        char *str;     /* Human readable representation of error, if one
                        * was provided by librdkafka.
                        * Else falls back on err2str(). */
        int fatal;     /**< Set to true if a fatal error. */
        int retriable; /**< Set to true if operation is retriable. */
        int txn_requires_abort; /**< Set to true if this is an abortable
                                 *   transaction error. */
} KafkaError;


static void cfl_PyErr_Fatal(rd_kafka_resp_err_t err, const char *reason);

static PyObject *KafkaError_code(KafkaError *self, PyObject *ignore) {
        return cfl_PyInt_FromInt(self->code);
}

static PyObject *KafkaError_str(KafkaError *self, PyObject *ignore) {
        if (self->str)
                return cfl_PyUnistr(_FromString(self->str));
        else
                return cfl_PyUnistr(_FromString(rd_kafka_err2str(self->code)));
}

static PyObject *KafkaError_name(KafkaError *self, PyObject *ignore) {
        /* FIXME: Pre-create name objects */
        return cfl_PyUnistr(_FromString(rd_kafka_err2name(self->code)));
}

static PyObject *KafkaError_fatal(KafkaError *self, PyObject *ignore) {
        PyObject *ret = self->fatal ? Py_True : Py_False;
        Py_INCREF(ret);
        return ret;
}

static PyObject *KafkaError_retriable(KafkaError *self, PyObject *ignore) {
        PyObject *ret = self->retriable ? Py_True : Py_False;
        Py_INCREF(ret);
        return ret;
}

static PyObject *KafkaError_txn_requires_abort(KafkaError *self,
                                               PyObject *ignore) {
        PyObject *ret = self->txn_requires_abort ? Py_True : Py_False;
        Py_INCREF(ret);
        return ret;
}

static PyObject *KafkaError_reduce(KafkaError *self,
                                   PyObject *Py_UNUSED(ignored)) {
        PyObject *KafkaError_type = NULL;
        PyObject *result          = NULL;
        PyObject *args            = NULL;
        PyObject *reason          = NULL;

        KafkaError_type =
            cfl_PyObject_lookup("confluent_kafka.cimpl", "KafkaError");

        if (self->str) {
                reason = cfl_PyUnistr(_FromString(self->str));
        } else {
                reason = NULL;
        }

        /* Build args tuple: (error, reason, fatal, retriable,
         * txn_requires_abort) */
        if (reason) {
                args = Py_BuildValue("(iOiii)", self->code, reason, self->fatal,
                                     self->retriable, self->txn_requires_abort);
                Py_DECREF(reason);
        } else {
                args = Py_BuildValue("(iziii)", self->code, NULL, self->fatal,
                                     self->retriable, self->txn_requires_abort);
        }
        if (!args) {
                Py_DECREF(KafkaError_type);
                return NULL;
        }

        result = Py_BuildValue("(OO)", KafkaError_type, args);

        Py_DECREF(KafkaError_type);
        Py_DECREF(args);
        return result;
}

static PyMethodDef KafkaError_methods[] = {
    {"code", (PyCFunction)KafkaError_code, METH_NOARGS,
     "  Returns the error/event code for comparison to"
     "KafkaError.<ERR_CONSTANTS>.\n"
     "\n"
     "  :returns: error/event code\n"
     "  :rtype: int\n"
     "\n"},
    {"str", (PyCFunction)KafkaError_str, METH_NOARGS,
     "  Returns the human-readable error/event string.\n"
     "\n"
     "  :returns: error/event message string\n"
     "  :rtype: str\n"
     "\n"},
    {"name", (PyCFunction)KafkaError_name, METH_NOARGS,
     "  Returns the enum name for error/event.\n"
     "\n"
     "  :returns: error/event enum name string\n"
     "  :rtype: str\n"
     "\n"},
    {"fatal", (PyCFunction)KafkaError_fatal, METH_NOARGS,
     "  :returns: True if this a fatal error, else False.\n"
     "  :rtype: bool\n"
     "\n"},
    {"retriable", (PyCFunction)KafkaError_retriable, METH_NOARGS,
     "  :returns: True if the operation that failed may be retried, "
     "else False.\n"
     "  :rtype: bool\n"
     "\n"},
    {"txn_requires_abort", (PyCFunction)KafkaError_txn_requires_abort,
     METH_NOARGS,
     "  :returns: True if the error is an abortable transaction error "
     "in which case application must abort the current transaction with "
     "abort_transaction() and start a new transaction with "
     "begin_transaction() if it wishes to proceed with "
     "transactional operations. "
     "This will only return true for errors from the transactional "
     "producer API.\n"
     "  :rtype: bool\n"
     "\n"},
    {"__reduce__", (PyCFunction)KafkaError_reduce, METH_NOARGS,
     " Function for serializing KafkaError using the pickle protocol."},
    {NULL}};


static void KafkaError_clear(PyObject *self0) {
        KafkaError *self = (KafkaError *)self0;
        if (self->str) {
                free(self->str);
                self->str = NULL;
        }
}

static void KafkaError_dealloc(PyObject *self0) {
        KafkaError *self = (KafkaError *)self0;
        KafkaError_clear(self0);
        ;
        PyObject_GC_UnTrack(self0);
        Py_TYPE(self)->tp_free(self0);
}



static int KafkaError_traverse(KafkaError *self, visitproc visit, void *arg) {
        return 0;
}

static PyObject *KafkaError_str0(KafkaError *self) {
        return cfl_PyUnistr(_FromFormat(
            "KafkaError{%scode=%s,val=%d,str=\"%s\"}",
            self->fatal ? "FATAL," : "", rd_kafka_err2name(self->code),
            self->code, self->str ? self->str : rd_kafka_err2str(self->code)));
}

static long KafkaError_hash(KafkaError *self) {
        return self->code;
}

static PyTypeObject KafkaErrorType;



static PyObject *
KafkaError_richcompare(KafkaError *self, PyObject *o2, int op) {
        int code2;
        int r;
        PyObject *result;

        if (Py_TYPE(o2) == &KafkaErrorType)
                code2 = ((KafkaError *)o2)->code;
        else
                code2 = cfl_PyInt_AsInt(o2);

        switch (op) {
        case Py_LT:
                r = self->code < code2;
                break;
        case Py_LE:
                r = self->code <= code2;
                break;
        case Py_EQ:
                r = self->code == code2;
                break;
        case Py_NE:
                r = self->code != code2;
                break;
        case Py_GT:
                r = self->code > code2;
                break;
        case Py_GE:
                r = self->code >= code2;
                break;
        default:
                r = 0;
                break;
        }

        result = r ? Py_True : Py_False;
        Py_INCREF(result);
        return result;
}

static PyObject *
KafkaError_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) {
        return type->tp_alloc(type, 0);
}

/**
 * @brief Initialize a KafkaError object.
 */
static void
KafkaError_init(KafkaError *self, rd_kafka_resp_err_t code, const char *str) {
        self->code               = code;
        self->fatal              = 0;
        self->retriable          = 0;
        self->txn_requires_abort = 0;
        if (str)
                self->str = strdup(str);
        else
                self->str = NULL;
}

static int
KafkaError_init0(PyObject *selfobj, PyObject *args, PyObject *kwargs) {
        KafkaError *self = (KafkaError *)selfobj;
        int code;
        int fatal = 0, retriable = 0, txn_requires_abort = 0;
        const char *reason = NULL;
        static char *kws[] = {
            "error", "reason", "fatal", "retriable", "txn_requires_abort",
            NULL};

        if (!PyArg_ParseTupleAndKeywords(args, kwargs, "i|ziii", kws, &code,
                                         &reason, &fatal, &retriable,
                                         &txn_requires_abort))
                return -1;

        KafkaError_init(self, code, reason ? reason : rd_kafka_err2str(code));
        self->fatal              = fatal;
        self->retriable          = retriable;
        self->txn_requires_abort = txn_requires_abort;

        return 0;
}

static PyTypeObject KafkaErrorType = {
    PyVarObject_HEAD_INIT(NULL,
                          0) "confluent_kafka.cimpl.KafkaError", /*tp_name*/
    sizeof(KafkaError),             /*tp_basicsize*/
    0,                              /*tp_itemsize*/
    (destructor)KafkaError_dealloc, /*tp_dealloc*/
    0,                              /*tp_print*/
    0,                              /*tp_getattr*/
    0,                              /*tp_setattr*/
    0,                              /*tp_compare*/
    (reprfunc)KafkaError_str0,      /*tp_repr*/
    0,                              /*tp_as_number*/
    0,                              /*tp_as_sequence*/
    0,                              /*tp_as_mapping*/
    (hashfunc)KafkaError_hash,      /*tp_hash */
    0,                              /*tp_call*/
    0,                              /*tp_str*/
    PyObject_GenericGetAttr,        /*tp_getattro*/
    0,                              /*tp_setattro*/
    0,                              /*tp_as_buffer*/
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_BASE_EXC_SUBCLASS |
        Py_TPFLAGS_HAVE_GC, /*tp_flags*/
    "Kafka error and event object\n"
    "\n"
    "  The KafkaError class serves multiple purposes\n"
    "\n"
    "  - Propagation of errors\n"
    "  - Propagation of events\n"
    "  - Exceptions\n"
    "\n"
    "Args:\n"
    "  error (KafkaError): Error code indicating the type of error.\n"
    "\n"
    "  reason (str): Alternative message to describe the error.\n"
    "\n"
    "  fatal (bool): Set to true if a fatal error.\n"
    "\n"
    "  retriable (bool): Set to true if operation is retriable.\n"
    "\n"
    "  txn_requires_abort (bool): Set to true if this is an abortable\n"
    "  transaction error.\n"
    "\n",                                /*tp_doc*/
    (traverseproc)KafkaError_traverse,   /* tp_traverse */
    (inquiry)KafkaError_clear,           /* tp_clear */
    (richcmpfunc)KafkaError_richcompare, /* tp_richcompare */
    0,                                   /* tp_weaklistoffset */
    0,                                   /* tp_iter */
    0,                                   /* tp_iternext */
    KafkaError_methods,                  /* tp_methods */
    0,                                   /* tp_members */
    0,                                   /* tp_getset */
    0,                                   /* tp_base */
    0,                                   /* tp_dict */
    0,                                   /* tp_descr_get */
    0,                                   /* tp_descr_set */
    0,                                   /* tp_dictoffset */
    KafkaError_init0,                    /* tp_init */
    0,                                   /* tp_alloc */
    KafkaError_new                       /* tp_new */
};

/**
 * @brief Internal factory to create KafkaError object.
 */
PyObject *KafkaError_new0(rd_kafka_resp_err_t err, const char *fmt, ...) {

        KafkaError *self;
        va_list ap;
        char buf[512];

        self = (KafkaError *)KafkaErrorType.tp_alloc(&KafkaErrorType, 0);
        if (!self)
                return NULL;

        if (fmt) {
                va_start(ap, fmt);
                vsnprintf(buf, sizeof(buf), fmt, ap);
                va_end(ap);
        }

        KafkaError_init(self, err, fmt ? buf : rd_kafka_err2str(err));

        return (PyObject *)self;
}

/**
 * @brief Internal factory to create KafkaError object.
 * @returns a new KafkaError object if \p err != 0, else a None object.
 */
PyObject *KafkaError_new_or_None(rd_kafka_resp_err_t err, const char *str) {
        if (!err)
                Py_RETURN_NONE;
        if (str)
                return KafkaError_new0(err, "%s", str);
        else
                return KafkaError_new0(err, NULL);
}


/**
 * @brief Create a KafkaError object from  an rd_kafka_error_t *
 *        and destroy the C object when done.
 */
PyObject *KafkaError_new_from_error_destroy(rd_kafka_error_t *error) {
        KafkaError *kerr;

        kerr = (KafkaError *)KafkaError_new0(rd_kafka_error_code(error), "%s",
                                             rd_kafka_error_string(error));

        kerr->fatal              = rd_kafka_error_is_fatal(error);
        kerr->retriable          = rd_kafka_error_is_retriable(error);
        kerr->txn_requires_abort = rd_kafka_error_txn_requires_abort(error);
        rd_kafka_error_destroy(error);

        return (PyObject *)kerr;
}


/**
 * @brief Raise exception from fatal error.
 */
static void cfl_PyErr_Fatal(rd_kafka_resp_err_t err, const char *reason) {
        PyObject *eo              = KafkaError_new0(err, "%s", reason);
        ((KafkaError *)eo)->fatal = 1;
        PyErr_SetObject(KafkaException, eo);
}


/****************************************************************************
 *
 *
 * Message
 *
 *
 *
 *
 ****************************************************************************/


/**
 * @returns a Message's error object, if any, else None.
 * @remark The error object refcount is increased by this function.
 */
PyObject *Message_error(Message *self, PyObject *ignore) {
        if (self->error) {
                Py_INCREF(self->error);
                return self->error;
        } else
                Py_RETURN_NONE;
}

static PyObject *Message_value(Message *self, PyObject *ignore) {
        if (self->value) {
                Py_INCREF(self->value);
                return self->value;
        } else
                Py_RETURN_NONE;
}


static PyObject *Message_key(Message *self, PyObject *ignore) {
        if (self->key) {
                Py_INCREF(self->key);
                return self->key;
        } else
                Py_RETURN_NONE;
}

static PyObject *Message_topic(Message *self, PyObject *ignore) {
        if (self->topic) {
                Py_INCREF(self->topic);
                return self->topic;
        } else
                Py_RETURN_NONE;
}

static PyObject *Message_partition(Message *self, PyObject *ignore) {
        if (self->partition != RD_KAFKA_PARTITION_UA)
                return cfl_PyInt_FromInt(self->partition);
        else
                Py_RETURN_NONE;
}


static PyObject *Message_offset(Message *self, PyObject *ignore) {
        if (self->offset >= 0)
                return PyLong_FromLongLong(self->offset);
        else
                Py_RETURN_NONE;
}

static PyObject *Message_leader_epoch(Message *self, PyObject *ignore) {
        if (self->leader_epoch >= 0)
                return cfl_PyInt_FromInt(self->leader_epoch);
        else
                Py_RETURN_NONE;
}


static PyObject *Message_timestamp(Message *self, PyObject *ignore) {
        return Py_BuildValue("iL", self->tstype, self->timestamp);
}

static PyObject *Message_latency(Message *self, PyObject *ignore) {
        if (self->latency == -1)
                Py_RETURN_NONE;
        return PyFloat_FromDouble((double)self->latency / 1000000.0);
}

static PyObject *Message_headers(Message *self, PyObject *ignore) {
#ifdef RD_KAFKA_V_HEADERS
        if (self->headers) {
                Py_INCREF(self->headers);
                return self->headers;
        } else if (self->c_headers) {
                self->headers = c_headers_to_py(self->c_headers);
                rd_kafka_headers_destroy(self->c_headers);
                self->c_headers = NULL;
                Py_INCREF(self->headers);
                return self->headers;
        } else {
                Py_RETURN_NONE;
        }
#else
        Py_RETURN_NONE;
#endif
}

static PyObject *Message_set_headers(Message *self, PyObject *new_headers) {
        if (self->headers)
                Py_DECREF(self->headers);
        self->headers = new_headers;
        Py_INCREF(self->headers);

        Py_RETURN_NONE;
}

static PyObject *Message_set_value(Message *self, PyObject *new_val) {
        if (self->value)
                Py_DECREF(self->value);
        self->value = new_val;
        Py_INCREF(self->value);

        Py_RETURN_NONE;
}

static PyObject *Message_set_key(Message *self, PyObject *new_key) {
        if (self->key)
                Py_DECREF(self->key);
        self->key = new_key;
        Py_INCREF(self->key);

        Py_RETURN_NONE;
}

static PyObject *Message_set_topic(Message *self, PyObject *new_topic) {
        if (self->topic)
                Py_DECREF(self->topic);
        self->topic = new_topic;
        Py_INCREF(self->topic);

        Py_RETURN_NONE;
}

static PyObject *Message_set_error(Message *self, PyObject *new_error) {
        if (self->error)
                Py_DECREF(self->error);
        self->error = new_error;
        Py_INCREF(self->error);

        Py_RETURN_NONE;
}

static PyObject *Message_reduce(Message *self, PyObject *Py_UNUSED(ignored)) {
        PyObject *Message_type = NULL;
        PyObject *result       = NULL;

#ifdef RD_KAFKA_V_HEADERS
        if (!self->headers && self->c_headers) {
                self->headers = c_headers_to_py(self->c_headers);
                rd_kafka_headers_destroy(self->c_headers);
                self->c_headers = NULL;
        }
#endif


        Message_type = cfl_PyObject_lookup("confluent_kafka.cimpl", "Message");

        PyObject *latency_obj = NULL;
        if (self->latency >= 0) {
                latency_obj =
                    PyFloat_FromDouble((double)self->latency / 1000000.0);
        } else {
                /* Return -1.0 for negative latency to match Message_init
                 * default */
                latency_obj = PyFloat_FromDouble(-1.0);
        }
        result = Py_BuildValue(
            "O(NiLNNNNOOi)", Message_type, Message_topic(self, NULL),
            self->partition, self->offset, Message_key(self, NULL),
            Message_value(self, NULL), Message_headers(self, NULL),
            Message_error(self, NULL), Message_timestamp(self, NULL),
            latency_obj, self->leader_epoch);
        Py_DECREF(latency_obj);


        Py_DECREF(Message_type);
        return result;
}

static PyMethodDef Message_methods[] = {
    {"error", (PyCFunction)Message_error, METH_NOARGS,
     "  The message object is also used to propagate errors and events, "
     "an application must check error() to determine if the Message "
     "is a proper message (error() returns None) or an error or event "
     "(error() returns a KafkaError object)\n"
     "\n"
     "  :rtype: None or :py:class:`KafkaError`\n"
     "\n"},

    {"value", (PyCFunction)Message_value, METH_NOARGS,
     "  :returns: message value (payload) or None if not available.\n"
     "  :rtype: str|bytes or None\n"
     "\n"},
    {"key", (PyCFunction)Message_key, METH_NOARGS,
     "  :returns: message key or None if not available.\n"
     "  :rtype: str|bytes or None\n"
     "\n"},
    {"topic", (PyCFunction)Message_topic, METH_NOARGS,
     "  :returns: topic name or None if not available.\n"
     "  :rtype: str or None\n"
     "\n"},
    {"partition", (PyCFunction)Message_partition, METH_NOARGS,
     "  :returns: partition number or None if not available.\n"
     "  :rtype: int or None\n"
     "\n"},
    {"offset", (PyCFunction)Message_offset, METH_NOARGS,
     "  :returns: message offset or None if not available.\n"
     "  :rtype: int or None\n"
     "\n"},
    {"leader_epoch", (PyCFunction)Message_leader_epoch, METH_NOARGS,
     "  :returns: message offset leader epoch or None if not available.\n"
     "  :rtype: int or None\n"
     "\n"},
    {"timestamp", (PyCFunction)Message_timestamp, METH_NOARGS,
     "Retrieve timestamp type and timestamp from message.\n"
     "The timestamp type is one of:\n\n"
     "  * :py:const:`TIMESTAMP_NOT_AVAILABLE` "
     "- Timestamps not supported by broker.\n"
     "  * :py:const:`TIMESTAMP_CREATE_TIME` "
     "- Message creation time (or source / producer time).\n"
     "  * :py:const:`TIMESTAMP_LOG_APPEND_TIME` "
     "- Broker receive time.\n"
     "\n"
     "  The returned timestamp should be ignored if the timestamp type is "
     ":py:const:`TIMESTAMP_NOT_AVAILABLE`.\n"
     "\n"
     "  The timestamp is the number of milliseconds since the epoch (UTC).\n"
     "\n"
     "  Timestamps require broker version 0.10.0.0 or later and "
     "``{'api.version.request': True}`` configured on the client.\n"
     "\n"
     "  :returns: tuple of message timestamp type, and timestamp.\n"
     "  :rtype: (int, int)\n"
     "\n"},
    {"latency", (PyCFunction)Message_latency, METH_NOARGS,
     "Retrieve the time it took to produce the message, from calling "
     "produce() to the time the acknowledgement was received from "
     "the broker.\n"
     "Must only be used with the producer for successfully produced "
     "messages.\n"
     "\n"
     "  :returns: latency as float seconds, or None if latency "
     "information is not available (such as for errored messages).\n"
     "  :rtype: float or None\n"
     "\n"},
    {"headers", (PyCFunction)Message_headers, METH_NOARGS,
     "  Retrieve the headers set on a message. Each header is a key value"
     "pair. Please note that header keys are ordered and can repeat.\n"
     "\n"
     "  :returns: list of two-tuples, one (key, value) pair for each header.\n"
     "  :rtype: [(str, bytes),...] or None.\n"
     "\n"},
    {"set_headers", (PyCFunction)Message_set_headers, METH_O,
     "  Set the field 'Message.headers' with new value.\n"
     "\n"
     "  :param object value: Message.headers.\n"
     "  :returns: None.\n"
     "  :rtype: None\n"
     "\n"},
    {"set_value", (PyCFunction)Message_set_value, METH_O,
     "  Set the field 'Message.value' with new value.\n"
     "\n"
     "  :param object value: Message.value.\n"
     "  :returns: None.\n"
     "  :rtype: None\n"
     "\n"},
    {"set_key", (PyCFunction)Message_set_key, METH_O,
     "  Set the field 'Message.key' with new value.\n"
     "\n"
     "  :param object value: Message.key.\n"
     "  :returns: None.\n"
     "  :rtype: None\n"
     "\n"},
    {"set_topic", (PyCFunction)Message_set_topic, METH_O,
     "  Set the field 'Message.topic' with new value.\n"
     "\n"
     "  :param object value: Message.topic.\n"
     "  :returns: None.\n"
     "  :rtype: None\n"
     "\n"},
    {"set_error", (PyCFunction)Message_set_error, METH_O,
     "  Set the field 'Message.error' with new value.\n"
     "\n"
     "  :param object value: Message.error.\n"
     "  :returns: None.\n"
     "  :rtype: None\n"
     "\n"},
    {"__reduce__", (PyCFunction)Message_reduce, METH_NOARGS,
     " Function for serializing Message using the pickle protocol."},
    {NULL}};

static int Message_clear(Message *self) {
        if (self->topic) {
                Py_DECREF(self->topic);
                self->topic = NULL;
        }
        if (self->value) {
                Py_DECREF(self->value);
                self->value = NULL;
        }
        if (self->key) {
                Py_DECREF(self->key);
                self->key = NULL;
        }
        if (self->error) {
                Py_DECREF(self->error);
                self->error = NULL;
        }
        if (self->headers) {
                Py_DECREF(self->headers);
                self->headers = NULL;
        }
#ifdef RD_KAFKA_V_HEADERS
        if (self->c_headers) {
                rd_kafka_headers_destroy(self->c_headers);
                self->c_headers = NULL;
        }
#endif
        return 0;
}

static void Message_dealloc(Message *self) {
        Message_clear(self);
        PyObject_GC_UnTrack(self);
        Py_TYPE(self)->tp_free((PyObject *)self);
}

static int Message_init(PyObject *self0, PyObject *args, PyObject *kwargs) {
        Message *self        = (Message *)self0;
        PyObject *topic      = NULL;
        int32_t partition    = RD_KAFKA_PARTITION_UA;
        int64_t offset       = RD_KAFKA_OFFSET_INVALID;
        PyObject *key        = NULL;
        PyObject *value      = NULL;
        PyObject *headers    = NULL;
        PyObject *error      = NULL;
        PyObject *timestamp  = NULL;
        double latency       = -1;
        int32_t leader_epoch = -1;

        static char *kws[] = {"topic",   "partition",    "offset", "key",
                              "value",   "headers",      "error",  "timestamp",
                              "latency", "leader_epoch", NULL};

        if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OiLOOOOOdi", kws,
                                         &topic, &partition, &offset, &key,
                                         &value, &headers, &error, &timestamp,
                                         &latency, &leader_epoch)) {
                return -1;
        }

        /* Initialize all PyObject fields to NULL first */
        self->topic   = NULL;
        self->value   = NULL;
        self->key     = NULL;
        self->headers = NULL;
#ifdef RD_KAFKA_V_HEADERS
        self->c_headers = NULL;
#endif
        self->error = NULL;

        /* Set topic (string) */
        if (topic && topic != Py_None) {
                Py_INCREF(topic);
                self->topic = topic;
        }

        /* Set key (bytes) */
        if (key && key != Py_None) {
                Py_INCREF(key);
                self->key = key;
        }

        /* Set value (bytes) */
        if (value && value != Py_None) {
                Py_INCREF(value);
                self->value = value;
        }

        /* Set headers (list of tuples) */
        if (headers && headers != Py_None) {
                Py_INCREF(headers);
                self->headers = headers;
        }

        /* Set error (KafkaError) */
        if (error && error != Py_None) {
                Py_INCREF(error);
                self->error = error;
        }

        /* Set timestamp (tuple of (timestamp_type, timestamp)) */
        if (timestamp && timestamp != Py_None) {
                if (!PyTuple_Check(timestamp) || PyTuple_Size(timestamp) != 2) {
                        PyErr_SetString(
                            PyExc_TypeError,
                            "timestamp must be a tuple of (int, int)");
                        return -1;
                }
                self->tstype = (rd_kafka_timestamp_type_t)cfl_PyInt_AsInt(
                    PyTuple_GET_ITEM(timestamp, 0));
                self->timestamp =
                    PyLong_AsLongLong(PyTuple_GET_ITEM(timestamp, 1));
        } else {
                self->tstype    = RD_KAFKA_TIMESTAMP_NOT_AVAILABLE;
                self->timestamp = 0;
        }

        self->partition    = partition < 0 ? RD_KAFKA_PARTITION_UA : partition;
        self->offset       = offset < 0 ? RD_KAFKA_OFFSET_INVALID : offset;
        self->leader_epoch = leader_epoch < 0 ? -1 : leader_epoch;
        self->latency      = (int64_t)(latency < 0 ? -1 : latency * 1000000.0);

        return 0;
}



static PyObject *
Message_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) {
        return type->tp_alloc(type, 0);
}


static int Message_traverse(Message *self, visitproc visit, void *arg) {
        if (self->topic)
                Py_VISIT(self->topic);
        if (self->value)
                Py_VISIT(self->value);
        if (self->key)
                Py_VISIT(self->key);
        if (self->error)
                Py_VISIT(self->error);
        if (self->headers)
                Py_VISIT(self->headers);
        return 0;
}

static PyObject *Message_richcompare(PyObject *self, PyObject *other, int op) {
        if (op != Py_EQ && op != Py_NE) {
                Py_INCREF(Py_NotImplemented);
                return Py_NotImplemented;
        }

        if (self == other) {
                if (op == Py_EQ)
                        Py_RETURN_TRUE;
                else
                        Py_RETURN_FALSE;
        }

        if (!PyObject_TypeCheck(other, &MessageType)) {
                if (op == Py_EQ)
                        Py_RETURN_FALSE;
                else
                        Py_RETURN_TRUE;
        }

        Message *msg_self  = (Message *)self;
        Message *msg_other = (Message *)other;

        int result;

#define _LOCAL_COMPARE(left, right)                                            \
        do {                                                                   \
                result = PyObject_RichCompareBool(left, right, Py_EQ);         \
                if (result < 0)                                                \
                        return NULL;                                           \
                if (result == 0) {                                             \
                        if (op == Py_EQ)                                       \
                                Py_RETURN_FALSE;                               \
                        else                                                   \
                                Py_RETURN_TRUE;                                \
                }                                                              \
        } while (0)
        _LOCAL_COMPARE(msg_self->topic, msg_other->topic);
        _LOCAL_COMPARE(msg_self->value, msg_other->value);
        _LOCAL_COMPARE(msg_self->key, msg_other->key);
        _LOCAL_COMPARE(msg_self->headers, msg_other->headers);
        _LOCAL_COMPARE(msg_self->error, msg_other->error);
#undef _LOCAL_COMPARE

#define _LOCAL_COMPARE(left, right)                                            \
        do {                                                                   \
                if (left != right) {                                           \
                        if (op == Py_EQ)                                       \
                                Py_RETURN_FALSE;                               \
                        else                                                   \
                                Py_RETURN_TRUE;                                \
                }                                                              \
        } while (0)
        _LOCAL_COMPARE(msg_self->partition, msg_other->partition);
        _LOCAL_COMPARE(msg_self->offset, msg_other->offset);
        _LOCAL_COMPARE(msg_self->leader_epoch, msg_other->leader_epoch);
        _LOCAL_COMPARE(msg_self->timestamp, msg_other->timestamp);
        // latency is skipped, it is a float and not that significant.
#undef _LOCAL_COMPARE

        Py_RETURN_TRUE;
}

static Py_ssize_t Message__len__(Message *self) {
        return self->value && self->value != Py_None
                   ? PyObject_Length(self->value)
                   : 0;
}

static PySequenceMethods Message_seq_methods = {
    (lenfunc)Message__len__ /* sq_length */
};


PyTypeObject MessageType = {
    PyVarObject_HEAD_INIT(NULL, 0) "confluent_kafka.cimpl.Message", /*tp_name*/
    sizeof(Message),             /*tp_basicsize*/
    0,                           /*tp_itemsize*/
    (destructor)Message_dealloc, /*tp_dealloc*/
    0,                           /*tp_print*/
    0,                           /*tp_getattr*/
    0,                           /*tp_setattr*/
    0,                           /*tp_compare*/
    0,                           /*tp_repr*/
    0,                           /*tp_as_number*/
    &Message_seq_methods,        /*tp_as_sequence*/
    0,                           /*tp_as_mapping*/
    0,                           /*tp_hash */
    0,                           /*tp_call*/
    0,                           /*tp_str*/
    PyObject_GenericGetAttr,     /*tp_getattro*/
    0,                           /*tp_setattro*/
    0,                           /*tp_as_buffer*/
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /*tp_flags*/
    "The Message object represents either a single consumed or "
    "produced message, or an event (:py:func:`error()` is not None).\n"
    "\n"
    "An application must check with :py:func:`error()` to see if the "
    "object is a proper message (error() returns None) or an "
    "error/event.\n"
    "\n"
    ".. py:function:: Message([topic], [partition], [offset], [key], [value], "
    "[headers], [error], [timestamp], [latency], [leader_epoch])\n"
    "\n"
    "  Instantiate a Message object.\n"
    "\n"
    "  .. warning::"
    "    This constructor is intended **only for testing and mock scenarios**."
    "    Do **not** use user instantiated Message object in mainstream APIs or "
    "in production."
    "    Using this constructor outside of test environments may result in"
    "    unexpected behavior, security issues, or inconsistent application "
    "state.\n"
    "\n"
    "  All parameters are optional.\n"
    "\n"
    "  :param string topic: Topic name\n"
    "  :param int partition: Partition number\n"
    "  :param int offset: Message offset\n"
    "  :param bytes key: Message key\n"
    "  :param bytes value: Message value\n"
    "  :param list headers: Message headers\n"
    "  :param KafkaError error: KafkaError object\n"
    "  :param tuple timestamp: (Timestamp type (int), Message epoch timestamp "
    "in seconds (int))\n"
    "  :param float latency: Message latency in seconds\n"
    "  :param int leader_epoch: Message leader epoch in seconds\n"
    "  :rtype: Message\n"
    "\n"
    ".. py:function:: len()\n"
    "\n"
    "  :returns: Message value (payload) size in bytes\n"
    "  :rtype: int\n"
    "\n",                             /*tp_doc*/
    (traverseproc)Message_traverse,   /* tp_traverse */
    (inquiry)Message_clear,           /* tp_clear */
    (richcmpfunc)Message_richcompare, /* tp_richcompare */
    0,                                /* tp_weaklistoffset */
    0,                                /* tp_iter */
    0,                                /* tp_iternext */
    Message_methods,                  /* tp_methods */
    0,                                /* tp_members */
    0,                                /* tp_getset */
    0,                                /* tp_base */
    0,                                /* tp_dict */
    0,                                /* tp_descr_get */
    0,                                /* tp_descr_set */
    0,                                /* tp_dictoffset */
    Message_init,                     /* tp_init */
    0,                                /* tp_alloc */
    Message_new                       /* tp_new */
};

/**
 * @brief Internal factory to create Message object from message_t
 */
PyObject *Message_new0(const Handle *handle, const rd_kafka_message_t *rkm) {
        Message *self;

        self = (Message *)MessageType.tp_alloc(&MessageType, 0);
        if (!self)
                return NULL;

        /* Only use message error string on Consumer, for Producers
         * it will contain the original message payload. */
        self->error = KafkaError_new_or_None(
            rkm->err, (rkm->err && handle->type != RD_KAFKA_PRODUCER)
                          ? rd_kafka_message_errstr(rkm)
                          : NULL);

        if (rkm->rkt)
                self->topic =
                    cfl_PyUnistr(_FromString(rd_kafka_topic_name(rkm->rkt)));
        if (rkm->payload)
                self->value =
                    cfl_PyBin(_FromStringAndSize(rkm->payload, rkm->len));
        if (rkm->key)
                self->key =
                    cfl_PyBin(_FromStringAndSize(rkm->key, rkm->key_len));

        self->partition    = rkm->partition;
        self->offset       = rkm->offset;
        self->leader_epoch = rd_kafka_message_leader_epoch(rkm);

        self->timestamp = rd_kafka_message_timestamp(rkm, &self->tstype);

        if (handle->type == RD_KAFKA_PRODUCER)
                self->latency = rd_kafka_message_latency(rkm);
        else
                self->latency = -1;

        return (PyObject *)self;
}

/****************************************************************************
 *
 *
 * Uuid
 *
 *
 *
 *
 ****************************************************************************/
static PyObject *Uuid_most_significant_bits(Uuid *self, PyObject *ignore) {
        if (self->cUuid) {
                return cfl_PyLong_FromLong(
                    rd_kafka_Uuid_most_significant_bits(self->cUuid));
        }
        Py_RETURN_NONE;
}

static PyObject *Uuid_least_significant_bits(Uuid *self, PyObject *ignore) {
        if (self->cUuid) {
                return cfl_PyLong_FromLong(
                    rd_kafka_Uuid_least_significant_bits(self->cUuid));
        }
        Py_RETURN_NONE;
}

static PyMethodDef Uuid_methods[] = {
    {"get_most_significant_bits", (PyCFunction)Uuid_most_significant_bits,
     METH_NOARGS,
     "  :returns: Most significant 64 bits of the 128 bits Uuid\n"
     "  :rtype: int\n"
     "\n"},
    {"get_least_significant_bits", (PyCFunction)Uuid_least_significant_bits,
     METH_NOARGS,
     "  :returns: Least significant 64 bits of the 128 bits Uuid\n"
     "  :rtype: int\n"
     "\n"},
    {NULL}};


static PyObject *Uuid_str0(Uuid *self) {
        if (self->cUuid) {
                const char *base64str = rd_kafka_Uuid_base64str(self->cUuid);
                if (base64str)
                        return cfl_PyUnistr(_FromString(base64str));
        }
        Py_RETURN_NONE;
}

static long Uuid_hash(Uuid *self) {

        return rd_kafka_Uuid_most_significant_bits(self->cUuid) ^
               rd_kafka_Uuid_least_significant_bits(self->cUuid);
}


static PyObject *
Uuid_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) {
        PyObject *self = type->tp_alloc(type, 1);
        return self;
}


static int Uuid_init(PyObject *self0, PyObject *args, PyObject *kwargs) {
        Uuid *self         = (Uuid *)self0;
        static char *kws[] = {"most_significant_bits", "least_significant_bits",
                              NULL};
        int64_t most_significant_bits;
        int64_t least_significant_bits;

        if (!PyArg_ParseTupleAndKeywords(args, kwargs, "LL", kws,
                                         &most_significant_bits,
                                         &least_significant_bits))
                return -1;

        self->cUuid =
            rd_kafka_Uuid_new(most_significant_bits, least_significant_bits);

        return 0;
}

static int Uuid_clear(Uuid *self) {
        if (self->cUuid) {
                rd_kafka_Uuid_destroy(self->cUuid);
                self->cUuid = NULL;
        }
        return 0;
}

static int Uuid_traverse(Uuid *self, visitproc visit, void *arg) {
        return 0;
}

static void Uuid_dealloc(Uuid *self) {
        PyObject_GC_UnTrack(self);
        Uuid_clear(self);

        Py_TYPE(self)->tp_free((PyObject *)self);
}

PyTypeObject UuidType = {
    PyVarObject_HEAD_INIT(NULL, 0) "cimpl.Uuid", /* tp_name */
    sizeof(Uuid),                                /* tp_basicsize */
    0,                                           /* tp_itemsize */
    (destructor)Uuid_dealloc,                    /* tp_dealloc */
    0,                                           /* tp_print */
    0,                                           /* tp_getattr */
    0,                                           /* tp_setattr */
    0,                                           /* tp_compare */
    (reprfunc)Uuid_str0,                         /* tp_repr */
    0,                                           /* tp_as_number */
    0,                                           /* tp_as_sequence */
    0,                                           /* tp_as_mapping */
    (hashfunc)Uuid_hash,                         /* tp_hash */
    0,                                           /* tp_call */
    0,                                           /* tp_str */
    PyObject_GenericGetAttr,                     /* tp_getattro */
    0,                                           /* tp_setattro */
    0,                                           /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
        Py_TPFLAGS_HAVE_GC, /* tp_flags */
    "Generic Uuid. Being used in various identifiers including topic_id.\n"
    "\n"
    ".. py:function:: Uuid(most_significant_bits, least_significant_bits)\n"
    "\n"
    "  Instantiate a Uuid object.\n"
    "\n"
    "  :param long most_significant_bits: Most significant 64 bits of the 128 "
    "bits Uuid.\n"
    "  :param long least_significant_bits: Least significant 64 bits of the "
    "128 bits Uuid.\n"
    "  :rtype: Uuid\n"
    "\n"
    "\n",                        /* tp_doc */
    (traverseproc)Uuid_traverse, /* tp_traverse */
    (inquiry)Uuid_clear,         /* tp_clear */
    0,                           /* tp_richcompare */
    0,                           /* tp_weaklistoffset */
    0,                           /* tp_iter */
    0,                           /* tp_iternext */
    Uuid_methods,                /* tp_methods */
    0,                           /* tp_members */
    0,                           /* tp_getset */
    0,                           /* tp_base */
    0,                           /* tp_dict */
    0,                           /* tp_descr_get */
    0,                           /* tp_descr_set */
    0,                           /* tp_dictoffset */
    Uuid_init,                   /* tp_init */
    0,                           /* tp_alloc */
    Uuid_new                     /* tp_new */
};


/****************************************************************************
 *
 *
 * TopicPartition
 *
 *
 *
 *
 ****************************************************************************/
static int TopicPartition_clear(TopicPartition *self) {
        if (self->topic) {
                free(self->topic);
                self->topic = NULL;
        }
        if (self->error) {
                Py_DECREF(self->error);
                self->error = NULL;
        }
        if (self->metadata) {
                free(self->metadata);
                self->metadata = NULL;
        }
        return 0;
}

static void TopicPartition_setup(TopicPartition *self,
                                 const char *topic,
                                 int partition,
                                 long long offset,
                                 int32_t leader_epoch,
                                 const char *metadata,
                                 rd_kafka_resp_err_t err) {
        self->topic     = strdup(topic);
        self->partition = partition;
        self->offset    = offset;

        if (leader_epoch < 0)
                leader_epoch = -1;
        self->leader_epoch = leader_epoch;

        if (metadata != NULL) {
                self->metadata = strdup(metadata);
        } else {
                self->metadata = NULL;
        }

        self->error = KafkaError_new_or_None(err, NULL);
}


static void TopicPartition_dealloc(TopicPartition *self) {
        PyObject_GC_UnTrack(self);

        TopicPartition_clear(self);

        Py_TYPE(self)->tp_free((PyObject *)self);
}


static int
TopicPartition_init(PyObject *self, PyObject *args, PyObject *kwargs) {
        const char *topic;
        int partition        = RD_KAFKA_PARTITION_UA;
        int32_t leader_epoch = -1;
        long long offset     = RD_KAFKA_OFFSET_INVALID;
        const char *metadata = NULL;

        static char *kws[] = {"topic",    "partition",    "offset",
                              "metadata", "leader_epoch", NULL};

        if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|iLsi", kws, &topic,
                                         &partition, &offset, &metadata,
                                         &leader_epoch)) {
                return -1;
        }

        TopicPartition_setup((TopicPartition *)self, topic, partition, offset,
                             leader_epoch, metadata, 0);
        return 0;
}


static PyObject *
TopicPartition_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) {
        PyObject *self = type->tp_alloc(type, 1);
        return self;
}



static int
TopicPartition_traverse(TopicPartition *self, visitproc visit, void *arg) {
        if (self->error)
                Py_VISIT(self->error);
        return 0;
}

static PyObject *TopicPartition_get_leader_epoch(TopicPartition *tp,
                                                 void *closure) {
        if (tp->leader_epoch >= 0) {
                return cfl_PyInt_FromInt(tp->leader_epoch);
        }
        Py_RETURN_NONE;
}


static PyMemberDef TopicPartition_members[] = {
    {"topic", T_STRING, offsetof(TopicPartition, topic), READONLY,
     ":attribute topic: Topic name (string)"},
    {"partition", T_INT, offsetof(TopicPartition, partition), 0,
     ":attribute partition: Partition number (int)"},
    {"offset", T_LONGLONG, offsetof(TopicPartition, offset), 0,
     ":attribute offset: Offset (long)\n"
     "\n"
     "Either an absolute offset (>=0) or a logical offset: "
     " :py:const:`OFFSET_BEGINNING`,"
     " :py:const:`OFFSET_END`,"
     " :py:const:`OFFSET_STORED`,"
     " :py:const:`OFFSET_INVALID`\n"},
    {"metadata", T_STRING, offsetof(TopicPartition, metadata), READONLY,
     "attribute metadata: Optional application metadata committed with the "
     "offset (string)"},
    {"error", T_OBJECT, offsetof(TopicPartition, error), READONLY,
     ":attribute error: Indicates an error (with :py:class:`KafkaError`) "
     "unless None."},
    {NULL}};

static PyGetSetDef TopicPartition_getters_and_setters[] = {
    {/* name */
     "leader_epoch", (getter)TopicPartition_get_leader_epoch, NULL,
     /* doc */
     ":attribute leader_epoch: Offset leader epoch (int), or None",
     /* closure */
     NULL},
    {NULL}};


static PyObject *TopicPartition_str0(TopicPartition *self) {
        PyObject *errstr     = NULL;
        PyObject *errstr8    = NULL;
        const char *c_errstr = NULL;
        PyObject *ret;
        char offset_str[40];
        char leader_epoch_str[12];

        snprintf(offset_str, sizeof(offset_str), "%" CFL_PRId64 "",
                 self->offset);
        if (self->leader_epoch >= 0)
                snprintf(leader_epoch_str, sizeof(leader_epoch_str),
                         "%" CFL_PRId32 "", self->leader_epoch);
        else
                snprintf(leader_epoch_str, sizeof(leader_epoch_str), "None");

        if (self->error != Py_None) {
                errstr   = cfl_PyObject_Unistr(self->error);
                c_errstr = cfl_PyUnistr_AsUTF8(errstr, &errstr8);
        }

        ret = cfl_PyUnistr(
            _FromFormat("TopicPartition{topic=%s,partition=%" CFL_PRId32
                        ",offset=%s,leader_epoch=%s,error=%s}",
                        self->topic, self->partition, offset_str,
                        leader_epoch_str, c_errstr ? c_errstr : "None"));
        Py_XDECREF(errstr8);
        Py_XDECREF(errstr);
        return ret;
}


static PyObject *
TopicPartition_richcompare(TopicPartition *self, PyObject *o2, int op) {
        TopicPartition *a = self, *b;
        int tr, pr;
        int r;
        PyObject *result;

        if (Py_TYPE(o2) != Py_TYPE(self)) {
                PyErr_SetNone(PyExc_NotImplementedError);
                return NULL;
        }

        b = (TopicPartition *)o2;

        tr = strcmp(a->topic, b->topic);
        pr = a->partition - b->partition;
        switch (op) {
        case Py_LT:
                r = tr < 0 || (tr == 0 && pr < 0);
                break;
        case Py_LE:
                r = tr < 0 || (tr == 0 && pr <= 0);
                break;
        case Py_EQ:
                r = (tr == 0 && pr == 0);
                break;
        case Py_NE:
                r = (tr != 0 || pr != 0);
                break;
        case Py_GT:
                r = tr > 0 || (tr == 0 && pr > 0);
                break;
        case Py_GE:
                r = tr > 0 || (tr == 0 && pr >= 0);
                break;
        default:
                r = 0;
                break;
        }

        result = r ? Py_True : Py_False;
        Py_INCREF(result);
        return result;
}


static long TopicPartition_hash(TopicPartition *self) {
        PyObject *topic = cfl_PyUnistr(_FromString(self->topic));
        long r          = PyObject_Hash(topic) ^ self->partition;
        Py_DECREF(topic);
        return r;
}


PyTypeObject TopicPartitionType = {
    PyVarObject_HEAD_INIT(NULL, 0) "cimpl.TopicPartition", /*tp_name*/
    sizeof(TopicPartition),                                /*tp_basicsize*/
    0,                                                     /*tp_itemsize*/
    (destructor)TopicPartition_dealloc,                    /*tp_dealloc*/
    0,                                                     /*tp_print*/
    0,                                                     /*tp_getattr*/
    0,                                                     /*tp_setattr*/
    0,                                                     /*tp_compare*/
    (reprfunc)TopicPartition_str0,                         /*tp_repr*/
    0,                                                     /*tp_as_number*/
    0,                                                     /*tp_as_sequence*/
    0,                                                     /*tp_as_mapping*/
    (hashfunc)TopicPartition_hash,                         /*tp_hash */
    0,                                                     /*tp_call*/
    0,                                                     /*tp_str*/
    PyObject_GenericGetAttr,                               /*tp_getattro*/
    0,                                                     /*tp_setattro*/
    0,                                                     /*tp_as_buffer*/
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /*tp_flags*/
    "TopicPartition is a generic type to hold a single partition and "
    "various information about it.\n"
    "\n"
    "It is typically used to provide a list of topics or partitions for "
    "various operations, such as :py:func:`Consumer.assign()`.\n"
    "\n"
    ".. py:function:: TopicPartition(topic, [partition], [offset],"
    " [metadata], [leader_epoch])\n"
    "\n"
    "  Instantiate a TopicPartition object.\n"
    "\n"
    "  :param string topic: Topic name\n"
    "  :param int partition: Partition id\n"
    "  :param int offset: Initial partition offset\n"
    "  :param string metadata: Offset metadata\n"
    "  :param int leader_epoch: Offset leader epoch\n"
    "  :rtype: TopicPartition\n"
    "\n"
    "\n",                                    /*tp_doc*/
    (traverseproc)TopicPartition_traverse,   /* tp_traverse */
    (inquiry)TopicPartition_clear,           /* tp_clear */
    (richcmpfunc)TopicPartition_richcompare, /* tp_richcompare */
    0,                                       /* tp_weaklistoffset */
    0,                                       /* tp_iter */
    0,                                       /* tp_iternext */
    0,                                       /* tp_methods */
    TopicPartition_members,                  /* tp_members */
    TopicPartition_getters_and_setters,      /* tp_getset */
    0,                                       /* tp_base */
    0,                                       /* tp_dict */
    0,                                       /* tp_descr_get */
    0,                                       /* tp_descr_set */
    0,                                       /* tp_dictoffset */
    TopicPartition_init,                     /* tp_init */
    0,                                       /* tp_alloc */
    TopicPartition_new                       /* tp_new */
};

/**
 * @brief Internal factory to create a TopicPartition object.
 */
static PyObject *TopicPartition_new0(const char *topic,
                                     int partition,
                                     long long offset,
                                     int32_t leader_epoch,
                                     const char *metadata,
                                     rd_kafka_resp_err_t err) {
        TopicPartition *self;

        self = (TopicPartition *)TopicPartitionType.tp_new(&TopicPartitionType,
                                                           NULL, NULL);

        TopicPartition_setup(self, topic, partition, offset, leader_epoch,
                             metadata, err);

        return (PyObject *)self;
}



PyObject *c_part_to_py(const rd_kafka_topic_partition_t *rktpar) {
        return TopicPartition_new0(
            rktpar->topic, rktpar->partition, rktpar->offset,
            rd_kafka_topic_partition_get_leader_epoch(rktpar), rktpar->metadata,
            rktpar->err);
}

/**
 * @brief Convert C rd_kafka_topic_partition_list_t to Python
 * list(TopicPartition).
 *
 * @returns The new Python list object.
 */
PyObject *c_parts_to_py(const rd_kafka_topic_partition_list_t *c_parts) {
        PyObject *parts;
        size_t i;

        parts = PyList_New(c_parts->cnt);

        for (i = 0; i < (size_t)c_parts->cnt; i++) {
                const rd_kafka_topic_partition_t *rktpar = &c_parts->elems[i];
                PyList_SET_ITEM(parts, i, c_part_to_py(rktpar));
        }

        return parts;
}

/**
 * @brief Convert Python list(TopicPartition) to C
 * rd_kafka_topic_partition_list_t.
 *
 * @returns The new C list on success or NULL on error.
 */
rd_kafka_topic_partition_list_t *py_to_c_parts(PyObject *plist) {
        rd_kafka_topic_partition_list_t *c_parts;
        size_t i;

        if (!PyList_Check(plist)) {
                PyErr_SetString(PyExc_TypeError,
                                "requires list of TopicPartition");
                return NULL;
        }

        c_parts = rd_kafka_topic_partition_list_new((int)PyList_Size(plist));

        for (i = 0; i < (size_t)PyList_Size(plist); i++) {
                rd_kafka_topic_partition_t *rktpar;
                TopicPartition *tp = (TopicPartition *)PyList_GetItem(plist, i);

                if (PyObject_Type((PyObject *)tp) !=
                    (PyObject *)&TopicPartitionType) {
                        PyErr_Format(PyExc_TypeError, "expected %s",
                                     TopicPartitionType.tp_name);
                        rd_kafka_topic_partition_list_destroy(c_parts);
                        return NULL;
                }

                rktpar = rd_kafka_topic_partition_list_add(c_parts, tp->topic,
                                                           tp->partition);
                rktpar->offset = tp->offset;
                rd_kafka_topic_partition_set_leader_epoch(rktpar,
                                                          tp->leader_epoch);
                if (tp->metadata != NULL) {
                        rktpar->metadata_size = strlen(tp->metadata) + 1;
                        rktpar->metadata      = strdup(tp->metadata);
                } else {
                        rktpar->metadata_size = 0;
                        rktpar->metadata      = NULL;
                }
        }

        return c_parts;
}

/**
 * @brief Convert C rd_kafka_topic_partition_result_t to Python
 * dict(TopicPartition, KafkaException).
 *
 * @returns The new Python dict object.
 */
PyObject *c_topic_partition_result_to_py_dict(
    const rd_kafka_topic_partition_result_t **partition_results,
    size_t cnt) {
        PyObject *result = NULL;
        size_t i;

        result = PyDict_New();

        for (i = 0; i < cnt; i++) {
                PyObject *key;
                PyObject *value;
                const rd_kafka_topic_partition_t *c_topic_partition;
                const rd_kafka_error_t *c_error;

                c_topic_partition = rd_kafka_topic_partition_result_partition(
                    partition_results[i]);
                c_error =
                    rd_kafka_topic_partition_result_error(partition_results[i]);

                value = KafkaError_new_or_None(rd_kafka_error_code(c_error),
                                               rd_kafka_error_string(c_error));
                key   = c_part_to_py(c_topic_partition);

                PyDict_SetItem(result, key, value);

                Py_DECREF(key);
                Py_DECREF(value);
        }

        return result;
}

#ifdef RD_KAFKA_V_HEADERS


/**
 * @brief Translate Python \p key and \p value to C types and set on
 *        provided \p rd_headers object.
 *
 * @returns 1 on success or 0 if an exception was raised.
 */
static int
py_header_to_c(rd_kafka_headers_t *rd_headers, PyObject *key, PyObject *value) {
        PyObject *ks, *ks8, *vo8 = NULL;
        const char *k;
        const void *v    = NULL;
        Py_ssize_t vsize = 0;
        rd_kafka_resp_err_t err;

        if (!(ks = cfl_PyObject_Unistr(key))) {
                PyErr_SetString(PyExc_TypeError,
                                "expected header key to be unicode "
                                "string");
                return 0;
        }

        k = cfl_PyUnistr_AsUTF8(ks, &ks8);

        if (value != Py_None) {
                if (cfl_PyBin(_Check(value))) {
                        /* Proper binary */
                        if (cfl_PyBin(_AsStringAndSize(value, (char **)&v,
                                                       &vsize)) == -1) {
                                Py_DECREF(ks);
                                Py_XDECREF(ks8);
                                return 0;
                        }
                } else if (cfl_PyUnistr(_Check(value))) {
                        /* Unicode string, translate to utf-8. */
                        v = cfl_PyUnistr_AsUTF8(value, &vo8);
                        if (!v) {
                                Py_DECREF(ks);
                                Py_XDECREF(ks8);
                                return 0;
                        }
                        vsize = (Py_ssize_t)strlen(v);
                } else {
                        PyErr_Format(
                            PyExc_TypeError,
                            "expected header value to be "
                            "None, binary, or unicode string, not %s",
                            ((PyTypeObject *)PyObject_Type(value))->tp_name);
                        Py_DECREF(ks);
                        Py_XDECREF(ks8);
                        return 0;
                }
        }

        if ((err = rd_kafka_header_add(rd_headers, k, -1, v, vsize))) {
                cfl_PyErr_Format(err,
                                 "Unable to add message header \"%s\": "
                                 "%s",
                                 k, rd_kafka_err2str(err));
                Py_DECREF(ks);
                Py_XDECREF(ks8);
                Py_XDECREF(vo8);
                return 0;
        }

        Py_DECREF(ks);
        Py_XDECREF(ks8);
        Py_XDECREF(vo8);

        return 1;
}

/**
 * @brief Convert Python list of tuples to rd_kafka_headers_t
 *
 * Header names must be unicode strong.
 * Header values may be None, binary or unicode string, the latter is
 * automatically encoded as utf-8.
 */
static rd_kafka_headers_t *py_headers_list_to_c(PyObject *hdrs) {
        int i, len;
        rd_kafka_headers_t *rd_headers = NULL;

        len        = (int)PyList_Size(hdrs);
        rd_headers = rd_kafka_headers_new(len);

        for (i = 0; i < len; i++) {
                PyObject *tuple = PyList_GET_ITEM(hdrs, i);

                if (!PyTuple_Check(tuple) || PyTuple_Size(tuple) != 2) {
                        rd_kafka_headers_destroy(rd_headers);
                        PyErr_SetString(PyExc_TypeError,
                                        "Headers are expected to be a "
                                        "list of (key, value) tuples");
                        return NULL;
                }

                if (!py_header_to_c(rd_headers, PyTuple_GET_ITEM(tuple, 0),
                                    PyTuple_GET_ITEM(tuple, 1))) {
                        rd_kafka_headers_destroy(rd_headers);
                        return NULL;
                }
        }
        return rd_headers;
}


/**
 * @brief Convert Python dict to rd_kafka_headers_t
 */
static rd_kafka_headers_t *py_headers_dict_to_c(PyObject *hdrs) {
        int len;
        Py_ssize_t pos                 = 0;
        rd_kafka_headers_t *rd_headers = NULL;
        PyObject *ko, *vo;

        len        = (int)PyDict_Size(hdrs);
        rd_headers = rd_kafka_headers_new(len);

        while (PyDict_Next(hdrs, &pos, &ko, &vo)) {

                if (!py_header_to_c(rd_headers, ko, vo)) {
                        rd_kafka_headers_destroy(rd_headers);
                        return NULL;
                }
        }

        return rd_headers;
}


/**
 * @brief Convert Python list[(header_key, header_value),...]) to C
 * rd_kafka_topic_partition_list_t.
 *
 * @returns The new Python list[(header_key, header_value),...] object.
 */
rd_kafka_headers_t *py_headers_to_c(PyObject *hdrs) {

        if (PyList_Check(hdrs)) {
                return py_headers_list_to_c(hdrs);
        } else if (PyDict_Check(hdrs)) {
                return py_headers_dict_to_c(hdrs);
        } else {
                PyErr_Format(PyExc_TypeError,
                             "expected headers to be "
                             "dict or list of (key, value) tuples, not %s",
                             ((PyTypeObject *)PyObject_Type(hdrs))->tp_name);
                return NULL;
        }
}


/**
 * @brief Convert rd_kafka_headers_t to Python list[(header_key,
 * header_value),...])
 *
 * @returns The new C headers on success or NULL on error.
 */
PyObject *c_headers_to_py(rd_kafka_headers_t *headers) {
        size_t idx         = 0;
        size_t header_size = 0;
        const char *header_key;
        const void *header_value;
        size_t header_value_size;
        PyObject *header_list;

        header_size = rd_kafka_header_cnt(headers);
        header_list = PyList_New(header_size);

        while (!rd_kafka_header_get_all(headers, idx++, &header_key,
                                        &header_value, &header_value_size)) {
                /* Create one (key, value) tuple for each header */
                PyObject *header_tuple = PyTuple_New(2);
                PyTuple_SetItem(header_tuple, 0,
                                cfl_PyUnistr(_FromString(header_key)));

                if (header_value) {
                        PyTuple_SetItem(header_tuple, 1,
                                        cfl_PyBin(_FromStringAndSize(
                                            header_value, header_value_size)));
                } else {
                        Py_INCREF(Py_None);
                        PyTuple_SetItem(header_tuple, 1, Py_None);
                }
                PyList_SET_ITEM(header_list, idx - 1, header_tuple);
        }

        return header_list;
}
#endif


/**
 * @brief Convert C rd_kafka_consumer_group_metadata_t to Python binary string
 *
 * @returns The new Python object, or NULL and raises an exception failure.
 */
PyObject *c_cgmd_to_py(const rd_kafka_consumer_group_metadata_t *cgmd) {
        PyObject *obj;
        void *buffer;
        size_t size;
        rd_kafka_error_t *error;

        error = rd_kafka_consumer_group_metadata_write(cgmd, &buffer, &size);
        if (error) {
                cfl_PyErr_from_error_destroy(error);
                return NULL;
        }

        obj = cfl_PyBin(_FromStringAndSize(buffer, (Py_ssize_t)size));
        rd_kafka_mem_free(NULL, buffer);

        return obj;
}

/**
 * @brief Convert Python bytes object to C rd_kafka_consumer_group_metadata_t.
 *
 * @returns The new C object, or NULL and raises an exception on failure.
 */
rd_kafka_consumer_group_metadata_t *py_to_c_cgmd(PyObject *obj) {
        rd_kafka_consumer_group_metadata_t *cgmd;
        rd_kafka_error_t *error;
        char *buffer;
        Py_ssize_t size;

        if (cfl_PyBin(_AsStringAndSize(obj, &buffer, &size)) == -1)
                return NULL;

        error = rd_kafka_consumer_group_metadata_read(
            &cgmd, (const void *)buffer, (size_t)size);
        if (error) {
                cfl_PyErr_from_error_destroy(error);
                return NULL;
        }

        return cgmd;
}

PyObject *c_Node_to_py(const rd_kafka_Node_t *c_node) {
        PyObject *node      = NULL;
        PyObject *Node_type = NULL;
        PyObject *args      = NULL;
        PyObject *kwargs    = NULL;
        const char *rack    = NULL;

        if (!c_node)
                Py_RETURN_NONE;

        Node_type = cfl_PyObject_lookup("confluent_kafka", "Node");
        if (!Node_type) {
                goto err;
        }

        kwargs = PyDict_New();

        cfl_PyDict_SetInt(kwargs, "id", rd_kafka_Node_id(c_node));
        cfl_PyDict_SetInt(kwargs, "port", rd_kafka_Node_port(c_node));
        if (rd_kafka_Node_host(c_node))
                cfl_PyDict_SetString(kwargs, "host",
                                     rd_kafka_Node_host(c_node));
        else
                PyDict_SetItemString(kwargs, "host", Py_None);
        if ((rack = rd_kafka_Node_rack(c_node)))
                cfl_PyDict_SetString(kwargs, "rack", rack);

        args = PyTuple_New(0);

        node = PyObject_Call(Node_type, args, kwargs);

        Py_DECREF(Node_type);
        Py_DECREF(args);
        Py_DECREF(kwargs);
        return node;

err:
        Py_XDECREF(Node_type);
        Py_XDECREF(args);
        Py_XDECREF(kwargs);
        Py_XDECREF(node);
        return NULL;
}

PyObject *c_Uuid_to_py(const rd_kafka_Uuid_t *c_uuid) {
        PyObject *uuid      = NULL;
        PyObject *Uuid_type = NULL;
        PyObject *args      = NULL;
        PyObject *kwargs    = NULL;

        if (!c_uuid)
                Py_RETURN_NONE;

        Uuid_type = cfl_PyObject_lookup("confluent_kafka", "Uuid");
        if (!Uuid_type) {
                goto err;
        }


        kwargs = PyDict_New();

        cfl_PyDict_SetLong(kwargs, "most_significant_bits",
                           rd_kafka_Uuid_most_significant_bits(c_uuid));
        cfl_PyDict_SetLong(kwargs, "least_significant_bits",
                           rd_kafka_Uuid_least_significant_bits(c_uuid));

        args = PyTuple_New(0);

        uuid = PyObject_Call(Uuid_type, args, kwargs);

        Py_DECREF(Uuid_type);
        Py_DECREF(args);
        Py_DECREF(kwargs);

        return uuid;

err:
        Py_XDECREF(Uuid_type);
        Py_XDECREF(args);
        Py_XDECREF(kwargs);
        Py_XDECREF(uuid);

        return NULL;
}


/****************************************************************************
 *
 *
 * Common callbacks
 *
 *
 *
 *
 ****************************************************************************/
static void
error_cb(rd_kafka_t *rk, int err, const char *reason, void *opaque) {
        Handle *h = opaque;
        PyObject *eo, *result;
        CallState *cs;

        cs = CallState_get(h);

        /* If the client raised a fatal error we'll raise an exception
         * rather than calling the error callback. */
        if (err == RD_KAFKA_RESP_ERR__FATAL) {
                char errstr[512];
                err = rd_kafka_fatal_error(rk, errstr, sizeof(errstr));
                cfl_PyErr_Fatal(err, errstr);
                goto crash;
        }

        if (!h->error_cb) {
                /* No callback defined */
                goto done;
        }

        eo     = KafkaError_new0(err, "%s", reason);
        result = PyObject_CallFunctionObjArgs(h->error_cb, eo, NULL);
        Py_DECREF(eo);

        if (result)
                Py_DECREF(result);
        else {
                CallState_fetch_exception(cs);
        crash:
                CallState_crash(cs);
                rd_kafka_yield(h->rk);
        }

done:
        CallState_resume(cs);
}

/**
 * @brief librdkafka throttle callback triggered by poll() or flush(), triggers
 * the corresponding Python throttle_cb
 */
static void throttle_cb(rd_kafka_t *rk,
                        const char *broker_name,
                        int32_t broker_id,
                        int throttle_time_ms,
                        void *opaque) {
        Handle *h = opaque;
        PyObject *ThrottleEvent_type, *throttle_event;
        PyObject *result, *args;
        CallState *cs;

        cs = CallState_get(h);
        if (!h->throttle_cb) {
                /* No callback defined */
                goto done;
        }

        ThrottleEvent_type =
            cfl_PyObject_lookup("confluent_kafka", "ThrottleEvent");

        if (!ThrottleEvent_type) {
                /* ThrottleEvent class not found */
                goto err;
        }

        args           = Py_BuildValue("(sid)", broker_name, broker_id,
                                       (double)throttle_time_ms / 1000);
        throttle_event = PyObject_Call(ThrottleEvent_type, args, NULL);

        Py_DECREF(args);
        Py_DECREF(ThrottleEvent_type);

        if (!throttle_event) {
                /* Failed to instantiate ThrottleEvent object */
                goto err;
        }

        result =
            PyObject_CallFunctionObjArgs(h->throttle_cb, throttle_event, NULL);

        Py_DECREF(throttle_event);

        if (result) {
                /* throttle_cb executed successfully */
                Py_DECREF(result);
                goto done;
        } else {
                CallState_fetch_exception(cs);
        }

        /**
         * Stop callback dispatcher, return err to application
         * fall-through to unlock GIL
         */
err:
        CallState_crash(cs);
        rd_kafka_yield(h->rk);
done:
        CallState_resume(cs);
}

static int stats_cb(rd_kafka_t *rk, char *json, size_t json_len, void *opaque) {
        Handle *h    = opaque;
        PyObject *eo = NULL, *result = NULL;
        CallState *cs = NULL;

        cs = CallState_get(h);
        if (json_len == 0) {
                /* No data returned*/
                goto done;
        }

        eo     = Py_BuildValue("s", json);
        result = PyObject_CallFunctionObjArgs(h->stats_cb, eo, NULL);
        Py_DECREF(eo);

        if (result)
                Py_DECREF(result);
        else {
                CallState_fetch_exception(cs);
                CallState_crash(cs);
                rd_kafka_yield(h->rk);
        }

done:
        CallState_resume(cs);
        return 0;
}

static void
log_cb(const rd_kafka_t *rk, int level, const char *fac, const char *buf) {
        Handle *h = rd_kafka_opaque(rk);
        PyObject *result;
        CallState *cs;
        static const int level_map[8] = {
            /* Map syslog levels to python logging levels */
            50, /* LOG_EMERG   -> logging.CRITICAL */
            50, /* LOG_ALERT   -> logging.CRITICAL */
            50, /* LOG_CRIT    -> logging.CRITICAL */
            40, /* LOG_ERR     -> logging.ERROR */
            30, /* LOG_WARNING -> logging.WARNING */
            20, /* LOG_NOTICE  -> logging.INFO */
            20, /* LOG_INFO    -> logging.INFO */
            10, /* LOG_DEBUG   -> logging.DEBUG */
        };

        cs = CallState_get(h);
        result =
            PyObject_CallMethod(h->logger, "log", "issss", level_map[level],
                                "%s [%s] %s", fac, rd_kafka_name(rk), buf);

        if (result)
                Py_DECREF(result);
        else {
                CallState_fetch_exception(cs);
                CallState_crash(cs);
                rd_kafka_yield(h->rk);
        }

        CallState_resume(cs);
}

/**
 * @brief Translate Python \p key and \p value to C types and set on
 *        provided \p extensions char* array at the provided index.
 *
 * @returns 1 on success or 0 if an exception was raised.
 */
static int py_extensions_to_c(char **extensions,
                              Py_ssize_t idx,
                              PyObject *key,
                              PyObject *value) {
        PyObject *ks, *ks8, *vo8 = NULL;
        const char *k;
        const char *v;
        Py_ssize_t ksize = 0;
        Py_ssize_t vsize = 0;

        if (!(ks = cfl_PyObject_Unistr(key))) {
                PyErr_SetString(PyExc_TypeError,
                                "expected extension key to be unicode "
                                "string");
                return 0;
        }

        k     = cfl_PyUnistr_AsUTF8(ks, &ks8);
        ksize = (Py_ssize_t)strlen(k);

        if (cfl_PyUnistr(_Check(value))) {
                /* Unicode string, translate to utf-8. */
                v = cfl_PyUnistr_AsUTF8(value, &vo8);
                if (!v) {
                        Py_DECREF(ks);
                        Py_XDECREF(ks8);
                        return 0;
                }
                vsize = (Py_ssize_t)strlen(v);
        } else {
                PyErr_Format(PyExc_TypeError,
                             "expected extension value to be "
                             "unicode string, not %s",
                             ((PyTypeObject *)PyObject_Type(value))->tp_name);
                Py_DECREF(ks);
                Py_XDECREF(ks8);
                return 0;
        }

        extensions[idx] = (char *)malloc(ksize + 1);
        snprintf(extensions[idx], ksize + 1, "%s", k);
        extensions[idx + 1] = (char *)malloc(vsize + 1);
        snprintf(extensions[idx + 1], vsize + 1, "%s", v);

        Py_DECREF(ks);
        Py_XDECREF(ks8);
        Py_XDECREF(vo8);

        return 1;
}


/**
 * @brief Waits for OAuth callback to set the token
 *
 * Useful during client init as we want to ensure we have the token before we
 * return back
 *
 * Returns 0 if token was set within the timeout period, -1 otherwise.
 */
int wait_for_oauth_token_set(Handle *h) {

        if (!h->oauth_cb)
                return 0;

        int max_wait_sec       = 10;
        int retry_interval_sec = 1; /* Check every 1 sec */
        int elapsed_sec        = 0;
        while (!h->oauth_token_set && elapsed_sec < max_wait_sec) {
                CallState cs;
                CallState_begin(h, &cs);
#ifdef _WIN32
                Sleep(retry_interval_sec * 1000);
#else
                sleep(retry_interval_sec);
#endif
                CallState_end(h, &cs);
                elapsed_sec += retry_interval_sec;
        }

        if (!h->oauth_token_set) {
                /* Token not set within timeout */
                cfl_PyErr_Format(
                    RD_KAFKA_RESP_ERR_SASL_AUTHENTICATION_FAILED,
                    "OAuth token not set within %d seconds timeout",
                    max_wait_sec);
                CallState cs;
                CallState_begin(h, &cs);
                rd_kafka_destroy(h->rk);
                h->rk = NULL;
                CallState_end(h, &cs);
                return -1;
        }
        return 0;
}

/**
 * @brief Callback invoked when a OAuth token needs to be refreshed.
 *
 * Note that this callback will be invoked by the background thread as
 * all client types have been configured to use background threads for sasl
 * events.
 */
static void
oauth_cb(rd_kafka_t *rk, const char *oauthbearer_config, void *opaque) {
        Handle *h = opaque;
        PyObject *eo, *result;
        PyGILState_STATE gstate;
        const char *token;
        double expiry;
        const char *principal         = "";
        PyObject *extensions          = NULL;
        char **rd_extensions          = NULL;
        Py_ssize_t rd_extensions_size = 0;
        char err_msg[2048];
        rd_kafka_resp_err_t err_code;

        gstate = PyGILState_Ensure();

        eo     = Py_BuildValue("s", oauthbearer_config);
        result = PyObject_CallFunctionObjArgs(h->oauth_cb, eo, NULL);
        Py_DECREF(eo);

        if (!result) {
                goto fail;
        }
        if (!PyArg_ParseTuple(result, "sd|sO!", &token, &expiry, &principal,
                              &PyDict_Type, &extensions)) {
                Py_DECREF(result);
                PyErr_SetString(PyExc_TypeError,
                                "expect returned value from oauth_cb "
                                "to be (token_str, expiry_time[, principal, "
                                "extensions]) tuple");
                goto err;
        }

        if (extensions) {
                int len        = (int)PyDict_Size(extensions);
                rd_extensions  = (char **)malloc(2 * len * sizeof(char *));
                Py_ssize_t pos = 0;
                PyObject *ko, *vo;
                while (PyDict_Next(extensions, &pos, &ko, &vo)) {
                        if (!py_extensions_to_c(rd_extensions,
                                                rd_extensions_size, ko, vo)) {
                                Py_DECREF(result);
                                free(rd_extensions);
                                goto err;
                        }
                        rd_extensions_size = rd_extensions_size + 2;
                }
        }

        err_code = rd_kafka_oauthbearer_set_token(
            h->rk, token, (int64_t)(expiry * 1000), principal,
            (const char **)rd_extensions, rd_extensions_size, err_msg,
            sizeof(err_msg));
        Py_DECREF(result);
        if (rd_extensions) {
                int i;
                for (i = 0; i < rd_extensions_size; i++) {
                        free(rd_extensions[i]);
                }
                free(rd_extensions);
        }

        if (err_code != RD_KAFKA_RESP_ERR_NO_ERROR) {
                PyErr_Format(PyExc_ValueError, "%s", err_msg);
                goto fail;
        }
        h->oauth_token_set = 1;
        goto done;

fail:
        err_code = rd_kafka_oauthbearer_set_token_failure(
            h->rk, "OAuth callback raised exception");
        if (err_code != RD_KAFKA_RESP_ERR_NO_ERROR) {
                PyErr_SetString(PyExc_ValueError,
                                "Failed to set token failure");
                goto err;
        }
        PyErr_Clear();
        goto done;
err:
        PyGILState_Release(gstate);
        rd_kafka_yield(h->rk);
done:
        PyGILState_Release(gstate);
}

/****************************************************************************
 *
 *
 * Common helpers
 *
 *
 *
 *
 ****************************************************************************/



/**
 * Clear Python object references in Handle
 */
void Handle_clear(Handle *h) {
        if (h->error_cb) {
                Py_DECREF(h->error_cb);
                h->error_cb = NULL;
        }

        if (h->throttle_cb) {
                Py_DECREF(h->throttle_cb);
                h->throttle_cb = NULL;
        }

        if (h->stats_cb) {
                Py_DECREF(h->stats_cb);
                h->stats_cb = NULL;
        }

        if (h->logger) {
                Py_DECREF(h->logger);
                h->logger = NULL;
        }

        if (h->initiated) {
#ifdef WITH_PY_TSS
                PyThread_tss_delete(&h->tlskey);
#else
                PyThread_delete_key(h->tlskey);
#endif
        }
}

/**
 * GC traversal for Python object references
 */
int Handle_traverse(Handle *h, visitproc visit, void *arg) {
        if (h->error_cb)
                Py_VISIT(h->error_cb);

        if (h->throttle_cb)
                Py_VISIT(h->throttle_cb);

        if (h->stats_cb)
                Py_VISIT(h->stats_cb);

        return 0;
}

/**
 * @brief Set single special producer config value.
 *
 * @returns 1 if handled, 0 if unknown, or -1 on failure (exception raised).
 */
static int producer_conf_set_special(Handle *self,
                                     rd_kafka_conf_t *conf,
                                     const char *name,
                                     PyObject *valobj) {

        if (!strcmp(name, "on_delivery")) {
                if (!PyCallable_Check(valobj)) {
                        cfl_PyErr_Format(RD_KAFKA_RESP_ERR__INVALID_ARG,
                                         "%s requires a callable "
                                         "object",
                                         name);
                        return -1;
                }

                self->u.Producer.default_dr_cb = valobj;
                Py_INCREF(self->u.Producer.default_dr_cb);

                return 1;

        } else if (!strcmp(name, "delivery.report.only.error")) {
                /* Since we allocate msgstate for each produced message
                 * with a callback we can't use delivery.report.only.error
                 * as-is, as we wouldn't be able to ever free those msgstates.
                 * Instead we shortcut this setting in the Python client,
                 * providing the same functionality from dr_msg_cb trampoline.
                 */

                if (!cfl_PyBool_get(valobj, name,
                                    &self->u.Producer.dr_only_error))
                        return -1;

                return 1;
        }

        return 0; /* Not handled */
}


/**
 * @brief Set single special consumer config value.
 *
 * @returns 1 if handled, 0 if unknown, or -1 on failure (exception raised).
 */
static int consumer_conf_set_special(Handle *self,
                                     rd_kafka_conf_t *conf,
                                     const char *name,
                                     PyObject *valobj) {

        if (!strcmp(name, "on_commit")) {
                if (!PyCallable_Check(valobj)) {
                        cfl_PyErr_Format(RD_KAFKA_RESP_ERR__INVALID_ARG,
                                         "%s requires a callable "
                                         "object",
                                         name);
                        return -1;
                }

                self->u.Consumer.on_commit = valobj;
                Py_INCREF(self->u.Consumer.on_commit);

                return 1;
        }

        return 0;
}

/**
 * @brief Call out to __init__.py _resolve_plugins() to see if any
 *        of the specified `plugin.library.paths` are found in the
 *        wheel's embedded library directory, and if so change the
 *        path to use these libraries.
 *
 * @returns a possibly updated plugin.library.paths string object which
 *          must be DECREF:ed, or NULL if an exception was raised.
 */
static PyObject *resolve_plugins(PyObject *plugins) {
        PyObject *resolved;
        PyObject *module, *function;

        module = PyImport_ImportModule("confluent_kafka");
        if (!module)
                return NULL;

        function = PyObject_GetAttrString(module, "_resolve_plugins");
        if (!function) {
                PyErr_SetString(PyExc_RuntimeError,
                                "confluent_kafka._resolve_plugins() not found");
                Py_DECREF(module);
                return NULL;
        }

        resolved = PyObject_CallFunctionObjArgs(function, plugins, NULL);

        Py_DECREF(function);
        Py_DECREF(module);

        if (!resolved) {
                PyErr_SetString(PyExc_RuntimeError,
                                "confluent_kafka._resolve_plugins() failed");
                return NULL;
        }

        return resolved;
}

/**
 * @brief Remove property from confidct and set rd_kafka_conf with its value
 *
 * @param vo The property value object
 *
 * @returns 1 on success or 0 on failure (exception raised).
 */
static int common_conf_set_special(PyObject *confdict,
                                   rd_kafka_conf_t *conf,
                                   const char *name,
                                   PyObject *vo) {
        const char *v;
        char errstr[256];
        PyObject *vs;
        PyObject *vs8 = NULL;

        if (!(vs = cfl_PyObject_Unistr(vo))) {
                PyErr_Format(PyExc_TypeError,
                             "expected configuration property %s "
                             "as type unicode string",
                             name);
                return 0;
        }

        v = cfl_PyUnistr_AsUTF8(vs, &vs8);
        if (rd_kafka_conf_set(conf, name, v, errstr, sizeof(errstr)) !=
            RD_KAFKA_CONF_OK) {
                cfl_PyErr_Format(RD_KAFKA_RESP_ERR__INVALID_ARG, "%s", errstr);
                Py_DECREF(vs);
                Py_XDECREF(vs8);
                return 0;
        }

        Py_DECREF(vs);
        Py_XDECREF(vs8);
        PyDict_DelItemString(confdict, name);
        return 1;
}


/**
 * @brief KIP-511: Set client.software.name and .version which is reported
 *          to the broker.
 *          Errors are ignored here since this is best-effort.
 */
static void common_conf_set_software(rd_kafka_conf_t *conf) {
        char version[128];

        rd_kafka_conf_set(conf, "client.software.name",
                          "confluent-kafka-python", NULL, 0);

        snprintf(version, sizeof(version), "%s-rdkafka-%s", CFL_VERSION_STR,
                 rd_kafka_version_str());
        rd_kafka_conf_set(conf, "client.software.version", version, NULL, 0);
}


/**
 * Common config setup for Kafka client handles.
 *
 * Returns a conf object on success or NULL on failure in which case
 * an exception has been raised.
 */
rd_kafka_conf_t *common_conf_setup(rd_kafka_type_t ktype,
                                   Handle *h,
                                   PyObject *args,
                                   PyObject *kwargs) {
        rd_kafka_conf_t *conf;
        Py_ssize_t pos = 0;
        PyObject *ko, *vo;
        PyObject *confdict = NULL;

        if (rd_kafka_version() < MIN_RD_KAFKA_VERSION) {
                PyErr_Format(PyExc_RuntimeError,
                             "%s: librdkafka version %s (0x%x) detected",
                             MIN_VER_ERRSTR, rd_kafka_version_str(),
                             rd_kafka_version());
                return NULL;
        }

        /* Supported parameter constellations:
         *  - kwargs (conf={..}, logger=..)
         *  - args and kwargs ({..}, logger=..)
         *  - args ({..})
         * When both args and kwargs are present the kwargs take
         * precedence in case of duplicate keys.
         * All keys map to configuration properties.
         *
         * Copy configuration dict to avoid manipulating application config.
         */
        if (args && PyTuple_Size(args)) {
                if (!PyTuple_Check(args) || PyTuple_Size(args) > 1) {
                        PyErr_SetString(
                            PyExc_TypeError,
                            "expected tuple containing single dict");
                        return NULL;
                } else if (PyTuple_Size(args) == 1 &&
                           !PyDict_Check(
                               (confdict = PyTuple_GetItem(args, 0)))) {
                        PyErr_SetString(PyExc_TypeError,
                                        "expected configuration dict");
                        return NULL;
                }
                confdict = PyDict_Copy(confdict);
        }

        if (!confdict) {
                if (!kwargs) {
                        PyErr_SetString(PyExc_TypeError,
                                        "expected configuration dict");
                        return NULL;
                }

                confdict = PyDict_Copy(kwargs);

        } else if (kwargs) {
                /* Update confdict with kwargs */
                PyDict_Update(confdict, kwargs);
        }

        if (ktype == RD_KAFKA_CONSUMER &&
            !PyDict_GetItemString(confdict, "group.id")) {

                PyErr_SetString(
                    PyExc_ValueError,
                    "Failed to create consumer: group.id must be set");
                Py_DECREF(confdict);
                return NULL;
        }

        conf = rd_kafka_conf_new();

        /* Set software name and verison prior to applying the confdict to
         * allow even higher-level clients to override it. */
        common_conf_set_software(conf);

        /*
         * Set debug contexts first to capture all events including plugin
         * loading
         */
        if ((vo = PyDict_GetItemString(confdict, "debug")) &&
            !common_conf_set_special(confdict, conf, "debug", vo))
                goto outer_err;

        /*
         * Plugins must be configured prior to handling any of their
         * configuration properties.
         * Dicts are unordered so we explicitly check for, set, and delete the
         * plugin paths here.
         * This ensures plugin configuration properties are handled in the
         * correct order.
         */
        if ((vo = PyDict_GetItemString(confdict, "plugin.library.paths"))) {
                /* Resolve plugin paths */
                PyObject *resolved;

                resolved = resolve_plugins(vo);
                if (!resolved)
                        goto outer_err;

                if (!common_conf_set_special(
                        confdict, conf, "plugin.library.paths", resolved)) {
                        Py_DECREF(resolved);
                        goto outer_err;
                }
                Py_DECREF(resolved);
        }

        if ((vo = PyDict_GetItemString(confdict, "default.topic.config"))) {
                /* TODO: uncomment for 1.0 release
                        PyErr_Warn(PyExc_DeprecationWarning,
                                     "default.topic.config has being deprecated,
                   " "set default topic configuration values in the global
                   dict");
                */
                if (PyDict_Update(confdict, vo) == -1) {
                        goto outer_err;
                }
                PyDict_DelItemString(confdict, "default.topic.config");
        }

        /* Convert config dict to config key-value pairs. */
        while (PyDict_Next(confdict, &pos, &ko, &vo)) {
                PyObject *ks;
                PyObject *ks8 = NULL;
                PyObject *vs = NULL, *vs8 = NULL;
                const char *k;
                const char *v;
                char errstr[256];
                int r = 0;

                if (!(ks = cfl_PyObject_Unistr(ko))) {
                        PyErr_SetString(PyExc_TypeError,
                                        "expected configuration property name "
                                        "as type unicode string");
                        goto inner_err;
                }

                k = cfl_PyUnistr_AsUTF8(ks, &ks8);
                if (!strcmp(k, "error_cb")) {
                        if (!PyCallable_Check(vo)) {
                                PyErr_SetString(PyExc_TypeError,
                                                "expected error_cb property "
                                                "as a callable function");
                                goto inner_err;
                        }
                        if (h->error_cb) {
                                Py_DECREF(h->error_cb);
                                h->error_cb = NULL;
                        }
                        if (vo != Py_None) {
                                h->error_cb = vo;
                                Py_INCREF(h->error_cb);
                        }
                        Py_XDECREF(ks8);
                        Py_DECREF(ks);
                        continue;
                } else if (!strcmp(k, "throttle_cb")) {
                        if (!PyCallable_Check(vo)) {
                                PyErr_SetString(PyExc_ValueError,
                                                "expected throttle_cb property "
                                                "as a callable function");
                                goto inner_err;
                        }
                        if (h->throttle_cb) {
                                Py_DECREF(h->throttle_cb);
                                h->throttle_cb = NULL;
                        }
                        if (vo != Py_None) {
                                h->throttle_cb = vo;
                                Py_INCREF(h->throttle_cb);
                        }
                        Py_XDECREF(ks8);
                        Py_DECREF(ks);
                        continue;
                } else if (!strcmp(k, "stats_cb")) {
                        if (!PyCallable_Check(vo)) {
                                PyErr_SetString(PyExc_TypeError,
                                                "expected stats_cb property "
                                                "as a callable function");
                                goto inner_err;
                        }

                        if (h->stats_cb) {
                                Py_DECREF(h->stats_cb);
                                h->stats_cb = NULL;
                        }
                        if (vo != Py_None) {
                                h->stats_cb = vo;
                                Py_INCREF(h->stats_cb);
                        }
                        Py_XDECREF(ks8);
                        Py_DECREF(ks);
                        continue;
                } else if (!strcmp(k, "logger")) {
                        if (h->logger) {
                                Py_DECREF(h->logger);
                                h->logger = NULL;
                        }

                        if (vo != Py_None) {
                                h->logger = vo;
                                Py_INCREF(h->logger);
                        }
                        Py_XDECREF(ks8);
                        Py_DECREF(ks);
                        continue;
                } else if (!strcmp(k, "oauth_cb")) {
                        if (!PyCallable_Check(vo)) {
                                PyErr_SetString(PyExc_TypeError,
                                                "expected oauth_cb property "
                                                "as a callable function");
                                goto inner_err;
                        }
                        if (h->oauth_cb) {
                                Py_DECREF(h->oauth_cb);
                                h->oauth_cb = NULL;
                        }

                        if (vo != Py_None) {
                                h->oauth_cb = vo;
                                Py_INCREF(h->oauth_cb);
                        }
                        Py_XDECREF(ks8);
                        Py_DECREF(ks);
                        continue;
                }

                /* Special handling for certain config keys. */
                if (ktype == RD_KAFKA_PRODUCER)
                        r = producer_conf_set_special(h, conf, k, vo);
                else if (ktype == RD_KAFKA_CONSUMER)
                        r = consumer_conf_set_special(h, conf, k, vo);
                if (r == -1) {
                        /* Error */
                        goto inner_err;
                } else if (r == 1) {
                        /* Handled */
                        continue;
                }


                /*
                 * Pass configuration property through to librdkafka.
                 */
                if (vo == Py_None) {
                        v = NULL;
                } else {
                        if (!(vs = cfl_PyObject_Unistr(vo))) {
                                PyErr_SetString(PyExc_TypeError,
                                                "expected configuration "
                                                "property value as type "
                                                "unicode string");
                                goto inner_err;
                        }
                        v = cfl_PyUnistr_AsUTF8(vs, &vs8);
                }

                if (rd_kafka_conf_set(conf, k, v, errstr, sizeof(errstr)) !=
                    RD_KAFKA_CONF_OK) {
                        cfl_PyErr_Format(RD_KAFKA_RESP_ERR__INVALID_ARG, "%s",
                                         errstr);
                        goto inner_err;
                }

                Py_XDECREF(vs8);
                Py_XDECREF(vs);
                Py_XDECREF(ks8);
                Py_DECREF(ks);
                continue;

        inner_err:
                Py_XDECREF(vs8);
                Py_XDECREF(vs);
                Py_XDECREF(ks8);
                Py_XDECREF(ks);
                goto outer_err;
        }

        Py_DECREF(confdict);

        rd_kafka_conf_set_error_cb(conf, error_cb);

        if (h->throttle_cb)
                rd_kafka_conf_set_throttle_cb(conf, throttle_cb);

        if (h->stats_cb)
                rd_kafka_conf_set_stats_cb(conf, stats_cb);

        if (h->logger) {
                /* Write logs to log queue (which is forwarded
                 * to the polled queue in the Producer/Consumer constructors) */
                rd_kafka_conf_set(conf, "log.queue", "true", NULL, 0);
                rd_kafka_conf_set_log_cb(conf, log_cb);
        }

        if (h->oauth_cb) {
                rd_kafka_conf_set_oauthbearer_token_refresh_cb(conf, oauth_cb);
                rd_kafka_conf_enable_sasl_queue(conf, 1);
        }

        rd_kafka_conf_set_opaque(conf, h);

#ifdef WITH_PY_TSS
        if (PyThread_tss_create(&h->tlskey)) {
                PyErr_SetString(PyExc_RuntimeError,
                                "Failed to initialize thread local storage");
                rd_kafka_conf_destroy(conf);
                return NULL;
        }
#else
        h->tlskey = PyThread_create_key();
#endif

        h->initiated = 1;

        return conf;

outer_err:
        Py_DECREF(confdict);
        rd_kafka_conf_destroy(conf);

        return NULL;
}



/**
 * @brief Initialiase a CallState and unlock the GIL prior to a
 *        possibly blocking external call.
 */
void CallState_begin(Handle *h, CallState *cs) {
        cs->thread_state = PyEval_SaveThread();
        assert(cs->thread_state != NULL);
        cs->crashed         = 0;
        cs->exception_value = NULL;
#ifdef WITH_PY_TSS
        PyThread_tss_set(&h->tlskey, cs);
#else
        PyThread_set_key_value(h->tlskey, cs);
#endif
}

/**
 * @brief Relock the GIL after external call is done.
 * @returns 0 if a Python signal was raised or a callback crashed, else 1.
 */
int CallState_end(Handle *h, CallState *cs) {
#ifdef WITH_PY_TSS
        PyThread_tss_set(&h->tlskey, NULL);
#else
        PyThread_delete_key_value(h->tlskey);
#endif

        PyEval_RestoreThread(cs->thread_state);

        if (PyErr_CheckSignals() == -1)
                return 0;

        if (cs->crashed) {
                /* Restore the saved exception if we have one */
                if (cs->exception_value) {
                        CallState_restore_exception(cs);
                }
                return 0;
        }

        return 1;
}


/**
 * @brief Get the current thread's CallState and re-locks the GIL.
 */
CallState *CallState_get(Handle *h) {
        CallState *cs;
#ifdef WITH_PY_TSS
        cs = PyThread_tss_get(&h->tlskey);
#else
        cs = PyThread_get_key_value(h->tlskey);
#endif
        assert(cs != NULL);
        assert(cs->thread_state != NULL);
        PyEval_RestoreThread(cs->thread_state);
        cs->thread_state = NULL;
        return cs;
}

/**
 * @brief Un-locks the GIL to resume blocking external call.
 */
void CallState_resume(CallState *cs) {
        assert(cs->thread_state == NULL);
        cs->thread_state = PyEval_SaveThread();
}

/**
 * @brief Indicate that call crashed.
 */
void CallState_crash(CallState *cs) {
        cs->crashed++;
}



/**
 * @brief Find class/type/object \p typename in \p modulename
 *
 * @returns a new reference to the object.
 *
 * @raises a TypeError exception if the type is not found.
 */

PyObject *cfl_PyObject_lookup(const char *modulename, const char *typename) {
        PyObject *module = PyImport_ImportModule(modulename);
        PyObject *obj;

        if (!module) {
                PyErr_Format(PyExc_ImportError,
                             "Module not found when looking up %s.%s",
                             modulename, typename);
                return NULL;
        }

        obj = PyObject_GetAttrString(module, typename);
        if (!obj) {
                Py_DECREF(module);
                PyErr_Format(PyExc_TypeError,
                             "No such class/type/object: %s.%s", modulename,
                             typename);
                return NULL;
        }

        return obj;
}


void cfl_PyDict_SetString(PyObject *dict, const char *name, const char *val) {
        PyObject *vo = cfl_PyUnistr(_FromString(val));
        PyDict_SetItemString(dict, name, vo);
        Py_DECREF(vo);
}

void cfl_PyDict_SetInt(PyObject *dict, const char *name, int val) {
        PyObject *vo = cfl_PyInt_FromInt(val);
        PyDict_SetItemString(dict, name, vo);
        Py_DECREF(vo);
}

void cfl_PyDict_SetLong(PyObject *dict, const char *name, long val) {
        PyObject *vo = cfl_PyLong_FromLong(val);
        PyDict_SetItemString(dict, name, vo);
        Py_DECREF(vo);
}

int cfl_PyObject_SetString(PyObject *o, const char *name, const char *val) {
        PyObject *vo = cfl_PyUnistr(_FromString(val));
        int r        = PyObject_SetAttrString(o, name, vo);
        Py_DECREF(vo);
        return r;
}

int cfl_PyObject_SetInt(PyObject *o, const char *name, int val) {
        PyObject *vo = cfl_PyInt_FromInt(val);
        int r        = PyObject_SetAttrString(o, name, vo);
        Py_DECREF(vo);
        return r;
}


/**
 * @brief Get attribute \p attr_name from \p object and verify it is
 *        of type \p py_type.
 *
 * @param py_type the value type of \p attr_name must match \p py_type, unless
 *                \p py_type is NULL.
 *
 * @returns 1 if \p valp was updated with the object (new reference) or NULL
 *          if not matched and not required, or
 *          0 if an exception was raised.
 */
int cfl_PyObject_GetAttr(PyObject *object,
                         const char *attr_name,
                         PyObject **valp,
                         const PyTypeObject *py_type,
                         int required,
                         int allow_None) {
        PyObject *o;

        o = PyObject_GetAttrString(object, attr_name);
        if (!o) {
                if (!required) {
                        *valp = NULL;
                        return 1;
                }

                PyErr_Format(PyExc_TypeError, "Required attribute .%s missing",
                             attr_name);
                return 0;
        }

        if (!(allow_None && o == Py_None) && py_type && Py_TYPE(o) != py_type) {
                Py_DECREF(o);
                PyErr_Format(PyExc_TypeError,
                             "Expected .%s to be %s type, not %s", attr_name,
                             py_type->tp_name,
                             ((PyTypeObject *)PyObject_Type(o))->tp_name);
                return 0;
        }

        *valp = o;

        return 1;
}

/**
 * @brief Get attribute \p attr_name from \p object and make sure it is
 *        an integer type.
 *
 * @returns 1 if \p valp was updated with either the object value, or \p defval.
 *          0 if an exception was raised.
 */
int cfl_PyObject_GetInt(PyObject *object,
                        const char *attr_name,
                        int *valp,
                        int defval,
                        int required) {
        PyObject *o;

        if (!cfl_PyObject_GetAttr(object, attr_name, &o,
#ifdef PY3
                                  &PyLong_Type,
#else
                                  &PyInt_Type,
#endif
                                  required, 0))
                return 0;

        if (!o) {
                *valp = defval;
                return 1;
        }

        *valp = cfl_PyInt_AsInt(o);
        Py_DECREF(o);

        return 1;
}


/**
 * @brief Checks that \p object is a bool (or boolable) and sets
 *        \p *valp according to the object.
 *
 * @returns 1 if \p valp was set, or 0 if \p object is not a boolable object.
 *          An exception is raised in the error case.
 */
int cfl_PyBool_get(PyObject *object, const char *name, int *valp) {
        if (!PyBool_Check(object)) {
                PyErr_Format(PyExc_TypeError,
                             "Expected %s to be bool type, not %s", name,
                             ((PyTypeObject *)PyObject_Type(object))->tp_name);
                return 0;
        }

        *valp = object == Py_True;

        return 1;
}

/**
 * @brief Get attribute \p attr_name from \p object and make sure it is
 *        a string type or None if \p allow_None is 1
 *
 * @returns 1 if \p valp was updated with a newly allocated copy of either the
 *          object value (UTF8), or \p defval or NULL if the attr is None
 *          0 if an exception was raised.
 */
int cfl_PyObject_GetString(PyObject *object,
                           const char *attr_name,
                           char **valp,
                           const char *defval,
                           int required,
                           int allow_None) {
        PyObject *o, *uo, *uop;

        if (!cfl_PyObject_GetAttr(object, attr_name, &o,
#ifdef PY3
                                  &PyUnicode_Type,
#else
                                  /* Python 2: support both str and unicode
                                   *           let PyObject_Unistr() do the
                                   *           proper conversion below. */
                                  NULL,
#endif
                                  required, allow_None))
                return 0;

        if (!o) {
                *valp = defval ? strdup(defval) : NULL;
                return 1;
        }

        if (o == Py_None) {
                Py_DECREF(o);
                *valp = NULL;
                return 1;
        }

        if (!(uo = cfl_PyObject_Unistr(o))) {
                Py_DECREF(o);
                PyErr_Format(PyExc_TypeError,
                             "Expected .%s to be a unicode string type, not %s",
                             attr_name,
                             ((PyTypeObject *)PyObject_Type(o))->tp_name);
                return 0;
        }
        Py_DECREF(o);

        *valp = (char *)cfl_PyUnistr_AsUTF8(uo, &uop);
        if (!*valp) {
                Py_DECREF(uo);
                Py_XDECREF(uop);
                return 0; /* exception raised by AsUTF8 */
        }

        *valp = strdup(*valp);
        Py_DECREF(uo);
        Py_XDECREF(uop);

        return 1;
}



/**
 * @returns a Python list of longs based on the input int32_t array
 */
PyObject *cfl_int32_array_to_py_list(const int32_t *arr, size_t cnt) {
        PyObject *list;
        size_t i;

        list = PyList_New((Py_ssize_t)cnt);
        if (!list)
                return NULL;

        for (i = 0; i < cnt; i++)
                PyList_SET_ITEM(list, (Py_ssize_t)i, cfl_PyInt_FromInt(arr[i]));

        return list;
}


/****************************************************************************
 *
 *
 * Methods common across all types of clients.
 *
 *
 *
 *
 ****************************************************************************/

const char set_sasl_credentials_doc[] = PyDoc_STR(
    ".. py:function:: set_sasl_credentials(username, password)\n"
    "\n"
    "  Sets the SASL credentials used for this client.\n"
    "  These credentials will overwrite the old ones, and will be used the "
    "next time the client needs to authenticate.\n"
    "  This method will not disconnect existing broker connections that have "
    "been established with the old credentials.\n"
    "  This method is applicable only to SASL PLAIN and SCRAM mechanisms.\n");


PyObject *set_sasl_credentials(Handle *self, PyObject *args, PyObject *kwargs) {
        const char *username = NULL;
        const char *password = NULL;
        rd_kafka_error_t *error;
        CallState cs;
        static char *kws[] = {"username", "password", NULL};

        if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ss", kws, &username,
                                         &password)) {
                return NULL;
        }

        CallState_begin(self, &cs);
        error = rd_kafka_sasl_set_credentials(self->rk, username, password);

        if (!CallState_end(self, &cs)) {
                if (error) /* Ignore error in favour of callstate exception */
                        rd_kafka_error_destroy(error);
                return NULL;
        }

        if (error) {
                cfl_PyErr_from_error_destroy(error);
                return NULL;
        }

        Py_RETURN_NONE;
}


/****************************************************************************
 *
 *
 * Base
 *
 *
 *
 *
 ****************************************************************************/


static PyObject *libversion(PyObject *self, PyObject *args) {
        return Py_BuildValue("si", rd_kafka_version_str(), rd_kafka_version());
}

/**
 * @brief confluent-kafka-python version.
 */
static PyObject *version(PyObject *self, PyObject *args) {
        return Py_BuildValue("s", CFL_VERSION_STR);
}


/****************************************************************************
 *
 *
 * Partitioner functions
 *
 *
 ****************************************************************************/

/**
 * @brief Type definition for librdkafka partitioner functions.
 */
typedef int32_t (*partitioner_func_t)(const rd_kafka_topic_t *rkt,
                                      const void *key,
                                      size_t keylen,
                                      int32_t partition_cnt,
                                      void *rkt_opaque,
                                      void *msg_opaque);

/**
 * @brief Helper function for partitioner wrappers.
 *
 * Handles common argument parsing, validation, and calling of
 * librdkafka partitioner functions.
 *
 * @param args Python arguments (key: bytes, partition_count: int)
 * @param partitioner_func librdkafka partitioner function to call
 *
 * @returns PyObject* - Partition ID as Python int, or NULL on error
 */
static PyObject *partitioner_helper(PyObject *args,
                                    partitioner_func_t partitioner_func) {
        const char *key;
        Py_ssize_t keylen;
        int partition_count;
        int32_t partition;

        if (!PyArg_ParseTuple(args, "y#i", &key, &keylen, &partition_count))
                return NULL;

        if (partition_count <= 0) {
                PyErr_Format(PyExc_ValueError,
                             "partition_count must be > 0, got %d",
                             partition_count);
                return NULL;
        }

        partition = partitioner_func(NULL, key, (size_t)keylen, partition_count,
                                     NULL, NULL);

        return cfl_PyInt_FromInt(partition);
}

/**
 * @brief Calculate partition using Murmur2 hash (Java-compatible).
 * Deterministic.
 */
static PyObject *murmur2(PyObject *self, PyObject *args) {
        return partitioner_helper(args, rd_kafka_msg_partitioner_murmur2);
}

/**
 * @brief Calculate partition using consistent hash (CRC32). Deterministic.
 */
static PyObject *consistent(PyObject *self, PyObject *args) {
        return partitioner_helper(args, rd_kafka_msg_partitioner_consistent);
}

/**
 * @brief Calculate partition using FNV-1a hash. Deterministic.
 */
static PyObject *fnv1a(PyObject *self, PyObject *args) {
        return partitioner_helper(args, rd_kafka_msg_partitioner_fnv1a);
}


static PyMethodDef cimpl_methods[] = {
    {"libversion", libversion, METH_NOARGS,
     "  Retrieve librdkafka version string and integer\n"
     "\n"
     "  :returns: version_string\n"
     "  :rtype: str\n"
     "\n"},
    {"version", version, METH_NOARGS,
     "  Retrieve module version string\n"
     "\n"
     "  :returns: version_string\n"
     "  :rtype: str\n"
     "\n"},
    {"murmur2", murmur2, METH_VARARGS,
     "  Calculate partition using Murmur2 hash (Java-compatible).\n"
     "\n"
     "  Deterministic function using Murmur2 hashing algorithm.\n"
     "\n"
     "  :param bytes key: The message key\n"
     "  :param int partition_count: Number of partitions (must be > 0)\n"
     "  :returns: Partition ID (0 to partition_count-1)\n"
     "  :rtype: int\n"
     "  :raises ValueError: If partition_count <= 0\n"
     "  :raises TypeError: If key is not bytes or partition_count is not int\n"
     "\n"
     "  Example::\n"
     "\n"
     "    partition = murmur2(b\"user_12345\", 10)\n"
     "\n"},
    {"consistent", consistent, METH_VARARGS,
     "  Calculate partition using CRC32-based consistent hash.\n"
     "\n"
     "  Deterministic function using CRC32 hashing algorithm.\n"
     "\n"
     "  :param bytes key: The message key\n"
     "  :param int partition_count: Number of partitions (must be > 0)\n"
     "  :returns: Partition ID (0 to partition_count-1)\n"
     "  :rtype: int\n"
     "  :raises ValueError: If partition_count <= 0\n"
     "  :raises TypeError: If key is not bytes or partition_count is not int\n"
     "\n"},
    {"fnv1a", fnv1a, METH_VARARGS,
     "  Calculate partition using FNV-1a hash.\n"
     "\n"
     "  Deterministic function using FNV-1a hashing algorithm.\n"
     "\n"
     "  :param bytes key: The message key\n"
     "  :param int partition_count: Number of partitions (must be > 0)\n"
     "  :returns: Partition ID (0 to partition_count-1)\n"
     "  :rtype: int\n"
     "  :raises ValueError: If partition_count <= 0\n"
     "  :raises TypeError: If key is not bytes or partition_count is not int\n"
     "\n"},
    {NULL}};


/**
 * @brief Add librdkafka error enums to KafkaError's type dict.
 * @returns an updated doc string containing all error constants.
 */
static char *KafkaError_add_errs(PyObject *dict, const char *origdoc) {
        const struct rd_kafka_err_desc *descs;
        size_t cnt;
        size_t i;
        char *doc;
        size_t dof = 0, dsize;
        /* RST grid table column widths */
#define _COL1_W 50
#define _COL2_W 100 /* Must be larger than COL1 */
        char dash[_COL2_W], eq[_COL2_W];

        rd_kafka_get_err_descs(&descs, &cnt);

        memset(dash, '-', sizeof(dash));
        memset(eq, '=', sizeof(eq));

        /* Setup output doc buffer. */
        dof   = strlen(origdoc);
        dsize = dof + 500 + (cnt * 200);
        doc   = malloc(dsize);
        memcpy(doc, origdoc, dof + 1);

#define _PRINT(...)                                                            \
        do {                                                                   \
                char tmpdoc[512];                                              \
                size_t _len;                                                   \
                _len = snprintf(tmpdoc, sizeof(tmpdoc), __VA_ARGS__);          \
                if (_len > sizeof(tmpdoc))                                     \
                        _len = sizeof(tmpdoc) - 1;                             \
                if (dof + _len >= dsize) {                                     \
                        dsize += 2;                                            \
                        doc = realloc(doc, dsize);                             \
                }                                                              \
                memcpy(doc + dof, tmpdoc, _len + 1);                           \
                dof += _len;                                                   \
        } while (0)

        /* Error constant table header (RST grid table) */
        _PRINT(
            "Error and event constants:\n\n"
            "+-%.*s-+-%.*s-+\n"
            "| %-*.*s | %-*.*s |\n"
            "+=%.*s=+=%.*s=+\n",
            _COL1_W, dash, _COL2_W, dash, _COL1_W, _COL1_W, "Constant", _COL2_W,
            _COL2_W, "Description", _COL1_W, eq, _COL2_W, eq);

        for (i = 0; i < cnt; i++) {
                PyObject *code;

                if (!descs[i].desc)
                        continue;

                code = cfl_PyInt_FromInt(descs[i].code);

                PyDict_SetItemString(dict, descs[i].name, code);

                Py_DECREF(code);

                _PRINT(
                    "| %-*.*s | %-*.*s |\n"
                    "+-%.*s-+-%.*s-+\n",
                    _COL1_W, _COL1_W, descs[i].name, _COL2_W, _COL2_W,
                    descs[i].desc, _COL1_W, dash, _COL2_W, dash);
        }

        _PRINT("\n");

        return doc; /* FIXME: leak */
}


#ifdef PY3
static struct PyModuleDef cimpl_moduledef = {
    PyModuleDef_HEAD_INIT,
    "cimpl", /* m_name */
    "Confluent's Python client for Apache Kafka (C implementation)", /* m_doc */
    -1,            /* m_size */
    cimpl_methods, /* m_methods */
};
#endif


static PyObject *_init_cimpl(void) {
        PyObject *m;

/* PyEval_InitThreads became deprecated in Python 3.9 and will be removed in
 * Python 3.11. Prior to Python 3.7, this call was required to initialize the
 * GIL. */
#if PY_VERSION_HEX < 0x03090000
        PyEval_InitThreads();
#endif

        if (PyType_Ready(&KafkaErrorType) < 0)
                return NULL;
        if (PyType_Ready(&MessageType) < 0)
                return NULL;
        if (PyType_Ready(&UuidType) < 0)
                return NULL;
        if (PyType_Ready(&TopicPartitionType) < 0)
                return NULL;
        if (PyType_Ready(&ProducerType) < 0)
                return NULL;
        if (PyType_Ready(&ConsumerType) < 0)
                return NULL;
        if (PyType_Ready(&AdminType) < 0)
                return NULL;
        if (AdminTypes_Ready() < 0)
                return NULL;

#ifdef PY3
        m = PyModule_Create(&cimpl_moduledef);
#else
        m = Py_InitModule3(
            "cimpl", cimpl_methods,
            "Confluent's Python client for Apache Kafka (C implementation)");
#endif
        if (!m)
                return NULL;

        Py_INCREF(&KafkaErrorType);
        KafkaErrorType.tp_doc =
            KafkaError_add_errs(KafkaErrorType.tp_dict, KafkaErrorType.tp_doc);
        PyModule_AddObject(m, "KafkaError", (PyObject *)&KafkaErrorType);

        Py_INCREF(&MessageType);
        PyModule_AddObject(m, "Message", (PyObject *)&MessageType);

        Py_INCREF(&UuidType);
        PyModule_AddObject(m, "Uuid", (PyObject *)&UuidType);

        Py_INCREF(&TopicPartitionType);
        PyModule_AddObject(m, "TopicPartition",
                           (PyObject *)&TopicPartitionType);

        Py_INCREF(&ProducerType);
        PyModule_AddObject(m, "Producer", (PyObject *)&ProducerType);

        Py_INCREF(&ConsumerType);
        PyModule_AddObject(m, "Consumer", (PyObject *)&ConsumerType);

        Py_INCREF(&AdminType);
        PyModule_AddObject(m, "_AdminClientImpl", (PyObject *)&AdminType);

        AdminTypes_AddObjects(m);

#if PY_VERSION_HEX >= 0x02070000
        KafkaException = PyErr_NewExceptionWithDoc(
            "cimpl.KafkaException",
            "Kafka exception that wraps the :py:class:`KafkaError` "
            "class.\n"
            "\n"
            "Use ``exception.args[0]`` to extract the "
            ":py:class:`KafkaError` object\n"
            "\n",
            NULL, NULL);
#else
        KafkaException = PyErr_NewException("cimpl.KafkaException", NULL, NULL);
#endif
        Py_INCREF(KafkaException);
        PyModule_AddObject(m, "KafkaException", KafkaException);

        PyModule_AddIntConstant(m, "TIMESTAMP_NOT_AVAILABLE",
                                RD_KAFKA_TIMESTAMP_NOT_AVAILABLE);
        PyModule_AddIntConstant(m, "TIMESTAMP_CREATE_TIME",
                                RD_KAFKA_TIMESTAMP_CREATE_TIME);
        PyModule_AddIntConstant(m, "TIMESTAMP_LOG_APPEND_TIME",
                                RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME);

        PyModule_AddIntConstant(m, "OFFSET_BEGINNING",
                                RD_KAFKA_OFFSET_BEGINNING);
        PyModule_AddIntConstant(m, "OFFSET_END", RD_KAFKA_OFFSET_END);
        PyModule_AddIntConstant(m, "OFFSET_STORED", RD_KAFKA_OFFSET_STORED);
        PyModule_AddIntConstant(m, "OFFSET_INVALID", RD_KAFKA_OFFSET_INVALID);

        return m;
}


#ifdef PY3
PyMODINIT_FUNC PyInit_cimpl(void) {
        return _init_cimpl();
}
#else
PyMODINIT_FUNC initcimpl(void) {
        _init_cimpl();
}
#endif
