// @generated by UpstreamBorrow.py.
// See the Buck target fbcode//cinderx/UpstreamBorrow:gen_borrowed.c.

// -*- mode: c++ -*- vi: set ft=cpp:
// Copyright (c) Meta Platforms, Inc. and affiliates.

// This file is processed by UpstreamBorrow.py. To see the generated output:
// buck build -m ovr_config//third-party/python/constraints:3.15 \
//     --out=- fbcode//cinderx/UpstreamBorrow:borrowed_gen_borrowed.c

// clang-format off

#include "cinderx/UpstreamBorrow/borrowed.h"
#include "cinderx/module_c_state.h"
#include "internal/pycore_intrinsics.h"
#include "internal/pycore_bitutils.h"
#include "internal/pycore_call.h"
#include "internal/pycore_ceval.h"
#include "internal/pycore_critical_section.h"
#include "internal/pycore_floatobject.h"
#include "internal/pycore_genobject.h"
#include "internal/pycore_interpframe.h"
#include "internal/pycore_freelist.h"
#include "internal/pycore_tuple.h"
#include "internal/pycore_unicodeobject.h"
#include "internal/pycore_weakref.h"
#include "cinderx/Interpreter/cinder_opcode.h"
#include "internal/pycore_function.h"
#include "internal/pycore_descrobject.h"
#include "internal/pycore_long.h"
#include "internal/pycore_optimizer.h"     // _PyExecutorObject
#include "internal/pycore_lazyimportobject.h"
#ifdef ENABLE_LAZY_IMPORTS
#include "pycore_import.h"        // _PyImport_LoadLazyImport()
#include "pycore_lazyimport.h"    // _PyLazyImport_New(), _PyLazyImport_GetName()
#endif
#include "opcode.h"

#define _PyOpcode_Caches _CiOpcode_Caches
#define _PyOpcode_Deopt _CiOpcode_Deopt
#define _Py_type_getattro PyType_Type.tp_getattro

#undef _PyObject_GC_TRACK
#define _PyObject_GC_TRACK PyObject_GC_Track

getattrofunc Ci_tp_getattr_hook, Ci_tp_getattro;

// _Py_slot_tp_getattr_hook is used when __getattr__ is defined
#define _Py_slot_tp_getattr_hook Ci_tp_getattr_hook
// _Py_slot_tp_getattro is used when __getattribute__ is defined
#define _Py_slot_tp_getattro Ci_tp_getattro

int unicode_eq(PyObject *str1, PyObject *str2)
{
    Py_ssize_t len = PyUnicode_GET_LENGTH(str1);
    if (PyUnicode_GET_LENGTH(str2) != len) {
        return 0;
    }

    int kind = PyUnicode_KIND(str1);
    if (PyUnicode_KIND(str2) != kind) {
        return 0;
    }

    const void *data1 = PyUnicode_DATA(str1);
    const void *data2 = PyUnicode_DATA(str2);
    return (memcmp(data1, data2, len * kind) == 0);
}

PyObject *
_PyType_LookupRefAndVersion(PyTypeObject *type, PyObject *name, unsigned int *version) {
  // This is a bit much to borrow as the dependency tree goes deep... but the version
  // is the version at the start before the MRO
  bool has_version = type->tp_version_tag != 0;
  if (!has_version && PyUnstable_Type_AssignVersionTag(type)) {
    has_version = true;
  }
  unsigned int start_version = type->tp_version_tag;
  PyObject *res = _PyType_LookupRef(type, name);
  *version = has_version ? start_version : 0;
  return res;
}

#ifdef ENABLE_PEP523_HOOK
_PyFrameEvalFunction Ci_EvalFrameFunc;
#else
#define Ci_EvalFrameFunc NULL
#endif

// In 3.12 _PyAsyncGenValueWrapperNew needs thread-state. As this is used from
// the JIT we could get the value from the thread-state register. This would be
// slightly more efficient, but quite a bit more work and async-generators are
// rare. So we just wrap it up here.
PyObject* Cix_PyAsyncGenValueWrapperNew(PyObject* value) {
  return _PyAsyncGenValueWrapperNew(PyThreadState_GET(), value);
}

#define _PyGen_CAST(op) \
    _Py_CAST(PyGenObject*, (op))

static PyObject *
compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame)
{
    _PyInterpreterFrame *frame = current_frame;
    /* First count how many frames we have */
    int frame_count = 0;
    for (; frame && frame_count < origin_depth; ++frame_count) {
        frame = _PyFrame_GetFirstComplete(frame->previous);
    }

    /* Now collect them */
    PyObject *cr_origin = PyTuple_New(frame_count);
    if (cr_origin == NULL) {
        return NULL;
    }
    frame = current_frame;
    for (int i = 0; i < frame_count; ++i) {
        PyCodeObject *code = _PyFrame_GetCode(frame);
        int line = PyUnstable_InterpreterFrame_GetLine(frame);
        PyObject *frameinfo = Py_BuildValue("OiO", code->co_filename, line,
                                            code->co_name);
        if (!frameinfo) {
            Py_DECREF(cr_origin);
            return NULL;
        }
        PyTuple_SET_ITEM(cr_origin, i, frameinfo);
        frame = _PyFrame_GetFirstComplete(frame->previous);
    }

    return cr_origin;
}
PyObject* Cix_compute_cr_origin(int origin_depth, _PyInterpreterFrame* current_frame) {
  return compute_cr_origin(origin_depth, current_frame);
}

uint32_t
_PyFunction_GetVersionForCurrentState(PyFunctionObject *func)
{
    return func->func_version;
}
const PyConfig*
_PyInterpreterState_GetConfig(PyInterpreterState *interp)
{
    return &interp->config;
}


#define PyDict_LOG_MINSIZE 3
#define PyDict_MINSIZE 8
#ifdef Py_GIL_DISABLED
#define ASSERT_DICT_LOCKED(op) ASSERT_DICT_LOCKED(_Py_CAST(PyObject*, op))
#define ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op)                         \
    if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) {       \
        ASSERT_DICT_LOCKED(op);                                         \
    }
#define ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(op)                         \
    if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) {      \
        _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);                 \
    }
#define IS_DICT_SHARED(mp) _PyObject_GC_IS_SHARED(mp)
#define SET_DICT_SHARED(mp) _PyObject_GC_SET_SHARED(mp)
#define LOAD_INDEX(keys, size, idx) _Py_atomic_load_int##size##_relaxed(&((const int##size##_t*)keys->dk_indices)[idx]);
#define STORE_INDEX(keys, size, idx, value) _Py_atomic_store_int##size##_relaxed(&((int##size##_t*)keys->dk_indices)[idx], (int##size##_t)value);
#define ASSERT_OWNED_OR_SHARED(mp) \
    assert(_Py_IsOwnedByCurrentThread((PyObject *)mp) || IS_DICT_SHARED(mp));
#define LOCK_KEYS_IF_SPLIT(keys, kind) \
        if (kind == DICT_KEYS_SPLIT) { \
            LOCK_KEYS(keys);           \
        }
#define UNLOCK_KEYS_IF_SPLIT(keys, kind) \
        if (kind == DICT_KEYS_SPLIT) {   \
            UNLOCK_KEYS(keys);           \
        }
#define LOCK_KEYS(keys) PyMutex_LockFlags(&keys->dk_mutex, _Py_LOCK_DONT_DETACH)
#define UNLOCK_KEYS(keys) PyMutex_Unlock(&keys->dk_mutex)
#define ASSERT_KEYS_LOCKED(keys) assert(PyMutex_IsLocked(&keys->dk_mutex))
#define LOAD_SHARED_KEY(key) _Py_atomic_load_ptr_acquire(&key)
#define STORE_SHARED_KEY(key, value) _Py_atomic_store_ptr_release(&key, value)
#define INCREF_KEYS(dk)  _Py_atomic_add_ssize(&dk->dk_refcnt, 1)
#define DECREF_KEYS(dk)  _Py_atomic_add_ssize(&dk->dk_refcnt, -1)
#define LOAD_KEYS_NENTRIES(keys) _Py_atomic_load_ssize_relaxed(&keys->dk_nentries)
#define INCREF_KEYS_FT(dk) dictkeys_incref(dk)
#define DECREF_KEYS_FT(dk, shared) dictkeys_decref(dk, shared)
#else /* Py_GIL_DISABLED */
#define ASSERT_DICT_LOCKED(op)
#define ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op)
#define ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(op)
#define LOCK_KEYS(keys)
#define UNLOCK_KEYS(keys)
#define ASSERT_KEYS_LOCKED(keys)
#define LOAD_SHARED_KEY(key) key
#define STORE_SHARED_KEY(key, value) key = value
#define INCREF_KEYS(dk)  dk->dk_refcnt++
#define DECREF_KEYS(dk)  dk->dk_refcnt--
#define LOAD_KEYS_NENTRIES(keys) keys->dk_nentries
#define INCREF_KEYS_FT(dk)
#define DECREF_KEYS_FT(dk, shared)
#define LOCK_KEYS_IF_SPLIT(keys, kind)
#define UNLOCK_KEYS_IF_SPLIT(keys, kind)
#define IS_DICT_SHARED(mp) (false)
#define SET_DICT_SHARED(mp)
#define LOAD_INDEX(keys, size, idx) ((const int##size##_t*)(keys->dk_indices))[idx]
#define STORE_INDEX(keys, size, idx, value) ((int##size##_t*)(keys->dk_indices))[idx] = (int##size##_t)value
#endif
#ifndef NDEBUG
#endif
#define _PyAnyDict_CAST(op) \
    (assert(PyAnyDict_Check(op)), _Py_CAST(PyDictObject*, op))
#define GET_USED(ep) FT_ATOMIC_LOAD_SSIZE_RELAXED((ep)->ma_used)
#define STORE_KEY(ep, key) FT_ATOMIC_STORE_PTR_RELEASE((ep)->me_key, key)
#define STORE_VALUE(ep, value) FT_ATOMIC_STORE_PTR_RELEASE((ep)->me_value, value)
#define STORE_SPLIT_VALUE(mp, idx, value) FT_ATOMIC_STORE_PTR_RELEASE(mp->ma_values->values[idx], value)
#define STORE_HASH(ep, hash) FT_ATOMIC_STORE_SSIZE_RELAXED((ep)->me_hash, hash)
#define STORE_KEYS_USABLE(keys, usable) FT_ATOMIC_STORE_SSIZE_RELAXED(keys->dk_usable, usable)
#define STORE_KEYS_NENTRIES(keys, nentries) FT_ATOMIC_STORE_SSIZE_RELAXED(keys->dk_nentries, nentries)
#define STORE_USED(mp, used) FT_ATOMIC_STORE_SSIZE_RELAXED(mp->ma_used, used)
#define PERTURB_SHIFT 5
#ifndef NDEBUG
#endif
#define DK_MASK(dk) (DK_SIZE(dk)-1)
#define _Py_DICT_IMMORTAL_INITIAL_REFCNT PY_SSIZE_T_MIN
#ifdef Py_REF_DEBUG
#endif
#ifdef Py_REF_DEBUG
#endif
#if SIZEOF_VOID_P > 4
#endif
#if SIZEOF_VOID_P > 4
#endif
#define USABLE_FRACTION(n) (((n) << 1)/3)
#if SIZEOF_LONG == SIZEOF_SIZE_T
#elif defined(_MSC_VER)
#else
#endif
#define GROWTH_RATE(d) ((d)->ma_used*3)
#ifdef Py_GIL_DISABLED
#endif
#define Py_EMPTY_KEYS &empty_keys_struct
#ifdef DEBUG_PYDICT
#  define ASSERT_CONSISTENT(op) assert(_PyDict_CheckConsistency((PyObject *)(op), 1))
#else
#  define ASSERT_CONSISTENT(op) assert(_PyDict_CheckConsistency((PyObject *)(op), 0))
#endif
#ifdef DEBUG_PYDICT
#endif
#define CHECK(expr) \
    do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0)
#undef CHECK
#if SIZEOF_VOID_P > 4
#endif
#ifdef Py_REF_DEBUG
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys)
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_REF_DEBUG
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#else   // Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_REF_DEBUG
#endif
#ifdef Py_DEBUG
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#define frozendict_does_not_support(WHAT) \
    PyErr_SetString(PyExc_TypeError, "frozendict object does " \
                    "not support item " WHAT)
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#else /* Py_GIL_DISABLED */
#endif  /* Py_GIL_DISABLED */
#ifdef Py_GIL_DISABLED
#else
#endif
#ifndef Py_GIL_DISABLED
#endif  /* Py_GIL_DISABLED */
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_STATS
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#if 0
#define CHECK(val) assert(val); if (!(val)) { return 0; }
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifndef NDEBUG
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifndef NDEBUG
#endif
// Several functions in dictobject.c check for exact pointer equality on Py_EMPTY_KEYS
// but the global it points to is only locally visible. Fortunately, we can extract
// it by making a new empty dictionary and extracting the keys. We do this on CinderX
// module initialization.
#undef Py_EMPTY_KEYS
PyDictKeysObject* ci_dict_empty_keys;
#define Py_EMPTY_KEYS ci_dict_empty_keys

// This was mostly generated with:
//   buck run fbcode//cinderx/UpstreamBorrow:callgraph -- --py_version 3.15 <(echo '{"Objects/dictobject.c": ["_PyObjectDict_SetItem", "_PyDict_LookupIndex", "_Py_dict_lookup",  "_PyDictKeys_StringLookup", "_PyDictKeys_StringLookupSplit", "_PyDictKeys_GetVersionForCurrentState", "_PyDictKeys_StringLookupAndVersion", "_PyDict_GetKeysVersionForCurrentState"]}')
// except compare_unicode_unicode, compare_unicode_generic, compare_generic.
// These are referred to by value rather than call expression and :callgraph
// does not handle this.
// We also had to explicitly remove PyDict_New() because we need the original
// version to initialize ci_dict_empty_keys.
//
static inline int
can_modify_dict(PyDictObject *mp)
{
    if (PyFrozenDict_Check(mp)) {
        // No locking required to modify a newly created frozendict
        // since it's only accessible from the current thread.
        return PyUnstable_Object_IsUniquelyReferenced(_PyObject_CAST(mp));
    }
    else {
        // Locking is only required if the dictionary is not
        // uniquely referenced.
        ASSERT_DICT_LOCKED(mp);
        return 1;
    }
}
static size_t
values_size_from_count(size_t count)
{
    assert(count >= 1);
    size_t suffix_size = _Py_SIZE_ROUND_UP(count, sizeof(PyObject *));
    assert(suffix_size < 128);
    assert(suffix_size % sizeof(PyObject *) == 0);
    return (count + 1) * sizeof(PyObject *) + suffix_size;
}
static inline PyDictValues*
new_values(size_t size)
{
    size_t n = values_size_from_count(size);
    PyDictValues *res = (PyDictValues *)PyMem_Malloc(n);
    if (res == NULL) {
        return NULL;
    }
    res->embedded = 0;
    res->size = 0;
    assert(size < 256);
    res->capacity = (uint8_t)size;
    return res;
}
static inline void
dictkeys_incref(PyDictKeysObject *dk)
{
    if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) < 0) {
        assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_DICT_IMMORTAL_INITIAL_REFCNT);
        return;
    }
#ifdef Py_REF_DEBUG
    _Py_IncRefTotal(_PyThreadState_GET());
#endif
    INCREF_KEYS(dk);
}
static void
free_keys_object(PyDictKeysObject *keys, bool use_qsbr)
{
#ifdef Py_GIL_DISABLED
    if (use_qsbr) {
        _PyMem_FreeDelayed(keys, _PyDict_KeysSize(keys));
        return;
    }
#endif
    if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE && keys->dk_kind == DICT_KEYS_UNICODE) {
        _Py_FREELIST_FREE(dictkeys, keys, PyMem_Free);
    }
    else {
        PyMem_Free(keys);
    }
}
static inline void
dictkeys_decref(PyDictKeysObject *dk, bool use_qsbr)
{
    if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) < 0) {
        assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_DICT_IMMORTAL_INITIAL_REFCNT);
        return;
    }
    assert(FT_ATOMIC_LOAD_SSIZE(dk->dk_refcnt) > 0);
#ifdef Py_REF_DEBUG
    _Py_DecRefTotal(_PyThreadState_GET());
#endif
    if (DECREF_KEYS(dk) == 1) {
        if (DK_IS_UNICODE(dk)) {
            PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dk);
            Py_ssize_t i, n;
            for (i = 0, n = dk->dk_nentries; i < n; i++) {
                Py_XDECREF(entries[i].me_key);
                Py_XDECREF(entries[i].me_value);
            }
        }
        else {
            PyDictKeyEntry *entries = DK_ENTRIES(dk);
            Py_ssize_t i, n;
            for (i = 0, n = dk->dk_nentries; i < n; i++) {
                Py_XDECREF(entries[i].me_key);
                Py_XDECREF(entries[i].me_value);
            }
        }
        free_keys_object(dk, use_qsbr);
    }
}
static inline void
free_values(PyDictValues *values, bool use_qsbr)
{
    assert(values->embedded == 0);
#ifdef Py_GIL_DISABLED
    if (use_qsbr) {
        _PyMem_FreeDelayed(values, values_size_from_count(values->capacity));
        return;
    }
#endif
    PyMem_Free(values);
}
static inline Py_ssize_t
dictkeys_get_index(const PyDictKeysObject *keys, Py_ssize_t i)
{
    int log2size = DK_LOG_SIZE(keys);
    Py_ssize_t ix;

    if (log2size < 8) {
        ix = LOAD_INDEX(keys, 8, i);
    }
    else if (log2size < 16) {
        ix = LOAD_INDEX(keys, 16, i);
    }
#if SIZEOF_VOID_P > 4
    else if (log2size >= 32) {
        ix = LOAD_INDEX(keys, 64, i);
    }
#endif
    else {
        ix = LOAD_INDEX(keys, 32, i);
    }
    assert(ix >= DKIX_DUMMY);
    return ix;
}
static inline Py_hash_t
unicode_get_hash(PyObject *o)
{
    return PyUnstable_Unicode_GET_CACHED_HASH(o);
}
static inline int
compare_unicode_unicode(PyDictObject *mp, PyDictKeysObject *dk,
                        void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash)
{
    PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix];
    PyObject *ep_key = FT_ATOMIC_LOAD_PTR_CONSUME(ep->me_key);
    assert(ep_key != NULL);
    assert(PyUnicode_CheckExact(ep_key));
    if (ep_key == key ||
            (unicode_get_hash(ep_key) == hash && unicode_eq(ep_key, key))) {
        return 1;
    }
    return 0;
}
static inline int
compare_unicode_generic(PyDictObject *mp, PyDictKeysObject *dk,
                        void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash)
{
    PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix];
    assert(ep->me_key != NULL);
    assert(PyUnicode_CheckExact(ep->me_key));
    assert(!PyUnicode_CheckExact(key));

    if (unicode_get_hash(ep->me_key) == hash) {
        PyObject *startkey = ep->me_key;
        Py_INCREF(startkey);
        int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ);
        Py_DECREF(startkey);
        if (cmp < 0) {
            return DKIX_ERROR;
        }
        if (dk == mp->ma_keys && ep->me_key == startkey) {
            return cmp;
        }
        else {
            /* The dict was mutated, restart */
            return DKIX_KEY_CHANGED;
        }
    }
    return 0;
}
static inline int
compare_generic(PyDictObject *mp, PyDictKeysObject *dk,
                void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash)
{
    PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix];
    assert(ep->me_key != NULL);
    if (ep->me_key == key) {
        return 1;
    }
    if (ep->me_hash == hash) {
        PyObject *startkey = ep->me_key;
        Py_INCREF(startkey);
        int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ);
        Py_DECREF(startkey);
        if (cmp < 0) {
            return DKIX_ERROR;
        }
        if (dk == mp->ma_keys && ep->me_key == startkey) {
            return cmp;
        }
        else {
            /* The dict was mutated, restart */
            return DKIX_KEY_CHANGED;
        }
    }
    return 0;
}
static inline int
get_index_from_order(PyDictObject *mp, Py_ssize_t i)
{
    assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE);
    assert(i < mp->ma_values->size);
    uint8_t *array = get_insertion_order_array(mp->ma_values);
    return array[i];
}
int
_PyDict_CheckConsistency(PyObject *op, int check_content)
{
    ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op);

#define CHECK(expr) \
    do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0)

    assert(op != NULL);
    CHECK(PyAnyDict_Check(op));
    PyDictObject *mp = (PyDictObject *)op;

    PyDictKeysObject *keys = mp->ma_keys;
    int splitted = _PyDict_HasSplitTable(mp);
    Py_ssize_t usable = USABLE_FRACTION(DK_SIZE(keys));

    // In the free-threaded build, shared keys may be concurrently modified,
    // so use atomic loads.
    Py_ssize_t dk_usable = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_usable);
    Py_ssize_t dk_nentries = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_nentries);

    CHECK(0 <= mp->ma_used && mp->ma_used <= usable);
    CHECK(0 <= dk_usable && dk_usable <= usable);
    CHECK(0 <= dk_nentries && dk_nentries <= usable);
    CHECK(dk_usable + dk_nentries <= usable);

    if (!splitted) {
        /* combined table */
        CHECK(keys->dk_kind != DICT_KEYS_SPLIT);
        CHECK(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS);
    }
    else {
        CHECK(keys->dk_kind == DICT_KEYS_SPLIT);
        CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE);
        if (mp->ma_values->embedded) {
            CHECK(mp->ma_values->embedded == 1);
            CHECK(mp->ma_values->valid == 1);
        }
    }

    if (check_content) {
        LOCK_KEYS_IF_SPLIT(keys, keys->dk_kind);
        for (Py_ssize_t i=0; i < DK_SIZE(keys); i++) {
            Py_ssize_t ix = dictkeys_get_index(keys, i);
            CHECK(DKIX_DUMMY <= ix && ix <= usable);
        }

        if (keys->dk_kind == DICT_KEYS_GENERAL) {
            PyDictKeyEntry *entries = DK_ENTRIES(keys);
            for (Py_ssize_t i=0; i < usable; i++) {
                PyDictKeyEntry *entry = &entries[i];
                PyObject *key = entry->me_key;

                if (key != NULL) {
                    /* test_dict fails if PyObject_Hash() is called again */
                    CHECK(entry->me_hash != -1);
                    CHECK(entry->me_value != NULL);

                    if (PyUnicode_CheckExact(key)) {
                        Py_hash_t hash = unicode_get_hash(key);
                        CHECK(entry->me_hash == hash);
                    }
                }
            }
        }
        else {
            PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys);
            for (Py_ssize_t i=0; i < usable; i++) {
                PyDictUnicodeEntry *entry = &entries[i];
                PyObject *key = entry->me_key;

                if (key != NULL) {
                    CHECK(PyUnicode_CheckExact(key));
                    Py_hash_t hash = unicode_get_hash(key);
                    CHECK(hash != -1);
                    if (!splitted) {
                        CHECK(entry->me_value != NULL);
                    }
                }

                if (splitted) {
                    CHECK(entry->me_value == NULL);
                }
            }
        }

        if (splitted) {
            CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE);
            /* splitted table */
            int duplicate_check = 0;
            for (Py_ssize_t i=0; i < mp->ma_used; i++) {
                int index = get_index_from_order(mp, i);
                CHECK((duplicate_check & (1<<index)) == 0);
                duplicate_check |= (1<<index);
                CHECK(mp->ma_values->values[index] != NULL);
            }
        }
        UNLOCK_KEYS_IF_SPLIT(keys, keys->dk_kind);
    }
    return 1;

#undef CHECK
}
static inline PyObject *
new_dict_impl(PyDictObject *mp, PyDictKeysObject *keys,
              PyDictValues *values, Py_ssize_t used,
              int free_values_on_failure)
{
    assert(keys != NULL);
    if (mp == NULL) {
        dictkeys_decref(keys, false);
        if (free_values_on_failure) {
            free_values(values, false);
        }
        return NULL;
    }

    mp->ma_keys = keys;
    mp->ma_values = values;
    mp->ma_used = used;
    mp->_ma_watcher_tag = 0;
    ASSERT_CONSISTENT(mp);
    _PyObject_GC_TRACK(mp);
    return (PyObject *)mp;
}
static PyObject *
new_dict(PyDictKeysObject *keys, PyDictValues *values,
         Py_ssize_t used, int free_values_on_failure)
{
    PyDictObject *mp = _Py_FREELIST_POP(PyDictObject, dicts);
    if (mp == NULL) {
        mp = PyObject_GC_New(PyDictObject, &PyDict_Type);
    }
    assert(mp == NULL || Py_IS_TYPE(mp, &PyDict_Type));

    return new_dict_impl(mp, keys, values, used, free_values_on_failure);
}
static PyObject *
new_dict_with_shared_keys(PyDictKeysObject *keys)
{
    size_t size = shared_keys_usable_size(keys);
    PyDictValues *values = new_values(size);
    if (values == NULL) {
        return PyErr_NoMemory();
    }
    dictkeys_incref(keys);
    for (size_t i = 0; i < size; i++) {
        values->values[i] = NULL;
    }
    return new_dict(keys, values, 0, 1);
}
static inline PyObject *
ensure_nonmanaged_dict(PyObject *obj, PyObject **dictptr)
{
    PyDictKeysObject *cached;

    PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*dictptr);
    if (dict == NULL) {
#ifdef Py_GIL_DISABLED
        Py_BEGIN_CRITICAL_SECTION(obj);
        dict = *dictptr;
        if (dict != NULL) {
            goto done;
        }
#endif
        PyTypeObject *tp = Py_TYPE(obj);
        if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) {
            assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES));
            dict = new_dict_with_shared_keys(cached);
        }
        else {
            dict = PyDict_New();
        }
        FT_ATOMIC_STORE_PTR_RELEASE(*dictptr, dict);
#ifdef Py_GIL_DISABLED
done:
        Py_END_CRITICAL_SECTION();
#endif
    }
    return dict;
}
static void
dict_unhashable_type(PyObject *op, PyObject *key)
{
    PyObject *exc = PyErr_GetRaisedException();
    assert(exc != NULL);
    if (!Py_IS_TYPE(exc, (PyTypeObject*)PyExc_TypeError)) {
        PyErr_SetRaisedException(exc);
        return;
    }

    const char *errmsg;
    if (PyFrozenDict_Check(op)) {
        errmsg = "cannot use '%T' as a frozendict key (%S)";
    }
    else {
        errmsg = "cannot use '%T' as a dict key (%S)";
    }
    PyErr_Format(PyExc_TypeError, errmsg, key, exc);
    Py_DECREF(exc);
}
static inline Py_ALWAYS_INLINE Py_ssize_t
do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash,
          int (*check_lookup)(PyDictObject *, PyDictKeysObject *, void *, Py_ssize_t ix, PyObject *key, Py_hash_t))
{
    void *ep0 = _DK_ENTRIES(dk);
    size_t mask = DK_MASK(dk);
    size_t perturb = hash;
    size_t i = (size_t)hash & mask;
    Py_ssize_t ix;
    for (;;) {
        ix = dictkeys_get_index(dk, i);
        if (ix >= 0) {
            int cmp = check_lookup(mp, dk, ep0, ix, key, hash);
            if (cmp < 0) {
                return cmp;
            } else if (cmp) {
                return ix;
            }
        }
        else if (ix == DKIX_EMPTY) {
            return DKIX_EMPTY;
        }
        perturb >>= PERTURB_SHIFT;
        i = mask & (i*5 + perturb + 1);

        // Manual loop unrolling
        ix = dictkeys_get_index(dk, i);
        if (ix >= 0) {
            int cmp = check_lookup(mp, dk, ep0, ix, key, hash);
            if (cmp < 0) {
                return cmp;
            } else if (cmp) {
                return ix;
            }
        }
        else if (ix == DKIX_EMPTY) {
            return DKIX_EMPTY;
        }
        perturb >>= PERTURB_SHIFT;
        i = mask & (i*5 + perturb + 1);
    }
    Py_UNREACHABLE();
}
static Py_ssize_t _Py_HOT_FUNCTION
unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash)
{
    return do_lookup(NULL, dk, key, hash, compare_unicode_unicode);
}
static Py_ssize_t
unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash)
{
    return do_lookup(mp, dk, key, hash, compare_unicode_generic);
}
static Py_ssize_t
dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash)
{
    return do_lookup(mp, dk, key, hash, compare_generic);
}
static Py_ssize_t
lookdict_index(PyDictKeysObject *k, Py_hash_t hash, Py_ssize_t index)
{
    size_t mask = DK_MASK(k);
    size_t perturb = (size_t)hash;
    size_t i = (size_t)hash & mask;

    for (;;) {
        Py_ssize_t ix = dictkeys_get_index(k, i);
        if (ix == index) {
            return i;
        }
        if (ix == DKIX_EMPTY) {
            return DKIX_EMPTY;
        }
        perturb >>= PERTURB_SHIFT;
        i = mask & (i*5 + perturb + 1);
    }
    Py_UNREACHABLE();
}
static void
delete_index_from_values(PyDictValues *values, Py_ssize_t ix)
{
    uint8_t *array = get_insertion_order_array(values);
    int size = values->size;
    assert(size <= values->capacity);
    int i;
    for (i = 0; array[i] != ix; i++) {
        assert(i < size);
    }
    assert(i < size);
    size--;
    for (; i < size; i++) {
        array[i] = array[i+1];
    }
    values->size = size;
}
static inline void
dictkeys_set_index(PyDictKeysObject *keys, Py_ssize_t i, Py_ssize_t ix)
{
    int log2size = DK_LOG_SIZE(keys);

    assert(ix >= DKIX_DUMMY);
    assert(keys->dk_version == 0);

    if (log2size < 8) {
        assert(ix <= 0x7f);
        STORE_INDEX(keys, 8, i, ix);
    }
    else if (log2size < 16) {
        assert(ix <= 0x7fff);
        STORE_INDEX(keys, 16, i, ix);
    }
#if SIZEOF_VOID_P > 4
    else if (log2size >= 32) {
        STORE_INDEX(keys, 64, i, ix);
    }
#endif
    else {
        assert(ix <= 0x7fffffff);
        STORE_INDEX(keys, 32, i, ix);
    }
}
static void
delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix,
               PyObject *old_value)
{
    assert(can_modify_dict(mp));

    PyObject *old_key;

    Py_ssize_t hashpos = lookdict_index(mp->ma_keys, hash, ix);
    assert(hashpos >= 0);

    STORE_USED(mp, mp->ma_used - 1);
    if (_PyDict_HasSplitTable(mp)) {
        assert(old_value == mp->ma_values->values[ix]);
        STORE_SPLIT_VALUE(mp, ix, NULL);
        assert(ix < SHARED_KEYS_MAX_SIZE);
        /* Update order */
        delete_index_from_values(mp->ma_values, ix);
        ASSERT_CONSISTENT(mp);
    }
    else {
        FT_ATOMIC_STORE_UINT32_RELAXED(mp->ma_keys->dk_version, 0);
        dictkeys_set_index(mp->ma_keys, hashpos, DKIX_DUMMY);
        if (DK_IS_UNICODE(mp->ma_keys)) {
            PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix];
            old_key = ep->me_key;
            STORE_KEY(ep, NULL);
            STORE_VALUE(ep, NULL);
        }
        else {
            PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix];
            old_key = ep->me_key;
            STORE_KEY(ep, NULL);
            STORE_VALUE(ep, NULL);
            STORE_HASH(ep, 0);
        }
        Py_DECREF(old_key);
    }
    Py_DECREF(old_value);

    ASSERT_CONSISTENT(mp);
}
static PyDictKeysObject*
new_keys_object(uint8_t log2_size, bool unicode)
{
    Py_ssize_t usable;
    int log2_bytes;
    size_t entry_size = unicode ? sizeof(PyDictUnicodeEntry) : sizeof(PyDictKeyEntry);

    assert(log2_size >= PyDict_LOG_MINSIZE);

    usable = USABLE_FRACTION((size_t)1<<log2_size);
    if (log2_size < 8) {
        log2_bytes = log2_size;
    }
    else if (log2_size < 16) {
        log2_bytes = log2_size + 1;
    }
#if SIZEOF_VOID_P > 4
    else if (log2_size >= 32) {
        log2_bytes = log2_size + 3;
    }
#endif
    else {
        log2_bytes = log2_size + 2;
    }

    PyDictKeysObject *dk = NULL;
    if (log2_size == PyDict_LOG_MINSIZE && unicode) {
        dk = _Py_FREELIST_POP_MEM(dictkeys);
    }
    if (dk == NULL) {
        dk = PyMem_Malloc(sizeof(PyDictKeysObject)
                          + ((size_t)1 << log2_bytes)
                          + entry_size * usable);
        if (dk == NULL) {
            PyErr_NoMemory();
            return NULL;
        }
    }
#ifdef Py_REF_DEBUG
    _Py_IncRefTotal(_PyThreadState_GET());
#endif
    dk->dk_refcnt = 1;
    dk->dk_log2_size = log2_size;
    dk->dk_log2_index_bytes = log2_bytes;
    dk->dk_kind = unicode ? DICT_KEYS_UNICODE : DICT_KEYS_GENERAL;
#ifdef Py_GIL_DISABLED
    dk->dk_mutex = (PyMutex){0};
#endif
    dk->dk_nentries = 0;
    dk->dk_usable = usable;
    dk->dk_version = 0;
    memset(&dk->dk_indices[0], 0xff, ((size_t)1 << log2_bytes));
    memset(&dk->dk_indices[(size_t)1 << log2_bytes], 0, entry_size * usable);
    return dk;
}
static int
insert_to_emptydict(PyDictObject *mp,
                    PyObject *key, Py_hash_t hash, PyObject *value)
{
    assert(can_modify_dict(mp));
    assert(mp->ma_keys == Py_EMPTY_KEYS);

    int unicode = PyUnicode_CheckExact(key);
    PyDictKeysObject *newkeys = new_keys_object(PyDict_LOG_MINSIZE, unicode);
    if (newkeys == NULL) {
        Py_DECREF(key);
        Py_DECREF(value);
        return -1;
    }
    _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, value);

    /* We don't decref Py_EMPTY_KEYS here because it is immortal. */
    assert(mp->ma_values == NULL);

    size_t hashpos = (size_t)hash & (PyDict_MINSIZE-1);
    dictkeys_set_index(newkeys, hashpos, 0);
    if (unicode) {
        PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(newkeys);
        ep->me_key = key;
        STORE_VALUE(ep, value);
    }
    else {
        PyDictKeyEntry *ep = DK_ENTRIES(newkeys);
        ep->me_key = key;
        ep->me_hash = hash;
        STORE_VALUE(ep, value);
    }
    STORE_USED(mp, mp->ma_used + 1);
    newkeys->dk_usable--;
    newkeys->dk_nentries++;
    // We store the keys last so no one can see them in a partially inconsistent
    // state so that we don't need to switch the keys to being shared yet for
    // the case where we're inserting from the non-owner thread.  We don't use
    // set_keys here because the transition from empty to non-empty is safe
    // as the empty keys will never be freed.
    FT_ATOMIC_STORE_PTR_RELEASE(mp->ma_keys, newkeys);
    return 0;
}
static inline void
ensure_shared_on_resize(PyDictObject *mp)
{
#ifdef Py_GIL_DISABLED
    _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp);

    if (!_Py_IsOwnedByCurrentThread((PyObject *)mp) && !IS_DICT_SHARED(mp)) {
        // We are writing to the dict from another thread that owns
        // it and we haven't marked it as shared which will ensure
        // that when we re-size ma_keys or ma_values that we will
        // free using QSBR.  We need to lock the dictionary to
        // contend with writes from the owning thread, mark it as
        // shared, and then we can continue with lock-free reads.
        // Technically this is a little heavy handed, we could just
        // free the individual old keys / old-values using qsbr
        SET_DICT_SHARED(mp);
    }
#endif
}
static void
build_indices_generic(PyDictKeysObject *keys, PyDictKeyEntry *ep, Py_ssize_t n)
{
    size_t mask = DK_MASK(keys);
    for (Py_ssize_t ix = 0; ix != n; ix++, ep++) {
        Py_hash_t hash = ep->me_hash;
        size_t i = hash & mask;
        for (size_t perturb = hash; dictkeys_get_index(keys, i) != DKIX_EMPTY;) {
            perturb >>= PERTURB_SHIFT;
            i = mask & (i*5 + perturb + 1);
        }
        dictkeys_set_index(keys, i, ix);
    }
}
static void
build_indices_unicode(PyDictKeysObject *keys, PyDictUnicodeEntry *ep, Py_ssize_t n)
{
    size_t mask = DK_MASK(keys);
    for (Py_ssize_t ix = 0; ix != n; ix++, ep++) {
        Py_hash_t hash = unicode_get_hash(ep->me_key);
        assert(hash != -1);
        size_t i = hash & mask;
        for (size_t perturb = hash; dictkeys_get_index(keys, i) != DKIX_EMPTY;) {
            perturb >>= PERTURB_SHIFT;
            i = mask & (i*5 + perturb + 1);
        }
        dictkeys_set_index(keys, i, ix);
    }
}
static inline void
set_keys(PyDictObject *mp, PyDictKeysObject *keys)
{
    mp->ma_keys = keys;
}
static inline void
set_values(PyDictObject *mp, PyDictValues *values)
{
    mp->ma_values = values;
}
static void
invalidate_and_clear_inline_values(PyDictValues *values)
{
    assert(values->embedded);
    FT_ATOMIC_STORE_UINT8(values->valid, 0);
    for (int i = 0; i < values->capacity; i++) {
        FT_ATOMIC_STORE_PTR_RELEASE(values->values[i], NULL);
    }
}
static int
dictresize(PyDictObject *mp,
           uint8_t log2_newsize, int unicode)
{
    assert(can_modify_dict(mp));

    PyDictKeysObject *oldkeys, *newkeys;
    PyDictValues *oldvalues;

    if (log2_newsize >= SIZEOF_SIZE_T*8) {
        PyErr_NoMemory();
        return -1;
    }
    assert(log2_newsize >= PyDict_LOG_MINSIZE);

    oldkeys = mp->ma_keys;
    oldvalues = mp->ma_values;

    if (!DK_IS_UNICODE(oldkeys)) {
        unicode = 0;
    }

    ensure_shared_on_resize(mp);
    /* NOTE: Current odict checks mp->ma_keys to detect resize happen.
     * So we can't reuse oldkeys even if oldkeys->dk_size == newsize.
     * TODO: Try reusing oldkeys when reimplement odict.
     */

    /* Allocate a new table. */
    newkeys = new_keys_object(log2_newsize, unicode);
    if (newkeys == NULL) {
        return -1;
    }
    // New table must be large enough.
    assert(newkeys->dk_usable >= mp->ma_used);

    Py_ssize_t numentries = mp->ma_used;

    if (oldvalues != NULL) {
        LOCK_KEYS(oldkeys);
        PyDictUnicodeEntry *oldentries = DK_UNICODE_ENTRIES(oldkeys);
        /* Convert split table into new combined table.
         * We must incref keys; we can transfer values.
         */
        if (newkeys->dk_kind == DICT_KEYS_GENERAL) {
            // split -> generic
            PyDictKeyEntry *newentries = DK_ENTRIES(newkeys);

            for (Py_ssize_t i = 0; i < numentries; i++) {
                int index = get_index_from_order(mp, i);
                PyDictUnicodeEntry *ep = &oldentries[index];
                assert(oldvalues->values[index] != NULL);
                newentries[i].me_key = Py_NewRef(ep->me_key);
                newentries[i].me_hash = unicode_get_hash(ep->me_key);
                newentries[i].me_value = oldvalues->values[index];
            }
            build_indices_generic(newkeys, newentries, numentries);
        }
        else { // split -> combined unicode
            PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(newkeys);

            for (Py_ssize_t i = 0; i < numentries; i++) {
                int index = get_index_from_order(mp, i);
                PyDictUnicodeEntry *ep = &oldentries[index];
                assert(oldvalues->values[index] != NULL);
                newentries[i].me_key = Py_NewRef(ep->me_key);
                newentries[i].me_value = oldvalues->values[index];
            }
            build_indices_unicode(newkeys, newentries, numentries);
        }
        UNLOCK_KEYS(oldkeys);
        set_keys(mp, newkeys);
        dictkeys_decref(oldkeys, IS_DICT_SHARED(mp));
        set_values(mp, NULL);
        if (oldvalues->embedded) {
            assert(oldvalues->embedded == 1);
            assert(oldvalues->valid == 1);
            invalidate_and_clear_inline_values(oldvalues);
        }
        else {
            free_values(oldvalues, IS_DICT_SHARED(mp));
        }
    }
    else {  // oldkeys is combined.
        if (oldkeys->dk_kind == DICT_KEYS_GENERAL) {
            // generic -> generic
            assert(newkeys->dk_kind == DICT_KEYS_GENERAL);
            PyDictKeyEntry *oldentries = DK_ENTRIES(oldkeys);
            PyDictKeyEntry *newentries = DK_ENTRIES(newkeys);
            if (oldkeys->dk_nentries == numentries) {
                memcpy(newentries, oldentries, numentries * sizeof(PyDictKeyEntry));
            }
            else {
                PyDictKeyEntry *ep = oldentries;
                for (Py_ssize_t i = 0; i < numentries; i++) {
                    while (ep->me_value == NULL)
                        ep++;
                    newentries[i] = *ep++;
                }
            }
            build_indices_generic(newkeys, newentries, numentries);
        }
        else {  // oldkeys is combined unicode
            PyDictUnicodeEntry *oldentries = DK_UNICODE_ENTRIES(oldkeys);
            if (unicode) { // combined unicode -> combined unicode
                PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(newkeys);
                if (oldkeys->dk_nentries == numentries && mp->ma_keys->dk_kind == DICT_KEYS_UNICODE) {
                    memcpy(newentries, oldentries, numentries * sizeof(PyDictUnicodeEntry));
                }
                else {
                    PyDictUnicodeEntry *ep = oldentries;
                    for (Py_ssize_t i = 0; i < numentries; i++) {
                        while (ep->me_value == NULL)
                            ep++;
                        newentries[i] = *ep++;
                    }
                }
                build_indices_unicode(newkeys, newentries, numentries);
            }
            else { // combined unicode -> generic
                PyDictKeyEntry *newentries = DK_ENTRIES(newkeys);
                PyDictUnicodeEntry *ep = oldentries;
                for (Py_ssize_t i = 0; i < numentries; i++) {
                    while (ep->me_value == NULL)
                        ep++;
                    newentries[i].me_key = ep->me_key;
                    newentries[i].me_hash = unicode_get_hash(ep->me_key);
                    newentries[i].me_value = ep->me_value;
                    ep++;
                }
                build_indices_generic(newkeys, newentries, numentries);
            }
        }

        set_keys(mp, newkeys);

        if (oldkeys != Py_EMPTY_KEYS) {
#ifdef Py_REF_DEBUG
            _Py_DecRefTotal(_PyThreadState_GET());
#endif
            assert(oldkeys->dk_kind != DICT_KEYS_SPLIT);
            assert(oldkeys->dk_refcnt == 1);
            free_keys_object(oldkeys, IS_DICT_SHARED(mp));
        }
    }

    STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - numentries);
    STORE_KEYS_NENTRIES(mp->ma_keys, numentries);
    ASSERT_CONSISTENT(mp);
    return 0;
}
static inline uint8_t
calculate_log2_keysize(Py_ssize_t minsize)
{
#if SIZEOF_LONG == SIZEOF_SIZE_T
    minsize = Py_MAX(minsize, PyDict_MINSIZE);
    return _Py_bit_length(minsize - 1);
#elif defined(_MSC_VER)
    // On 64bit Windows, sizeof(long) == 4. We cannot use _Py_bit_length.
    minsize = Py_MAX(minsize, PyDict_MINSIZE);
    unsigned long msb;
    _BitScanReverse64(&msb, (uint64_t)minsize - 1);
    return (uint8_t)(msb + 1);
#else
    uint8_t log2_size;
    for (log2_size = PyDict_LOG_MINSIZE;
            (((Py_ssize_t)1) << log2_size) < minsize;
            log2_size++)
        ;
    return log2_size;
#endif
}
static int
insertion_resize(PyDictObject *mp, int unicode)
{
    return dictresize(mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode);
}
static inline int
is_unusable_slot(Py_ssize_t ix)
{
#ifdef Py_GIL_DISABLED
    return ix >= 0 || ix == DKIX_DUMMY;
#else
    return ix >= 0;
#endif
}
static Py_ssize_t
find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash)
{
    assert(keys != NULL);

    const size_t mask = DK_MASK(keys);
    size_t i = hash & mask;
    Py_ssize_t ix = dictkeys_get_index(keys, i);
    for (size_t perturb = hash; is_unusable_slot(ix);) {
        perturb >>= PERTURB_SHIFT;
        i = (i*5 + perturb + 1) & mask;
        ix = dictkeys_get_index(keys, i);
    }
    return i;
}
static inline void split_keys_entry_added(PyDictKeysObject *keys)
{
    keys->dk_usable--;
    keys->dk_nentries++;
}
static Py_ssize_t
insert_split_key(PyDictKeysObject *keys, PyObject *key, Py_hash_t hash)
{
    assert(PyUnicode_CheckExact(key));
    Py_ssize_t ix;


#ifdef Py_GIL_DISABLED
    ix = unicodekeys_lookup_unicode_threadsafe(keys, key, hash);
    if (ix >= 0) {
        return ix;
    }
#endif

    LOCK_KEYS(keys);
    ix = unicodekeys_lookup_unicode(keys, key, hash);
    if (ix == DKIX_EMPTY && keys->dk_usable > 0) {
        // Insert into new slot
        FT_ATOMIC_STORE_UINT32_RELAXED(keys->dk_version, 0);
        Py_ssize_t hashpos = find_empty_slot(keys, hash);
        ix = keys->dk_nentries;
        dictkeys_set_index(keys, hashpos, ix);
        PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(keys)[ix];
        STORE_SHARED_KEY(ep->me_key, Py_NewRef(key));
        split_keys_entry_added(keys);
    }
    assert (ix < SHARED_KEYS_MAX_SIZE);
    UNLOCK_KEYS(keys);
    return ix;
}
static void
insert_split_value(PyDictObject *mp, PyObject *key, PyObject *value, Py_ssize_t ix)
{
    assert(can_modify_dict(mp));
    assert(PyUnicode_CheckExact(key));

    PyObject *old_value = mp->ma_values->values[ix];
    if (old_value == NULL) {
        _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, value);
        STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value));
        _PyDictValues_AddToInsertionOrder(mp->ma_values, ix);
        STORE_USED(mp, mp->ma_used + 1);
    }
    else {
        _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, mp, key, value);
        STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value));
        // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault,
        // when dict only holds the strong reference to value in ep->me_value.
        Py_DECREF(old_value);
    }
    ASSERT_CONSISTENT(mp);
}
static inline int
insert_combined_dict(PyDictObject *mp,
                     Py_hash_t hash, PyObject *key, PyObject *value)
{
    // gh-140551: If dict was cleared in _Py_dict_lookup,
    // we have to resize one more time to force general key kind.
    if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) {
        if (insertion_resize(mp, 0) < 0)
            return -1;
        assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL);
    }

    if (mp->ma_keys->dk_usable <= 0) {
        /* Need to resize. */
        if (insertion_resize(mp, 1) < 0) {
            return -1;
        }
    }

    _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, value);
    FT_ATOMIC_STORE_UINT32_RELAXED(mp->ma_keys->dk_version, 0);

    Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash);
    dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries);

    if (DK_IS_UNICODE(mp->ma_keys)) {
        PyDictUnicodeEntry *ep;
        ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries];
        STORE_KEY(ep, key);
        STORE_VALUE(ep, value);
    }
    else {
        PyDictKeyEntry *ep;
        ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries];
        STORE_KEY(ep, key);
        STORE_VALUE(ep, value);
        STORE_HASH(ep, hash);
    }
    STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - 1);
    STORE_KEYS_NENTRIES(mp->ma_keys, mp->ma_keys->dk_nentries + 1);
    assert(mp->ma_keys->dk_usable >= 0);
    return 0;
}
static int
insertdict(PyDictObject *mp,
           PyObject *key, Py_hash_t hash, PyObject *value)
{
    assert(can_modify_dict(mp));

    PyObject *old_value = NULL;
    Py_ssize_t ix;

    if (_PyDict_HasSplitTable(mp) && PyUnicode_CheckExact(key)) {
        ix = insert_split_key(mp->ma_keys, key, hash);
        if (ix != DKIX_EMPTY) {
            insert_split_value(mp, key, value, ix);
            Py_DECREF(key);
            Py_DECREF(value);
            return 0;
        }
        // No space in shared keys. Go to insert_combined_dict() below.
    }
    else {
        ix = _Py_dict_lookup(mp, key, hash, &old_value);
        if (ix == DKIX_ERROR)
            goto Fail;
    }

    if (old_value == NULL) {
        // insert_combined_dict() will convert from non DICT_KEYS_GENERAL table
        // into DICT_KEYS_GENERAL table if key is not Unicode.
        // We don't convert it before _Py_dict_lookup because non-Unicode key
        // may change generic table into Unicode table.
        //
        // NOTE: ix may not be DKIX_EMPTY because split table may have key
        // without value.
        if (insert_combined_dict(mp, hash, key, value) < 0) {
            goto Fail;
        }
        STORE_USED(mp, mp->ma_used + 1);
        ASSERT_CONSISTENT(mp);
        return 0;
    }

    if (old_value != value) {
        _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, mp, key, value);
        assert(old_value != NULL);
        if (DK_IS_UNICODE(mp->ma_keys)) {
            if (_PyDict_HasSplitTable(mp)) {
                STORE_SPLIT_VALUE(mp, ix, value);
            }
            else {
                PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix];
                STORE_VALUE(ep, value);
            }
        }
        else {
            PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix];
            STORE_VALUE(ep, value);
        }
    }
    Py_XDECREF(old_value); /* which **CAN** re-enter (see issue #22653) */
    ASSERT_CONSISTENT(mp);
    Py_DECREF(key);
    return 0;

Fail:
    Py_DECREF(value);
    Py_DECREF(key);
    return -1;
}
static int
setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value)
{
    assert(PyAnyDict_Check(mp));
    assert(can_modify_dict(mp));
    assert(key);
    assert(value);

    Py_hash_t hash = _PyObject_HashFast(key);
    if (hash == -1) {
        dict_unhashable_type((PyObject*)mp, key);
        Py_DECREF(key);
        Py_DECREF(value);
        return -1;
    }

    if (mp->ma_keys == Py_EMPTY_KEYS) {
        return insert_to_emptydict(mp, key, hash, value);
    }
    /* insertdict() handles any resizing that might be necessary */
    return insertdict(mp, key, hash, value);
}
static int
setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value)
{
    assert(key);
    assert(value);
    return setitem_take2_lock_held(mp,
                                   Py_NewRef(key), Py_NewRef(value));
}
int
_PyDict_DelItem_KnownHash_LockHeld(PyObject *op, PyObject *key, Py_hash_t hash)
{
    if (!PyDict_Check(op)) {
        if (PyFrozenDict_Check(op)) {
            frozendict_does_not_support("deletion");
        }
        else {
            PyErr_BadInternalCall();
        }
        return -1;
    }

    Py_ssize_t ix;
    PyObject *old_value;
    PyDictObject *mp = (PyDictObject *)op;
    assert(can_modify_dict(mp));

    assert(key);
    assert(hash != -1);
    ix = _Py_dict_lookup(mp, key, hash, &old_value);
    if (ix == DKIX_ERROR)
        return -1;
    if (ix == DKIX_EMPTY || old_value == NULL) {
        _PyErr_SetKeyError(key);
        return -1;
    }

    _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL);
    delitem_common(mp, hash, ix, old_value);
    return 0;
}
int
_PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value)
{
    if (!PyDict_Check(dict)) {
        if (PyFrozenDict_Check(dict)) {
            if (value == NULL) {
                frozendict_does_not_support("deletion");
            }
            else {
                frozendict_does_not_support("assignment");
            }
        }
        else {
            PyErr_BadInternalCall();
        }
        return -1;
    }

    if (value == NULL) {
        Py_hash_t hash = _PyObject_HashFast(name);
        if (hash == -1) {
            dict_unhashable_type((PyObject*)dict, name);
            return -1;
        }
        return _PyDict_DelItem_KnownHash_LockHeld((PyObject *)dict, name, hash);
    } else {
        return setitem_lock_held(dict, name, value);
    }
}

Py_ssize_t
_Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr)
{
    PyDictKeysObject *dk;
    DictKeysKind kind;
    Py_ssize_t ix;

    _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp);
start:
    dk = mp->ma_keys;
    kind = dk->dk_kind;

    if (kind != DICT_KEYS_GENERAL) {
        if (PyUnicode_CheckExact(key)) {
#ifdef Py_GIL_DISABLED
            if (kind == DICT_KEYS_SPLIT) {
                ix = unicodekeys_lookup_split(dk, key, hash);
            }
            else {
                ix = unicodekeys_lookup_unicode(dk, key, hash);
            }
#else
            ix = unicodekeys_lookup_unicode(dk, key, hash);
#endif
        }
        else {
            INCREF_KEYS_FT(dk);
            LOCK_KEYS_IF_SPLIT(dk, kind);

            ix = unicodekeys_lookup_generic(mp, dk, key, hash);

            UNLOCK_KEYS_IF_SPLIT(dk, kind);
            DECREF_KEYS_FT(dk, IS_DICT_SHARED(mp));
            if (ix == DKIX_KEY_CHANGED) {
                goto start;
            }
        }

        if (ix >= 0) {
            if (kind == DICT_KEYS_SPLIT) {
                *value_addr = mp->ma_values->values[ix];
            }
            else {
                *value_addr = DK_UNICODE_ENTRIES(dk)[ix].me_value;
            }
        }
        else {
            *value_addr = NULL;
        }
    }
    else {
        ix = dictkeys_generic_lookup(mp, dk, key, hash);
        if (ix == DKIX_KEY_CHANGED) {
            goto start;
        }
        if (ix >= 0) {
            *value_addr = DK_ENTRIES(dk)[ix].me_value;
        }
        else {
            *value_addr = NULL;
        }
    }

    return ix;
}
Py_ssize_t
_PyDict_LookupIndexAndValue(PyDictObject *mp, PyObject *key, PyObject **value)
{
    // TODO: Thread safety
    assert(PyDict_CheckExact((PyObject*)mp));
    assert(PyUnicode_CheckExact(key));

    Py_hash_t hash = _PyObject_HashFast(key);
    if (hash == -1) {
        dict_unhashable_type((PyObject*)mp, key);
        return -1;
    }

    return _Py_dict_lookup(mp, key, hash, value);
}
Py_ssize_t
_PyDict_LookupIndex(PyDictObject *mp, PyObject *key)
{
    PyObject *value; // discarded
    return _PyDict_LookupIndexAndValue(mp, key, &value);
}
static bool
check_keys_unicode(PyDictKeysObject *dk, PyObject *key)
{
    return PyUnicode_CheckExact(key) && (dk->dk_kind != DICT_KEYS_GENERAL);
}
static Py_ssize_t
hash_unicode_key(PyObject *key)
{
    assert(PyUnicode_CheckExact(key));
    Py_hash_t hash = unicode_get_hash(key);
    if (hash == -1) {
        hash = PyUnicode_Type.tp_hash(key);
        assert(hash != -1);
    }
    return hash;
}
Py_ssize_t
_PyDictKeys_StringLookup(PyDictKeysObject* dk, PyObject *key)
{
    if (!check_keys_unicode(dk, key)) {
        return DKIX_ERROR;
    }
    Py_hash_t hash = hash_unicode_key(key);
    return unicodekeys_lookup_unicode(dk, key, hash);
}
static Py_ssize_t
unicodekeys_lookup_split(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash)
{
    Py_ssize_t ix;
    assert(dk->dk_kind == DICT_KEYS_SPLIT);
    assert(PyUnicode_CheckExact(key));

#ifdef Py_GIL_DISABLED
    // A split dictionaries keys can be mutated by other dictionaries
    // but if we have a unicode key we can avoid locking the shared
    // keys.
    ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash);
    if (ix == DKIX_KEY_CHANGED) {
        LOCK_KEYS(dk);
        ix = unicodekeys_lookup_unicode(dk, key, hash);
        UNLOCK_KEYS(dk);
    }
#else
    ix = unicodekeys_lookup_unicode(dk, key, hash);
#endif
    return ix;
}
Py_ssize_t
_PyDictKeys_StringLookupSplit(PyDictKeysObject* dk, PyObject *key)
{
    assert(dk->dk_kind == DICT_KEYS_SPLIT);
    assert(PyUnicode_CheckExact(key));
    Py_hash_t hash = unicode_get_hash(key);
    if (hash == -1) {
        hash = PyUnicode_Type.tp_hash(key);
        if (hash == -1) {
            PyErr_Clear();
            return DKIX_ERROR;
        }
    }
    return unicodekeys_lookup_split(dk, key, hash);
}
static inline uint32_t
get_next_dict_keys_version(PyInterpreterState *interp)
{
#ifdef Py_GIL_DISABLED
    uint32_t v;
    do {
        v = _Py_atomic_load_uint32_relaxed(
            &interp->dict_state.next_keys_version);
        if (v == 0) {
            return 0;
        }
    } while (!_Py_atomic_compare_exchange_uint32(
        &interp->dict_state.next_keys_version, &v, v + 1));
#else
    if (interp->dict_state.next_keys_version == 0) {
        return 0;
    }
    uint32_t v = interp->dict_state.next_keys_version++;
#endif
    return v;
}
uint32_t
_PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp,
                                      PyDictKeysObject *dictkeys)
{
    uint32_t dk_version = FT_ATOMIC_LOAD_UINT32_RELAXED(dictkeys->dk_version);
    if (dk_version != 0) {
        return dk_version;
    }
    dk_version = get_next_dict_keys_version(interp);
    FT_ATOMIC_STORE_UINT32_RELAXED(dictkeys->dk_version, dk_version);
    return dk_version;
}
Py_ssize_t
_PyDictKeys_StringLookupAndVersion(PyDictKeysObject *dk, PyObject *key, uint32_t *version)
{
    if (!check_keys_unicode(dk, key)) {
        return DKIX_ERROR;
    }
    Py_ssize_t ix;
    Py_hash_t hash = hash_unicode_key(key);
    LOCK_KEYS(dk);
    ix = unicodekeys_lookup_unicode(dk, key, hash);
    *version = _PyDictKeys_GetVersionForCurrentState(_PyInterpreterState_GET(), dk);
    UNLOCK_KEYS(dk);
    return ix;
}
static inline void
ensure_shared_on_keys_version_assignment(PyDictObject *mp)
{
    ASSERT_DICT_LOCKED((PyObject *) mp);
    #ifdef Py_GIL_DISABLED
    if (!IS_DICT_SHARED(mp)) {
        // This ensures that a concurrent resize operation will delay
        // freeing the old keys or values using QSBR, which is necessary to
        // safely allow concurrent reads without locking.
        SET_DICT_SHARED(mp);
    }
    #endif
}
uint32_t
_PyDict_GetKeysVersionForCurrentState(PyInterpreterState *interp,
                                      PyDictObject *dict)
{
    ASSERT_DICT_LOCKED((PyObject *) dict);
    uint32_t dk_version =
        _PyDictKeys_GetVersionForCurrentState(interp, dict->ma_keys);
    ensure_shared_on_keys_version_assignment(dict);
    return dk_version;
}

void Cix_dict_insert_split_value(
    PyInterpreterState *interp,
    PyDictObject *mp,
    PyObject *key,
    PyObject *value,
    Py_ssize_t ix) {
#if defined(__clang__)
  [[clang::always_inline]]
#elif defined(__GNUC__)
  [[gnu::always_inline]]
#endif
  insert_split_value(mp, key, value, ix);
}

#define _PyObject_SetAttributeErrorContext _CiPyObject_SetAttributeErrorContext
int
_PyObject_SetAttributeErrorContext(PyObject* v, PyObject* name)
{
    assert(PyErr_Occurred());
    if (!PyErr_ExceptionMatches(PyExc_AttributeError)){
        return 0;
    }
    // Intercept AttributeError exceptions and augment them to offer suggestions later.
    PyObject *exc = PyErr_GetRaisedException();
    if (!PyErr_GivenExceptionMatches(exc, PyExc_AttributeError)) {
        goto restore;
    }
    PyAttributeErrorObject* the_exc = (PyAttributeErrorObject*) exc;
    // Check if this exception was already augmented
    if (the_exc->name || the_exc->obj) {
        goto restore;
    }
    // Augment the exception with the name and object
    if (PyObject_SetAttr(exc, &_Py_ID(name), name) ||
        PyObject_SetAttr(exc, &_Py_ID(obj), v)) {
        Py_DECREF(exc);
        return 1;
    }
restore:
    PyErr_SetRaisedException(exc);
    return 0;
}

// Wrapper as set_attribute_error_context is declared "static inline".
int
Cix_set_attribute_error_context(PyObject *v, PyObject *name) {
  return _PyObject_SetAttributeErrorContext(v, name);
}

static const uint8_t DE_INSTRUMENT[256] = {
    [INSTRUMENTED_RESUME] = RESUME,
    [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE,
    [INSTRUMENTED_CALL] = CALL,
    [INSTRUMENTED_CALL_KW] = CALL_KW,
    [INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX,
    [INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE,
    [INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD,
    [INSTRUMENTED_JUMP_BACKWARD] = JUMP_BACKWARD,
    [INSTRUMENTED_POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE,
    [INSTRUMENTED_POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE,
    [INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE,
    [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE,
    [INSTRUMENTED_FOR_ITER] = FOR_ITER,
    [INSTRUMENTED_POP_ITER] = POP_ITER,
    [INSTRUMENTED_END_FOR] = END_FOR,
    [INSTRUMENTED_END_SEND] = END_SEND,
    [INSTRUMENTED_LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR,
    [INSTRUMENTED_NOT_TAKEN] = NOT_TAKEN,
    [INSTRUMENTED_END_ASYNC_FOR] = END_ASYNC_FOR,
};
uint8_t
Cix_DEINSTRUMENT(uint8_t op) {
  return DE_INSTRUMENT[op];
}


static inline uint8_t
get_original_opcode(_PyCoLineInstrumentationData *line_data, int index)
{
    return line_data->data[index*line_data->bytes_per_entry];
}
uint8_t Cix_GetOriginalOpcode(
    _PyCoLineInstrumentationData* line_data,
    int index) {
  return get_original_opcode(line_data, index);
}

_PyErr_StackItem *
_PyErr_GetTopmostException(PyThreadState *tstate)
{
    _PyErr_StackItem *exc_info = tstate->exc_info;
    assert(exc_info);

    while (exc_info->exc_value == NULL && exc_info->previous_item != NULL)
    {
        exc_info = exc_info->previous_item;
    }
    assert(!Py_IsNone(exc_info->exc_value));
    return exc_info;
}

// Internal dependencies for gc_freeze_impl.
#ifdef Py_GIL_DISABLED
#endif
#define ADD_INT(NAME) if (PyModule_AddIntConstant(module, #NAME, _PyGC_ ## NAME) < 0) { return -1; }
#undef ADD_INT
#ifndef Py_GIL_DISABLED
#ifdef Py_DEBUG
#  define GC_DEBUG
#endif
#define GC_NEXT _PyGCHead_NEXT
#define GC_PREV _PyGCHead_PREV
#define PREV_MASK_COLLECTING   _PyGC_PREV_MASK_COLLECTING
#define NEXT_MASK_UNREACHABLE  2
#define AS_GC(op) _Py_AS_GC(op)
#define FROM_GC(gc) _Py_FROM_GC(gc)
#define GENERATION_AUTO (-1)
#define INIT_HEAD(GEN) \
    do { \
        GEN.head._gc_next = (uintptr_t)&GEN.head; \
        GEN.head._gc_prev = (uintptr_t)&GEN.head; \
    } while (0)
#undef INIT_HEAD
#ifdef GC_DEBUG
#else
#define validate_list(x, y) do{}while(0)
#endif
#ifdef GC_EXTRA_DEBUG
#else
#define validate_spaces(g) do{}while(0)
#define validate_consistent_old_space(l) do{}while(0)
#define gc_list_validate_space(l, s) do{}while(0)
#endif
#define SCAN_RATE_DIVISOR 10
#ifndef NDEBUG
#endif
#ifdef Py_STATS
#endif
#ifdef Py_DEBUG
#endif
#ifdef Py_DEBUG
#endif
#ifdef Py_DEBUG
#endif
#endif  // Py_GIL_DISABLED
typedef struct _gc_runtime_state GCState;
static inline void
gc_list_init(PyGC_Head *list)
{
    // List header must not have flags.
    // We can assign pointer by simple cast.
    list->_gc_prev = (uintptr_t)list;
    list->_gc_next = (uintptr_t)list;
}
static inline int
gc_list_is_empty(PyGC_Head *list)
{
    return (list->_gc_next == (uintptr_t)list);
}
static GCState *
get_gc_state(void)
{
    PyInterpreterState *interp = _PyInterpreterState_GET();
    return &interp->gc;
}
static inline int
gc_old_space(PyGC_Head *g)
{
    return g->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1;
}
static void
gc_list_merge(PyGC_Head *from, PyGC_Head *to)
{
    assert(from != to);
    if (!gc_list_is_empty(from)) {
        PyGC_Head *to_tail = GC_PREV(to);
        PyGC_Head *from_head = GC_NEXT(from);
        PyGC_Head *from_tail = GC_PREV(from);
        assert(from_head != from);
        assert(from_tail != from);
        assert(gc_list_is_empty(to) ||
            gc_old_space(to_tail) == gc_old_space(from_tail));

        _PyGCHead_SET_NEXT(to_tail, from_head);
        _PyGCHead_SET_PREV(from_head, to_tail);

        _PyGCHead_SET_NEXT(from_tail, to);
        _PyGCHead_SET_PREV(to, from_tail);
    }
    gc_list_init(from);
}
static inline void
gc_set_old_space(PyGC_Head *g, int space)
{
    assert(space == 0 || space == _PyGC_NEXT_MASK_OLD_SPACE_1);
    g->_gc_next &= ~_PyGC_NEXT_MASK_OLD_SPACE_1;
    g->_gc_next |= space;
}
static inline Py_ssize_t
gc_list_set_space(PyGC_Head *list, int space)
{
    Py_ssize_t size = 0;
    PyGC_Head *gc;
    for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) {
        gc_set_old_space(gc, space);
        size++;
    }
    return size;
}
// End internal dependencies.

#define _PyGC_Freeze _CiGC_Freeze
void
_PyGC_Freeze(PyInterpreterState *interp)
{
    GCState *gcstate = &interp->gc;
    /* The permanent_generation must be visited */
    gc_list_set_space(&gcstate->young.head, gcstate->visited_space);
    gc_list_merge(&gcstate->young.head, &gcstate->permanent_generation.head);
    gcstate->young.count = 0;
    PyGC_Head*old0 = &gcstate->old[0].head;
    PyGC_Head*old1 = &gcstate->old[1].head;
    if (gcstate->visited_space) {
        gc_list_set_space(old0, 1);
    }
    else {
        gc_list_set_space(old1, 0);
    }
    gc_list_merge(old0, &gcstate->permanent_generation.head);
    gcstate->old[0].count = 0;
    gc_list_merge(old1, &gcstate->permanent_generation.head);
    gcstate->old[1].count = 0;
    validate_spaces(gcstate);
}
static PyObject *
gc_freeze_impl(PyObject *module)
/*[clinic end generated code: output=502159d9cdc4c139 input=11fb59b0a75dcf3d]*/
{
    PyInterpreterState *interp = _PyInterpreterState_GET();
    _PyGC_Freeze(interp);
    Py_RETURN_NONE;
}
PyObject* Cix_gc_freeze_impl(PyObject* mod) {
  return gc_freeze_impl(mod);
}

// Recreate builtin_next_impl (removed in https://github.com/python/cpython/pull/130371)

PyObject* builtin_next_impl(PyObject *it, PyObject* def)
{
    PyObject *res;

    if (!PyIter_Check(it)) {
        PyErr_Format(PyExc_TypeError,
                "'%.200s' object is not an iterator",
                Py_TYPE(it)->tp_name);
        return NULL;
    }

    res = (*Py_TYPE(it)->tp_iternext)(it);
    if (res != NULL) {
        return res;
    } else if (def != NULL) {
        if (PyErr_Occurred()) {
            if(!PyErr_ExceptionMatches(PyExc_StopIteration))
                return NULL;
            PyErr_Clear();
        }
        return Py_NewRef(def);
    } else if (PyErr_Occurred()) {
        return NULL;
    } else {
        PyErr_SetNone(PyExc_StopIteration);
        return NULL;
    }
}

PyObject* Ci_Builtin_Next_Core(PyObject* it, PyObject* def) {
    return builtin_next_impl(it, def);
}

PyObject *Cix_monitoring_disable, *Cix_monitoring_missing;

int init_upstream_borrow(void) {
  // Nothing to do here; retained for consistency with 3.10
  PyObject *sys = PyImport_ImportModule("sys");
  if (sys == NULL) {
    return -1;
  }
  PyObject *monitoring = PyObject_GetAttrString(sys, "monitoring");
  Py_DECREF(sys);
  if (monitoring == NULL) {
    return -1;
  }
  Cix_monitoring_disable = PyObject_GetAttr(monitoring, PyUnicode_FromString("DISABLE"));
  Cix_monitoring_missing = PyObject_GetAttr(monitoring, PyUnicode_FromString("MISSING"));
  Py_DECREF(monitoring);
  if (Cix_monitoring_disable == NULL || Cix_monitoring_missing == NULL) {
    return -1;
  }

  // Create a couple of classes to find the interpreters special functions for __getattr__
  // and __getattribute__
  const char *code_str =
    "class GetAttr:\n"
    "    def __getattr__(self, name): pass\n"
    "class GetAttribute:\n"
    "    def __getattribute__(self, name): pass\n";

  PyObject *code = NULL, *globals = NULL, *inst = NULL;
  PyDictObject* empty_dict = NULL;
  int result = -1;
  code = Py_CompileString(code_str, "cinderx_getattr_init.py", Py_file_input);
  if (code == NULL) {
    goto error;
  }
  globals = PyDict_New();
  if (globals == NULL) {
    goto error;
  }

  PyObject *eval_result = PyEval_EvalCode(code, globals, globals);
  if (eval_result == NULL) {
    goto error;
  }
  Py_DECREF(eval_result);

  PyObject *getattr = PyDict_GetItemString(globals, "GetAttr");
  PyObject *getattribute = PyDict_GetItemString(globals, "GetAttribute");
  if (getattr == NULL ||
      getattribute == NULL ||
      Py_TYPE(getattr) != &PyType_Type ||
      Py_TYPE(getattribute) != &PyType_Type) {
    PyErr_SetString(PyExc_RuntimeError,
                    "failed to initialize GetAttr and GetAttribute: classes not defined");
    goto error;
  }

  // We need to make an instance and access an attribute to initialize _Py_slot_tp_getattro
  inst = PyObject_CallNoArgs(getattribute);
  if (inst == NULL) {
    PyErr_SetString(PyExc_RuntimeError,
                    "failed to initialize GetAttr and GetAttribute: creating instance failed");
    goto error;
  }

  Py_XDECREF(PyObject_GetAttrString(inst, "foo"));

  Ci_tp_getattr_hook = ((PyTypeObject*)getattr)->tp_getattro;
  Ci_tp_getattro = ((PyTypeObject*)getattribute)->tp_getattro;
  if (Ci_tp_getattr_hook == NULL || Ci_tp_getattro == NULL) {
    PyErr_SetString(PyExc_RuntimeError,
                    "failed to initialize GetAttr and GetAttribute: got NULL values");
    goto error;
  }

  // Extract the empty keys object which several borrowed functions from
  // dictobject.c user in pointer equality tests.
  empty_dict = (PyDictObject*)PyDict_New();
  ci_dict_empty_keys = empty_dict->ma_keys;

  result = 0;

error:
  Py_XDECREF(code);
  Py_XDECREF(globals);
  Py_XDECREF(inst);
  Py_XDECREF(empty_dict);
  return result;
}


#define MCACHE_MAX_ATTR_SIZE    100
#define MCACHE_HASH(version, name_hash)                                 \
        (((unsigned int)(version) ^ (unsigned int)(name_hash))          \
         & ((1 << MCACHE_SIZE_EXP) - 1))
#define MCACHE_HASH_METHOD(type, name)                                  \
    MCACHE_HASH(FT_ATOMIC_LOAD_UINT_RELAXED((type)->tp_version_tag),   \
                ((Py_ssize_t)(name)) >> 3)
#define MCACHE_CACHEABLE_NAME(name)                             \
        (PyUnicode_CheckExact(name) &&                           \
         (PyUnicode_GET_LENGTH(name) <= MCACHE_MAX_ATTR_SIZE))
#define NEXT_VERSION_TAG(interp) \
    (interp)->types.next_version_tag
#ifdef Py_GIL_DISABLED
#define TYPE_LOCK &_PyInterpreterState_GET()->types.mutex
#define BEGIN_TYPE_LOCK() Py_BEGIN_CRITICAL_SECTION_MUTEX(TYPE_LOCK)
#define END_TYPE_LOCK() Py_END_CRITICAL_SECTION()
#define BEGIN_TYPE_DICT_LOCK(d) \
    Py_BEGIN_CRITICAL_SECTION2_MUTEX(TYPE_LOCK, &_PyObject_CAST(d)->ob_mutex)
#define END_TYPE_DICT_LOCK() Py_END_CRITICAL_SECTION2()
#if !defined(NDEBUG) || defined(Py_DEBUG)
#endif
#if SIZEOF_VOID_P > 4
#define TYPE_IS_REVEALED(tp) ((((PyObject *)(tp))->ob_flags & _Py_TYPE_REVEALED_FLAG) != 0)
#else
#define TYPE_IS_REVEALED(tp) 0
#endif
#ifdef Py_DEBUG
#define ASSERT_TYPE_LOCK_HELD() \
    if (!types_world_is_stopped()) { _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(TYPE_LOCK); }
#define ASSERT_WORLD_STOPPED_OR_NEW_TYPE(tp) \
    assert(!TYPE_IS_REVEALED(tp) || types_world_is_stopped())
#define ASSERT_NEW_TYPE_OR_LOCKED(tp) \
    if (TYPE_IS_REVEALED(tp)) { ASSERT_TYPE_LOCK_HELD(); }
#else
#define ASSERT_TYPE_LOCK_HELD()
#define ASSERT_WORLD_STOPPED_OR_NEW_TYPE(tp)
#define ASSERT_NEW_TYPE_OR_LOCKED(tp)
#endif
#else
#define BEGIN_TYPE_LOCK()
#define END_TYPE_LOCK()
#define BEGIN_TYPE_DICT_LOCK(d)
#define END_TYPE_DICT_LOCK()
#define ASSERT_TYPE_LOCK_HELD()
#define TYPE_IS_REVEALED(tp) 0
#define ASSERT_WORLD_STOPPED_OR_NEW_TYPE(tp)
#define ASSERT_NEW_TYPE_OR_LOCKED(tp)
#define types_world_is_stopped() 1
#define types_stop_world()
#define types_start_world()
#define type_lock_prevent_release()
#define type_lock_allow_release()
#endif
#define PyTypeObject_CAST(op)   ((PyTypeObject *)(op))
#ifndef NDEBUG
#endif
#define SIGNATURE_END_MARKER         ")\n--\n\n"
#define SIGNATURE_END_MARKER_LENGTH  6
#define CHECK(expr) \
    do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG((PyObject *)type, Py_STRINGIFY(expr)); } } while (0)
#undef CHECK
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifndef Py_GIL_DISABLED
#else
#endif
#ifndef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#define MAX_VERSIONS_PER_CLASS 1000
#if _Py_ATTR_CACHE_UNUSED < MAX_VERSIONS_PER_CLASS
#endif
#ifdef Py_DEBUG
#endif
#ifdef Py_GIL_DISABLED
#define SLOT_UPDATE_CHUNK_SIZE 30
#else
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#if defined(Py_GIL_DISABLED) && defined(Py_DEBUG) && SIZEOF_VOID_P > 4
#endif
#ifdef Py_GIL_DISABLED
#endif
#if defined(Py_GIL_DISABLED) && defined(Py_DEBUG) && SIZEOF_VOID_P > 4
#endif
#if Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#if Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifndef NDEBUG
#else
#endif
#ifdef Py_GIL_DISABLED
#endif
#define COPYVAL(SLOT) \
    if (type->SLOT == 0) { type->SLOT = base->SLOT; }
#undef COPYVAL
#undef SLOTDEFINED
#undef COPYSLOT
#undef COPYNUM
#undef COPYSEQ
#undef COPYMAP
#undef COPYBUF
#define SLOTDEFINED(SLOT) \
    (base->SLOT != 0 && \
     (basebase == NULL || base->SLOT != basebase->SLOT))
#define COPYSLOT(SLOT) \
    if (!type->SLOT && SLOTDEFINED(SLOT)) type->SLOT = base->SLOT
#define COPYASYNC(SLOT) COPYSLOT(tp_as_async->SLOT)
#define COPYNUM(SLOT) COPYSLOT(tp_as_number->SLOT)
#define COPYSEQ(SLOT) COPYSLOT(tp_as_sequence->SLOT)
#define COPYMAP(SLOT) COPYSLOT(tp_as_mapping->SLOT)
#define COPYBUF(SLOT) COPYSLOT(tp_as_buffer->SLOT)
#define COLLECTION_FLAGS (Py_TPFLAGS_SEQUENCE | Py_TPFLAGS_MAPPING)
#ifdef Py_TRACE_REFS
#endif
#undef RICHCMP_WRAPPER
#define RICHCMP_WRAPPER(NAME, OP) \
static PyObject * \
richcmp_##NAME(PyObject *self, PyObject *args, void *wrapped) \
{ \
    return wrap_richcmpfunc(self, args, wrapped, OP); \
}
#define SLOT0(FUNCNAME, DUNDER) \
static PyObject * \
FUNCNAME(PyObject *self) \
{ \
    PyObject* stack[1] = {self}; \
    return vectorcall_method(&_Py_ID(DUNDER), stack, 1); \
}
#define SLOT1(FUNCNAME, DUNDER, ARG1TYPE) \
static PyObject * \
FUNCNAME(PyObject *self, ARG1TYPE arg1) \
{ \
    PyObject* stack[2] = {self, arg1}; \
    return vectorcall_method(&_Py_ID(DUNDER), stack, 2); \
}
#define SLOT1BINFULL(FUNCNAME, TESTFUNC, SLOTNAME, DUNDER, RDUNDER) \
static PyObject * \
FUNCNAME(PyObject *self, PyObject *other) \
{ \
    PyObject* stack[2]; \
    PyThreadState *tstate = _PyThreadState_GET(); \
    int do_other = !Py_IS_TYPE(self, Py_TYPE(other)) && \
        Py_TYPE(other)->tp_as_number != NULL && \
        Py_TYPE(other)->tp_as_number->SLOTNAME == TESTFUNC; \
    if (Py_TYPE(self)->tp_as_number != NULL && \
        Py_TYPE(self)->tp_as_number->SLOTNAME == TESTFUNC) { \
        PyObject *r; \
        if (do_other && PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self))) { \
            int ok = method_is_overloaded(self, other, &_Py_ID(RDUNDER)); \
            if (ok < 0) { \
                return NULL; \
            } \
            if (ok) { \
                stack[0] = other; \
                stack[1] = self; \
                r = vectorcall_maybe(tstate, &_Py_ID(RDUNDER), stack, 2); \
                if (r != Py_NotImplemented) \
                    return r; \
                Py_DECREF(r); \
                do_other = 0; \
            } \
        } \
        stack[0] = self; \
        stack[1] = other; \
        r = vectorcall_maybe(tstate, &_Py_ID(DUNDER), stack, 2); \
        if (r != Py_NotImplemented || \
            Py_IS_TYPE(other, Py_TYPE(self))) \
            return r; \
        Py_DECREF(r); \
    } \
    if (do_other) { \
        stack[0] = other; \
        stack[1] = self; \
        return vectorcall_maybe(tstate, &_Py_ID(RDUNDER), stack, 2); \
    } \
    Py_RETURN_NOTIMPLEMENTED; \
}
#define SLOT1BIN(FUNCNAME, SLOTNAME, DUNDER, RDUNDER) \
    SLOT1BINFULL(FUNCNAME, FUNCNAME, SLOTNAME, DUNDER, RDUNDER)
#define slot_mp_length slot_sq_length
#ifndef Py_GIL_DISABLED
#endif
#ifndef Py_GIL_DISABLED
#endif
#define PyBufferWrapper_CAST(op)    ((PyBufferWrapper *)(op))
#undef TPSLOT
#undef FLSLOT
#undef BUFSLOT
#undef AMSLOT
#undef ETSLOT
#undef SQSLOT
#undef MPSLOT
#undef NBSLOT
#undef UNSLOT
#undef IBSLOT
#undef BINSLOT
#undef RBINSLOT
#define TPSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
    {#NAME, offsetof(PyTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, \
     PyDoc_STR(DOC), .name_strobj = &_Py_ID(NAME)}
#define FLSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC, FLAGS) \
    {#NAME, offsetof(PyTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, \
     PyDoc_STR(DOC), FLAGS, .name_strobj = &_Py_ID(NAME) }
#define ETSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
    {#NAME, offsetof(PyHeapTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, \
     PyDoc_STR(DOC), .name_strobj = &_Py_ID(NAME) }
#define BUFSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
    ETSLOT(NAME, as_buffer.SLOT, FUNCTION, WRAPPER, DOC)
#define AMSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
    ETSLOT(NAME, as_async.SLOT, FUNCTION, WRAPPER, DOC)
#define SQSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
    ETSLOT(NAME, as_sequence.SLOT, FUNCTION, WRAPPER, DOC)
#define MPSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
    ETSLOT(NAME, as_mapping.SLOT, FUNCTION, WRAPPER, DOC)
#define NBSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
    ETSLOT(NAME, as_number.SLOT, FUNCTION, WRAPPER, DOC)
#define UNSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
    ETSLOT(NAME, as_number.SLOT, FUNCTION, WRAPPER, \
           #NAME "($self, /)\n--\n\n" DOC)
#define IBSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \
    ETSLOT(NAME, as_number.SLOT, FUNCTION, WRAPPER, \
           #NAME "($self, value, /)\n--\n\nReturn self" DOC "value.")
#define BINSLOT(NAME, SLOT, FUNCTION, DOC) \
    ETSLOT(NAME, as_number.SLOT, FUNCTION, wrap_binaryfunc_l, \
           #NAME "($self, value, /)\n--\n\nReturn self" DOC "value.")
#define RBINSLOT(NAME, SLOT, FUNCTION, DOC) \
    ETSLOT(NAME, as_number.SLOT, FUNCTION, wrap_binaryfunc_r, \
           #NAME "($self, value, /)\n--\n\nReturn value" DOC "self.")
#define BINSLOTNOTINFIX(NAME, SLOT, FUNCTION, DOC) \
    ETSLOT(NAME, as_number.SLOT, FUNCTION, wrap_binaryfunc_l, \
           #NAME "($self, value, /)\n--\n\n" DOC)
#define RBINSLOTNOTINFIX(NAME, SLOT, FUNCTION, DOC) \
    ETSLOT(NAME, as_number.SLOT, FUNCTION, wrap_binaryfunc_r, \
           #NAME "($self, value, /)\n--\n\n" DOC)
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#define superobject_CAST(op)    ((superobject *)(op))
static inline int
managed_static_type_index_is_set(PyTypeObject *self)
{
    return self->tp_subclasses != NULL;
}
static inline size_t
managed_static_type_index_get(PyTypeObject *self)
{
    assert(managed_static_type_index_is_set(self));
    /* We store a 1-based index so 0 can mean "not initialized". */
    return (size_t)self->tp_subclasses - 1;
}
static managed_static_type_state *
managed_static_type_state_get(PyInterpreterState *interp, PyTypeObject *self)
{
    // It's probably a builtin type.
    size_t index = managed_static_type_index_get(self);
    managed_static_type_state *state =
            &(interp->types.builtins.initialized[index]);
    if (state->type == self) {
        return state;
    }
    if (index > _Py_MAX_MANAGED_STATIC_EXT_TYPES) {
        return state;
    }
    return &(interp->types.for_extensions.initialized[index]);
}
managed_static_type_state *
_PyStaticType_GetState(PyInterpreterState *interp, PyTypeObject *self)
{
    assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN);
    return managed_static_type_state_get(interp, self);
}
int
_PyType_CacheInitForSpecialization(PyHeapTypeObject *type, PyObject *init,
                                   unsigned int tp_version)
{
    if (!init || !tp_version) {
        return 0;
    }
    int can_cache;
    BEGIN_TYPE_LOCK();
    can_cache = ((PyTypeObject*)type)->tp_version_tag == tp_version;
    #ifdef Py_GIL_DISABLED
    can_cache = can_cache && _PyObject_HasDeferredRefcount(init);
    #endif
    if (can_cache) {
        FT_ATOMIC_STORE_PTR_RELEASE(type->_spec_cache.init, init);
    }
    END_TYPE_LOCK();
    return can_cache;
}
int
_PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version)
{
    if (!descriptor || !tp_version) {
        return 0;
    }
    int can_cache;
    BEGIN_TYPE_LOCK();
    can_cache = ((PyTypeObject*)ht)->tp_version_tag == tp_version;
    // This pointer is invalidated by PyType_Modified (see the comment on
    // struct _specialization_cache):
    PyFunctionObject *func = (PyFunctionObject *)descriptor;
    uint32_t version = _PyFunction_GetVersionForCurrentState(func);
    can_cache = can_cache && _PyFunction_IsVersionValid(version);
#ifdef Py_GIL_DISABLED
    can_cache = can_cache && _PyObject_HasDeferredRefcount(descriptor);
#endif
    if (can_cache) {
        FT_ATOMIC_STORE_PTR_RELEASE(ht->_spec_cache.getitem, descriptor);
        FT_ATOMIC_STORE_UINT32_RELAXED(ht->_spec_cache.getitem_version, version);
    }
    END_TYPE_LOCK();
    return can_cache;
}
static inline PyObject *
lookup_tp_bases(PyTypeObject *self)
{
    return self->tp_bases;
}
static inline unsigned int
next_global_version_tag(void)
{
    unsigned int old;
    do {
        old = _Py_atomic_load_uint_relaxed(&_PyRuntime.types.next_version_tag);
        if (old >= _Py_MAX_GLOBAL_TYPE_VERSION_TAG) {
            return 0;
        }
    } while (!_Py_atomic_compare_exchange_uint(&_PyRuntime.types.next_version_tag, &old, old + 1));
    return old + 1;
}
static void
set_version_unlocked(PyTypeObject *tp, unsigned int version)
{
    assert(version == 0 || (tp->tp_versions_used != _Py_ATTR_CACHE_UNUSED));
#ifndef Py_GIL_DISABLED
    PyInterpreterState *interp = _PyInterpreterState_GET();
    // lookup the old version and set to null
    if (tp->tp_version_tag != 0) {
        PyTypeObject **slot =
            interp->types.type_version_cache
            + (tp->tp_version_tag % TYPE_VERSION_CACHE_SIZE);
        *slot = NULL;
    }
    if (version) {
        tp->tp_versions_used++;
    }
#else
    if (version) {
        _Py_atomic_add_uint16(&tp->tp_versions_used, 1);
    }
#endif
    FT_ATOMIC_STORE_UINT_RELAXED(tp->tp_version_tag, version);
#ifndef Py_GIL_DISABLED
    if (version != 0) {
        PyTypeObject **slot =
            interp->types.type_version_cache
            + (version % TYPE_VERSION_CACHE_SIZE);
        *slot = tp;
    }
#endif
}
static int
assign_version_tag(PyInterpreterState *interp, PyTypeObject *type)
{
    ASSERT_TYPE_LOCK_HELD();

    /* Ensure that the tp_version_tag is valid.
     * To respect the invariant, this must first be done on all super classes.
     * Return 0 if this cannot be done, 1 if tp_version_tag is set.
    */
    if (type->tp_version_tag != 0) {
        return 1;
    }
    if (!_PyType_HasFeature(type, Py_TPFLAGS_READY)) {
        return 0;
    }
    if (type->tp_versions_used >= MAX_VERSIONS_PER_CLASS) {
        /* (this includes `tp_versions_used == _Py_ATTR_CACHE_UNUSED`) */
        return 0;
    }

    PyObject *bases = lookup_tp_bases(type);
    Py_ssize_t n = PyTuple_GET_SIZE(bases);
    for (Py_ssize_t i = 0; i < n; i++) {
        PyObject *b = PyTuple_GET_ITEM(bases, i);
        if (!assign_version_tag(interp, _PyType_CAST(b))) {
            return 0;
        }
    }
    if (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) {
        /* static types */
        unsigned int next_version_tag = next_global_version_tag();
        if (next_version_tag == 0) {
            /* We have run out of version numbers */
            return 0;
        }
        set_version_unlocked(type, next_version_tag);
        assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG);
    }
    else {
        /* heap types */
        if (NEXT_VERSION_TAG(interp) == 0) {
            /* We have run out of version numbers */
            return 0;
        }
        set_version_unlocked(type, NEXT_VERSION_TAG(interp)++);
        assert (type->tp_version_tag != 0);
    }
    return 1;
}
int
_PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version)
{
    int err;
    BEGIN_TYPE_LOCK();
    err = validate(ty);
    if (!err) {
        if(assign_version_tag(_PyInterpreterState_GET(), ty)) {
            *tp_version = ty->tp_version_tag;
        }
        else {
            err = -1;
        }
    }
    END_TYPE_LOCK();
    return err;
}

#define _PyFrame_New_NoTrack _CiFrame_New_NoTrack
PyFrameObject*
_PyFrame_New_NoTrack(PyCodeObject *code)
{
    CALL_STAT_INC(frame_objects_created);
    int slots = code->co_nlocalsplus + code->co_stacksize;
    PyFrameObject *f = PyObject_GC_NewVar(PyFrameObject, &PyFrame_Type, slots);
    if (f == NULL) {
        return NULL;
    }
    f->f_back = NULL;
    f->f_trace = NULL;
    f->f_trace_lines = 1;
    f->f_trace_opcodes = 0;
    f->f_lineno = 0;
    f->f_extra_locals = NULL;
    f->f_locals_cache = NULL;
    f->f_overwritten_fast_locals = NULL;
    return f;
}
PyFrameObject *
_PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame)
{
    assert(frame->frame_obj == NULL);
    PyObject *exc = PyErr_GetRaisedException();

    PyFrameObject *f = _PyFrame_New_NoTrack(_PyFrame_GetCode(frame));
    if (f == NULL) {
        Py_XDECREF(exc);
        return NULL;
    }
    PyErr_SetRaisedException(exc);

    // GH-97002: There was a time when a frame object could be created when we
    // are allocating the new frame object f above, so frame->frame_obj would
    // be assigned already. That path does not exist anymore. We won't call any
    // Python code in this function and garbage collection will not run.
    // Notice that _PyFrame_New_NoTrack() can potentially raise a MemoryError,
    // but it won't allocate a traceback until the frame unwinds, so we are safe
    // here.
    assert(frame->frame_obj == NULL);
    assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT);
    f->f_frame = frame;
    frame->frame_obj = f;
    return f;
}
void
_PyFrame_ClearLocals(_PyInterpreterFrame *frame)
{
    assert(frame->stackpointer != NULL);
    _PyStackRef *sp = frame->stackpointer;
    _PyStackRef *locals = frame->localsplus;
    frame->stackpointer = locals;
    while (sp > locals) {
        sp--;
        PyStackRef_XCLOSE(*sp);
    }
    Py_CLEAR(frame->f_locals);
}
static void
take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame)
{
    Py_BEGIN_CRITICAL_SECTION(f);
    assert(frame->owner < FRAME_OWNED_BY_INTERPRETER);
    assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT);
    _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)f->_f_frame_data;
    _PyFrame_Copy(frame, new_frame);
    // _PyFrame_Copy takes the reference to the executable,
    // so we need to restore it.
    new_frame->f_executable = PyStackRef_DUP(new_frame->f_executable);
    f->f_frame = new_frame;
    new_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT;
    if (_PyFrame_IsIncomplete(new_frame)) {
        // This may be a newly-created generator or coroutine frame. Since it's
        // dead anyways, just pretend that the first RESUME ran:
        PyCodeObject *code = _PyFrame_GetCode(new_frame);
        new_frame->instr_ptr =
            _PyFrame_GetBytecode(new_frame) + code->_co_firsttraceable + 1;
    }
    assert(!_PyFrame_IsIncomplete(new_frame));
    assert(f->f_back == NULL);
    _PyInterpreterFrame *prev = _PyFrame_GetFirstComplete(frame->previous);
    if (prev) {
        assert(prev->owner < FRAME_OWNED_BY_INTERPRETER);
        PyObject *exc = PyErr_GetRaisedException();
        /* Link PyFrameObjects.f_back and remove link through _PyInterpreterFrame.previous */
        PyFrameObject *back = _PyFrame_GetFrameObject(prev);
        if (back == NULL) {
            /* Memory error here. */
            assert(PyErr_ExceptionMatches(PyExc_MemoryError));
            /* Nothing we can do about it */
            PyErr_Clear();
        }
        else {
            f->f_back = (PyFrameObject *)Py_NewRef(back);
        }
        PyErr_SetRaisedException(exc);
    }
    if (!_PyObject_GC_IS_TRACKED((PyObject *)f)) {
        _PyObject_GC_TRACK((PyObject *)f);
    }
    Py_END_CRITICAL_SECTION();
}
void
_PyFrame_ClearExceptCode(_PyInterpreterFrame *frame)
{
    /* It is the responsibility of the owning generator/coroutine
     * to have cleared the enclosing generator, if any. */
    assert(frame->owner != FRAME_OWNED_BY_GENERATOR ||
           FT_ATOMIC_LOAD_INT8_RELAXED(_PyGen_GetGeneratorFromFrame(frame)->gi_frame_state) == FRAME_CLEARED);
    // GH-99729: Clearing this frame can expose the stack (via finalizers). It's
    // crucial that this frame has been unlinked, and is no longer visible:
    assert(_PyThreadState_GET()->current_frame != frame);
    if (frame->frame_obj) {
        PyFrameObject *f = frame->frame_obj;
        frame->frame_obj = NULL;
        if (!_PyObject_IsUniquelyReferenced((PyObject *)f)) {
            take_ownership(f, frame);
            Py_DECREF(f);
            return;
        }
        Py_DECREF(f);
    }
    _PyFrame_ClearLocals(frame);
    PyStackRef_CLEAR(frame->f_funcobj);
}


int
_PyObject_HasLen(PyObject *o) {
    return (Py_TYPE(o)->tp_as_sequence && Py_TYPE(o)->tp_as_sequence->sq_length) ||
        (Py_TYPE(o)->tp_as_mapping && Py_TYPE(o)->tp_as_mapping->mp_length);
}


#ifdef HAVE_DLOPEN
#  ifdef HAVE_DLFCN_H
#  endif
#  if !HAVE_DECL_RTLD_LAZY
#    define RTLD_LAZY 1
#  endif
#endif
#define tstate_verify_not_active(tstate) \
    if (tstate == current_fast_get()) { \
        _Py_FatalErrorFormat(__func__, "tstate %p is still current", tstate); \
    }
#ifndef NDEBUG
#endif  // !NDEBUG
#ifdef PY_HAVE_THREAD_NATIVE_ID
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifndef HAVE_PTHREAD_STUBS
#endif
#ifdef PY_HAVE_THREAD_NATIVE_ID
#endif
#define LOCKS_INIT(runtime) \
    { \
        &(runtime)->interpreters.mutex, \
        &(runtime)->xi.data_lookup.registry.mutex, \
        &(runtime)->unicode_state.ids.mutex, \
        &(runtime)->imports.extensions.mutex, \
        &(runtime)->ceval.pending_mainthread.mutex, \
        &(runtime)->atexit.mutex, \
        &(runtime)->audit_hooks.mutex, \
        &(runtime)->allocators.mutex, \
        &(runtime)->_main_interpreter.types.mutex, \
        &(runtime)->_main_interpreter.code_state.mutex, \
    }
#ifdef Py_REF_DEBUG
#endif
#ifdef HAVE_FORK
#ifdef Py_GIL_DISABLED
#endif
#endif
#ifdef Py_STATS
#endif
#ifndef NDEBUG
#endif
#ifdef Py_GIL_DISABLED
#endif
#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
#  ifdef Py_STACKREF_CLOSE_DEBUG
#  endif
#endif
#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef HAVE_FORK
#endif
#ifdef _Py_TIER2
#endif
#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
#  ifdef Py_STACKREF_CLOSE_DEBUG
#  endif
#endif
#ifdef Py_REF_DEBUG
#endif
#ifdef HAVE_FORK
#endif
#ifndef NDEBUG
#endif
#if LLONG_MAX > INT64_MAX
#endif
#ifndef NDEBUG
#endif
#ifdef Py_STATS
#endif
#ifdef Py_DEBUG
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef _Py_TIER2
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_STATS
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef _Py_TIER2
#endif
#if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED)
#endif
#if _Py_TIER2
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_STATS
#endif
#ifdef Py_GIL_DISABLED
#endif
#if Py_STATS
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#if defined(Py_DEBUG)
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_STATS
#endif
#if defined(Py_DEBUG)
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#define _Py_FOR_EACH_STW_INTERP(stw, i)                                     \
    for (PyInterpreterState *i = interp_for_stop_the_world((stw));          \
            i != NULL; i = ((stw->is_global) ? i->next : NULL))
#endif  // Py_GIL_DISABLED
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#if defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API)
#endif
#ifndef NDEBUG
#endif
#ifdef _Py_TIER2
#endif
#define MINIMUM_OVERHEAD 1000
#ifndef NDEBUG
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef WITH_MIMALLOC
#endif
#if defined(Py_GIL_DISABLED) && !defined(WITH_MIMALLOC)
#endif
#undef  uint
#define uint pymem_uint
#ifdef WITH_MIMALLOC
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#define QSBR_PAGE_MEM_LIMIT 4096*20
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#else
#endif
#endif // WITH_MIMALLOC
#define MALLOC_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree}
#ifdef WITH_MIMALLOC
#  define MIMALLOC_ALLOC    {NULL, _PyMem_MiMalloc, _PyMem_MiCalloc, _PyMem_MiRealloc, _PyMem_MiFree}
#  define MIMALLOC_RAWALLOC {NULL, _PyMem_MiRawMalloc, _PyMem_MiRawCalloc, _PyMem_MiRawRealloc, _PyMem_MiRawFree}
#  define MIMALLOC_OBJALLOC {NULL, _PyObject_MiMalloc, _PyObject_MiCalloc, _PyObject_MiRealloc, _PyObject_MiFree}
#endif
#if defined(WITH_PYMALLOC)
#  define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free}
#endif  // WITH_PYMALLOC
#if defined(Py_GIL_DISABLED)
#  define PYRAW_ALLOC MIMALLOC_RAWALLOC
#  define PYMEM_ALLOC MIMALLOC_ALLOC
#  define PYOBJ_ALLOC MIMALLOC_OBJALLOC
#elif defined(WITH_PYMALLOC)
#  define PYRAW_ALLOC MALLOC_ALLOC
#  define PYMEM_ALLOC PYMALLOC_ALLOC
#  define PYOBJ_ALLOC PYMALLOC_ALLOC
#else
#  define PYRAW_ALLOC MALLOC_ALLOC
#  define PYMEM_ALLOC MALLOC_ALLOC
#  define PYOBJ_ALLOC MALLOC_ALLOC
#endif
#define PYDBGRAW_ALLOC \
    {&_PyRuntime.allocators.debug.raw, _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree}
#define PYDBGMEM_ALLOC \
    {&_PyRuntime.allocators.debug.mem, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree}
#define PYDBGOBJ_ALLOC \
    {&_PyRuntime.allocators.debug.obj, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree}
#ifdef Py_DEBUG
#else
#endif
#ifdef Py_DEBUG
#else
#endif
#ifdef Py_DEBUG
#else
#endif
#ifdef Py_DEBUG
#else
#endif
#ifdef WITH_PYMALLOC
#  ifdef MS_WINDOWS
#  elif defined(HAVE_MMAP)
#    ifdef MAP_ANONYMOUS
#      define ARENAS_USE_MMAP
#    endif
#  endif
#endif
#if defined(PYMALLOC_USE_HUGEPAGES) && defined(ARENAS_USE_MMAP) && defined(MAP_HUGETLB)
#ifdef __linux__
#endif
#endif
#ifdef MS_WINDOWS
#  ifdef PYMALLOC_USE_HUGEPAGES
#  endif
#elif defined(ARENAS_USE_MMAP)
#  ifdef PYMALLOC_USE_HUGEPAGES
#    ifdef MAP_HUGETLB
#    endif
#  endif
#else
#endif
#if defined(ARENAS_USE_MMAP)
#else
#endif
#ifdef MS_WINDOWS
#elif defined(ARENAS_USE_MMAP)
#else
#endif
#define ALLOCATORS_MUTEX (_PyRuntime.allocators.mutex)
#define _PyMem_Raw (_PyRuntime.allocators.standard.raw)
#define _PyMem (_PyRuntime.allocators.standard.mem)
#define _PyObject (_PyRuntime.allocators.standard.obj)
#define _PyMem_Debug (_PyRuntime.allocators.debug)
#define _PyObject_Arena (_PyRuntime.allocators.obj_arena)
#ifdef Py_DEBUG
#else
#endif
#if defined(WITH_PYMALLOC) && !defined(Py_GIL_DISABLED)
#endif
#ifdef WITH_MIMALLOC
#endif
#ifndef Py_GIL_DISABLED
#endif
#ifdef WITH_PYMALLOC
#endif
#ifdef WITH_MIMALLOC
#endif
#ifdef WITH_PYMALLOC
#endif
#ifdef WITH_MIMALLOC
#endif
#ifdef WITH_PYMALLOC
#endif
#ifdef WITH_MIMALLOC
#endif
#ifdef WITH_PYMALLOC
#endif
#ifdef WITH_MIMALLOC
#endif
#ifdef WITH_PYMALLOC
#ifdef WITH_MIMALLOC
#ifdef Py_GIL_DISABLED
#else
#endif
#endif  // WITH_MIMALLOC
#endif  // WITH_PYMALLOC
#define WORK_ITEMS_PER_CHUNK 254
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#define QSBR_DEFERRED_LIMIT 127
#define QSBR_FREE_MEM_LIMIT 1024*1024
#endif
#ifndef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#if (defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 2))) && defined(__OPTIMIZE__)
#  define UNLIKELY(value) __builtin_expect((value), 0)
#  define LIKELY(value) __builtin_expect((value), 1)
#else
#  define UNLIKELY(value) (value)
#  define LIKELY(value) (value)
#endif
#ifdef WITH_PYMALLOC
#ifdef WITH_VALGRIND
#endif
#define usedpools (state->pools.used)
#define allarenas (state->mgmt.arenas)
#define maxarenas (state->mgmt.maxarenas)
#define unused_arena_objects (state->mgmt.unused_arena_objects)
#define usable_arenas (state->mgmt.usable_arenas)
#define nfp2lasta (state->mgmt.nfp2lasta)
#define narenas_currently_allocated (state->mgmt.narenas_currently_allocated)
#define ntimes_arena_allocated (state->mgmt.ntimes_arena_allocated)
#define narenas_highwater (state->mgmt.narenas_highwater)
#define raw_allocated_blocks (state->mgmt.raw_allocated_blocks)
#ifdef WITH_MIMALLOC
#ifdef Py_GIL_DISABLED
#else
#endif
#endif
#ifdef WITH_MIMALLOC
#endif
#ifdef Py_DEBUG
#else
#endif
#ifdef WITH_MIMALLOC
#endif
#ifdef Py_DEBUG
#endif
#ifdef Py_DEBUG
#endif
#ifdef Py_DEBUG
#endif
#if WITH_PYMALLOC_RADIX_TREE
#define arena_map_root (state->usage.arena_map_root)
#ifdef USE_INTERIOR_NODES
#define arena_map_mid_count (state->usage.arena_map_mid_count)
#define arena_map_bot_count (state->usage.arena_map_bot_count)
#endif
#ifdef USE_INTERIOR_NODES
#else
#endif
#endif /* WITH_PYMALLOC_RADIX_TREE */
#if SIZEOF_SIZE_T <= SIZEOF_INT
#endif
#if WITH_PYMALLOC_RADIX_TREE
#endif
#if WITH_PYMALLOC_RADIX_TREE
#else
#endif /* !WITH_PYMALLOC_RADIX_TREE */
#ifdef WITH_MEMORY_LIMITS
#endif
#ifdef WITH_VALGRIND
#endif
#if WITH_PYMALLOC_RADIX_TREE
#endif
#ifdef WITH_VALGRIND
#endif
#ifdef WITH_VALGRIND
#endif
#else   /* ! WITH_PYMALLOC */
#endif /* WITH_PYMALLOC */
#ifdef PYMEM_DEBUG_SERIALNO
#endif
#define SST SIZEOF_SIZE_T
#ifdef PYMEM_DEBUG_SERIALNO
#  define PYMEM_DEBUG_EXTRA_BYTES 4 * SST
#else
#  define PYMEM_DEBUG_EXTRA_BYTES 3 * SST
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef PYMEM_DEBUG_SERIALNO
#endif
#ifdef PYMEM_DEBUG_SERIALNO
#endif
#define ERASED_SIZE 64
#ifdef PYMEM_DEBUG_SERIALNO
#endif
#ifndef Py_GIL_DISABLED
#endif
#ifdef PYMEM_DEBUG_SERIALNO
#endif
#ifdef PYMEM_DEBUG_SERIALNO
#endif
#ifndef Py_GIL_DISABLED
#endif
#ifndef Py_GIL_DISABLED
#else
#endif
#ifdef PYMEM_DEBUG_SERIALNO
#endif
#if WITH_PYMALLOC
#else
#endif
#ifdef WITH_PYMALLOC
#endif /* WITH_PYMALLOC */
#ifdef WITH_PYMALLOC
#endif /* WITH_PYMALLOC */
#ifdef WITH_PYMALLOC
#if WITH_PYMALLOC_RADIX_TREE
#ifdef USE_INTERIOR_NODES
#endif
#endif
#ifdef Py_DEBUG
#endif
#ifdef WITH_MIMALLOC
#endif
#ifdef Py_DEBUG
#endif
#ifdef Py_DEBUG
#endif
#ifdef PYMEM_DEBUG_SERIALNO
#endif
#if WITH_PYMALLOC_RADIX_TREE
#ifdef USE_INTERIOR_NODES
#endif
#ifdef USE_INTERIOR_NODES
#endif
#endif
#ifdef WITH_MIMALLOC
#endif
#endif /* #ifdef WITH_PYMALLOC */
void *
_PyObject_VirtualAlloc(size_t size)
{
    return _PyObject_Arena.alloc(_PyObject_Arena.ctx, size);
}
static _PyStackChunk*
allocate_chunk(int size_in_bytes, _PyStackChunk* previous)
{
    assert(size_in_bytes % sizeof(PyObject **) == 0);
    _PyStackChunk *res = _PyObject_VirtualAlloc(size_in_bytes);
    if (res == NULL) {
        return NULL;
    }
    res->previous = previous;
    res->size = size_in_bytes;
    res->top = 0;
    return res;
}
static PyObject **
push_chunk(PyThreadState *tstate, int size)
{
    int allocate_size = _PY_DATA_STACK_CHUNK_SIZE;
    while (allocate_size < (int)sizeof(PyObject*)*(size + MINIMUM_OVERHEAD)) {
        allocate_size *= 2;
    }
    _PyStackChunk *new;
    if (tstate->datastack_cached_chunk != NULL
        && (size_t)allocate_size <= tstate->datastack_cached_chunk->size)
    {
        new = tstate->datastack_cached_chunk;
        tstate->datastack_cached_chunk = NULL;
        new->previous = tstate->datastack_chunk;
        new->top = 0;
    }
    else {
        new = allocate_chunk(allocate_size, tstate->datastack_chunk);
        if (new == NULL) {
            return NULL;
        }
    }
    if (tstate->datastack_chunk) {
        tstate->datastack_chunk->top = tstate->datastack_top -
                                       &tstate->datastack_chunk->data[0];
    }
    tstate->datastack_chunk = new;
    tstate->datastack_limit = (PyObject **)(((char *)new) + allocate_size);
    // When new is the "root" chunk (i.e. new->previous == NULL), we can keep
    // _PyThreadState_PopFrame from freeing it later by "skipping" over the
    // first element:
    PyObject **res = &new->data[new->previous == NULL];
    tstate->datastack_top = res + size;
    return res;
}
_PyInterpreterFrame *
_PyThreadState_PushFrame(PyThreadState *tstate, size_t size)
{
    assert(size < INT_MAX/sizeof(PyObject *));
    if (_PyThreadState_HasStackSpace(tstate, (int)size)) {
        _PyInterpreterFrame *res = (_PyInterpreterFrame *)tstate->datastack_top;
        tstate->datastack_top += size;
        return res;
    }
    return (_PyInterpreterFrame *)push_chunk(tstate, (int)size);
}

void _PyErr_SetObject(PyThreadState* tstate, PyObject* type, PyObject* value) {
  PyErr_SetObject(type, value);
}

#define _PyErr_NoMemory(tstate) PyErr_NoMemory()

void
_PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs,
                         PyObject *kwnames)
{
    /* Only decref kwargs values, positional args are borrowed */
    Py_ssize_t nkwargs = PyTuple_GET_SIZE(kwnames);
    for (Py_ssize_t i = 0; i < nkwargs; i++) {
        Py_DECREF(stack[nargs + i]);
    }
    _PyStack_UnpackDict_FreeNoDecRef(stack, kwnames);
}

PyObject *
_PyExc_CreateExceptionGroup(const char *msg_str, PyObject *excs)
{
    PyObject *msg = PyUnicode_FromString(msg_str);
    if (!msg) {
        return NULL;
    }
    PyObject *args = _PyTuple_FromPairSteal(msg, Py_NewRef(excs));
    if (!args) {
        return NULL;
    }
    PyObject *result = PyObject_CallObject(PyExc_BaseExceptionGroup, args);
    Py_DECREF(args);
    return result;
}


static PySliceObject *
_PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step)
{
    assert(start != NULL && stop != NULL && step != NULL);
    PySliceObject *obj = _Py_FREELIST_POP(PySliceObject, slices);
    if (obj == NULL) {
        obj = PyObject_GC_New(PySliceObject, &PySlice_Type);
        if (obj == NULL) {
            goto error;
        }
    }

    obj->start = start;
    obj->stop = stop;
    obj->step = Py_NewRef(step);

    _PyObject_GC_TRACK(obj);
    return obj;
error:
    Py_DECREF(start);
    Py_DECREF(stop);
    return NULL;
}
PyObject *
_PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop)
{
    assert(start != NULL && stop != NULL);
    return (PyObject *)_PyBuildSlice_Consume2(start, stop, Py_None);
}


PyObject *
_PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs)
{
    return PyNumber_Power(lhs, rhs, Py_None);
}
PyObject *
_PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs)
{
    return PyNumber_InPlacePower(lhs, rhs, Py_None);
}

#if Py_STATS
#define SPECIALIZATION_FAIL(opcode, kind) \
do { \
    PyStats *s = _PyStats_GET(); \
    if (s) { \
        int _kind = (kind); \
        assert(_kind < SPECIALIZATION_FAILURE_KINDS); \
        s->opcode_stats[opcode].specialization.failure_kinds[_kind]++; \
    } \
} while (0)
#else
#  define SPECIALIZATION_FAIL(opcode, kind) ((void)0)
#endif  // Py_STATS
#define SIMPLE_FUNCTION 0
#define SPEC_FAIL_OTHER 0
#define SPEC_FAIL_NO_DICT 1
#define SPEC_FAIL_OVERRIDDEN 2
#define SPEC_FAIL_OUT_OF_VERSIONS 3
#define SPEC_FAIL_OUT_OF_RANGE 4
#define SPEC_FAIL_EXPECTED_ERROR 5
#define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 6
#define SPEC_FAIL_CODE_COMPLEX_PARAMETERS 7
#define SPEC_FAIL_CODE_NOT_OPTIMIZED 8
#define SPEC_FAIL_LOAD_GLOBAL_NON_DICT 17
#define SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT 18
#define SPEC_FAIL_SUPER_BAD_CLASS 9
#define SPEC_FAIL_SUPER_SHADOWED 10
#define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 9
#define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 10
#define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 11
#define SPEC_FAIL_ATTR_METHOD 12
#define SPEC_FAIL_ATTR_MUTABLE_CLASS 13
#define SPEC_FAIL_ATTR_PROPERTY 14
#define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 15
#define SPEC_FAIL_ATTR_READ_ONLY 16
#define SPEC_FAIL_ATTR_AUDITED_SLOT 17
#define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18
#define SPEC_FAIL_ATTR_NON_STRING 19
#define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20
#define SPEC_FAIL_ATTR_SHADOWED 21
#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22
#define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23
#define SPEC_FAIL_ATTR_OBJECT_SLOT 24
#define SPEC_FAIL_ATTR_MODULE_LAZY_VALUE 25
#define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26
#define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27
#define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28
#define SPEC_FAIL_ATTR_NOT_IN_KEYS 29
#define SPEC_FAIL_ATTR_NOT_IN_DICT 30
#define SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE 31
#define SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR 32
#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33
#define SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN 34
#define SPEC_FAIL_ATTR_SPLIT_DICT 35
#define SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED 36
#define SPEC_FAIL_ATTR_SLOT_AFTER_ITEMS 37
#define SPEC_FAIL_SUBSCR_ARRAY_INT 9
#define SPEC_FAIL_SUBSCR_ARRAY_SLICE 10
#define SPEC_FAIL_SUBSCR_LIST_SLICE 11
#define SPEC_FAIL_SUBSCR_BUFFER_INT 12
#define SPEC_FAIL_SUBSCR_BUFFER_SLICE 13
#define SPEC_FAIL_SUBSCR_BYTEARRAY_INT 18
#define SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE 19
#define SPEC_FAIL_SUBSCR_PY_SIMPLE 20
#define SPEC_FAIL_SUBSCR_PY_OTHER 21
#define SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE 22
#define SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE 23
#define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES          9
#define SPEC_FAIL_BINARY_OP_ADD_OTHER                   10
#define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES         11
#define SPEC_FAIL_BINARY_OP_AND_INT                     12
#define SPEC_FAIL_BINARY_OP_AND_OTHER                   13
#define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE                14
#define SPEC_FAIL_BINARY_OP_LSHIFT                      15
#define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY             16
#define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES    17
#define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER              18
#define SPEC_FAIL_BINARY_OP_OR                          19
#define SPEC_FAIL_BINARY_OP_POWER                       20
#define SPEC_FAIL_BINARY_OP_REMAINDER                   21
#define SPEC_FAIL_BINARY_OP_RSHIFT                      22
#define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES    23
#define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER              24
#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 25
#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT           26
#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER           27
#define SPEC_FAIL_BINARY_OP_XOR                         28
#define SPEC_FAIL_BINARY_OP_OR_INT                      29
#define SPEC_FAIL_BINARY_OP_OR_DIFFERENT_TYPES          30
#define SPEC_FAIL_BINARY_OP_XOR_INT                     31
#define SPEC_FAIL_BINARY_OP_XOR_DIFFERENT_TYPES         32
#define SPEC_FAIL_BINARY_OP_SUBSCR                      33
#define SPEC_FAIL_BINARY_OP_SUBSCR_LIST_SLICE           34
#define SPEC_FAIL_BINARY_OP_SUBSCR_TUPLE_SLICE          35
#define SPEC_FAIL_BINARY_OP_SUBSCR_STRING_SLICE         36
#define SPEC_FAIL_BINARY_OP_SUBSCR_NOT_HEAP_TYPE        37
#define SPEC_FAIL_BINARY_OP_SUBSCR_OTHER_SLICE          38
#define SPEC_FAIL_BINARY_OP_SUBSCR_MAPPINGPROXY         39
#define SPEC_FAIL_BINARY_OP_SUBSCR_RE_MATCH             40
#define SPEC_FAIL_BINARY_OP_SUBSCR_ARRAY                41
#define SPEC_FAIL_BINARY_OP_SUBSCR_DEQUE                42
#define SPEC_FAIL_BINARY_OP_SUBSCR_ENUMDICT             43
#define SPEC_FAIL_BINARY_OP_SUBSCR_STACKSUMMARY         44
#define SPEC_FAIL_BINARY_OP_SUBSCR_DEFAULTDICT          45
#define SPEC_FAIL_BINARY_OP_SUBSCR_COUNTER              46
#define SPEC_FAIL_BINARY_OP_SUBSCR_ORDEREDDICT          47
#define SPEC_FAIL_BINARY_OP_SUBSCR_BYTES                48
#define SPEC_FAIL_BINARY_OP_SUBSCR_STRUCTTIME           49
#define SPEC_FAIL_BINARY_OP_SUBSCR_RANGE                50
#define SPEC_FAIL_CALL_INSTANCE_METHOD 11
#define SPEC_FAIL_CALL_CMETHOD 12
#define SPEC_FAIL_CALL_CFUNC_VARARGS 13
#define SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS 14
#define SPEC_FAIL_CALL_CFUNC_NOARGS 15
#define SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS 16
#define SPEC_FAIL_CALL_METH_DESCR_VARARGS 17
#define SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS 18
#define SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS 19
#define SPEC_FAIL_CALL_BAD_CALL_FLAGS 20
#define SPEC_FAIL_CALL_INIT_NOT_PYTHON 21
#define SPEC_FAIL_CALL_PEP_523 22
#define SPEC_FAIL_CALL_BOUND_METHOD 23
#define SPEC_FAIL_CALL_VECTORCALL 24
#define SPEC_FAIL_CALL_CLASS_MUTABLE 26
#define SPEC_FAIL_CALL_METHOD_WRAPPER 28
#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29
#define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30
#define SPEC_FAIL_CALL_METACLASS 31
#define SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES 32
#define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12
#define SPEC_FAIL_COMPARE_OP_STRING 13
#define SPEC_FAIL_COMPARE_OP_BIG_INT 14
#define SPEC_FAIL_COMPARE_OP_BYTES 15
#define SPEC_FAIL_COMPARE_OP_TUPLE 16
#define SPEC_FAIL_COMPARE_OP_LIST 17
#define SPEC_FAIL_COMPARE_OP_SET 18
#define SPEC_FAIL_COMPARE_OP_BOOL 19
#define SPEC_FAIL_COMPARE_OP_BASEOBJECT 20
#define SPEC_FAIL_COMPARE_OP_FLOAT_LONG 21
#define SPEC_FAIL_COMPARE_OP_LONG_FLOAT 22
#define SPEC_FAIL_ITER_GENERATOR 10
#define SPEC_FAIL_ITER_COROUTINE 11
#define SPEC_FAIL_ITER_ASYNC_GENERATOR 12
#define SPEC_FAIL_ITER_LIST 13
#define SPEC_FAIL_ITER_TUPLE 14
#define SPEC_FAIL_ITER_SET 15
#define SPEC_FAIL_ITER_STRING 16
#define SPEC_FAIL_ITER_BYTES 17
#define SPEC_FAIL_ITER_RANGE 18
#define SPEC_FAIL_ITER_ITERTOOLS 19
#define SPEC_FAIL_ITER_DICT_KEYS 20
#define SPEC_FAIL_ITER_DICT_ITEMS 21
#define SPEC_FAIL_ITER_DICT_VALUES 22
#define SPEC_FAIL_ITER_ENUMERATE 23
#define SPEC_FAIL_ITER_MAP 24
#define SPEC_FAIL_ITER_ZIP 25
#define SPEC_FAIL_ITER_SEQ_ITER 26
#define SPEC_FAIL_ITER_REVERSED_LIST 27
#define SPEC_FAIL_ITER_CALLABLE 28
#define SPEC_FAIL_ITER_ASCII_STRING 29
#define SPEC_FAIL_ITER_ASYNC_GENERATOR_SEND 30
#define SPEC_FAIL_ITER_SELF 31
#define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 9
#define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 10
#define SPEC_FAIL_TO_BOOL_BYTEARRAY    9
#define SPEC_FAIL_TO_BOOL_BYTES       10
#define SPEC_FAIL_TO_BOOL_DICT        11
#define SPEC_FAIL_TO_BOOL_FLOAT       12
#define SPEC_FAIL_TO_BOOL_MAPPING     13
#define SPEC_FAIL_TO_BOOL_MEMORY_VIEW 14
#define SPEC_FAIL_TO_BOOL_NUMBER      15
#define SPEC_FAIL_TO_BOOL_SEQUENCE    16
#define SPEC_FAIL_TO_BOOL_SET         17
#define SPEC_FAIL_TO_BOOL_TUPLE       18
#define SPEC_FAIL_CONTAINS_OP_STR        9
#define SPEC_FAIL_CONTAINS_OP_TUPLE      10
#define SPEC_FAIL_CONTAINS_OP_LIST       11
#define SPEC_FAIL_CONTAINS_OP_USER_CLASS 12
#ifdef Py_GIL_DISABLED
#else
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_STATS
#endif   // Py_STATS
#ifdef Py_STATS
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_STATS
#endif
#ifdef Py_STATS
#endif
#define BITWISE_LONGS_ACTION(NAME, OP) \
    static PyObject * \
    (NAME)(PyObject *lhs, PyObject *rhs) \
    { \
        Py_ssize_t rhs_val = _PyLong_CompactValue((PyLongObject *)rhs); \
        Py_ssize_t lhs_val = _PyLong_CompactValue((PyLongObject *)lhs); \
        return PyLong_FromSsize_t(lhs_val OP rhs_val); \
    }
#undef BITWISE_LONGS_ACTION
#define FLOAT_LONG_ACTION(NAME, OP) \
    static PyObject * \
    (NAME)(PyObject *lhs, PyObject *rhs) \
    { \
        double lhs_val = PyFloat_AS_DOUBLE(lhs); \
        Py_ssize_t rhs_val = _PyLong_CompactValue((PyLongObject *)rhs); \
        return PyFloat_FromDouble(lhs_val OP rhs_val); \
    }
#undef FLOAT_LONG_ACTION
#define LONG_FLOAT_ACTION(NAME, OP) \
    static PyObject * \
    (NAME)(PyObject *lhs, PyObject *rhs) \
    { \
        double rhs_val = PyFloat_AS_DOUBLE(rhs); \
        Py_ssize_t lhs_val = _PyLong_CompactValue((PyLongObject *)lhs); \
        return PyFloat_FromDouble(lhs_val OP rhs_val); \
    }
#undef LONG_FLOAT_ACTION
#ifdef Py_STATS
#endif   // Py_STATS
#ifdef Py_STATS
#endif   // Py_STATS
#ifdef Py_STATS
#endif   // Py_STATS
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_STATS
#endif  // Py_STATS
#ifdef Py_STATS
#endif
#ifdef Py_STATS
#endif
#define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3)
#ifdef Py_GIL_DISABLED
#endif
#ifdef Py_GIL_DISABLED
#endif
typedef enum {
    OVERRIDING, /* Is an overriding descriptor, and will remain so. */
    METHOD, /* Attribute has Py_TPFLAGS_METHOD_DESCRIPTOR set */
    PROPERTY, /* Is a property */
    OBJECT_SLOT, /* Is an object slot descriptor */
    OTHER_SLOT, /* Is a slot descriptor of another type */
    NON_OVERRIDING, /* Is another non-overriding descriptor, and is an instance of an immutable class*/
    BUILTIN_CLASSMETHOD, /* Builtin methods with METH_CLASS */
    PYTHON_CLASSMETHOD, /* Python classmethod(func) object */
    NON_DESCRIPTOR, /* Is not a descriptor, and is an instance of an immutable class */
    MUTABLE,   /* Instance of a mutable class; might, or might not, be a descriptor */
    ABSENT, /* Attribute is not present on the class */
    DUNDER_CLASS, /* __class__ attribute */
    GETSET_OVERRIDDEN, /* __getattribute__ or __setattr__ has been overridden */
    GETATTRIBUTE_IS_PYTHON_FUNCTION  /* Descriptor requires calling a Python __getattribute__ */
} DescriptorClassification;
static inline int
set_opcode(_Py_CODEUNIT *instr, uint8_t opcode)
{
#ifdef Py_GIL_DISABLED
    uint8_t old_op = _Py_atomic_load_uint8_relaxed(&instr->op.code);
    if (old_op >= MIN_INSTRUMENTED_OPCODE) {
        /* Lost race with instrumentation */
        return 0;
    }
    if (!_Py_atomic_compare_exchange_uint8(&instr->op.code, &old_op, opcode)) {
        /* Lost race with instrumentation */
        assert(old_op >= MIN_INSTRUMENTED_OPCODE);
        return 0;
    }
    return 1;
#else
    instr->op.code = opcode;
    return 1;
#endif
}
static inline void
set_counter(_Py_BackoffCounter *counter, _Py_BackoffCounter value)
{
    FT_ATOMIC_STORE_UINT16_RELAXED(counter->value_and_backoff,
                                   value.value_and_backoff);
}
static inline void
specialize(_Py_CODEUNIT *instr, uint8_t specialized_opcode)
{
    assert(!PyErr_Occurred());
    if (!set_opcode(instr, specialized_opcode)) {
        STAT_INC(_PyOpcode_Deopt[specialized_opcode], failure);
        SPECIALIZATION_FAIL(_PyOpcode_Deopt[specialized_opcode],
                            SPEC_FAIL_OTHER);
        return;
    }
    STAT_INC(_PyOpcode_Deopt[specialized_opcode], success);
    set_counter((_Py_BackoffCounter *)instr + 1, adaptive_counter_cooldown());
}
static inline _Py_BackoffCounter
load_counter(_Py_BackoffCounter *counter)
{
    _Py_BackoffCounter result = {
        .value_and_backoff =
            FT_ATOMIC_LOAD_UINT16_RELAXED(counter->value_and_backoff)};
    return result;
}
static inline void
unspecialize(_Py_CODEUNIT *instr)
{
    assert(!PyErr_Occurred());
    uint8_t opcode = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.code);
    uint8_t generic_opcode = _PyOpcode_Deopt[opcode];
    STAT_INC(generic_opcode, failure);
    if (!set_opcode(instr, generic_opcode)) {
        SPECIALIZATION_FAIL(generic_opcode, SPEC_FAIL_OTHER);
        return;
    }
    _Py_BackoffCounter *counter = (_Py_BackoffCounter *)instr + 1;
    _Py_BackoffCounter cur = load_counter(counter);
    set_counter(counter, adaptive_counter_backoff(cur));
}
Py_NO_INLINE void
_Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _Py_CODEUNIT *instr, int load_method) {
    PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
    PyObject *cls = PyStackRef_AsPyObjectBorrow(cls_st);

    assert(ENABLE_SPECIALIZATION);
    assert(_PyOpcode_Caches[LOAD_SUPER_ATTR] == INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR);
    if (global_super != (PyObject *)&PySuper_Type) {
        SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_SHADOWED);
        goto fail;
    }
    if (!PyType_Check(cls)) {
        SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_BAD_CLASS);
        goto fail;
    }
    uint8_t load_code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR;
    specialize(instr, load_code);
    return;
fail:
    unspecialize(instr);
}
static int
specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name)
{
    _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
    if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_STRING);
        return -1;
    }
    PyObject *value;
    Py_ssize_t index = _PyDict_LookupIndexAndValue(dict, name, &value);
    if (value != NULL && PyLazyImport_CheckExact(value)) {
        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MODULE_LAZY_VALUE);
        return -1;
    }
    assert(index != DKIX_ERROR);
    if (index != (uint16_t)index) {
        SPECIALIZATION_FAIL(LOAD_ATTR,
                            index == DKIX_EMPTY ?
                            SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND :
                            SPEC_FAIL_OUT_OF_RANGE);
        return -1;
    }
    uint32_t keys_version = _PyDict_GetKeysVersionForCurrentState(
            _PyInterpreterState_GET(), dict);
    if (keys_version == 0) {
        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
        return -1;
    }
#ifdef Py_GIL_DISABLED
    maybe_enable_deferred_ref_count(value);
#endif
    write_u32(cache->version, keys_version);
    cache->index = (uint16_t)index;
    specialize(instr, LOAD_ATTR_MODULE);
    return 0;
}
static int
specialize_module_load_attr(
    PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
{
    PyModuleObject *m = (PyModuleObject *)owner;
    assert((Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
    PyDictObject *dict = (PyDictObject *)m->md_dict;
    if (dict == NULL) {
        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT);
        return -1;
    }
    int result;
    Py_BEGIN_CRITICAL_SECTION(dict);
    result = specialize_module_load_attr_lock_held(dict, instr, name);
    Py_END_CRITICAL_SECTION();
    return result;
}
static DescriptorClassification
classify_descriptor(PyObject *descriptor, bool has_getattr)
{
    if (descriptor == NULL) {
        return ABSENT;
    }
    PyTypeObject *desc_cls = Py_TYPE(descriptor);
    if (!(desc_cls->tp_flags & Py_TPFLAGS_IMMUTABLETYPE)) {
        return MUTABLE;
    }
    if (desc_cls->tp_descr_set) {
        if (desc_cls == &PyMemberDescr_Type) {
            PyMemberDescrObject *member = (PyMemberDescrObject *)descriptor;
            struct PyMemberDef *dmem = member->d_member;
            if (dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT) {
                return OBJECT_SLOT;
            }
            return OTHER_SLOT;
        }
        if (desc_cls == &PyProperty_Type) {
            /* We can't detect at runtime whether an attribute exists
               with property. So that means we may have to call
               __getattr__. */
            return has_getattr ? GETSET_OVERRIDDEN : PROPERTY;
        }
        return OVERRIDING;
    }
    if (desc_cls->tp_descr_get) {
        if (desc_cls->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR) {
            return METHOD;
        }
        if (Py_IS_TYPE(descriptor, &PyClassMethodDescr_Type)) {
            return BUILTIN_CLASSMETHOD;
        }
        if (Py_IS_TYPE(descriptor, &PyClassMethod_Type)) {
            return PYTHON_CLASSMETHOD;
        }
        return NON_OVERRIDING;
    }
    return NON_DESCRIPTOR;
}
static bool
descriptor_is_class(PyObject *descriptor, PyObject *name)
{
    return ((PyUnicode_CompareWithASCIIString(name, "__class__") == 0) &&
            (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)));
}
static DescriptorClassification
analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) {
    bool has_getattr = false;
    bool have_ga_version = false;
    unsigned int ga_version;
    getattrofunc getattro_slot = type->tp_getattro;
    if (getattro_slot == PyObject_GenericGetAttr) {
        /* Normal attribute lookup; */
        has_getattr = false;
    }
    else if (getattro_slot == _Py_slot_tp_getattr_hook ||
        getattro_slot == _Py_slot_tp_getattro) {
        /* One or both of __getattribute__ or __getattr__ may have been
         overridden See typeobject.c for why these functions are special. */
        PyObject *getattribute = _PyType_LookupRefAndVersion(type,
                &_Py_ID(__getattribute__), &ga_version);
        have_ga_version = true;
        PyInterpreterState *interp = _PyInterpreterState_GET();
        bool has_custom_getattribute = getattribute != NULL &&
            getattribute != interp->callable_cache.object__getattribute__;
        PyObject *getattr = _PyType_Lookup(type, &_Py_ID(__getattr__));
        has_getattr = getattr != NULL;
        if (has_custom_getattribute) {
            if (!has_getattr &&
                Py_IS_TYPE(getattribute, &PyFunction_Type)) {
                *descr = getattribute;
                *tp_version = ga_version;
                return GETATTRIBUTE_IS_PYTHON_FUNCTION;
            }
            /* Potentially both __getattr__ and __getattribute__ are set.
               Too complicated */
            Py_DECREF(getattribute);
            *descr = NULL;
            *tp_version = ga_version;
            return GETSET_OVERRIDDEN;
        }
        /* Potentially has __getattr__ but no custom __getattribute__.
           Fall through to usual descriptor analysis.
           Usual attribute lookup should only be allowed at runtime
           if we can guarantee that there is no way an exception can be
           raised. This means some specializations, e.g. specializing
           for property() isn't safe.
        */
        Py_XDECREF(getattribute);
    }
    else {
        *descr = NULL;
        *tp_version = FT_ATOMIC_LOAD_UINT_RELAXED(type->tp_version_tag);
        return GETSET_OVERRIDDEN;
    }
    unsigned int descr_version;
    PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, &descr_version);
    *descr = descriptor;
    *tp_version = have_ga_version ? ga_version : descr_version;
    if (descriptor_is_class(descriptor, name)) {
        return DUNDER_CLASS;
    }
    return classify_descriptor(descriptor, has_getattr);
}
static int
specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
                             PyObject *name)
{
    assert(PyType_Check(owner));
    PyTypeObject *cls = (PyTypeObject *)owner;
    _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
    if (Py_TYPE(cls)->tp_getattro != _Py_type_getattro) {
        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN);
        return -1;
    }
    unsigned int meta_version = 0;
    PyObject *metadescriptor = _PyType_LookupRefAndVersion(Py_TYPE(cls), name, &meta_version);
    DescriptorClassification metakind = classify_descriptor(metadescriptor, false);
    Py_XDECREF(metadescriptor);
    switch (metakind) {
        case METHOD:
        case NON_DESCRIPTOR:
        case NON_OVERRIDING:
        case BUILTIN_CLASSMETHOD:
        case PYTHON_CLASSMETHOD:
        case ABSENT:
            break;
        default:
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE);
            return -1;
    }
    PyObject *descr = NULL;
    DescriptorClassification kind = 0;
    unsigned int tp_version = 0;
    kind = analyze_descriptor_load(cls, name, &descr, &tp_version);
    if (tp_version == 0) {
        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
        Py_XDECREF(descr);
        return -1;
    }
    bool metaclass_check = false;
    if ((Py_TYPE(cls)->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) == 0) {
        metaclass_check = true;
        if (meta_version == 0) {
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
            Py_XDECREF(descr);
            return -1;
        }
    }
    switch (kind) {
        case METHOD:
        case NON_DESCRIPTOR:
            #ifdef Py_GIL_DISABLED
            if (!_PyObject_HasDeferredRefcount(descr)) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
                Py_XDECREF(descr);
                return -1;
            }
            #endif
            write_u32(cache->type_version, tp_version);
            write_ptr(cache->descr, descr);
            if (metaclass_check) {
                write_u32(cache->keys_version, meta_version);
                specialize(instr, LOAD_ATTR_CLASS_WITH_METACLASS_CHECK);
            }
            else {
                specialize(instr, LOAD_ATTR_CLASS);
            }
            Py_XDECREF(descr);
            return 0;
#ifdef Py_STATS
        case ABSENT:
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
            Py_XDECREF(descr);
            return -1;
#endif
        default:
            SPECIALIZATION_FAIL(LOAD_ATTR, load_attr_fail_kind(kind));
            Py_XDECREF(descr);
            return -1;
    }
}
static bool
instance_has_key(PyObject *obj, PyObject *name, uint32_t *shared_keys_version)
{
    PyTypeObject *cls = Py_TYPE(obj);
    if ((cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
        return false;
    }
    if (cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
        PyDictKeysObject *keys = ((PyHeapTypeObject *)cls)->ht_cached_keys;
        Py_ssize_t index =
            _PyDictKeys_StringLookupAndVersion(keys, name, shared_keys_version);
        return index >= 0;
    }
    PyDictObject *dict = _PyObject_GetManagedDict(obj);
    if (dict == NULL || !PyDict_CheckExact(dict)) {
        return false;
    }
    bool result;
    Py_BEGIN_CRITICAL_SECTION(dict);
    if (dict->ma_values) {
        result = false;
    }
    else {
        result = (_PyDict_LookupIndex(dict, name) >= 0);
    }
    Py_END_CRITICAL_SECTION();
    return result;
}
static int
specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr,
                              PyObject *name, PyObject *descr,
                              unsigned int tp_version,
                              DescriptorClassification kind, bool is_method,
                              uint32_t shared_keys_version)
{
    _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
    PyTypeObject *owner_cls = Py_TYPE(owner);

    assert(descr != NULL);
    assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR));

    #ifdef Py_GIL_DISABLED
    if (!_PyObject_HasDeferredRefcount(descr)) {
        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
        return 0;
    }
    #endif

    unsigned long tp_flags = PyType_GetFlags(owner_cls);
    if (tp_flags & Py_TPFLAGS_INLINE_VALUES) {
        #ifndef Py_GIL_DISABLED
        assert(_PyDictKeys_StringLookup(
                   ((PyHeapTypeObject *)owner_cls)->ht_cached_keys, name) < 0);
        #endif
        if (shared_keys_version == 0) {
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
            return 0;
        }
        write_u32(cache->keys_version, shared_keys_version);
        specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
    }
    else {
        Py_ssize_t dictoffset;
        if (tp_flags & Py_TPFLAGS_MANAGED_DICT) {
            dictoffset = MANAGED_DICT_OFFSET;
        }
        else {
            dictoffset = owner_cls->tp_dictoffset;
            if (dictoffset < 0 || dictoffset > INT16_MAX + MANAGED_DICT_OFFSET) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
                return 0;
            }
        }
        if (dictoffset == 0) {
            specialize(instr, is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT);
        }
        else if (is_method) {
            PyObject **addr = (PyObject **)((char *)owner + dictoffset);
            PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*addr);
            if (dict) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
                return 0;
            }
            /* Cache entries must be unsigned values, so we offset the
             * dictoffset by MANAGED_DICT_OFFSET.
             * We do the reverse offset in LOAD_ATTR_METHOD_LAZY_DICT */
            dictoffset -= MANAGED_DICT_OFFSET;
            assert(((uint16_t)dictoffset) == dictoffset);
            cache->dict_offset = (uint16_t)dictoffset;
            specialize(instr, LOAD_ATTR_METHOD_LAZY_DICT);
        }
        else {
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE);
            return 0;
        }
    }
    /* `descr` is borrowed. This is safe for methods (even inherited ones from
    *  super classes!) as long as tp_version_tag is validated for two main reasons:
    *
    *  1. The class will always hold a reference to the method so it will
    *  usually not be GC-ed. Should it be deleted in Python, e.g.
    *  `del obj.meth`, tp_version_tag will be invalidated, because of reason 2.
    *
    *  2. The pre-existing type method cache (MCACHE) uses the same principles
    *  of caching a borrowed descriptor. The MCACHE infrastructure does all the
    *  heavy lifting for us. E.g. it invalidates tp_version_tag on any MRO
    *  modification, on any type object change along said MRO, etc. (see
    *  PyType_Modified usages in typeobject.c). The MCACHE has been
    *  working since Python 2.6 and it's battle-tested.
    */
    write_u32(cache->type_version, tp_version);
    write_ptr(cache->descr, descr);
    return 1;
}
static int
function_kind(PyCodeObject *code) {
    int flags = code->co_flags;
    if ((flags & (CO_VARKEYWORDS | CO_VARARGS)) || code->co_kwonlyargcount) {
        return SPEC_FAIL_CODE_COMPLEX_PARAMETERS;
    }
    if ((flags & CO_OPTIMIZED) == 0) {
        return SPEC_FAIL_CODE_NOT_OPTIMIZED;
    }
    return SIMPLE_FUNCTION;
}
static bool
function_check_args(PyObject *o, int expected_argcount, int opcode)
{
    assert(Py_IS_TYPE(o, &PyFunction_Type));
    PyFunctionObject *func = (PyFunctionObject *)o;
    PyCodeObject *fcode = (PyCodeObject *)func->func_code;
    int kind = function_kind(fcode);
    if (kind != SIMPLE_FUNCTION) {
        SPECIALIZATION_FAIL(opcode, kind);
        return false;
    }
    if (fcode->co_argcount != expected_argcount) {
        SPECIALIZATION_FAIL(opcode, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
        return false;
    }
    return true;
}
static uint32_t
function_get_version(PyObject *o, int opcode)
{
    assert(Py_IS_TYPE(o, &PyFunction_Type));
    PyFunctionObject *func = (PyFunctionObject *)o;
    uint32_t version = _PyFunction_GetVersionForCurrentState(func);
    if (!_PyFunction_IsVersionValid(version)) {
        SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS);
        return 0;
    }
    return version;
}
static int
specialize_dict_access_inline(
    PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
    PyObject *name, unsigned int tp_version,
    int base_op, int values_op)
{
    _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
    PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
    assert(PyUnicode_CheckExact(name));
    _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(owner);
    Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name);
    assert (index != DKIX_ERROR);
    if (index == DKIX_EMPTY) {
        SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS);
        return 0;
    }
    assert(index >= 0);
    assert(_PyObject_InlineValues(owner)->valid);
    char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index];
    Py_ssize_t offset = value_addr - (char *)owner;
    if (offset != (uint16_t)offset) {
        SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
        return 0;
    }
    cache->index = (uint16_t)offset;
    write_u32(cache->version, tp_version);
    specialize(instr, values_op);
    return 1;
}
static int
specialize_dict_access_hint(
    PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type,
    PyObject *name, unsigned int tp_version,
    int base_op, int hint_op)
{
    _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);

    _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict);
#ifdef Py_GIL_DISABLED
    _PyDict_EnsureSharedOnRead(dict);
#endif

    // We found an instance with a __dict__.
    if (_PyDict_HasSplitTable(dict)) {
        SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT);
        return 0;
    }
    Py_ssize_t index = _PyDict_LookupIndex(dict, name);
    if (index != (uint16_t)index) {
        SPECIALIZATION_FAIL(base_op,
                            index == DKIX_EMPTY ?
                            SPEC_FAIL_ATTR_NOT_IN_DICT :
                            SPEC_FAIL_OUT_OF_RANGE);
        return 0;
    }
    cache->index = (uint16_t)index;
    write_u32(cache->version, tp_version);
    specialize(instr, hint_op);
    return 1;
}
static int
specialize_dict_access(
    PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
    DescriptorClassification kind, PyObject *name, unsigned int tp_version,
    int base_op, int values_op, int hint_op)
{
    assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT ||
        kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD ||
        kind == METHOD);
    // No descriptor, or non overriding.
    if ((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
        SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
        return 0;
    }
    if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES &&
        FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner)->valid) &&
        !(base_op == STORE_ATTR && _PyObject_GetManagedDict(owner) != NULL))
    {
        int res;
        Py_BEGIN_CRITICAL_SECTION(owner);
        PyDictObject *dict = _PyObject_GetManagedDict(owner);
        if (dict == NULL) {
            // managed dict, not materialized, inline values valid
            res = specialize_dict_access_inline(owner, instr, type, name,
                                                tp_version, base_op, values_op);
        }
        else {
            // lost race and dict was created, fail specialization
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER);
            res = 0;
        }
        Py_END_CRITICAL_SECTION();
        return res;
    }
    else {
        PyDictObject *dict = _PyObject_GetManagedDict(owner);
        if (dict == NULL || !PyDict_CheckExact(dict)) {
            SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
            return 0;
        }
        int res;
        Py_BEGIN_CRITICAL_SECTION(dict);
        // materialized managed dict
        res = specialize_dict_access_hint(dict, instr, type, name,
                                          tp_version, base_op, hint_op);
        Py_END_CRITICAL_SECTION();
        return res;
    }
}
static int
do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name,
                                 bool shadow, uint32_t shared_keys_version,
                                 DescriptorClassification kind, PyObject *descr, unsigned int tp_version)
{
    _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
    PyTypeObject *type = Py_TYPE(owner);
    if (tp_version == 0) {
        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
        return -1;
    }
    uint8_t oparg = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.arg);
    switch(kind) {
        case OVERRIDING:
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
            return -1;
        case METHOD:
        {
            if (shadow) {
                goto try_instance;
            }
            if (oparg & 1) {
                if (specialize_attr_loadclassattr(owner, instr, name, descr,
                                                  tp_version, kind, true,
                                                  shared_keys_version)) {
                    return 0;
                }
                else {
                    return -1;
                }
            }
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
            return -1;
        }
        case PROPERTY:
        {
            _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1);
            assert(Py_TYPE(descr) == &PyProperty_Type);
            PyObject *fget = ((_PyPropertyObject *)descr)->prop_get;
            if (fget == NULL) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
                return -1;
            }
            if (!Py_IS_TYPE(fget, &PyFunction_Type)) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION);
                return -1;
            }
            if (!function_check_args(fget, 1, LOAD_ATTR)) {
                return -1;
            }
            if (oparg & 1) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
                return -1;
            }
            /* Don't specialize if PEP 523 is active */
            if ((_PyInterpreterState_GET()->eval_frame != NULL && _PyInterpreterState_GET()->eval_frame != Ci_EvalFrameFunc)) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
                return -1;
            }
            #ifdef Py_GIL_DISABLED
            if (!_PyObject_HasDeferredRefcount(fget)) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
                return -1;
            }
            #endif
            assert(tp_version != 0);
            write_u32(lm_cache->type_version, tp_version);
            /* borrowed */
            write_ptr(lm_cache->descr, fget);
            specialize(instr, LOAD_ATTR_PROPERTY);
            return 0;
        }
        case OBJECT_SLOT:
        {
            PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
            struct PyMemberDef *dmem = member->d_member;
            Py_ssize_t offset = dmem->offset;
            if (!PyObject_TypeCheck(owner, member->d_common.d_type)) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
                return -1;
            }
            if (dmem->flags & _Py_AFTER_ITEMS) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SLOT_AFTER_ITEMS);
                return -1;
            }
            if (dmem->flags & Py_AUDIT_READ) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_AUDITED_SLOT);
                return -1;
            }
            if (offset != (uint16_t)offset) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
                return -1;
            }
            assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT);
            assert(offset > 0);
            cache->index = (uint16_t)offset;
            write_u32(cache->version, tp_version);
            specialize(instr, LOAD_ATTR_SLOT);
            return 0;
        }
        case DUNDER_CLASS:
        {
            Py_ssize_t offset = offsetof(PyObject, ob_type);
            assert(offset == (uint16_t)offset);
            cache->index = (uint16_t)offset;
            write_u32(cache->version, tp_version);
            specialize(instr, LOAD_ATTR_SLOT);
            return 0;
        }
        case OTHER_SLOT:
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);
            return -1;
        case MUTABLE:
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);
            return -1;
        case GETSET_OVERRIDDEN:
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OVERRIDDEN);
            return -1;
        case GETATTRIBUTE_IS_PYTHON_FUNCTION:
        {
            assert(Py_IS_TYPE(descr, &PyFunction_Type));
            _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1);
            if (!function_check_args(descr, 2, LOAD_ATTR)) {
                return -1;
            }
            if (oparg & 1) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
                return -1;
            }
            uint32_t version = function_get_version(descr, LOAD_ATTR);
            if (version == 0) {
                return -1;
            }
            /* Don't specialize if PEP 523 is active */
            if ((_PyInterpreterState_GET()->eval_frame != NULL && _PyInterpreterState_GET()->eval_frame != Ci_EvalFrameFunc)) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
                return -1;
            }
            #ifdef Py_GIL_DISABLED
            if (!_PyObject_HasDeferredRefcount(descr)) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
                return -1;
            }
            #endif
            write_u32(lm_cache->keys_version, version);
            /* borrowed */
            write_ptr(lm_cache->descr, descr);
            write_u32(lm_cache->type_version, tp_version);
            specialize(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN);
            return 0;
        }
        case BUILTIN_CLASSMETHOD:
        case PYTHON_CLASSMETHOD:
        case NON_OVERRIDING:
            if (shadow) {
                goto try_instance;
            }
            return -1;
        case NON_DESCRIPTOR:
            if (shadow) {
                goto try_instance;
            }
            if ((oparg & 1) == 0) {
                if (specialize_attr_loadclassattr(owner, instr, name, descr,
                                                  tp_version, kind, false,
                                                  shared_keys_version)) {
                    return 0;
                }
            }
            return -1;
        case ABSENT:
            if (shadow) {
                goto try_instance;
            }
            set_counter((_Py_BackoffCounter*)instr + 1, adaptive_counter_cooldown());
            return 0;
    }
    Py_UNREACHABLE();
try_instance:
    if (specialize_dict_access(owner, instr, type, kind, name, tp_version,
                               LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT))
    {
        return 0;
    }
    return -1;
}
static int
specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name)
{
    // 0 is not a valid version
    uint32_t shared_keys_version = 0;
    bool shadow = instance_has_key(owner, name, &shared_keys_version);
    PyObject *descr = NULL;
    unsigned int tp_version = 0;
    PyTypeObject *type = Py_TYPE(owner);
    DescriptorClassification kind = analyze_descriptor_load(type, name, &descr, &tp_version);
    int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version);
    Py_XDECREF(descr);
    return result;
}
Py_NO_INLINE void
_Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name)
{
    PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st);

    assert(ENABLE_SPECIALIZATION);
    assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR);
    PyTypeObject *type = Py_TYPE(owner);
    bool fail;
    if (!_PyType_IsReady(type)) {
        // We *might* not really need this check, but we inherited it from
        // PyObject_GenericGetAttr and friends... and this way we still do the
        // right thing if someone forgets to call PyType_Ready(type):
        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
        fail = true;
    }
    else if (Py_TYPE(owner)->tp_getattro == PyModule_Type.tp_getattro) {
        fail = specialize_module_load_attr(owner, instr, name);
    }
    else if (PyType_Check(owner)) {
        fail = specialize_class_load_attr(owner, instr, name);
    }
    else {
        fail = specialize_instance_load_attr(owner, instr, name);
    }

    if (fail) {
        unspecialize(instr);
    }
}
static DescriptorClassification
analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version)
{
    if (type->tp_setattro != PyObject_GenericSetAttr) {
        *descr = NULL;
        return GETSET_OVERRIDDEN;
    }
    PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, tp_version);
    *descr = descriptor;
    if (descriptor_is_class(descriptor, name)) {
        return DUNDER_CLASS;
    }
    return classify_descriptor(descriptor, false);
}
Py_NO_INLINE void
_Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name)
{
    PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st);

    assert(ENABLE_SPECIALIZATION);
    assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR);
    PyObject *descr = NULL;
    _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
    PyTypeObject *type = Py_TYPE(owner);
    if (!_PyType_IsReady(type)) {
        // We *might* not really need this check, but we inherited it from
        // PyObject_GenericSetAttr and friends... and this way we still do the
        // right thing if someone forgets to call PyType_Ready(type):
        SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER);
        goto fail;
    }
    if (PyModule_CheckExact(owner)) {
        SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
        goto fail;
    }
    unsigned int tp_version = 0;
    DescriptorClassification kind = analyze_descriptor_store(type, name, &descr, &tp_version);
    if (tp_version == 0) {
        goto fail;
    }
    assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
    switch(kind) {
        case OVERRIDING:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
            goto fail;
        case METHOD:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_METHOD);
            goto fail;
        case PROPERTY:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_PROPERTY);
            goto fail;
        case OBJECT_SLOT:
        {
            PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
            struct PyMemberDef *dmem = member->d_member;
            Py_ssize_t offset = dmem->offset;
            if (!PyObject_TypeCheck(owner, member->d_common.d_type)) {
                SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_EXPECTED_ERROR);
                goto fail;
            }
            if (dmem->flags & _Py_AFTER_ITEMS) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SLOT_AFTER_ITEMS);
                goto fail;
            }
            if (dmem->flags & Py_READONLY) {
                SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_READ_ONLY);
                goto fail;
            }
            if (offset != (uint16_t)offset) {
                SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OUT_OF_RANGE);
                goto fail;
            }
            assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT);
            assert(offset > 0);
            cache->index = (uint16_t)offset;
            write_u32(cache->version, tp_version);
            specialize(instr, STORE_ATTR_SLOT);
            goto success;
        }
        case DUNDER_CLASS:
        case OTHER_SLOT:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);
            goto fail;
        case MUTABLE:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);
            goto fail;
        case GETATTRIBUTE_IS_PYTHON_FUNCTION:
        case GETSET_OVERRIDDEN:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
            goto fail;
        case BUILTIN_CLASSMETHOD:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ);
            goto fail;
        case PYTHON_CLASSMETHOD:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ);
            goto fail;
        case NON_OVERRIDING:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR);
            goto fail;
        case NON_DESCRIPTOR:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE);
            goto fail;
        case ABSENT:
            if (specialize_dict_access(owner, instr, type, kind, name, tp_version,
                                       STORE_ATTR, STORE_ATTR_INSTANCE_VALUE,
                                       STORE_ATTR_WITH_HINT)) {
                goto success;
            }
    }
fail:
    Py_XDECREF(descr);
    unspecialize(instr);
    return;
success:
    Py_XDECREF(descr);
    return;
}
static void
specialize_load_global_lock_held(
    PyObject *globals, PyObject *builtins,
    _Py_CODEUNIT *instr, PyObject *name)
{
    assert(ENABLE_SPECIALIZATION);
    assert(_PyOpcode_Caches[LOAD_GLOBAL] == INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
    /* Use inline cache */
    _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)(instr + 1);
    assert(PyUnicode_CheckExact(name));
    if (!PyDict_CheckExact(globals)) {
        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_DICT);
        goto fail;
    }
    PyDictKeysObject * globals_keys = ((PyDictObject *)globals)->ma_keys;
    if (globals_keys->dk_kind != DICT_KEYS_UNICODE) {
        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT);
        goto fail;
    }
    PyObject *value;
    Py_ssize_t index = _PyDict_LookupIndexAndValue((PyDictObject *)globals, name, &value);
    if (index == DKIX_ERROR) {
        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR);
        goto fail;
    }
    if (value != NULL && PyLazyImport_CheckExact(value)) {
        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_ATTR_MODULE_LAZY_VALUE);
        goto fail;
    }
    PyInterpreterState *interp = _PyInterpreterState_GET();
    if (index != DKIX_EMPTY) {
        if (index != (uint16_t)index) {
            SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
            goto fail;
        }
        uint32_t keys_version = _PyDict_GetKeysVersionForCurrentState(
                interp, (PyDictObject*) globals);
        if (keys_version == 0) {
            SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS);
            goto fail;
        }
        if (keys_version != (uint16_t)keys_version) {
            SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
            goto fail;
        }
#ifdef Py_GIL_DISABLED
        maybe_enable_deferred_ref_count(value);
#endif
        cache->index = (uint16_t)index;
        cache->module_keys_version = (uint16_t)keys_version;
        specialize(instr, LOAD_GLOBAL_MODULE);
        return;
    }
    if (!PyDict_CheckExact(builtins)) {
        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_DICT);
        goto fail;
    }
    PyDictKeysObject * builtin_keys = ((PyDictObject *)builtins)->ma_keys;
    if (builtin_keys->dk_kind != DICT_KEYS_UNICODE) {
        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT);
        goto fail;
    }
    index = _PyDictKeys_StringLookup(builtin_keys, name);
    if (index == DKIX_ERROR) {
        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR);
        goto fail;
    }
    if (index != (uint16_t)index) {
        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
        goto fail;
    }
    uint32_t globals_version = _PyDict_GetKeysVersionForCurrentState(
            interp, (PyDictObject*) globals);
    if (globals_version == 0) {
        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS);
        goto fail;
    }
    if (globals_version != (uint16_t)globals_version) {
        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
        goto fail;
    }
    uint32_t builtins_version = _PyDict_GetKeysVersionForCurrentState(
            interp, (PyDictObject*) builtins);
    if (builtins_version == 0) {
        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS);
        goto fail;
    }
    if (builtins_version > UINT16_MAX) {
        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
        goto fail;
    }
    cache->index = (uint16_t)index;
    cache->module_keys_version = (uint16_t)globals_version;
    cache->builtin_keys_version = (uint16_t)builtins_version;
    specialize(instr, LOAD_GLOBAL_BUILTIN);
    return;
fail:
    unspecialize(instr);
}
Py_NO_INLINE void
_Py_Specialize_LoadGlobal(
    PyObject *globals, PyObject *builtins,
    _Py_CODEUNIT *instr, PyObject *name)
{
    Py_BEGIN_CRITICAL_SECTION2(globals, builtins);
    specialize_load_global_lock_held(globals, builtins, instr, name);
    Py_END_CRITICAL_SECTION2();
}
Py_NO_INLINE void
_Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr)
{
    PyObject *container = PyStackRef_AsPyObjectBorrow(container_st);
    PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);

    assert(ENABLE_SPECIALIZATION);
    PyTypeObject *container_type = Py_TYPE(container);
    if (container_type == &PyList_Type) {
        if (PyLong_CheckExact(sub)) {
            if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)
                && ((PyLongObject *)sub)->long_value.ob_digit[0] < (size_t)PyList_GET_SIZE(container))
            {
                specialize(instr, STORE_SUBSCR_LIST_INT);
                return;
            }
            else {
                SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
                unspecialize(instr);
                return;
            }
        }
        else if (PySlice_Check(sub)) {
            SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_LIST_SLICE);
            unspecialize(instr);
            return;
        }
        else {
            SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
            unspecialize(instr);
            return;
        }
    }
    if (container_type == &PyDict_Type) {
        specialize(instr, STORE_SUBSCR_DICT);
        return;
    }
    SPECIALIZATION_FAIL(STORE_SUBSCR, store_subscr_fail_kind(container, sub));
    unspecialize(instr);
}
static int
specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs)
{
    if (PyCFunction_GET_FUNCTION(callable) == NULL) {
        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OTHER);
        return 1;
    }
    switch (PyCFunction_GET_FLAGS(callable) &
        (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
        METH_KEYWORDS | METH_METHOD)) {
        case METH_O: {
            if (nargs != 1) {
                SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
                return 1;
            }
            /* len(o) */
            PyInterpreterState *interp = _PyInterpreterState_GET();
            if (callable == interp->callable_cache.len && instr->op.arg == 1) {
                specialize(instr, CALL_LEN);
                return 0;
            }
            specialize(instr, CALL_BUILTIN_O);
            return 0;
        }
        case METH_FASTCALL: {
            if (nargs == 2) {
                /* isinstance(o1, o2) */
                PyInterpreterState *interp = _PyInterpreterState_GET();
                if (callable == interp->callable_cache.isinstance && instr->op.arg == 2) {
                    specialize(instr, CALL_ISINSTANCE);
                    return 0;
                }
            }
            specialize(instr, CALL_BUILTIN_FAST);
            return 0;
        }
        case METH_FASTCALL | METH_KEYWORDS: {
            specialize(instr, CALL_BUILTIN_FAST_WITH_KEYWORDS);
            return 0;
        }
        default:
            specialize(instr, CALL_NON_PY_GENERAL);
            return 0;
    }
}
static int
specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs,
                   bool bound_method)
{
    _PyCallCache *cache = (_PyCallCache *)(instr + 1);
    PyCodeObject *code = (PyCodeObject *)func->func_code;
    int kind = function_kind(code);
    /* Don't specialize if PEP 523 is active */
    if ((_PyInterpreterState_GET()->eval_frame != NULL && _PyInterpreterState_GET()->eval_frame != Ci_EvalFrameFunc)) {
        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523);
        return -1;
    }
    if (func->vectorcall != _PyFunction_Vectorcall) {
        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_VECTORCALL);
        return -1;
    }
    int argcount = -1;
    if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) {
        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED);
        return -1;
    }
    if (kind == SIMPLE_FUNCTION) {
        argcount = code->co_argcount;
    }
    int version = _PyFunction_GetVersionForCurrentState(func);
    if (!_PyFunction_IsVersionValid(version)) {
        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS);
        return -1;
    }
    write_u32(cache->func_version, version);
    uint8_t opcode;
    if (argcount == nargs + bound_method) {
        opcode =
            bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS;
    }
    else {
        opcode = bound_method ? CALL_BOUND_METHOD_GENERAL : CALL_PY_GENERAL;
    }
    specialize(instr, opcode);
    return 0;
}
static PyObject *
get_init_for_simple_managed_python_class(PyTypeObject *tp, unsigned int *tp_version)
{
    assert(tp->tp_new == PyBaseObject_Type.tp_new);
    if (tp->tp_alloc != PyType_GenericAlloc) {
        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OVERRIDDEN);
        return NULL;
    }
    unsigned long tp_flags = PyType_GetFlags(tp);
    if (!(tp_flags & Py_TPFLAGS_HEAPTYPE)) {
        /* Is this possible? */
        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_EXPECTED_ERROR);
        return NULL;
    }
    PyObject *init = _PyType_LookupRefAndVersion(tp, &_Py_ID(__init__), tp_version);
    if (init == NULL || !PyFunction_Check(init)) {
        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_PYTHON);
        Py_XDECREF(init);
        return NULL;
    }
    int kind = function_kind((PyCodeObject *)PyFunction_GET_CODE(init));
    if (kind != SIMPLE_FUNCTION) {
        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_SIMPLE);
        Py_DECREF(init);
        return NULL;
    }
    return init;
}
static int
specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs)
{
    assert(PyType_Check(callable));
    PyTypeObject *tp = _PyType_CAST(callable);
    if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) {
        int oparg = instr->op.arg;
        if (nargs == 1 && oparg == 1) {
            if (tp == &PyUnicode_Type) {
                specialize(instr, CALL_STR_1);
                return 0;
            }
            else if (tp == &PyType_Type) {
                specialize(instr, CALL_TYPE_1);
                return 0;
            }
            else if (tp == &PyTuple_Type) {
                specialize(instr, CALL_TUPLE_1);
                return 0;
            }
        }
        if (tp->tp_vectorcall != NULL) {
            specialize(instr, CALL_BUILTIN_CLASS);
            return 0;
        }
        goto generic;
    }
    if (Py_TYPE(tp) != &PyType_Type) {
        goto generic;
    }
    if (tp->tp_new == PyBaseObject_Type.tp_new) {
        unsigned int tp_version = 0;
        PyObject *init = get_init_for_simple_managed_python_class(tp, &tp_version);
        if (!tp_version) {
            SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS);
            Py_XDECREF(init);
            return -1;
        }
        if (init != NULL && _PyType_CacheInitForSpecialization(
                                (PyHeapTypeObject *)tp, init, tp_version)) {
            _PyCallCache *cache = (_PyCallCache *)(instr + 1);
            write_u32(cache->func_version, tp_version);
            specialize(instr, CALL_ALLOC_AND_ENTER_INIT);
            Py_DECREF(init);
            return 0;
        }
        Py_XDECREF(init);
    }
generic:
    specialize(instr, CALL_NON_PY_GENERAL);
    return 0;
}
static int
specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null,
                             _Py_CODEUNIT *instr, int nargs)
{
    switch (descr->d_method->ml_flags &
        (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
        METH_KEYWORDS | METH_METHOD)) {
        case METH_NOARGS: {
            if (nargs != 1) {
                SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
                return -1;
            }
            specialize(instr, CALL_METHOD_DESCRIPTOR_NOARGS);
            return 0;
        }
        case METH_O: {
            if (nargs != 2) {
                SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
                return -1;
            }
            PyInterpreterState *interp = _PyInterpreterState_GET();
            PyObject *list_append = interp->callable_cache.list_append;
            int oparg = instr->op.arg;
            if ((PyObject *)descr == list_append && oparg == 1) {
                assert(self_or_null != NULL);
                if (PyList_CheckExact(self_or_null)) {
                    specialize(instr, CALL_LIST_APPEND);
                    return 0;
                }
            }
            specialize(instr, CALL_METHOD_DESCRIPTOR_O);
            return 0;
        }
        case METH_FASTCALL: {
            specialize(instr, CALL_METHOD_DESCRIPTOR_FAST);
            return 0;
        }
        case METH_FASTCALL | METH_KEYWORDS: {
            specialize(instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS);
            return 0;
        }
    }
    specialize(instr, CALL_NON_PY_GENERAL);
    return 0;
}
static int
specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs,
                   bool bound_method)
{
    _PyCallCache *cache = (_PyCallCache *)(instr + 1);
    PyCodeObject *code = (PyCodeObject *)func->func_code;
    int kind = function_kind(code);
    /* Don't specialize if PEP 523 is active */
    if ((_PyInterpreterState_GET()->eval_frame != NULL && _PyInterpreterState_GET()->eval_frame != Ci_EvalFrameFunc)) {
        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523);
        return -1;
    }
    if (func->vectorcall != _PyFunction_Vectorcall) {
        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_VECTORCALL);
        return -1;
    }
    if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) {
        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED);
        return -1;
    }
    int version = _PyFunction_GetVersionForCurrentState(func);
    if (!_PyFunction_IsVersionValid(version)) {
        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS);
        return -1;
    }
    write_u32(cache->func_version, version);
    specialize(instr, bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY);
    return 0;
}
Py_NO_INLINE void
_Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs)
{
    PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st);

    assert(ENABLE_SPECIALIZATION);
    assert(_PyOpcode_Caches[CALL_KW] == INLINE_CACHE_ENTRIES_CALL_KW);
    assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL_KW);
    int fail;
    if (PyFunction_Check(callable)) {
        fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false);
    }
    else if (PyMethod_Check(callable)) {
        PyObject *func = ((PyMethodObject *)callable)->im_func;
        if (PyFunction_Check(func)) {
            fail = specialize_py_call_kw((PyFunctionObject *)func, instr, nargs, true);
        }
        else {
            SPECIALIZATION_FAIL(CALL_KW, SPEC_FAIL_CALL_BOUND_METHOD);
            fail = -1;
        }
    }
    else {
        specialize(instr, CALL_KW_NON_PY);
        fail = 0;
    }
    if (fail) {
        unspecialize(instr);
    }
}
Py_NO_INLINE void
_Py_Specialize_CallFunctionEx(_PyStackRef func_st, _Py_CODEUNIT *instr)
{
    PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);

    assert(ENABLE_SPECIALIZATION);
    assert(_PyOpcode_Caches[CALL_FUNCTION_EX] == INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX);

    if (Py_TYPE(func) == &PyFunction_Type &&
        ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
        if ((_PyInterpreterState_GET()->eval_frame != NULL && _PyInterpreterState_GET()->eval_frame != Ci_EvalFrameFunc)) {
            goto failure;
        }
        specialize(instr, CALL_EX_PY);
        return;
    }
    specialize(instr, CALL_EX_NON_PY_GENERAL);
    return;
failure:
    unspecialize(instr);
}

static inline int
is_compactlong(PyObject *v)
{
    return PyLong_CheckExact(v) &&
           _PyLong_IsCompact((PyLongObject *)v);
}
static int
compactlongs_guard(PyObject *lhs, PyObject *rhs)
{
    return (is_compactlong(lhs) && is_compactlong(rhs));
}

#define BITWISE_LONGS_ACTION(NAME, OP) \
    static PyObject * \
    (NAME)(PyObject *lhs, PyObject *rhs) \
    { \
        Py_ssize_t rhs_val = _PyLong_CompactValue((PyLongObject *)rhs); \
        Py_ssize_t lhs_val = _PyLong_CompactValue((PyLongObject *)lhs); \
        return PyLong_FromSsize_t(lhs_val OP rhs_val); \
    }
BITWISE_LONGS_ACTION(compactlongs_or, |)
BITWISE_LONGS_ACTION(compactlongs_and, &)
BITWISE_LONGS_ACTION(compactlongs_xor, ^)
#undef BITWISE_LONGS_ACTION

static inline int
float_compactlong_guard(PyObject *lhs, PyObject *rhs)
{
    return (
        PyFloat_CheckExact(lhs) &&
        !isnan(PyFloat_AS_DOUBLE(lhs)) &&
        PyLong_CheckExact(rhs) &&
        _PyLong_IsCompact((PyLongObject *)rhs)
    );
}
static inline int
nonzero_float_compactlong_guard(PyObject *lhs, PyObject *rhs)
{
    return (
        float_compactlong_guard(lhs, rhs) && !_PyLong_IsZero((PyLongObject*)rhs)
    );
}

#define FLOAT_LONG_ACTION(NAME, OP) \
    static PyObject * \
    (NAME)(PyObject *lhs, PyObject *rhs) \
    { \
        double lhs_val = PyFloat_AsDouble(lhs); \
        Py_ssize_t rhs_val = _PyLong_CompactValue((PyLongObject *)rhs); \
        return PyFloat_FromDouble(lhs_val OP rhs_val); \
    }
FLOAT_LONG_ACTION(float_compactlong_add, +)
FLOAT_LONG_ACTION(float_compactlong_subtract, -)
FLOAT_LONG_ACTION(float_compactlong_multiply, *)
FLOAT_LONG_ACTION(float_compactlong_true_div, /)
#undef FLOAT_LONG_ACTION

static inline int
compactlong_float_guard(PyObject *lhs, PyObject *rhs)
{
    return (
        PyLong_CheckExact(lhs) &&
        _PyLong_IsCompact((PyLongObject *)lhs) &&
        PyFloat_CheckExact(rhs) &&
        !isnan(PyFloat_AS_DOUBLE(rhs))
    );
}
static inline int
nonzero_compactlong_float_guard(PyObject *lhs, PyObject *rhs)
{
    return (
        compactlong_float_guard(lhs, rhs) && PyFloat_AS_DOUBLE(rhs) != 0.0
    );
}

#define LONG_FLOAT_ACTION(NAME, OP) \
    static PyObject * \
    (NAME)(PyObject *lhs, PyObject *rhs) \
    { \
        double rhs_val = PyFloat_AsDouble(rhs); \
        Py_ssize_t lhs_val = _PyLong_CompactValue((PyLongObject *)lhs); \
        return PyFloat_FromDouble(lhs_val OP rhs_val); \
    }
LONG_FLOAT_ACTION(compactlong_float_add, +)
LONG_FLOAT_ACTION(compactlong_float_subtract, -)
LONG_FLOAT_ACTION(compactlong_float_multiply, *)
LONG_FLOAT_ACTION(compactlong_float_true_div, /)
#undef LONG_FLOAT_ACTION

static _PyBinaryOpSpecializationDescr binaryop_extend_descrs[] = {
    /* long-long arithmetic */
    {NB_OR, compactlongs_guard, compactlongs_or},
    {NB_AND, compactlongs_guard, compactlongs_and},
    {NB_XOR, compactlongs_guard, compactlongs_xor},
    {NB_INPLACE_OR, compactlongs_guard, compactlongs_or},
    {NB_INPLACE_AND, compactlongs_guard, compactlongs_and},
    {NB_INPLACE_XOR, compactlongs_guard, compactlongs_xor},

    /* float-long arithemetic */
    {NB_ADD, float_compactlong_guard, float_compactlong_add},
    {NB_SUBTRACT, float_compactlong_guard, float_compactlong_subtract},
    {NB_TRUE_DIVIDE, nonzero_float_compactlong_guard, float_compactlong_true_div},
    {NB_MULTIPLY, float_compactlong_guard, float_compactlong_multiply},

    /* float-float arithmetic */
    {NB_ADD, compactlong_float_guard, compactlong_float_add},
    {NB_SUBTRACT, compactlong_float_guard, compactlong_float_subtract},
    {NB_TRUE_DIVIDE, nonzero_compactlong_float_guard, compactlong_float_true_div},
    {NB_MULTIPLY, compactlong_float_guard, compactlong_float_multiply},
};
static int
binary_op_extended_specialization(PyObject *lhs, PyObject *rhs, int oparg,
                                  _PyBinaryOpSpecializationDescr **descr)
{
    size_t n = sizeof(binaryop_extend_descrs)/sizeof(_PyBinaryOpSpecializationDescr);
    for (size_t i = 0; i < n; i++) {
        _PyBinaryOpSpecializationDescr *d = &binaryop_extend_descrs[i];
        if (d->oparg == oparg && d->guard(lhs, rhs)) {
            *descr = d;
            return 1;
        }
    }
    return 0;
}
Py_NO_INLINE void
_Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr,
                        int oparg, _PyStackRef *locals)
{
    PyObject *lhs = PyStackRef_AsPyObjectBorrow(lhs_st);
    PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st);
    assert(ENABLE_SPECIALIZATION);
    assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP);

    _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1);
    if (instr->op.code == BINARY_OP_EXTEND) {
        write_ptr(cache->external_cache, NULL);
    }

    switch (oparg) {
        case NB_ADD:
        case NB_INPLACE_ADD:
            if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
                break;
            }
            if (PyUnicode_CheckExact(lhs)) {
                _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_BINARY_OP + 1];
                bool to_store = (next.op.code == STORE_FAST);
                if (to_store && PyStackRef_AsPyObjectBorrow(locals[next.op.arg]) == lhs) {
                    specialize(instr, BINARY_OP_INPLACE_ADD_UNICODE);
                    return;
                }
                specialize(instr, BINARY_OP_ADD_UNICODE);
                return;
            }
            if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs)) {
                specialize(instr, BINARY_OP_ADD_INT);
                return;
            }
            if (PyFloat_CheckExact(lhs)) {
                specialize(instr, BINARY_OP_ADD_FLOAT);
                return;
            }
            break;
        case NB_MULTIPLY:
        case NB_INPLACE_MULTIPLY:
            if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
                break;
            }
            if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs)) {
                specialize(instr, BINARY_OP_MULTIPLY_INT);
                return;
            }
            if (PyFloat_CheckExact(lhs)) {
                specialize(instr, BINARY_OP_MULTIPLY_FLOAT);
                return;
            }
            break;
        case NB_SUBTRACT:
        case NB_INPLACE_SUBTRACT:
            if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
                break;
            }
            if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs)) {
                specialize(instr, BINARY_OP_SUBTRACT_INT);
                return;
            }
            if (PyFloat_CheckExact(lhs)) {
                specialize(instr, BINARY_OP_SUBTRACT_FLOAT);
                return;
            }
            break;
        case NB_SUBSCR:
            if (PyLong_CheckExact(rhs) && _PyLong_IsNonNegativeCompact((PyLongObject *)rhs)) {
                if (PyList_CheckExact(lhs)) {
                    specialize(instr, BINARY_OP_SUBSCR_LIST_INT);
                    return;
                }
                if (PyTuple_CheckExact(lhs)) {
                    specialize(instr, BINARY_OP_SUBSCR_TUPLE_INT);
                    return;
                }
                if (PyUnicode_CheckExact(lhs) && _PyLong_IsNonNegativeCompact((PyLongObject*)rhs)) {
                    if (PyUnicode_IS_COMPACT_ASCII(lhs)) {
                        specialize(instr, BINARY_OP_SUBSCR_STR_INT);
                        return;
                    } else {
                        specialize(instr, BINARY_OP_SUBSCR_USTR_INT);
                        return;
                    }
                }
            }
            if (PyAnyDict_CheckExact(lhs)) {
                specialize(instr, BINARY_OP_SUBSCR_DICT);
                return;
            }
            if (PyList_CheckExact(lhs) && PySlice_Check(rhs)) {
                specialize(instr, BINARY_OP_SUBSCR_LIST_SLICE);
                return;
            }
            unsigned int tp_version;
            PyTypeObject *container_type = Py_TYPE(lhs);
            PyObject *descriptor = _PyType_LookupRefAndVersion(container_type, &_Py_ID(__getitem__), &tp_version);
            if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type &&
                container_type->tp_flags & Py_TPFLAGS_HEAPTYPE)
            {
                PyFunctionObject *func = (PyFunctionObject *)descriptor;
                PyCodeObject *fcode = (PyCodeObject *)func->func_code;
                int kind = function_kind(fcode);
                PyHeapTypeObject *ht = (PyHeapTypeObject *)container_type;
                if (kind == SIMPLE_FUNCTION &&
                    fcode->co_argcount == 2 &&
                    !(_PyInterpreterState_GET()->eval_frame != NULL && _PyInterpreterState_GET()->eval_frame != Ci_EvalFrameFunc) && /* Don't specialize if PEP 523 is active */
                    _PyType_CacheGetItemForSpecialization(ht, descriptor, (uint32_t)tp_version))
                {
                    specialize(instr, BINARY_OP_SUBSCR_GETITEM);
                    Py_DECREF(descriptor);
                    return;
                }
            }
            Py_XDECREF(descriptor);
            break;
    }

    _PyBinaryOpSpecializationDescr *descr;
    if (binary_op_extended_specialization(lhs, rhs, oparg, &descr)) {
        specialize(instr, BINARY_OP_EXTEND);
        write_ptr(cache->external_cache, (void*)descr);
        return;
    }

    SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs));
    unspecialize(instr);
    return;
}
Py_NO_INLINE void
_Py_Specialize_CompareOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr,
                         int oparg)
{
    PyObject *lhs = PyStackRef_AsPyObjectBorrow(lhs_st);
    PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st);
    uint8_t specialized_op;

    assert(ENABLE_SPECIALIZATION);
    assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP);
    // All of these specializations compute boolean values, so they're all valid
    // regardless of the fifth-lowest oparg bit.
    if (Py_TYPE(lhs) != Py_TYPE(rhs)) {
        SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs));
        goto failure;
    }
    if (PyFloat_CheckExact(lhs)) {
        specialized_op = COMPARE_OP_FLOAT;
        goto success;
    }
    if (PyLong_CheckExact(lhs)) {
        if (_PyLong_IsCompact((PyLongObject *)lhs) && _PyLong_IsCompact((PyLongObject *)rhs)) {
            specialized_op = COMPARE_OP_INT;
            goto success;
        }
        else {
            SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_BIG_INT);
            goto failure;
        }
    }
    if (PyUnicode_CheckExact(lhs)) {
        int cmp = oparg >> 5;
        if (cmp != Py_EQ && cmp != Py_NE) {
            SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_STRING);
            goto failure;
        }
        else {
            specialized_op = COMPARE_OP_STR;
            goto success;
        }
    }
    SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs));
failure:
    unspecialize(instr);
    return;
success:
    specialize(instr, specialized_op);
}
Py_NO_INLINE void
_Py_Specialize_UnpackSequence(_PyStackRef seq_st, _Py_CODEUNIT *instr, int oparg)
{
    PyObject *seq = PyStackRef_AsPyObjectBorrow(seq_st);

    assert(ENABLE_SPECIALIZATION);
    assert(_PyOpcode_Caches[UNPACK_SEQUENCE] ==
           INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
    if (PyTuple_CheckExact(seq)) {
        if (PyTuple_GET_SIZE(seq) != oparg) {
            SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR);
            unspecialize(instr);
            return;
        }
        if (PyTuple_GET_SIZE(seq) == 2) {
            specialize(instr, UNPACK_SEQUENCE_TWO_TUPLE);
            return;
        }
        specialize(instr, UNPACK_SEQUENCE_TUPLE);
        return;
    }
    if (PyList_CheckExact(seq)) {
        if (PyList_GET_SIZE(seq) != oparg) {
            SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR);
            unspecialize(instr);
            return;
        }
        specialize(instr, UNPACK_SEQUENCE_LIST);
        return;
    }
    SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq));
    unspecialize(instr);
}
Py_NO_INLINE void
_Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT *instr, int oparg)
{
    assert(ENABLE_SPECIALIZATION);
    assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER);
    PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
    PyTypeObject *tp = Py_TYPE(iter_o);

    if (PyStackRef_IsNull(null_or_index)) {
        if (tp == &PyRangeIter_Type) {
#ifdef Py_GIL_DISABLED
            // Only specialize for uniquely referenced iterators, so that we know
            // they're only referenced by this one thread. This is more limiting
            // than we need (even `it = iter(mylist); for item in it:` won't get
            // specialized) but we don't have a way to check whether we're the only
            // _thread_ who has access to the object.
            if (!_PyObject_IsUniquelyReferenced(iter_o)) {
                goto failure;
            }
#endif
            specialize(instr, FOR_ITER_RANGE);
            return;
        }
        else if (tp == &PyGen_Type && oparg <= SHRT_MAX) {
            assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR  ||
                instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR
            );
            /* Don't specialize if PEP 523 is active */
            if ((_PyInterpreterState_GET()->eval_frame != NULL && _PyInterpreterState_GET()->eval_frame != Ci_EvalFrameFunc)) {
                goto failure;
            }
            specialize(instr, FOR_ITER_GEN);
            return;
        }
    }
    else {
        if (tp == &PyList_Type) {
#ifdef Py_GIL_DISABLED
            // Only specialize for lists owned by this thread or shared
            if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) {
                goto failure;
            }
#endif
            specialize(instr, FOR_ITER_LIST);
            return;
        }
        else if (tp == &PyTuple_Type) {
            specialize(instr, FOR_ITER_TUPLE);
            return;
        }
    }
failure:
    SPECIALIZATION_FAIL(FOR_ITER,
                        _PySpecialization_ClassifyIterator(iter_o));
    unspecialize(instr);
}
Py_NO_INLINE void
_Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr)
{
    PyObject *receiver = PyStackRef_AsPyObjectBorrow(receiver_st);

    assert(ENABLE_SPECIALIZATION);
    assert(_PyOpcode_Caches[SEND] == INLINE_CACHE_ENTRIES_SEND);
    PyTypeObject *tp = Py_TYPE(receiver);
    if (tp == &PyGen_Type || tp == &PyCoro_Type) {
        /* Don't specialize if PEP 523 is active */
        if ((_PyInterpreterState_GET()->eval_frame != NULL && _PyInterpreterState_GET()->eval_frame != Ci_EvalFrameFunc)) {
            SPECIALIZATION_FAIL(SEND, SPEC_FAIL_OTHER);
            goto failure;
        }
        specialize(instr, SEND_GEN);
        return;
    }
    SPECIALIZATION_FAIL(SEND,
                        _PySpecialization_ClassifyIterator(receiver));
failure:
    unspecialize(instr);
}
static int
check_type_always_true(PyTypeObject *ty)
{
    PyNumberMethods *nb = ty->tp_as_number;
    if (nb && nb->nb_bool) {
        return SPEC_FAIL_TO_BOOL_NUMBER;
    }
    PyMappingMethods *mp = ty->tp_as_mapping;
    if (mp && mp->mp_length) {
        return SPEC_FAIL_TO_BOOL_MAPPING;
    }
    PySequenceMethods *sq = ty->tp_as_sequence;
    if (sq && sq->sq_length) {
      return SPEC_FAIL_TO_BOOL_SEQUENCE;
    }
    return 0;
}
Py_NO_INLINE void
_Py_Specialize_ToBool(_PyStackRef value_o, _Py_CODEUNIT *instr)
{
    assert(ENABLE_SPECIALIZATION);
    assert(_PyOpcode_Caches[TO_BOOL] == INLINE_CACHE_ENTRIES_TO_BOOL);
    _PyToBoolCache *cache = (_PyToBoolCache *)(instr + 1);
    PyObject *value = PyStackRef_AsPyObjectBorrow(value_o);
    uint8_t specialized_op;
    if (PyBool_Check(value)) {
        specialized_op = TO_BOOL_BOOL;
        goto success;
    }
    if (PyLong_CheckExact(value)) {
        specialized_op = TO_BOOL_INT;
        goto success;
    }
    if (PyList_CheckExact(value)) {
        specialized_op = TO_BOOL_LIST;
        goto success;
    }
    if (Py_IsNone(value)) {
        specialized_op = TO_BOOL_NONE;
        goto success;
    }
    if (PyUnicode_CheckExact(value)) {
        specialized_op = TO_BOOL_STR;
        goto success;
    }
    if (PyType_HasFeature(Py_TYPE(value), Py_TPFLAGS_HEAPTYPE)) {
        unsigned int version = 0;
        int err = _PyType_Validate(Py_TYPE(value), check_type_always_true, &version);
        if (err < 0) {
            SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_OUT_OF_VERSIONS);
            goto failure;
        }
        else if (err > 0) {
            SPECIALIZATION_FAIL(TO_BOOL, err);
            goto failure;
        }

        assert(err == 0);
        assert(version);
        write_u32(cache->version, version);
        specialized_op = TO_BOOL_ALWAYS_TRUE;
        goto success;
    }

    SPECIALIZATION_FAIL(TO_BOOL, to_bool_fail_kind(value));
failure:
    unspecialize(instr);
    return;
success:
    specialize(instr, specialized_op);
}
Py_NO_INLINE void
_Py_Specialize_ContainsOp(_PyStackRef value_st, _Py_CODEUNIT *instr)
{
    PyObject *value = PyStackRef_AsPyObjectBorrow(value_st);

    assert(ENABLE_SPECIALIZATION);
    assert(_PyOpcode_Caches[CONTAINS_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP);
    if (PyAnyDict_CheckExact(value)) {
        specialize(instr, CONTAINS_OP_DICT);
        return;
    }
    if (PySet_CheckExact(value) || PyFrozenSet_CheckExact(value)) {
        specialize(instr, CONTAINS_OP_SET);
        return;
    }

    SPECIALIZATION_FAIL(CONTAINS_OP, containsop_fail_kind(value));
    unspecialize(instr);
    return;
}

#define MAX_CANDIDATE_ITEMS 750
#define MAX_STRING_SIZE 40
#define MOVE_COST 2
#define CASE_COST 1
#define LEAST_FIVE_BITS(n) ((n) & 31)
static inline int
substitution_cost(char a, char b)
{
    if (LEAST_FIVE_BITS(a) != LEAST_FIVE_BITS(b)) {
        // Not the same, not a case flip.
        return MOVE_COST;
    }
    if (a == b) {
        return 0;
    }
    if ('A' <= a && a <= 'Z') {
        a += ('a' - 'A');
    }
    if ('A' <= b && b <= 'Z') {
        b += ('a' - 'A');
    }
    if (a == b) {
        return CASE_COST;
    }
    return MOVE_COST;
}
static Py_ssize_t
levenshtein_distance(const char *a, size_t a_size,
                     const char *b, size_t b_size,
                     size_t max_cost, size_t *buffer)
{
    // Both strings are the same (by identity)
    if (a == b) {
        return 0;
    }

    // Trim away common affixes.
    while (a_size && b_size && a[0] == b[0]) {
        a++; a_size--;
        b++; b_size--;
    }
    while (a_size && b_size && a[a_size-1] == b[b_size-1]) {
        a_size--;
        b_size--;
    }
    if (a_size == 0 || b_size == 0) {
        return (a_size + b_size) * MOVE_COST;
    }
    if (a_size > MAX_STRING_SIZE || b_size > MAX_STRING_SIZE) {
        return max_cost + 1;
    }

    // Prefer shorter buffer
    if (b_size < a_size) {
        const char *t = a; a = b; b = t;
        size_t t_size = a_size; a_size = b_size; b_size = t_size;
    }

    // quick fail when a match is impossible.
    if ((b_size - a_size) * MOVE_COST > max_cost) {
        return max_cost + 1;
    }

    // Instead of producing the whole traditional len(a)-by-len(b)
    // matrix, we can update just one row in place.
    // Initialize the buffer row
    size_t tmp = MOVE_COST;
    for (size_t i = 0; i < a_size; i++) {
        // cost from b[:0] to a[:i+1]
        buffer[i] = tmp;
        tmp += MOVE_COST;
    }

    size_t result = 0;
    for (size_t b_index = 0; b_index < b_size; b_index++) {
        char code = b[b_index];
        // cost(b[:b_index], a[:0]) == b_index * MOVE_COST
        size_t distance = result = b_index * MOVE_COST;
        size_t minimum = SIZE_MAX;
        for (size_t index = 0; index < a_size; index++) {

            // cost(b[:b_index+1], a[:index+1]) = min(
            //     // 1) substitute
            //     cost(b[:b_index], a[:index])
            //         + substitution_cost(b[b_index], a[index]),
            //     // 2) delete from b
            //     cost(b[:b_index], a[:index+1]) + MOVE_COST,
            //     // 3) delete from a
            //     cost(b[:b_index+1], a[index]) + MOVE_COST
            // )

            // 1) Previous distance in this row is cost(b[:b_index], a[:index])
            size_t substitute = distance + substitution_cost(code, a[index]);
            // 2) cost(b[:b_index], a[:index+1]) from previous row
            distance = buffer[index];
            // 3) existing result is cost(b[:b_index+1], a[index])

            size_t insert_delete = Py_MIN(result, distance) + MOVE_COST;
            result = Py_MIN(insert_delete, substitute);

            // cost(b[:b_index+1], a[:index+1])
            buffer[index] = result;
            if (result < minimum) {
                minimum = result;
            }
        }
        if (minimum > max_cost) {
            // Everything in this row is too big, so bail early.
            return max_cost + 1;
        }
    }
    return result;
}
PyObject *
_Py_CalculateSuggestions(PyObject *dir,
                      PyObject *name)
{
    assert(!PyErr_Occurred());
    assert(PyList_CheckExact(dir));

    Py_ssize_t dir_size = PyList_GET_SIZE(dir);
    if (dir_size >= MAX_CANDIDATE_ITEMS) {
        return NULL;
    }

    Py_ssize_t suggestion_distance = PY_SSIZE_T_MAX;
    PyObject *suggestion = NULL;
    Py_ssize_t name_size;
    const char *name_str = PyUnicode_AsUTF8AndSize(name, &name_size);
    if (name_str == NULL) {
        return NULL;
    }
    size_t *buffer = PyMem_New(size_t, MAX_STRING_SIZE);
    if (buffer == NULL) {
        return PyErr_NoMemory();
    }
    for (Py_ssize_t i = 0; i < dir_size; ++i) {
        PyObject *item = PyList_GET_ITEM(dir, i);
        if (_PyUnicode_Equal(name, item)) {
            continue;
        }
        Py_ssize_t item_size;
        const char *item_str = PyUnicode_AsUTF8AndSize(item, &item_size);
        if (item_str == NULL) {
            PyMem_Free(buffer);
            return NULL;
        }
        // No more than 1/3 of the involved characters should need changed.
        Py_ssize_t max_distance = (name_size + item_size + 3) * MOVE_COST / 6;
        // Don't take matches we've already beaten.
        max_distance = Py_MIN(max_distance, suggestion_distance - 1);
        Py_ssize_t current_distance =
            levenshtein_distance(name_str, name_size, item_str,
                                 item_size, max_distance, buffer);
        if (current_distance > max_distance) {
            continue;
        }
        if (!suggestion || current_distance < suggestion_distance) {
            suggestion = item;
            suggestion_distance = current_distance;
        }
    }
    PyMem_Free(buffer);
    return Py_XNewRef(suggestion);
}

static PyObject *
tb_create_raw(PyTracebackObject *next, PyFrameObject *frame, int lasti,
              int lineno)
{
    PyTracebackObject *tb;
    if ((next != NULL && !PyTraceBack_Check(next)) ||
                    frame == NULL || !PyFrame_Check(frame)) {
        PyErr_BadInternalCall();
        return NULL;
    }
    tb = PyObject_GC_New(PyTracebackObject, &PyTraceBack_Type);
    if (tb != NULL) {
        tb->tb_next = (PyTracebackObject*)Py_XNewRef(next);
        tb->tb_frame = (PyFrameObject*)Py_XNewRef(frame);
        tb->tb_lasti = lasti;
        tb->tb_lineno = lineno;
        PyObject_GC_Track(tb);
    }
    return (PyObject *)tb;
}
PyObject*
_PyTraceBack_FromFrame(PyObject *tb_next, PyFrameObject *frame)
{
    assert(tb_next == NULL || PyTraceBack_Check(tb_next));
    assert(frame != NULL);
    int addr = _PyInterpreterFrame_LASTI(frame->f_frame) * sizeof(_Py_CODEUNIT);
    return tb_create_raw((PyTracebackObject *)tb_next, frame, addr, -1);
}

// Internal dependencies for gen_dealloc.
static void
gen_clear_frame(PyGenObject *gen)
{
    assert(FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state) == FRAME_CLEARED);
    _PyInterpreterFrame *frame = &gen->gi_iframe;
    frame->previous = NULL;
    _PyFrame_ClearExceptCode(frame);
    _PyErr_ClearExcState(&gen->gi_exc_state);
}
// End internal dependencies.
// Use our own memory deallocation which handles generators that might be on
// our custom free-list.
#define PyObject_GC_Del(x) Ci_free_jit_list_gen(x)
static void
gen_dealloc(PyObject *self)
{
    PyGenObject *gen = _PyGen_CAST(self);

    _PyObject_GC_UNTRACK(gen);

    FT_CLEAR_WEAKREFS(self, gen->gi_weakreflist);

    _PyObject_GC_TRACK(self);

    if (PyObject_CallFinalizerFromDealloc(self))
        return;                     /* resurrected.  :( */

    _PyObject_GC_UNTRACK(self);
    if (PyAsyncGen_CheckExact(gen)) {
        /* We have to handle this case for asynchronous generators
           right here, because this code has to be between UNTRACK
           and GC_Del. */
        Py_CLEAR(((PyAsyncGenObject*)gen)->ag_origin_or_finalizer);
    }
    if (PyCoro_CheckExact(gen)) {
        Py_CLEAR(((PyCoroObject *)gen)->cr_origin_or_finalizer);
    }
    if (gen->gi_frame_state != FRAME_CLEARED) {
        gen->gi_frame_state = FRAME_CLEARED;
        gen_clear_frame(gen);
    }
    assert(gen->gi_exc_state.exc_value == NULL);
    PyStackRef_CLEAR(gen->gi_iframe.f_executable);
    Py_CLEAR(gen->gi_name);
    Py_CLEAR(gen->gi_qualname);

    PyObject_GC_Del(gen);
}
#undef PyObject_GC_Del
void Cix_gen_dealloc_with_custom_free(PyObject* obj) {
    gen_dealloc(obj);
}

static PyObject *
gen_getyieldfrom(PyObject *self, void *Py_UNUSED(ignored))
{
    PyGenObject *gen = _PyGen_CAST(self);
#ifdef Py_GIL_DISABLED
    int8_t frame_state = _Py_atomic_load_int8_relaxed(&gen->gi_frame_state);
    do {
        if (frame_state != FRAME_SUSPENDED_YIELD_FROM &&
            frame_state != FRAME_SUSPENDED_YIELD_FROM_LOCKED)
        {
            Py_RETURN_NONE;
        }
    } while (!_Py_GEN_TRY_SET_FRAME_STATE(gen, frame_state, FRAME_SUSPENDED_YIELD_FROM_LOCKED));

    PyObject *result = PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(&gen->gi_iframe));
    _Py_atomic_store_int8_release(&gen->gi_frame_state, FRAME_SUSPENDED_YIELD_FROM);
    return result;
#else
    int8_t frame_state = gen->gi_frame_state;
    if (frame_state != FRAME_SUSPENDED_YIELD_FROM) {
        Py_RETURN_NONE;
    }
    return PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(&gen->gi_iframe));
#endif
}
PyObject * _PyGen_yf(PyGenObject *gen) {
  PyObject *res = gen_getyieldfrom((PyObject *)gen, NULL);
  if (res == Py_None) {
    return NULL;
  }
  return res;
}

void
_PyTuple_MaybeUntrack(PyObject *op)
{
    PyTupleObject *t;
    Py_ssize_t i, n;

    if (!PyTuple_CheckExact(op) || !_PyObject_GC_IS_TRACKED(op))
        return;
    t = (PyTupleObject *) op;
    n = Py_SIZE(t);
    for (i = 0; i < n; i++) {
        PyObject *elt = PyTuple_GET_ITEM(t, i);
        /* Tuple with NULL elements aren't
           fully constructed, don't untrack
           them yet. */
        if (!elt ||
            _PyObject_GC_MAY_BE_TRACKED(elt))
            return;
    }
    _PyObject_GC_UNTRACK(op);
}
