/*
** Garbage collector.
** Copyright (C) 2005-2014 Mike Pall. See Copyright Notice in luajit.h
**
** Major portions taken verbatim or adapted from the Lua interpreter.
** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
*/

#define lj_gc_c
#define LUA_CORE

#include "lj_obj.h"
#include "lj_gc.h"
#include "lj_err.h"
#include "lj_str.h"
#include "lj_tab.h"
#include "lj_func.h"
#include "lj_udata.h"
#include "lj_meta.h"
#include "lj_state.h"
#include "lj_frame.h"
#if LJ_HASFFI
    #include "lj_ctype.h"
    #include "lj_cdata.h"
#endif
#include "lj_trace.h"
#include "lj_vm.h"

#define GCSTEPSIZE 1024u
#define GCSWEEPMAX 40
#define GCSWEEPCOST 10
#define GCFINALIZECOST 100

/* Macros to set GCobj colors and flags. */
#define white2gray(x) ((x)->gch.marked &= (uint8_t)~LJ_GC_WHITES)
#define gray2black(x) ((x)->gch.marked |= LJ_GC_BLACK)
#define isfinalized(u) ((u)->marked & LJ_GC_FINALIZED)

/* -- Mark phase ---------------------------------------------------------- */

/* Mark a TValue (if needed). */
#define gc_marktv(g, tv)                                                       \
    {                                                                          \
        lua_assert(!tvisgcv(tv) || (~itype(tv) == gcval(tv)->gch.gct));        \
        if (tviswhite(tv))                                                     \
            gc_mark(g, gcV(tv));                                               \
    }

/* Mark a GCobj (if needed). */
#define gc_markobj(g, o)                                                       \
    {                                                                          \
        if (iswhite(obj2gco(o)))                                               \
            gc_mark(g, obj2gco(o));                                            \
    }

/* Mark a string object. */
#define gc_mark_str(s) ((s)->marked &= (uint8_t)~LJ_GC_WHITES)

/* Mark a white GCobj. */
static void gc_mark(global_State *g, GCobj *o)
{
    int gct = o->gch.gct;
    lua_assert(iswhite(o) && !isdead(g, o));
    white2gray(o);
    if (LJ_UNLIKELY(gct == ~LJ_TUDATA)) {
        GCtab *mt = tabref(gco2ud(o)->metatable);
        gray2black(o); /* Userdata are never gray. */
        if (mt)
            gc_markobj(g, mt);
        gc_markobj(g, tabref(gco2ud(o)->env));
    }
    else if (LJ_UNLIKELY(gct == ~LJ_TUPVAL)) {
        GCupval *uv = gco2uv(o);
        gc_marktv(g, uvval(uv));
        if (uv->closed)
            gray2black(o); /* Closed upvalues are never gray. */
    }
    else if (gct != ~LJ_TSTR && gct != ~LJ_TCDATA) {
        lua_assert(gct == ~LJ_TFUNC || gct == ~LJ_TTAB || gct == ~LJ_TTHREAD ||
                   gct == ~LJ_TPROTO);
        setgcrefr(o->gch.gclist, g->gc.gray);
        setgcref(g->gc.gray, o);
    }
}

/* Mark GC roots. */
static void gc_mark_gcroot(global_State *g)
{
    ptrdiff_t i;
    for (i = 0; i < GCROOT_MAX; i++)
        if (gcref(g->gcroot[i]) != NULL)
            gc_markobj(g, gcref(g->gcroot[i]));
}

/* Start a GC cycle and mark the root set. */
static void gc_mark_start(global_State *g)
{
    setgcrefnull(g->gc.gray);
    setgcrefnull(g->gc.grayagain);
    setgcrefnull(g->gc.weak);
    gc_markobj(g, mainthread(g));
    gc_markobj(g, tabref(mainthread(g)->env));
    gc_marktv(g, &g->registrytv);
    gc_mark_gcroot(g);
    g->gc.state = GCSpropagate;
}

/* Mark open upvalues. */
static void gc_mark_uv(global_State *g)
{
    GCupval *uv;
    for (uv = uvnext(&g->uvhead); uv != &g->uvhead; uv = uvnext(uv)) {
        lua_assert(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv);
        if (isgray(obj2gco(uv)))
            gc_marktv(g, uvval(uv));
    }
}

/* Mark userdata in mmudata list. */
static void gc_mark_mmudata(global_State *g)
{
    GCobj *root = gcref(g->gc.mmudata);
    GCobj *u = root;
    if (u) {
        do {
            u = gcnext(u);
            makewhite(g, u); /* Could be from previous GC. */
            gc_mark(g, u);
        } while (u != root);
    }
}

/* Separate userdata objects to be finalized to mmudata list. */
size_t lj_gc_separateudata(global_State *g, int all)
{
    size_t m = 0;
    GCRef *p = &mainthread(g)->nextgc;
    GCobj *o;
    while ((o = gcref(*p)) != NULL) {
        if (!(iswhite(o) || all) || isfinalized(gco2ud(o))) {
            p = &o->gch.nextgc; /* Nothing to do. */
        }
        else if (!lj_meta_fastg(g, tabref(gco2ud(o)->metatable), MM_gc)) {
            markfinalized(o); /* Done, as there's no __gc metamethod. */
            p = &o->gch.nextgc;
        }
        else { /* Otherwise move userdata to be finalized to mmudata list. */
            m += sizeudata(gco2ud(o));
            markfinalized(o);
            *p = o->gch.nextgc;
            if (gcref(g->gc.mmudata)) { /* Link to end of mmudata list. */
                GCobj *root = gcref(g->gc.mmudata);
                setgcrefr(o->gch.nextgc, root->gch.nextgc);
                setgcref(root->gch.nextgc, o);
                setgcref(g->gc.mmudata, o);
            }
            else { /* Create circular list. */
                setgcref(o->gch.nextgc, o);
                setgcref(g->gc.mmudata, o);
            }
        }
    }
    return m;
}

/* -- Propagation phase --------------------------------------------------- */

/* Traverse a table. */
static int gc_traverse_tab(global_State *g, GCtab *t)
{
    int weak = 0;
    cTValue *mode;
    GCtab *mt = tabref(t->metatable);
    if (mt)
        gc_markobj(g, mt);
    mode = lj_meta_fastg(g, mt, MM_mode);
    if (mode && tvisstr(mode)) { /* Valid __mode field? */
        const char *modestr = strVdata(mode);
        int c;
        while ((c = *modestr++)) {
            if (c == 'k')
                weak |= LJ_GC_WEAKKEY;
            else if (c == 'v')
                weak |= LJ_GC_WEAKVAL;
            else if (c == 'K')
                weak = (int)(~0u & ~LJ_GC_WEAKVAL);
        }
        if (weak > 0) { /* Weak tables are cleared in the atomic phase. */
            t->marked = (uint8_t)((t->marked & ~LJ_GC_WEAK) | weak);
            setgcrefr(t->gclist, g->gc.weak);
            setgcref(g->gc.weak, obj2gco(t));
        }
    }
    if (weak == LJ_GC_WEAK) /* Nothing to mark if both keys/values are weak. */
        return 1;
    if (!(weak & LJ_GC_WEAKVAL)) { /* Mark array part. */
        MSize i, asize = t->asize;
        for (i = 0; i < asize; i++)
            gc_marktv(g, arrayslot(t, i));
    }
    if (t->hmask > 0) { /* Mark hash part. */
        Node *node = noderef(t->node);
        MSize i, hmask = t->hmask;
        for (i = 0; i <= hmask; i++) {
            Node *n = &node[i];
            if (!tvisnil(&n->val)) { /* Mark non-empty slot. */
                lua_assert(!tvisnil(&n->key));
                if (!(weak & LJ_GC_WEAKKEY))
                    gc_marktv(g, &n->key);
                if (!(weak & LJ_GC_WEAKVAL))
                    gc_marktv(g, &n->val);
            }
        }
    }
    return weak;
}

/* Traverse a function. */
static void gc_traverse_func(global_State *g, GCfunc *fn)
{
    gc_markobj(g, tabref(fn->c.env));
    if (isluafunc(fn)) {
        uint32_t i;
        lua_assert(fn->l.nupvalues <= funcproto(fn)->sizeuv);
        gc_markobj(g, funcproto(fn));
        for (i = 0; i < fn->l.nupvalues; i++) /* Mark Lua function upvalues. */
            gc_markobj(g, &gcref(fn->l.uvptr[i])->uv);
    }
    else {
        uint32_t i;
        for (i = 0; i < fn->c.nupvalues; i++) /* Mark C function upvalues. */
            gc_marktv(g, &fn->c.upvalue[i]);
    }
}

#if LJ_HASJIT
/* Mark a trace. */
static void gc_marktrace(global_State *g, TraceNo traceno)
{
    GCobj *o = obj2gco(traceref(G2J(g), traceno));
    lua_assert(traceno != G2J(g)->cur.traceno);
    if (iswhite(o)) {
        white2gray(o);
        setgcrefr(o->gch.gclist, g->gc.gray);
        setgcref(g->gc.gray, o);
    }
}

/* Traverse a trace. */
static void gc_traverse_trace(global_State *g, GCtrace *T)
{
    IRRef ref;
    if (T->traceno == 0)
        return;
    for (ref = T->nk; ref < REF_TRUE; ref++) {
        IRIns *ir = &T->ir[ref];
        if (ir->o == IR_KGC)
            gc_markobj(g, ir_kgc(ir));
    }
    if (T->link)
        gc_marktrace(g, T->link);
    if (T->nextroot)
        gc_marktrace(g, T->nextroot);
    if (T->nextside)
        gc_marktrace(g, T->nextside);
    gc_markobj(g, gcref(T->startpt));
}

    /* The current trace is a GC root while not anchored in the prototype (yet). */
    #define gc_traverse_curtrace(g) gc_traverse_trace(g, &G2J(g)->cur)
#else
    #define gc_traverse_curtrace(g) UNUSED(g)
#endif

/* Traverse a prototype. */
static void gc_traverse_proto(global_State *g, GCproto *pt)
{
    ptrdiff_t i;
    gc_mark_str(proto_chunkname(pt));
    for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) /* Mark collectable consts. */
        gc_markobj(g, proto_kgc(pt, i));
#if LJ_HASJIT
    if (pt->trace)
        gc_marktrace(g, pt->trace);
#endif
}

/* Traverse the frame structure of a stack. */
static MSize gc_traverse_frames(global_State *g, lua_State *th)
{
    TValue *frame, *top = th->top - 1, *bot = tvref(th->stack);
    /* Note: extra vararg frame not skipped, marks function twice (harmless). */
    for (frame = th->base - 1; frame > bot; frame = frame_prev(frame)) {
        GCfunc *fn = frame_func(frame);
        TValue *ftop = frame;
        if (isluafunc(fn))
            ftop += funcproto(fn)->framesize;
        if (ftop > top)
            top = ftop;
        gc_markobj(g, fn); /* Need to mark hidden function (or L). */
    }
    top++; /* Correct bias of -1 (frame == base-1). */
    if (top > tvref(th->maxstack))
        top = tvref(th->maxstack);
    return (MSize)(top - bot); /* Return minimum needed stack size. */
}

/* Traverse a thread object. */
static void gc_traverse_thread(global_State *g, lua_State *th)
{
    TValue *o, *top = th->top;
    for (o = tvref(th->stack) + 1; o < top; o++)
        gc_marktv(g, o);
    if (g->gc.state == GCSatomic) {
        top = tvref(th->stack) + th->stacksize;
        for (; o < top; o++) /* Clear unmarked slots. */
            setnilV(o);
    }
    gc_markobj(g, tabref(th->env));
    lj_state_shrinkstack(th, gc_traverse_frames(g, th));
}

/* Propagate one gray object. Traverse it and turn it black. */
static size_t propagatemark(global_State *g)
{
    GCobj *o = gcref(g->gc.gray);
    int gct = o->gch.gct;
    lua_assert(isgray(o));
    gray2black(o);
    setgcrefr(g->gc.gray, o->gch.gclist); /* Remove from gray list. */
    if (LJ_LIKELY(gct == ~LJ_TTAB)) {
        GCtab *t = gco2tab(o);
        if (gc_traverse_tab(g, t) > 0)
            black2gray(o); /* Keep weak tables gray. */
        return sizeof(GCtab) + sizeof(TValue) * t->asize +
               sizeof(Node) * (t->hmask + 1);
    }
    else if (LJ_LIKELY(gct == ~LJ_TFUNC)) {
        GCfunc *fn = gco2func(o);
        gc_traverse_func(g, fn);
        return isluafunc(fn) ? sizeLfunc((MSize)fn->l.nupvalues)
                             : sizeCfunc((MSize)fn->c.nupvalues);
    }
    else if (LJ_LIKELY(gct == ~LJ_TPROTO)) {
        GCproto *pt = gco2pt(o);
        gc_traverse_proto(g, pt);
        return pt->sizept;
    }
    else if (LJ_LIKELY(gct == ~LJ_TTHREAD)) {
        lua_State *th = gco2th(o);
        setgcrefr(th->gclist, g->gc.grayagain);
        setgcref(g->gc.grayagain, o);
        black2gray(o); /* Threads are never black. */
        gc_traverse_thread(g, th);
        return sizeof(lua_State) + sizeof(TValue) * th->stacksize;
    }
    else {
#if LJ_HASJIT
        GCtrace *T = gco2trace(o);
        gc_traverse_trace(g, T);
        return ((sizeof(GCtrace) + 7) & ~7) +
               (T->nins - T->nk) * sizeof(IRIns) + T->nsnap * sizeof(SnapShot) +
               T->nsnapmap * sizeof(SnapEntry);
#else
        lua_assert(0);
        return 0;
#endif
    }
}

/* Propagate all gray objects. */
static size_t gc_propagate_gray(global_State *g)
{
    size_t m = 0;
    while (gcref(g->gc.gray) != NULL)
        m += propagatemark(g);
    return m;
}

/* -- Sweep phase --------------------------------------------------------- */

/* Try to shrink some common data structures. */
static void gc_shrink(global_State *g, lua_State *L)
{
    if (g->strnum <= (g->strmask >> 2) && g->strmask > LJ_MIN_STRTAB * 2 - 1)
        lj_str_resize(L, g->strmask >> 1); /* Shrink string table. */
    if (g->tmpbuf.sz > LJ_MIN_SBUF * 2)
        lj_str_resizebuf(L, &g->tmpbuf,
                         g->tmpbuf.sz >> 1); /* Shrink temp buf. */
}

/* Type of GC free functions. */
typedef void(LJ_FASTCALL *GCFreeFunc)(global_State *g, GCobj *o);

/* GC free functions for LJ_TSTR .. LJ_TUDATA. ORDER LJ_T */
static const GCFreeFunc gc_freefunc[] = {
    (GCFreeFunc)lj_str_free,   (GCFreeFunc)lj_func_freeuv,
    (GCFreeFunc)lj_state_free, (GCFreeFunc)lj_func_freeproto,
    (GCFreeFunc)lj_func_free,
#if LJ_HASJIT
    (GCFreeFunc)lj_trace_free,
#else
  (GCFreeFunc)0,
#endif
#if LJ_HASFFI
    (GCFreeFunc)lj_cdata_free,
#else
  (GCFreeFunc)0,
#endif
    (GCFreeFunc)lj_tab_free,   (GCFreeFunc)lj_udata_free};

/* Full sweep of a GC list. */
#define gc_fullsweep(g, p) gc_sweep(g, (p), LJ_MAX_MEM)

/* Partial sweep of a GC list. */
static GCRef *gc_sweep(global_State *g, GCRef *p, uint32_t lim)
{
    /* Mask with other white and LJ_GC_FIXED. Or LJ_GC_SFIXED on shutdown. */
    int ow = otherwhite(g);
    GCobj *o;
    while ((o = gcref(*p)) != NULL && lim-- > 0) {
        if (o->gch.gct == ~LJ_TTHREAD) /* Need to sweep open upvalues, too. */
            gc_fullsweep(g, &gco2th(o)->openupval);
        if (((o->gch.marked ^ LJ_GC_WHITES) &
             ow)) { /* Black or current white? */
            lua_assert(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED));
            makewhite(g, o); /* Value is alive, change to the current white. */
            p = &o->gch.nextgc;
        }
        else { /* Otherwise value is dead, free it. */
            lua_assert(isdead(g, o) || ow == LJ_GC_SFIXED);
            setgcrefr(*p, o->gch.nextgc);
            if (o == gcref(g->gc.root))
                setgcrefr(g->gc.root, o->gch.nextgc); /* Adjust list anchor. */
            gc_freefunc[o->gch.gct - ~LJ_TSTR](g, o);
        }
    }
    return p;
}

/* Check whether we can clear a key or a value slot from a table. */
static int gc_mayclear(cTValue *o, int val)
{
    if (tvisgcv(o)) {     /* Only collectable objects can be weak references. */
        if (tvisstr(o)) { /* But strings cannot be used as weak references. */
            gc_mark_str(strV(o)); /* And need to be marked. */
            return 0;
        }
        if (iswhite(gcV(o)))
            return 1; /* Object is about to be collected. */
        if (tvisudata(o) && val && isfinalized(udataV(o)))
            return 1; /* Finalized userdata is dropped only from values. */
    }
    return 0; /* Cannot clear. */
}

/* Clear collected entries from weak tables. */
static void gc_clearweak(GCobj *o)
{
    while (o) {
        GCtab *t = gco2tab(o);
        lua_assert((t->marked & LJ_GC_WEAK));
        if ((t->marked & LJ_GC_WEAKVAL)) {
            MSize i, asize = t->asize;
            for (i = 0; i < asize; i++) {
                /* Clear array slot when value is about to be collected. */
                TValue *tv = arrayslot(t, i);
                if (gc_mayclear(tv, 1))
                    setnilV(tv);
            }
        }
        if (t->hmask > 0) {
            Node *node = noderef(t->node);
            MSize i, hmask = t->hmask;
            for (i = 0; i <= hmask; i++) {
                Node *n = &node[i];
                /* Clear hash slot when key or value is about to be collected. */
                if (!tvisnil(&n->val) &&
                    (gc_mayclear(&n->key, 0) || gc_mayclear(&n->val, 1)))
                    setnilV(&n->val);
            }
        }
        o = gcref(t->gclist);
    }
}

/* Call a userdata or cdata finalizer. */
static void gc_call_finalizer(global_State *g, lua_State *L, cTValue *mo,
                              GCobj *o)
{
    /* Save and restore lots of state around the __gc callback. */
    uint8_t oldh = hook_save(g);
    MSize oldt = g->gc.threshold;
    int errcode;
    TValue *top;
    lj_trace_abort(g);
    top = L->top;
    L->top = top + 2;
    hook_entergc(g); /* Disable hooks and new traces during __gc. */
    g->gc.threshold = LJ_MAX_MEM; /* Prevent GC steps. */
    copyTV(L, top, mo);
    setgcV(L, top + 1, o, ~o->gch.gct);
    errcode = lj_vm_pcall(L, top + 1, 1 + 0, -1); /* Stack: |mo|o| -> | */
    hook_restore(g, oldh);
    g->gc.threshold = oldt; /* Restore GC threshold. */
    if (errcode)
        lj_err_throw(L, errcode); /* Propagate errors. */
}

/* Finalize one userdata or cdata object from the mmudata list. */
static void gc_finalize(lua_State *L)
{
    global_State *g = G(L);
    GCobj *o = gcnext(gcref(g->gc.mmudata));
    cTValue *mo;
    lua_assert(gcref(g->jit_L) == NULL); /* Must not be called on trace. */
    /* Unchain from list of userdata to be finalized. */
    if (o == gcref(g->gc.mmudata))
        setgcrefnull(g->gc.mmudata);
    else
        setgcrefr(gcref(g->gc.mmudata)->gch.nextgc, o->gch.nextgc);
#if LJ_HASFFI
    if (o->gch.gct == ~LJ_TCDATA) {
        TValue tmp, *tv;
        /* Add cdata back to the GC list and make it white. */
        setgcrefr(o->gch.nextgc, g->gc.root);
        setgcref(g->gc.root, o);
        makewhite(g, o);
        o->gch.marked &= (uint8_t)~LJ_GC_CDATA_FIN;
        /* Resolve finalizer. */
        setcdataV(L, &tmp, gco2cd(o));
        tv = lj_tab_set(L, ctype_ctsG(g)->finalizer, &tmp);
        if (!tvisnil(tv)) {
            g->gc.nocdatafin = 0;
            copyTV(L, &tmp, tv);
            setnilV(tv); /* Clear entry in finalizer table. */
            gc_call_finalizer(g, L, &tmp, o);
        }
        return;
    }
#endif
    /* Add userdata back to the main userdata list and make it white. */
    setgcrefr(o->gch.nextgc, mainthread(g)->nextgc);
    setgcref(mainthread(g)->nextgc, o);
    makewhite(g, o);
    /* Resolve the __gc metamethod. */
    mo = lj_meta_fastg(g, tabref(gco2ud(o)->metatable), MM_gc);
    if (mo)
        gc_call_finalizer(g, L, mo, o);
}

/* Finalize all userdata objects from mmudata list. */
void lj_gc_finalize_udata(lua_State *L)
{
    while (gcref(G(L)->gc.mmudata) != NULL)
        gc_finalize(L);
}

#if LJ_HASFFI
/* Finalize all cdata objects from finalizer table. */
void lj_gc_finalize_cdata(lua_State *L)
{
    global_State *g = G(L);
    CTState *cts = ctype_ctsG(g);
    if (cts) {
        GCtab *t = cts->finalizer;
        Node *node = noderef(t->node);
        ptrdiff_t i;
        setgcrefnull(t->metatable); /* Mark finalizer table as disabled. */
        for (i = (ptrdiff_t)t->hmask; i >= 0; i--)
            if (!tvisnil(&node[i].val) && tviscdata(&node[i].key)) {
                GCobj *o = gcV(&node[i].key);
                TValue tmp;
                makewhite(g, o);
                o->gch.marked &= (uint8_t)~LJ_GC_CDATA_FIN;
                copyTV(L, &tmp, &node[i].val);
                setnilV(&node[i].val);
                gc_call_finalizer(g, L, &tmp, o);
            }
    }
}
#endif

/* Free all remaining GC objects. */
void lj_gc_freeall(global_State *g)
{
    MSize i, strmask;
    /* Free everything, except super-fixed objects (the main thread). */
    g->gc.currentwhite = LJ_GC_WHITES | LJ_GC_SFIXED;
    gc_fullsweep(g, &g->gc.root);
    strmask = g->strmask;
    for (i = 0; i <= strmask; i++) /* Free all string hash chains. */
        gc_fullsweep(g, &g->strhash[i]);
}

/* -- Collector ----------------------------------------------------------- */

/* Atomic part of the GC cycle, transitioning from mark to sweep phase. */
static void atomic(global_State *g, lua_State *L)
{
    size_t udsize;

    gc_mark_uv(g); /* Need to remark open upvalues (the thread may be dead). */
    gc_propagate_gray(g); /* Propagate any left-overs. */

    setgcrefr(g->gc.gray, g->gc.weak); /* Empty the list of weak tables. */
    setgcrefnull(g->gc.weak);
    lua_assert(!iswhite(obj2gco(mainthread(g))));
    gc_markobj(g, L);        /* Mark running thread. */
    gc_traverse_curtrace(g); /* Traverse current trace. */
    gc_mark_gcroot(g);       /* Mark GC roots (again). */
    gc_propagate_gray(g);    /* Propagate all of the above. */

    setgcrefr(g->gc.gray, g->gc.grayagain); /* Empty the 2nd chance list. */
    setgcrefnull(g->gc.grayagain);
    gc_propagate_gray(g); /* Propagate it. */

    udsize = lj_gc_separateudata(g, 0); /* Separate userdata to be finalized. */
    gc_mark_mmudata(g);                 /* Mark them. */
    udsize += gc_propagate_gray(g);     /* And propagate the marks. */

    /* All marking done, clear weak tables. */
    gc_clearweak(gcref(g->gc.weak));

    /* Prepare for sweep phase. */
    g->gc.currentwhite = (uint8_t)otherwhite(g); /* Flip current white. */
    g->strempty.marked = g->gc.currentwhite;
    setmref(g->gc.sweep, &g->gc.root);
    g->gc.estimate = g->gc.total - (MSize)udsize; /* Initial estimate. */
}

/* GC state machine. Returns a cost estimate for each step performed. */
static size_t gc_onestep(lua_State *L)
{
    global_State *g = G(L);
    switch (g->gc.state) {
    case GCSpause:
        gc_mark_start(g); /* Start a new GC cycle by marking all GC roots. */
        return 0;
    case GCSpropagate:
        if (gcref(g->gc.gray) != NULL)
            return propagatemark(g); /* Propagate one gray object. */
        g->gc.state = GCSatomic;     /* End of mark phase. */
        return 0;
    case GCSatomic:
        if (gcref(g->jit_L)) /* Don't run atomic phase on trace. */
            return LJ_MAX_MEM;
        atomic(g, L);
        g->gc.state = GCSsweepstring; /* Start of sweep phase. */
        g->gc.sweepstr = 0;
        return 0;
    case GCSsweepstring: {
        MSize old = g->gc.total;
        gc_fullsweep(g, &g->strhash[g->gc.sweepstr++]); /* Sweep one chain. */
        if (g->gc.sweepstr > g->strmask)
            g->gc.state = GCSsweep; /* All string hash chains sweeped. */
        lua_assert(old >= g->gc.total);
        g->gc.estimate -= old - g->gc.total;
        return GCSWEEPCOST;
    }
    case GCSsweep: {
        MSize old = g->gc.total;
        setmref(g->gc.sweep, gc_sweep(g, mref(g->gc.sweep, GCRef), GCSWEEPMAX));
        if (gcref(*mref(g->gc.sweep, GCRef)) == NULL) {
            gc_shrink(g, L);
            if (gcref(g->gc.mmudata)) { /* Need any finalizations? */
                g->gc.state = GCSfinalize;
#if LJ_HASFFI
                g->gc.nocdatafin = 1;
#endif
            }
            else { /* Otherwise skip this phase to help the JIT. */
                g->gc.state = GCSpause; /* End of GC cycle. */
                g->gc.debt = 0;
            }
        }
        lua_assert(old >= g->gc.total);
        g->gc.estimate -= old - g->gc.total;
        return GCSWEEPMAX * GCSWEEPCOST;
    }
    case GCSfinalize:
        if (gcref(g->gc.mmudata) != NULL) {
            if (gcref(g->jit_L)) /* Don't call finalizers on trace. */
                return LJ_MAX_MEM;
            gc_finalize(L); /* Finalize one userdata object. */
            if (g->gc.estimate > GCFINALIZECOST)
                g->gc.estimate -= GCFINALIZECOST;
            return GCFINALIZECOST;
        }
#if LJ_HASFFI
        if (!g->gc.nocdatafin)
            lj_tab_rehash(L, ctype_ctsG(g)->finalizer);
#endif
        g->gc.state = GCSpause; /* End of GC cycle. */
        g->gc.debt = 0;
        return 0;
    default:
        lua_assert(0);
        return 0;
    }
}

/* Perform a limited amount of incremental GC steps. */
int LJ_FASTCALL lj_gc_step(lua_State *L)
{
    global_State *g = G(L);
    MSize lim;
    int32_t ostate = g->vmstate;
    setvmstate(g, GC);
    lim = (GCSTEPSIZE / 100) * g->gc.stepmul;
    if (lim == 0)
        lim = LJ_MAX_MEM;
    if (g->gc.total > g->gc.threshold)
        g->gc.debt += g->gc.total - g->gc.threshold;
    do {
        lim -= (MSize)gc_onestep(L);
        if (g->gc.state == GCSpause) {
            g->gc.threshold = (g->gc.estimate / 100) * g->gc.pause;
            g->vmstate = ostate;
            return 1; /* Finished a GC cycle. */
        }
    } while ((int32_t)lim > 0);
    if (g->gc.debt < GCSTEPSIZE) {
        g->gc.threshold = g->gc.total + GCSTEPSIZE;
        g->vmstate = ostate;
        return -1;
    }
    else {
        g->gc.debt -= GCSTEPSIZE;
        g->gc.threshold = g->gc.total;
        g->vmstate = ostate;
        return 0;
    }
}

/* Ditto, but fix the stack top first. */
void LJ_FASTCALL lj_gc_step_fixtop(lua_State *L)
{
    if (curr_funcisL(L))
        L->top = curr_topL(L);
    lj_gc_step(L);
}

#if LJ_HASJIT
/* Perform multiple GC steps. Called from JIT-compiled code. */
int LJ_FASTCALL lj_gc_step_jit(global_State *g, MSize steps)
{
    lua_State *L = gco2th(gcref(g->jit_L));
    L->base = mref(G(L)->jit_base, TValue);
    L->top = curr_topL(L);
    while (steps-- > 0 && lj_gc_step(L) == 0)
        ;
    /* Return 1 to force a trace exit. */
    return (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize);
}
#endif

/* Perform a full GC cycle. */
void lj_gc_fullgc(lua_State *L)
{
    global_State *g = G(L);
    int32_t ostate = g->vmstate;
    setvmstate(g, GC);
    if (g->gc.state <= GCSatomic) { /* Caught somewhere in the middle. */
        setmref(g->gc.sweep,
                &g->gc.root);     /* Sweep everything (preserving it). */
        setgcrefnull(g->gc.gray); /* Reset lists from partial propagation. */
        setgcrefnull(g->gc.grayagain);
        setgcrefnull(g->gc.weak);
        g->gc.state = GCSsweepstring; /* Fast forward to the sweep phase. */
        g->gc.sweepstr = 0;
    }
    while (g->gc.state == GCSsweepstring || g->gc.state == GCSsweep)
        gc_onestep(L); /* Finish sweep. */
    lua_assert(g->gc.state == GCSfinalize || g->gc.state == GCSpause);
    /* Now perform a full GC. */
    g->gc.state = GCSpause;
    do {
        gc_onestep(L);
    } while (g->gc.state != GCSpause);
    g->gc.threshold = (g->gc.estimate / 100) * g->gc.pause;
    g->vmstate = ostate;
}

/* -- Write barriers ------------------------------------------------------ */

/* Move the GC propagation frontier forward. */
void lj_gc_barrierf(global_State *g, GCobj *o, GCobj *v)
{
    lua_assert(isblack(o) && iswhite(v) && !isdead(g, v) && !isdead(g, o));
    lua_assert(g->gc.state != GCSfinalize && g->gc.state != GCSpause);
    lua_assert(o->gch.gct != ~LJ_TTAB);
    /* Preserve invariant during propagation. Otherwise it doesn't matter. */
    if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic)
        gc_mark(g, v); /* Move frontier forward. */
    else
        makewhite(g, o); /* Make it white to avoid the following barrier. */
}

/* Specialized barrier for closed upvalue. Pass &uv->tv. */
void LJ_FASTCALL lj_gc_barrieruv(global_State *g, TValue *tv)
{
#define TV2MARKED(x)                                                           \
    (*((uint8_t *)(x)-offsetof(GCupval, tv) + offsetof(GCupval, marked)))
    if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic)
        gc_mark(g, gcV(tv));
    else
        TV2MARKED(tv) = (TV2MARKED(tv) & (uint8_t)~LJ_GC_COLORS) | curwhite(g);
#undef TV2MARKED
}

/* Close upvalue. Also needs a write barrier. */
void lj_gc_closeuv(global_State *g, GCupval *uv)
{
    GCobj *o = obj2gco(uv);
    /* Copy stack slot to upvalue itself and point to the copy. */
    copyTV(mainthread(g), &uv->tv, uvval(uv));
    setmref(uv->v, &uv->tv);
    uv->closed = 1;
    setgcrefr(o->gch.nextgc, g->gc.root);
    setgcref(g->gc.root, o);
    if (isgray(o)) { /* A closed upvalue is never gray, so fix this. */
        if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic) {
            gray2black(o); /* Make it black and preserve invariant. */
            if (tviswhite(&uv->tv))
                lj_gc_barrierf(g, o, gcV(&uv->tv));
        }
        else {
            makewhite(g, o); /* Make it white, i.e. sweep the upvalue. */
            lua_assert(g->gc.state != GCSfinalize && g->gc.state != GCSpause);
        }
    }
}

#if LJ_HASJIT
/* Mark a trace if it's saved during the propagation phase. */
void lj_gc_barriertrace(global_State *g, uint32_t traceno)
{
    if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic)
        gc_marktrace(g, traceno);
}
#endif

/* -- Allocator ----------------------------------------------------------- */

/* Call pluggable memory allocator to allocate or resize a fragment. */
void *lj_mem_realloc(lua_State *L, void *p, MSize osz, MSize nsz)
{
    global_State *g = G(L);
    lua_assert((osz == 0) == (p == NULL));
    p = g->allocf(g->allocd, p, osz, nsz);
    if (p == NULL && nsz > 0)
        lj_err_mem(L);
    lua_assert((nsz == 0) == (p == NULL));
    lua_assert(checkptr32(p));
    g->gc.total = (g->gc.total - osz) + nsz;
    return p;
}

/* Allocate new GC object and link it to the root set. */
void *LJ_FASTCALL lj_mem_newgco(lua_State *L, MSize size)
{
    global_State *g = G(L);
    GCobj *o = (GCobj *)g->allocf(g->allocd, NULL, 0, size);
    if (o == NULL)
        lj_err_mem(L);
    lua_assert(checkptr32(o));
    g->gc.total += size;
    setgcrefr(o->gch.nextgc, g->gc.root);
    setgcref(g->gc.root, o);
    newwhite(g, o);
    return o;
}

/* Resize growable vector. */
void *lj_mem_grow(lua_State *L, void *p, MSize *szp, MSize lim, MSize esz)
{
    MSize sz = (*szp) << 1;
    if (sz < LJ_MIN_VECSZ)
        sz = LJ_MIN_VECSZ;
    if (sz > lim)
        sz = lim;
    p = lj_mem_realloc(L, p, (*szp) * esz, sz * esz);
    *szp = sz;
    return p;
}
