New control for reentrancy of emergency collections

Instead of assuming that shrinking a block may be an emergency
collection, use an explicit field ('gcstopem') to stop emergency
collections while GC is working.

lgc.c

@@ -1575,52 +1575,64 @@ static int sweepstep (lua_State *L, global_State *g,
 
 static lu_mem singlestep (lua_State *L) {
   global_State *g = G(L);
+  lu_mem work;
+  lua_assert(!g->gcstopem);  /* collector is not reentrant */
+  g->gcstopem = 1;  /* no emergency collections while collecting */
   switch (g->gcstate) {
     case GCSpause: {
       restartcollection(g);
       g->gcstate = GCSpropagate;
-      return 1;
+      work = 1;
+      break;
     }
     case GCSpropagate: {
       if (g->gray == NULL) {  /* no more gray objects? */
         g->gcstate = GCSenteratomic;  /* finish propagate phase */
-        return 0;
+        work = 0;
       }
       else
-        return propagatemark(g);  /* traverse one gray object */
+        work = propagatemark(g);  /* traverse one gray object */
+      break;
     }
     case GCSenteratomic: {
-      lu_mem work = atomic(L);  /* work is what was traversed by 'atomic' */
+      work = atomic(L);  /* work is what was traversed by 'atomic' */
       entersweep(L);
       g->GCestimate = gettotalbytes(g);  /* first estimate */;
-      return work;
+      break;
     }
     case GCSswpallgc: {  /* sweep "regular" objects */
-      return sweepstep(L, g, GCSswpfinobj, &g->finobj);
+      work = sweepstep(L, g, GCSswpfinobj, &g->finobj);
+      break;
     }
     case GCSswpfinobj: {  /* sweep objects with finalizers */
-      return sweepstep(L, g, GCSswptobefnz, &g->tobefnz);
+      work = sweepstep(L, g, GCSswptobefnz, &g->tobefnz);
+      break;
     }
     case GCSswptobefnz: {  /* sweep objects to be finalized */
-      return sweepstep(L, g, GCSswpend, NULL);
+      work = sweepstep(L, g, GCSswpend, NULL);
+      break;
     }
     case GCSswpend: {  /* finish sweeps */
       checkSizes(L, g);
       g->gcstate = GCScallfin;
-      return 0;
+      work = 0;
+      break;
     }
     case GCScallfin: {  /* call remaining finalizers */
       if (g->tobefnz && !g->gcemergency) {
-        int n = runafewfinalizers(L, GCFINMAX);
-        return n * GCFINALIZECOST;
+        g->gcstopem = 0;  /* ok collections during finalizers */
+        work = runafewfinalizers(L, GCFINMAX) * GCFINALIZECOST;
       }
       else {  /* emergency mode or no more finalizers */
         g->gcstate = GCSpause;  /* finish collection */
-        return 0;
+        work = 0;
       }
+      break;
     }
     default: lua_assert(0); return 0;
   }
+  g->gcstopem = 0;
+  return work;
 }
 
 

lmem.c

@@ -24,12 +24,12 @@
 
 #if defined(EMERGENCYGCTESTS)
 /*
-** First allocation will fail whenever not building initial state
-** and not shrinking a block. (This fail will trigger 'tryagain' and
-** a full GC cycle at every allocation.)
+** First allocation will fail whenever not building initial state.
+** (This fail will trigger 'tryagain' and a full GC cycle at every
+** allocation.)
 */
 static void *firsttry (global_State *g, void *block, size_t os, size_t ns) {
-  if (completestate(g) && ns > os)
+  if (completestate(g) && ns > 0)  /* frees never fail */
     return NULL;  /* fail */
   else  /* normal allocation */
     return (*g->frealloc)(g->ud, block, os, ns);
@@ -138,15 +138,17 @@ void luaM_free_ (lua_State *L, void *block, size_t osize) {
 
 
 /*
-** In case of allocation fail, this function will call the GC to try
-** to free some memory and then try the allocation again.
-** (It should not be called when shrinking a block, because then the
-** interpreter may be in the middle of a collection step.)
+** In case of allocation fail, this function will do an emergency
+** collection to free some memory and then try the allocation again.
+** The GC should not be called while state is not fully built, as the
+** collector is not yet fully initialized. Also, it should not be called
+** when 'gcstopem' is true, because then the interpreter is in the
+** middle of a collection step.
 */
 static void *tryagain (lua_State *L, void *block,
                        size_t osize, size_t nsize) {
   global_State *g = G(L);
-  if (completestate(g)) {  /* is state fully build? */
+  if (completestate(g) && !g->gcstopem) {
     luaC_fullgc(L, 1);  /* try to free some memory... */
     return (*g->frealloc)(g->ud, block, osize, nsize);  /* try again */
   }
@@ -156,8 +158,6 @@ static void *tryagain (lua_State *L, void *block,
 
 /*
 ** Generic allocation routine.
-** If allocation fails while shrinking a block, do not try again; the
-** GC shrinks some blocks and it is not reentrant.
 */
 void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) {
   void *newblock;
@@ -165,8 +165,7 @@ void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) {
   lua_assert((osize == 0) == (block == NULL));
   newblock = firsttry(g, block, osize, nsize);
   if (l_unlikely(newblock == NULL && nsize > 0)) {
-    if (nsize > osize)  /* not shrinking a block? */
-      newblock = tryagain(L, block, osize, nsize);
+    newblock = tryagain(L, block, osize, nsize);
     if (newblock == NULL)  /* still no memory? */
       return NULL;  /* do not update 'GCdebt' */
   }

lstate.c

@@ -379,6 +379,7 @@ LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) {
   g->panic = NULL;
   g->gcstate = GCSpause;
   g->gckind = KGC_INC;
+  g->gcstopem = 0;
   g->gcemergency = 0;
   g->finobj = g->tobefnz = g->fixedgc = NULL;
   g->firstold1 = g->survival = g->old1 = g->reallyold = NULL;

lstate.h

@@ -260,6 +260,7 @@ typedef struct global_State {
   lu_byte currentwhite;
   lu_byte gcstate;  /* state of garbage collector */
   lu_byte gckind;  /* kind of GC running */
+  lu_byte gcstopem;  /* stops emergency collections */
   lu_byte genminormul;  /* control for minor generational collections */
   lu_byte genmajormul;  /* control for major generational collections */
   lu_byte gcrunning;  /* true if GC is running */

testes/gc.lua

@@ -676,6 +676,14 @@ end
 -- just to make sure
 assert(collectgarbage'isrunning')
 
+do    -- check that the collector is reentrant in incremental mode
+  setmetatable({}, {__gc = function ()
+    collectgarbage()
+  end})
+  collectgarbage()
+end
+
+
 collectgarbage(oldmode)
 
 print('OK')