calling a vararg function needs to check GC

(because it creates a new table)

ldo.c

@@ -1,5 +1,5 @@
 /*
-** $Id: ldo.c,v 2.185 2017/12/29 15:44:51 roberto Exp roberto $
+** $Id: ldo.c,v 2.186 2018/01/10 19:19:27 roberto Exp roberto $
 ** Stack and Call structure of Lua
 ** See Copyright Notice in lua.h
 */
@@ -417,14 +417,14 @@ void luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, int n) {
   checkstackp(L, fsize, func);
   for (; i <= p->numparams; i++)
     setnilvalue(s2v(ci->func + i));  /* complete missing arguments */
-  if (p->is_vararg) {
-    L->top -= (func - ci->func);  /* move down top */
-    luaT_adjustvarargs(L, p, n - 1);
-  }
   ci->top = ci->func + 1 + fsize;  /* top for new function */
   lua_assert(ci->top <= L->stack_last);
   ci->u.l.savedpc = p->code;  /* starting point */
   ci->callstatus |= CIST_TAIL;
+  if (p->is_vararg) {
+    L->top -= (func - ci->func);  /* move down top */
+    luaT_adjustvarargs(L, p, n - 1);
+  }
   if (L->hookmask)
     hookcall(L, ci, 1);
 }
@@ -471,8 +471,6 @@ void luaD_call (lua_State *L, StkId func, int nresults) {
       checkstackp(L, fsize, func);
       for (; n < p->numparams; n++)
         setnilvalue(s2v(L->top++));  /* complete missing arguments */
-      if (p->is_vararg)
-        luaT_adjustvarargs(L, p, n);
       ci = next_ci(L);  /* now 'enter' new function */
       ci->nresults = nresults;
       ci->func = func;
@@ -480,6 +478,8 @@ void luaD_call (lua_State *L, StkId func, int nresults) {
       lua_assert(ci->top <= L->stack_last);
       ci->u.l.savedpc = p->code;  /* starting point */
       ci->callstatus = 0;
+      if (p->is_vararg)
+        luaT_adjustvarargs(L, p, n);  /* may invoke GC */
       if (L->hookmask)
         hookcall(L, ci, 0);
       luaV_execute(L, ci);  /* run the function */

ltm.c

@@ -1,5 +1,5 @@
 /*
-** $Id: ltm.c,v 2.55 2017/12/20 14:58:05 roberto Exp roberto $
+** $Id: ltm.c,v 2.56 2017/12/28 15:42:57 roberto Exp roberto $
 ** Tag methods
 ** See Copyright Notice in lua.h
 */
@@ -16,6 +16,7 @@
 
 #include "ldebug.h"
 #include "ldo.h"
+#include "lgc.h"
 #include "lobject.h"
 #include "lstate.h"
 #include "lstring.h"
@@ -231,6 +232,7 @@ void luaT_adjustvarargs (lua_State *L, Proto *p, int actual) {
   setivalue(luaH_set(L, vtab, &nname), actual);  /* store counter there */
   L->top -= actual;  /* remove extra elements from the stack */
   sethvalue2s(L, L->top - 1, vtab);  /* move table to new top */
+  luaC_checkGC(L);
 }