Implement yield from C hooks.

Get number of multiple results from C frame.
Add lj_cont_hook: restores multres and dispatch to static ins.
Can use fastcall for lj_dispatch_ins() now.

doc/changes.html

@@ -55,6 +55,7 @@ to see whether newer versions are available.
 <div class="major" style="background: #d0d0d0;">
 <h2 id="snap">Development Snapshot</h2>
 <ul>
+<li>Implement yield from C hooks.</li>
 <li>Add abstract C call handling to IR.</li>
 <li>Improve KNUM fuse vs. load heuristics.</li>
 <li>Drive the GC forward on string allocations in the parser.</li>

src/buildvm_x86.dasc

@@ -618,13 +618,11 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   |->vm_unwind_c:			// Unwind C stack, return from vm_pcall.
   |  // (void *cframe, int errcode)
   |.if X64
-  |  and CARG1, CFRAME_RAWMASK
   |  mov eax, CARG2d			// Error return status for vm_pcall.
   |  mov rsp, CARG1
   |.else
   |  mov ecx, [esp+4]
   |  mov eax, [esp+8]			// Error return status for vm_pcall.
-  |  and ecx, CFRAME_RAWMASK
   |  mov esp, ecx
   |.endif
   |  mov L:RB, SAVE_L
@@ -2618,24 +2616,15 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   |  test RDL, LUA_MASKLINE
   |  jz >5
   |1:
-  |.if X64
-  |  mov L:RB, SAVE_L
-  |  mov L:RB->base, BASE		// Caveat: CARG2d/CARG3d may be BASE.
-  |  mov CARG3d, NRESULTS		// Dynamic top for *M instructions.
-  |  mov CARG2d, PC
-  |  mov CARG1d, L:RB
-  |.else
   |  mov L:RB, SAVE_L
-  |  mov RD, NRESULTS			// Dynamic top for *M instructions.
-  |  mov ARG3, RD
   |  mov L:RB->base, BASE
-  |  mov ARG2, PC
-  |  mov ARG1, L:RB
-  |.endif
+  |  mov FCARG2, PC			// Caveat: FCARG2 == BASE
+  |  mov FCARG1, L:RB
   |  // SAVE_PC must hold the _previous_ PC. The callee updates it with PC.
-  |  call extern lj_dispatch_ins  // (lua_State *L, BCIns *pc, int nres)
-  |4:
+  |  call extern lj_dispatch_ins@8	// (lua_State *L, BCIns *pc)
+  |3:
   |  mov BASE, L:RB->base
+  |4:
   |  movzx RA, PC_RA
   |5:
   |  movzx OP, PC_OP
@@ -2646,6 +2635,12 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   |  jmp aword [DISPATCH+OP*4+GG_DISP_STATIC*4]	// Re-dispatch to static ins.
   |.endif
   |
+  |->cont_hook:				// Continue from hook yield.
+  |  add PC, 4
+  |  mov RA, [RB-24]
+  |  mov NRESULTS, RA			// Restore NRESULTS for *M ins.
+  |  jmp <4
+  |
   |->vm_hotloop:			// Hot loop counter underflow.
 #if LJ_HASJIT
   |.if X64
@@ -2658,7 +2653,7 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   |  mov [DISPATCH+DISPATCH_J(L)], L:RB
   |  mov SAVE_PC, PC
   |  call extern lj_trace_hot@8		// (jit_State *J, const BCIns *pc)
-  |  jmp <4
+  |  jmp <3
   |.endif
 #endif
   |

src/lj_api.c

@@ -1056,19 +1056,32 @@ LUALIB_API int luaL_callmeta(lua_State *L, int idx, const char *field)
 LUA_API int lua_yield(lua_State *L, int nresults)
 {
   void *cf = L->cframe;
-  cTValue *f;
-  if (!cframe_canyield(cf))
-    lj_err_msg(L, LJ_ERR_CYIELD);
-  f = L->top - nresults;
-  if (f > L->base) {
-    TValue *t = L->base;
-    while (--nresults >= 0) copyTV(L, t++, f++);
-    L->top = t;
+  global_State *g = G(L);
+  if (cframe_canyield(cf)) {
+    cf = cframe_raw(cf);
+    if (!hook_active(g)) {  /* Regular yield: move results down if needed. */
+      cTValue *f = L->top - nresults;
+      if (f > L->base) {
+	TValue *t = L->base;
+	while (--nresults >= 0) copyTV(L, t++, f++);
+	L->top = t;
+      }
+    } else {  /* Yield from hook: add a pseudo-frame. */
+      TValue *top = L->top;
+      hook_leave(g);
+      top->u64 = cframe_multres(cf);
+      setcont(top+1, lj_cont_hook);
+      setframe_pc(top+1, cframe_pc(cf)-1);
+      setframe_gc(top+2, obj2gco(L));
+      top[2].fr.tp.ftsz = cast_int((char *)(top+3)-(char *)L->base)+FRAME_CONT;
+      L->top = L->base = top+3;
+    }
+    L->cframe = NULL;
+    L->status = LUA_YIELD;
+    lj_vm_unwind_c(cf, LUA_YIELD);
   }
-  L->cframe = NULL;
-  L->status = LUA_YIELD;
-  lj_vm_unwind_c(cf, LUA_YIELD);
-  return -1;  /* unreachable */
+  lj_err_msg(L, LJ_ERR_CYIELD);
+  return 0;  /* unreachable */
 }
 
 LUA_API int lua_resume(lua_State *L, int nargs)

src/lj_dispatch.c

@@ -258,15 +258,16 @@ static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres)
 }
 
 /* Instruction dispatch callback for instr/line hooks or when recording. */
-void lj_dispatch_ins(lua_State *L, const BCIns *pc, uint32_t nres)
+void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
 {
   GCfunc *fn = curr_func(L);
   GCproto *pt = funcproto(fn);
-  BCReg slots = cur_topslot(pt, pc, nres);
-  global_State *g = G(L);
   void *cf = cframe_raw(L->cframe);
   const BCIns *oldpc = cframe_pc(cf);
+  global_State *g = G(L);
+  BCReg slots;
   setcframe_pc(cf, pc);
+  slots = cur_topslot(pt, pc, cframe_multres(cf));
   L->top = L->base + slots;  /* Fix top. */
 #if LJ_HASJIT
   {

src/lj_dispatch.h

@@ -59,6 +59,6 @@ LJ_FUNC void lj_dispatch_init(GG_State *GG);
 LJ_FUNC void lj_dispatch_update(global_State *g);
 
 /* Instruction dispatch callback for instr/line hooks or when recording. */
-LJ_FUNCA void lj_dispatch_ins(lua_State *L, const BCIns *pc, uint32_t nres);
+LJ_FUNCA void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc);
 
 #endif

src/lj_err.c

@@ -470,7 +470,7 @@ uncaught:
   L->cframe = NULL;
   if (cframe_canyield(cf)) {  /* Resume? */
     unwindstack(L, L->top, errcode);
-    lj_vm_unwind_c(cf, errcode);
+    lj_vm_unwind_c(cframe_raw(cf), errcode);
   }
   /* Better rethrow on main thread than panic. */
   {

src/lj_frame.h

@@ -63,6 +63,7 @@ enum {
 #define CFRAME_OFS_PREV		(13*4)
 #define CFRAME_OFS_L		(12*4)
 #define CFRAME_OFS_PC		(6*4)
+#define CFRAME_OFS_MULTRES	(5*4)
 #define CFRAME_SIZE		(12*4)
 #elif LJ_TARGET_X64
 #if _WIN64
@@ -71,6 +72,7 @@ enum {
 #define CFRAME_OFS_L		(32*4)
 #define CFRAME_OFS_ERRF		(31*4)
 #define CFRAME_OFS_NRES		(30*4)
+#define CFRAME_OFS_MULTRES	(29*4)
 #define CFRAME_SIZE		(14*8)
 #else
 #define CFRAME_OFS_PREV		(4*8)
@@ -78,6 +80,7 @@ enum {
 #define CFRAME_OFS_L		(4*4)
 #define CFRAME_OFS_ERRF		(3*4)
 #define CFRAME_OFS_NRES		(2*4)
+#define CFRAME_OFS_MULTRES	(1*4)
 #define CFRAME_SIZE		(12*8)
 #endif
 #else
@@ -91,6 +94,7 @@ enum {
 #define cframe_errfunc(cf)	(*(int32_t *)(((char *)(cf))+CFRAME_OFS_ERRF))
 #define cframe_nres(cf)		(*(int32_t *)(((char *)(cf))+CFRAME_OFS_NRES))
 #define cframe_prev(cf)		(*(void **)(((char *)(cf))+CFRAME_OFS_PREV))
+#define cframe_multres(cf)  (*(uint32_t *)(((char *)(cf))+CFRAME_OFS_MULTRES))
 #define cframe_L(cf) \
   (&gcref(*(GCRef *)(((char *)(cf))+CFRAME_OFS_L))->th)
 #define cframe_pc(cf) \

src/lj_vm.h

@@ -54,6 +54,7 @@ LJ_ASMF void lj_cont_ra(void);  /* Store result in RA from instruction. */
 LJ_ASMF void lj_cont_nop(void);  /* Do nothing, just continue execution. */
 LJ_ASMF void lj_cont_condt(void);  /* Branch if result is true. */
 LJ_ASMF void lj_cont_condf(void);  /* Branch if result is false. */
+LJ_ASMF void lj_cont_hook(void);  /* Continue from hook yield. */
 
 /* Start of the ASM code. */
 LJ_ASMF char lj_vm_asm_begin[];