Move dispatch tables out of GG_State struct.

src/Makefile.dep

@@ -22,11 +22,11 @@ lib_debug.o: lib_debug.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
 lib_init.o: lib_init.c lua.h luaconf.h lauxlib.h lualib.h
 lib_io.o: lib_io.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
   lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_ff.h lj_ffdef.h \
-  lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_bc.h lj_traceerr.h \
-  lj_lib.h lj_libdef.h
+  lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_traceerr.h lj_lib.h \
+  lj_libdef.h
 lib_jit.o: lib_jit.c lua.h luaconf.h lauxlib.h lualib.h lj_arch.h \
-  lj_obj.h lj_def.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_ir.h \
-  lj_jit.h lj_iropt.h lj_dispatch.h lj_bc.h lj_vm.h lj_vmevent.h lj_lib.h \
+  lj_obj.h lj_def.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_bc.h \
+  lj_ir.h lj_jit.h lj_iropt.h lj_dispatch.h lj_vm.h lj_vmevent.h lj_lib.h \
   luajit.h lj_libdef.h
 lib_math.o: lib_math.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
   lj_def.h lj_arch.h lj_lib.h lj_libdef.h
@@ -59,8 +59,8 @@ lj_err.o: lj_err.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_err.h \
   lj_errmsg.h lj_str.h lj_tab.h lj_func.h lj_state.h lj_frame.h lj_bc.h \
   lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_traceerr.h lj_vm.h
 lj_func.o: lj_func.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
-  lj_func.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_bc.h \
-  lj_traceerr.h lj_vm.h
+  lj_func.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h lj_traceerr.h \
+  lj_vm.h
 lj_gc.o: lj_gc.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
   lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_func.h lj_udata.h lj_meta.h \
   lj_state.h lj_frame.h lj_bc.h lj_trace.h lj_jit.h lj_ir.h lj_dispatch.h \
@@ -70,14 +70,14 @@ lj_gdbjit.o: lj_gdbjit.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
   lj_ir.h lj_dispatch.h
 lj_ir.o: lj_ir.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
   lj_str.h lj_tab.h lj_ir.h lj_jit.h lj_iropt.h lj_trace.h lj_dispatch.h \
-  lj_bc.h lj_traceerr.h lj_lib.h
+  lj_traceerr.h lj_lib.h
 lj_lex.o: lj_lex.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
   lj_err.h lj_errmsg.h lj_str.h lj_lex.h lj_parse.h lj_ctype.h
 lj_lib.o: lj_lib.c lauxlib.h lua.h luaconf.h lj_obj.h lj_def.h lj_arch.h \
   lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_func.h lj_vm.h \
   lj_lib.h
 lj_mcode.o: lj_mcode.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
-  lj_gc.h lj_jit.h lj_ir.h lj_mcode.h lj_trace.h lj_dispatch.h lj_bc.h \
+  lj_gc.h lj_jit.h lj_ir.h lj_mcode.h lj_trace.h lj_dispatch.h \
   lj_traceerr.h
 lj_meta.o: lj_meta.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
   lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_meta.h lj_bc.h lj_vm.h
@@ -85,11 +85,11 @@ lj_obj.o: lj_obj.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h
 lj_opt_dce.o: lj_opt_dce.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
   lj_ir.h lj_jit.h lj_iropt.h
 lj_opt_fold.o: lj_opt_fold.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
-  lj_str.h lj_ir.h lj_jit.h lj_iropt.h lj_trace.h lj_dispatch.h lj_bc.h \
+  lj_str.h lj_ir.h lj_jit.h lj_iropt.h lj_trace.h lj_dispatch.h \
   lj_traceerr.h lj_vm.h lj_folddef.h
 lj_opt_loop.o: lj_opt_loop.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
   lj_err.h lj_errmsg.h lj_str.h lj_ir.h lj_jit.h lj_iropt.h lj_trace.h \
-  lj_dispatch.h lj_bc.h lj_traceerr.h lj_snap.h lj_vm.h
+  lj_dispatch.h lj_traceerr.h lj_snap.h lj_vm.h
 lj_opt_mem.o: lj_opt_mem.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
   lj_tab.h lj_ir.h lj_jit.h lj_iropt.h
 lj_opt_narrow.o: lj_opt_narrow.c lj_obj.h lua.h luaconf.h lj_def.h \
@@ -122,8 +122,8 @@ lj_trace.o: lj_trace.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
 lj_udata.o: lj_udata.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
   lj_gc.h lj_udata.h
 lj_vmevent.o: lj_vmevent.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
-  lj_str.h lj_tab.h lj_state.h lj_dispatch.h lj_bc.h lj_jit.h lj_ir.h \
-  lj_vm.h lj_vmevent.h
+  lj_str.h lj_tab.h lj_state.h lj_dispatch.h lj_jit.h lj_ir.h lj_vm.h \
+  lj_vmevent.h
 ljamalg.o: ljamalg.c lua.h luaconf.h lauxlib.h lj_gc.c lj_obj.h lj_def.h \
   lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_func.h \
   lj_udata.h lj_meta.h lj_state.h lj_frame.h lj_bc.h lj_trace.h lj_jit.h \

src/buildvm_x64.h

@@ -1063,7 +1063,7 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   dasm_put(Dst, 1461, Dt1(->base), Dt1(->base), Dt1(->top), FRAME_CONT, Dt7(->gate), Dt1(->base), Dt1(->base));
   dasm_put(Dst, 1656, -BCBIAS_J*4, LJ_TISTRUECOND, LJ_TISTRUECOND, Dt1(->base));
   dasm_put(Dst, 1764, Dt1(->base), Dt1(->base), FRAME_CONT, LJ_TFUNC);
-  dasm_put(Dst, 1887, Dt7(->gate), Dt1(->base), Dt1(->base), Dt1(->base), Dt1(->base), Dt7(->gate), Dt1(->base), Dt1(->base), GG_DISP_STATIC*8, 1+1);
+  dasm_put(Dst, 1887, Dt7(->gate), Dt1(->base), Dt1(->base), Dt1(->base), Dt1(->base), Dt7(->gate), Dt1(->base), Dt1(->base), BC__MAX*8, 1+1);
   dasm_put(Dst, 2042, LJ_TISTRUECOND, 1+1, ~LJ_TNUMX);
   if (cmov) {
   dasm_put(Dst, 2136);
@@ -1346,7 +1346,7 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   dasm_put(Dst, 7442, DISPATCH_GL(hookmask), HOOK_VMEVENT, HOOK_ACTIVE, LUA_MASKLINE|LUA_MASKCOUNT, DISPATCH_GL(hookcount));
 #endif
   dasm_put(Dst, 7475, DISPATCH_GL(hookmask), HOOK_ACTIVE, LUA_MASKLINE|LUA_MASKCOUNT, DISPATCH_GL(hookcount), LUA_MASKLINE, Dt1(->base), Dt1(->base));
-  dasm_put(Dst, 7542, GG_DISP_STATIC*8);
+  dasm_put(Dst, 7542, BC__MAX*8);
 #if LJ_HASJIT
   dasm_put(Dst, 7579);
 #endif

src/buildvm_x64win.h

@@ -1060,7 +1060,7 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   dasm_put(Dst, 1419, Dt1(->base), Dt1(->base), Dt1(->top), FRAME_CONT, Dt7(->gate), Dt1(->base), Dt1(->base));
   dasm_put(Dst, 1615, -BCBIAS_J*4, LJ_TISTRUECOND, LJ_TISTRUECOND, Dt1(->base));
   dasm_put(Dst, 1724, Dt1(->base), Dt1(->base), FRAME_CONT, LJ_TFUNC);
-  dasm_put(Dst, 1849, Dt7(->gate), Dt1(->base), Dt1(->base), Dt1(->base), Dt1(->base), Dt7(->gate), Dt1(->base), Dt1(->base), GG_DISP_STATIC*8, 1+1);
+  dasm_put(Dst, 1849, Dt7(->gate), Dt1(->base), Dt1(->base), Dt1(->base), Dt1(->base), Dt7(->gate), Dt1(->base), Dt1(->base), BC__MAX*8, 1+1);
   dasm_put(Dst, 2005, LJ_TISTRUECOND, 1+1, ~LJ_TNUMX);
   if (cmov) {
   dasm_put(Dst, 2101);
@@ -1343,7 +1343,7 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   dasm_put(Dst, 7419, DISPATCH_GL(hookmask), HOOK_VMEVENT, HOOK_ACTIVE, LUA_MASKLINE|LUA_MASKCOUNT, DISPATCH_GL(hookcount));
 #endif
   dasm_put(Dst, 7450, DISPATCH_GL(hookmask), HOOK_ACTIVE, LUA_MASKLINE|LUA_MASKCOUNT, DISPATCH_GL(hookcount), LUA_MASKLINE, Dt1(->base), Dt1(->base));
-  dasm_put(Dst, 7516, GG_DISP_STATIC*8);
+  dasm_put(Dst, 7516, BC__MAX*8);
 #if LJ_HASJIT
   dasm_put(Dst, 7552);
 #endif

src/buildvm_x86.dasc

@@ -1271,9 +1271,9 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   |  movzx OP, RCL
   |  shr RC, 16
   |.if X64
-  |  jmp aword [DISPATCH+OP*8+GG_DISP_STATIC*8]	// Retry FORI or JFORI.
+  |  jmp aword [DISPATCH+OP*8+BC__MAX*8]	// Retry FORI or JFORI.
   |.else
-  |  jmp aword [DISPATCH+OP*4+GG_DISP_STATIC*4]	// Retry FORI or JFORI.
+  |  jmp aword [DISPATCH+OP*4+BC__MAX*4]	// Retry FORI or JFORI.
   |.endif
   |
   |//-----------------------------------------------------------------------
@@ -2629,9 +2629,9 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   |  movzx OP, PC_OP
   |  movzx RD, PC_RD
   |.if X64
-  |  jmp aword [DISPATCH+OP*8+GG_DISP_STATIC*8]	// Re-dispatch to static ins.
+  |  jmp aword [DISPATCH+OP*8+BC__MAX*8]	// Re-dispatch to static ins.
   |.else
-  |  jmp aword [DISPATCH+OP*4+GG_DISP_STATIC*4]	// Re-dispatch to static ins.
+  |  jmp aword [DISPATCH+OP*4+BC__MAX*4]	// Re-dispatch to static ins.
   |.endif
   |
   |->cont_hook:				// Continue from hook yield.

src/buildvm_x86.h

@@ -1140,7 +1140,7 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   dasm_put(Dst, 1393, Dt1(->base), Dt1(->base), Dt1(->top), FRAME_CONT, Dt7(->gate), Dt1(->base), Dt1(->base));
   dasm_put(Dst, 1600, -BCBIAS_J*4, LJ_TISTRUECOND, LJ_TISTRUECOND, Dt1(->base));
   dasm_put(Dst, 1711, Dt1(->base), Dt1(->base), FRAME_CONT, LJ_TFUNC);
-  dasm_put(Dst, 1840, Dt7(->gate), Dt1(->base), Dt1(->base), Dt1(->base), Dt1(->base), Dt7(->gate), Dt1(->base), Dt1(->base), GG_DISP_STATIC*4, 1+1);
+  dasm_put(Dst, 1840, Dt7(->gate), Dt1(->base), Dt1(->base), Dt1(->base), Dt1(->base), Dt7(->gate), Dt1(->base), Dt1(->base), BC__MAX*4, 1+1);
   dasm_put(Dst, 2000, LJ_TISTRUECOND, 1+1, ~LJ_TNUMX);
   if (cmov) {
   dasm_put(Dst, 2096);
@@ -1458,7 +1458,7 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   dasm_put(Dst, 8406, DISPATCH_GL(hookmask), HOOK_VMEVENT, HOOK_ACTIVE, LUA_MASKLINE|LUA_MASKCOUNT, DISPATCH_GL(hookcount));
 #endif
   dasm_put(Dst, 8437, DISPATCH_GL(hookmask), HOOK_ACTIVE, LUA_MASKLINE|LUA_MASKCOUNT, DISPATCH_GL(hookcount), LUA_MASKLINE, Dt1(->base), Dt1(->base));
-  dasm_put(Dst, 8503, GG_DISP_STATIC*4);
+  dasm_put(Dst, 8503, BC__MAX*4);
 #if LJ_HASJIT
   dasm_put(Dst, 8538, Dt1(->base), GG_DISP2J, DISPATCH_J(L));
 #endif

src/lib_jit.c

@@ -15,6 +15,7 @@
 #include "lj_err.h"
 #include "lj_str.h"
 #include "lj_tab.h"
+#include "lj_bc.h"
 #if LJ_HASJIT
 #include "lj_ir.h"
 #include "lj_jit.h"

src/lj_asm.c

@@ -906,7 +906,7 @@ static MCode *asm_exitstub_gen(ASMState *as, ExitNo group)
   *mxp++ = MODRM(XM_OFS8, 0, RID_ESP);
   *mxp++ = MODRM(XM_SCALE1, RID_ESP, RID_ESP);
   *mxp++ = 2*sizeof(void *);
-  *(int32_t *)mxp = ptr2addr(J2GG(as->J)->dispatch); mxp += 4;
+  *(int32_t *)mxp = ptr2addr(GG2DISP(J2GG(as->J))); mxp += 4;
   /* Jump to exit handler which fills in the ExitState. */
   *mxp++ = XI_JMP; mxp += 4;
   *((int32_t *)(mxp-4)) = (int32_t)((MCode *)lj_vm_exit_handler - mxp);
@@ -3066,7 +3066,7 @@ static void asm_tail_sync(ASMState *as)
 
   if (as->T->link == TRACE_INTERP) {
     /* Setup fixed registers for exit to interpreter. */
-    emit_loada(as, RID_DISPATCH, J2GG(as->J)->dispatch);
+    emit_loada(as, RID_DISPATCH, GG2DISP(J2GG(as->J)));
     emit_loadi(as, RID_PC, (int32_t)map[nent]);
   } else if (newbase) {
     /* Save modified BASE for linking to trace with higher start frame. */

src/lj_dispatch.c

@@ -19,13 +19,15 @@
 #include "lj_vm.h"
 #include "luajit.h"
 
+#define GG_DISP_STATIC		BC__MAX
+
 /* -- Dispatch table management ------------------------------------------- */
 
 /* Initialize instruction dispatch table and hot counters. */
 void lj_dispatch_init(GG_State *GG)
 {
   uint32_t i;
-  ASMFunction *disp = GG->dispatch;
+  ASMFunction *disp = GG2DISP(GG);
   for (i = 0; i < BC__MAX; i++)
     disp[GG_DISP_STATIC+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]);
   /* The JIT engine is off by default. luaopen_jit() turns it on. */
@@ -57,7 +59,7 @@ void lj_dispatch_update(global_State *g)
 #endif
   mode |= (g->hookmask & HOOK_EVENTMASK) ? 2 : 0;
   if (oldmode != mode) {  /* Mode changed? */
-    ASMFunction *disp = G2GG(g)->dispatch;
+    ASMFunction *disp = GG2DISP(G2GG(g));
     ASMFunction f_forl, f_iterl, f_loop;
     g->dispatchmode = mode;
     if ((mode & 5) == 1) {  /* Hotcount if JIT is on, but not when recording. */

src/lj_dispatch.h

@@ -7,7 +7,6 @@
 #define _LJ_DISPATCH_H
 
 #include "lj_obj.h"
-#include "lj_bc.h"
 #if LJ_HASJIT
 #include "lj_jit.h"
 #endif
@@ -30,24 +29,22 @@ typedef struct GG_State {
   jit_State J;				/* JIT state. */
   HotCount hotcount[HOTCOUNT_SIZE];	/* Hot counters. */
 #endif
-  ASMFunction dispatch[2*BC__MAX];	/* Instruction dispatch tables. */
+  /* Instruction dispatch tables follow. */
 } GG_State;
 
-#define GG_DISP_STATIC	BC__MAX
-
 #define GG_OFS(field)	((int)offsetof(GG_State, field))
-#define G2GG(gl) \
-  ((GG_State *)(((char *)(gl))-((char *)(&((GG_State *)0)->g))))
-#define J2GG(j) \
-  ((GG_State *)(((char *)(j))-((char *)(&((GG_State *)0)->J))))
-#define L2GG(L)		G2GG(G(L))
+#define GG_OFS_DISP	((int)sizeof(GG_State))
+#define GG2DISP(gg)	((ASMFunction *)((char *)(gg) + GG_OFS_DISP))
+#define G2GG(gl)	((GG_State *)((char *)(gl) - GG_OFS(g)))
+#define J2GG(j)		((GG_State *)((char *)(j) - GG_OFS(J)))
+#define L2GG(L)		(G2GG(G(L)))
 #define J2G(J)		(&J2GG(J)->g)
 #define G2J(gl)		(&G2GG(gl)->J)
 #define L2J(L)		(&L2GG(L)->J)
-#define GG_G2DISP	(GG_OFS(dispatch) - GG_OFS(g))
-#define GG_DISP2G	(GG_OFS(g) - GG_OFS(dispatch))
-#define GG_DISP2J	(GG_OFS(J) - GG_OFS(dispatch))
-#define GG_DISP2HOT	(GG_OFS(hotcount) - GG_OFS(dispatch))
+#define GG_G2DISP	(GG_OFS_DISP - GG_OFS(g))
+#define GG_DISP2G	(GG_OFS(g) - GG_OFS_DISP)
+#define GG_DISP2J	(GG_OFS(J) - GG_OFS_DISP)
+#define GG_DISP2HOT	(GG_OFS(hotcount) - GG_OFS_DISP)
 
 #define hotcount_get(gg, pc) \
   (gg)->hotcount[(u32ptr(pc)>>2) & (HOTCOUNT_SIZE-1)]

src/lj_state.c

@@ -119,6 +119,8 @@ static void stack_init(lua_State *L1, lua_State *L)
 
 /* -- State handling ------------------------------------------------------ */
 
+#define GG_SIZE		(sizeof(GG_State)+(BC__MAX*2)*sizeof(ASMFunction))
+
 /* Open parts that may cause memory-allocation errors. */
 static TValue *cpluaopen(lua_State *L, lua_CFunction dummy, void *ud)
 {
@@ -154,8 +156,8 @@ static void close_state(lua_State *L)
     lj_mem_freevec(g, g->strhash, g->strmask+1, GCRef);
     lj_str_freebuf(g, &g->tmpbuf);
     lj_mem_freevec(g, L->stack, L->stacksize, TValue);
-    lua_assert(g->gc.total == sizeof(GG_State));
-    g->allocf(g->allocd, G2GG(g), sizeof(GG_State), 0);
+    lua_assert(g->gc.total == GG_SIZE);
+    g->allocf(g->allocd, G2GG(g), GG_SIZE, 0);
   }
 }
 
@@ -165,7 +167,7 @@ lua_State *lj_state_newstate(lua_Alloc f, void *ud)
 LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud)
 #endif
 {
-  GG_State *GG = cast(GG_State *, f(ud, NULL, 0, sizeof(GG_State)));
+  GG_State *GG = cast(GG_State *, f(ud, NULL, 0, GG_SIZE));
   lua_State *L = &GG->L;
   global_State *g = &GG->g;
   if (GG == NULL || !checkptr32(GG)) return NULL;
@@ -188,7 +190,7 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud)
   g->gc.state = GCSpause;
   setgcref(g->gc.root, obj2gco(L));
   g->gc.sweep = &g->gc.root;
-  g->gc.total = sizeof(GG_State);
+  g->gc.total = GG_SIZE;
   g->gc.pause = LUAI_GCPAUSE;
   g->gc.stepmul = LUAI_GCMUL;
   lj_dispatch_init((GG_State *)L);