|
|
@@ -23,28 +23,50 @@
|
|
|
/* Some local macros to save typing. Undef'd at the end. */
|
|
|
#define IR(ref) (&J->cur.ir[(ref)])
|
|
|
|
|
|
+/* -- Snapshot buffer allocation ------------------------------------------ */
|
|
|
+
|
|
|
+/* Grow snapshot buffer. */
|
|
|
+void lj_snap_grow_buf_(jit_State *J, MSize need)
|
|
|
+{
|
|
|
+ MSize maxsnap = (MSize)J->param[JIT_P_maxsnap];
|
|
|
+ if (need > maxsnap)
|
|
|
+ lj_trace_err(J, LJ_TRERR_SNAPOV);
|
|
|
+ lj_mem_growvec(J->L, J->snapbuf, J->sizesnap, maxsnap, SnapShot);
|
|
|
+ J->cur.snap = J->snapbuf;
|
|
|
+}
|
|
|
+
|
|
|
+/* Grow snapshot map buffer. */
|
|
|
+void lj_snap_grow_map_(jit_State *J, MSize need)
|
|
|
+{
|
|
|
+ if (need < 2*J->sizesnapmap)
|
|
|
+ need = 2*J->sizesnapmap;
|
|
|
+ else if (need < 64)
|
|
|
+ need = 64;
|
|
|
+ J->snapmapbuf = (SnapEntry *)lj_mem_realloc(J->L, J->snapmapbuf,
|
|
|
+ J->sizesnapmap*sizeof(SnapEntry), need*sizeof(SnapEntry));
|
|
|
+ J->cur.snapmap = J->snapmapbuf;
|
|
|
+ J->sizesnapmap = need;
|
|
|
+}
|
|
|
+
|
|
|
/* -- Snapshot generation ------------------------------------------------- */
|
|
|
|
|
|
-/* NYI: Snapshots are in need of a redesign. The current storage model for
|
|
|
-** snapshot maps is too wasteful. They could be compressed (1D or 2D) and
|
|
|
-** made more flexible at the same time. Iterators should no longer need to
|
|
|
-** skip unmodified slots. IR_FRAME should be eliminated, too.
|
|
|
-*/
|
|
|
+/* NYI: IR_FRAME should be eliminated, too. */
|
|
|
|
|
|
/* Add all modified slots to the snapshot. */
|
|
|
static MSize snapshot_slots(jit_State *J, SnapEntry *map, BCReg nslots)
|
|
|
{
|
|
|
BCReg s;
|
|
|
+ MSize n = 0;
|
|
|
for (s = 0; s < nslots; s++) {
|
|
|
IRRef ref = tref_ref(J->slot[s]);
|
|
|
if (ref) {
|
|
|
IRIns *ir = IR(ref);
|
|
|
- if (ir->o == IR_SLOAD && ir->op1 == s && !(ir->op2 & IRSLOAD_INHERIT))
|
|
|
- ref = 0;
|
|
|
+ if (!(ir->o == IR_SLOAD && ir->op1 == s &&
|
|
|
+ !(ir->op2 & IRSLOAD_INHERIT)))
|
|
|
+ map[n++] = SNAP(s, ir->o == IR_FRAME ? SNAP_FRAME : 0, ref);
|
|
|
}
|
|
|
- map[s] = (SnapEntry)ref;
|
|
|
}
|
|
|
- return nslots;
|
|
|
+ return n;
|
|
|
}
|
|
|
|
|
|
/* Add frame links at the end of the snapshot. */
|
|
|
@@ -53,17 +75,17 @@ static MSize snapshot_framelinks(jit_State *J, SnapEntry *map)
|
|
|
cTValue *frame = J->L->base - 1;
|
|
|
cTValue *lim = J->L->base - J->baseslot;
|
|
|
MSize f = 0;
|
|
|
- map[f++] = u32ptr(J->pc);
|
|
|
- while (frame > lim) {
|
|
|
+ map[f++] = SNAP_MKPC(J->pc); /* The current PC is always the first entry. */
|
|
|
+ while (frame > lim) { /* Backwards traversal of all frames above base. */
|
|
|
if (frame_islua(frame)) {
|
|
|
- map[f++] = u32ptr(frame_pc(frame));
|
|
|
+ map[f++] = SNAP_MKPC(frame_pc(frame));
|
|
|
frame = frame_prevl(frame);
|
|
|
} else if (frame_ispcall(frame)) {
|
|
|
- map[f++] = (uint32_t)frame_ftsz(frame);
|
|
|
+ map[f++] = SNAP_MKFTSZ(frame_ftsz(frame));
|
|
|
frame = frame_prevd(frame);
|
|
|
} else if (frame_iscont(frame)) {
|
|
|
- map[f++] = (uint32_t)frame_ftsz(frame);
|
|
|
- map[f++] = u32ptr(frame_contpc(frame));
|
|
|
+ map[f++] = SNAP_MKFTSZ(frame_ftsz(frame));
|
|
|
+ map[f++] = SNAP_MKPC(frame_contpc(frame));
|
|
|
frame = frame_prevd(frame);
|
|
|
} else {
|
|
|
lua_assert(0);
|
|
|
@@ -76,28 +98,19 @@ static MSize snapshot_framelinks(jit_State *J, SnapEntry *map)
|
|
|
static void snapshot_stack(jit_State *J, SnapShot *snap, MSize nsnapmap)
|
|
|
{
|
|
|
BCReg nslots = J->baseslot + J->maxslot;
|
|
|
- MSize nsm, nframelinks;
|
|
|
+ MSize nent, nframelinks;
|
|
|
SnapEntry *p;
|
|
|
/* Conservative estimate. Continuation frames need 2 slots. */
|
|
|
- nsm = nsnapmap + nslots + (uint32_t)J->framedepth*2+1;
|
|
|
- if (LJ_UNLIKELY(nsm > J->sizesnapmap)) { /* Need to grow snapshot map? */
|
|
|
- if (nsm < 2*J->sizesnapmap)
|
|
|
- nsm = 2*J->sizesnapmap;
|
|
|
- else if (nsm < 64)
|
|
|
- nsm = 64;
|
|
|
- J->snapmapbuf = (SnapEntry *)lj_mem_realloc(J->L, J->snapmapbuf,
|
|
|
- J->sizesnapmap*sizeof(SnapEntry), nsm*sizeof(SnapEntry));
|
|
|
- J->cur.snapmap = J->snapmapbuf;
|
|
|
- J->sizesnapmap = nsm;
|
|
|
- }
|
|
|
+ lj_snap_grow_map(J, nsnapmap + nslots + (MSize)J->framedepth*2+1);
|
|
|
p = &J->cur.snapmap[nsnapmap];
|
|
|
- nslots = snapshot_slots(J, p, nslots);
|
|
|
- nframelinks = snapshot_framelinks(J, p + nslots);
|
|
|
- J->cur.nsnapmap = (uint16_t)(nsnapmap + nslots + nframelinks);
|
|
|
+ nent = snapshot_slots(J, p, nslots);
|
|
|
+ nframelinks = snapshot_framelinks(J, p + nent);
|
|
|
+ J->cur.nsnapmap = (uint16_t)(nsnapmap + nent + nframelinks);
|
|
|
snap->mapofs = (uint16_t)nsnapmap;
|
|
|
snap->ref = (IRRef1)J->cur.nins;
|
|
|
- snap->nslots = (uint8_t)nslots;
|
|
|
+ snap->nent = (uint8_t)nent;
|
|
|
snap->nframelinks = (uint8_t)nframelinks;
|
|
|
+ snap->nslots = (uint8_t)nslots;
|
|
|
snap->count = 0;
|
|
|
}
|
|
|
|
|
|
@@ -111,14 +124,7 @@ void lj_snap_add(jit_State *J)
|
|
|
(nsnap > 0 && J->cur.snap[nsnap-1].ref == J->cur.nins)) {
|
|
|
nsnapmap = J->cur.snap[--nsnap].mapofs;
|
|
|
} else {
|
|
|
- /* Need to grow snapshot buffer? */
|
|
|
- if (LJ_UNLIKELY(nsnap >= J->sizesnap)) {
|
|
|
- MSize maxsnap = (MSize)J->param[JIT_P_maxsnap];
|
|
|
- if (nsnap >= maxsnap)
|
|
|
- lj_trace_err(J, LJ_TRERR_SNAPOV);
|
|
|
- lj_mem_growvec(J->L, J->snapbuf, J->sizesnap, maxsnap, SnapShot);
|
|
|
- J->cur.snap = J->snapbuf;
|
|
|
- }
|
|
|
+ lj_snap_grow_buf(J, nsnap+1);
|
|
|
J->cur.nsnap = (uint16_t)(nsnap+1);
|
|
|
}
|
|
|
J->mergesnap = 0;
|
|
|
@@ -131,14 +137,21 @@ void lj_snap_shrink(jit_State *J)
|
|
|
{
|
|
|
BCReg nslots = J->baseslot + J->maxslot;
|
|
|
SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
|
|
|
- SnapEntry *oflinks = &J->cur.snapmap[snap->mapofs + snap->nslots];
|
|
|
- SnapEntry *nflinks = &J->cur.snapmap[snap->mapofs + nslots];
|
|
|
- uint32_t s, nframelinks = snap->nframelinks;
|
|
|
+ SnapEntry *map = &J->cur.snapmap[snap->mapofs];
|
|
|
+ MSize nent = snap->nent;
|
|
|
lua_assert(nslots < snap->nslots);
|
|
|
snap->nslots = (uint8_t)nslots;
|
|
|
- J->cur.nsnapmap = (uint16_t)(snap->mapofs + nslots + nframelinks);
|
|
|
- for (s = 0; s < nframelinks; s++) /* Move frame links down. */
|
|
|
- nflinks[s] = oflinks[s];
|
|
|
+ if (nent > 0 && snap_slot(map[nent-1]) >= nslots) {
|
|
|
+ MSize s, delta, nframelinks = snap->nframelinks;
|
|
|
+ for (nent--; nent > 0 && snap_slot(map[nent-1]) >= nslots; nent--)
|
|
|
+ ;
|
|
|
+ delta = snap->nent - nent;
|
|
|
+ snap->nent = (uint8_t)nent;
|
|
|
+ J->cur.nsnapmap = (uint16_t)(snap->mapofs + nent + nframelinks);
|
|
|
+ map += nent;
|
|
|
+ for (s = 0; s < nframelinks; s++) /* Move frame links down. */
|
|
|
+ map[s] = map[s+delta];
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
/* -- Snapshot access ----------------------------------------------------- */
|
|
|
@@ -167,21 +180,24 @@ static RegSP snap_renameref(Trace *T, SnapNo lim, IRRef ref, RegSP rs)
|
|
|
return rs;
|
|
|
}
|
|
|
|
|
|
-/* Convert a snapshot into a linear slot -> RegSP map. */
|
|
|
+/* Convert a snapshot into a linear slot -> RegSP map.
|
|
|
+** Note: unused slots are not initialized!
|
|
|
+*/
|
|
|
void lj_snap_regspmap(uint16_t *rsmap, Trace *T, SnapNo snapno)
|
|
|
{
|
|
|
SnapShot *snap = &T->snap[snapno];
|
|
|
- BCReg s, nslots = snap->nslots;
|
|
|
+ MSize n, nent = snap->nent;
|
|
|
SnapEntry *map = &T->snapmap[snap->mapofs];
|
|
|
BloomFilter rfilt = snap_renamefilter(T, snapno);
|
|
|
- for (s = 0; s < nslots; s++) {
|
|
|
- IRRef ref = snap_ref(map[s]);
|
|
|
+ for (n = 0; n < nent; n++) {
|
|
|
+ SnapEntry sn = map[n];
|
|
|
+ IRRef ref = snap_ref(sn);
|
|
|
if (!irref_isk(ref)) {
|
|
|
IRIns *ir = &T->ir[ref];
|
|
|
uint32_t rs = ir->prev;
|
|
|
if (bloomtest(rfilt, ref))
|
|
|
rs = snap_renameref(T, snapno, ref, rs);
|
|
|
- rsmap[s] = (uint16_t)rs;
|
|
|
+ rsmap[snap_slot(sn)] = (uint16_t)rs;
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -193,89 +209,88 @@ void lj_snap_restore(jit_State *J, void *exptr)
|
|
|
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
|
|
|
Trace *T = J->trace[J->parent];
|
|
|
SnapShot *snap = &T->snap[snapno];
|
|
|
- BCReg s, nslots = snap->nslots;
|
|
|
+ MSize n, nent = snap->nent;
|
|
|
SnapEntry *map = &T->snapmap[snap->mapofs];
|
|
|
- SnapEntry *flinks = map + nslots + snap->nframelinks;
|
|
|
- TValue *o, *newbase, *ntop;
|
|
|
+ SnapEntry *flinks = map + nent + snap->nframelinks;
|
|
|
+ BCReg nslots = snap->nslots;
|
|
|
+ TValue *frame;
|
|
|
BloomFilter rfilt = snap_renamefilter(T, snapno);
|
|
|
lua_State *L = J->L;
|
|
|
|
|
|
/* Make sure the stack is big enough for the slots from the snapshot. */
|
|
|
- if (L->base + nslots >= L->maxstack) {
|
|
|
+ if (LJ_UNLIKELY(L->base + nslots > L->maxstack)) {
|
|
|
L->top = curr_topL(L);
|
|
|
lj_state_growstack(L, nslots - curr_proto(L)->framesize);
|
|
|
}
|
|
|
|
|
|
/* Fill stack slots with data from the registers and spill slots. */
|
|
|
- newbase = NULL;
|
|
|
- ntop = L->base;
|
|
|
- for (s = 0, o = L->base-1; s < nslots; s++, o++) {
|
|
|
- IRRef ref = snap_ref(map[s]);
|
|
|
- if (ref) {
|
|
|
- IRIns *ir = &T->ir[ref];
|
|
|
- if (irref_isk(ref)) { /* Restore constant slot. */
|
|
|
- lj_ir_kvalue(L, o, ir);
|
|
|
- } else {
|
|
|
- IRType1 t = ir->t;
|
|
|
- RegSP rs = ir->prev;
|
|
|
- if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
|
|
|
- rs = snap_renameref(T, snapno, ref, rs);
|
|
|
- if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
|
|
|
- int32_t *sps = &ex->spill[regsp_spill(rs)];
|
|
|
- if (irt_isinteger(t)) {
|
|
|
- setintV(o, *sps);
|
|
|
- } else if (irt_isnum(t)) {
|
|
|
- o->u64 = *(uint64_t *)sps;
|
|
|
- } else {
|
|
|
- lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
|
|
|
- setgcrefi(o->gcr, *sps);
|
|
|
- setitype(o, irt_toitype(t));
|
|
|
- }
|
|
|
- } else if (ra_hasreg(regsp_reg(rs))) { /* Restore from register. */
|
|
|
- Reg r = regsp_reg(rs);
|
|
|
- if (irt_isinteger(t)) {
|
|
|
- setintV(o, ex->gpr[r-RID_MIN_GPR]);
|
|
|
- } else if (irt_isnum(t)) {
|
|
|
- setnumV(o, ex->fpr[r-RID_MIN_FPR]);
|
|
|
- } else {
|
|
|
- if (!irt_ispri(t))
|
|
|
- setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
|
|
|
- setitype(o, irt_toitype(t));
|
|
|
- }
|
|
|
- } else { /* Restore frame slot. */
|
|
|
- lua_assert(ir->o == IR_FRAME);
|
|
|
- /* This works for both PTR and FUNC IR_FRAME. */
|
|
|
- setgcrefp(o->fr.func, mref(T->ir[ir->op2].ptr, void));
|
|
|
- if (s != 0) /* Do not overwrite link to previous frame. */
|
|
|
- o->fr.tp.ftsz = (int32_t)*--flinks;
|
|
|
- if (irt_isfunc(ir->t)) {
|
|
|
- GCfunc *fn = gco2func(gcref(T->ir[ir->op2].gcr));
|
|
|
- if (isluafunc(fn)) {
|
|
|
- TValue *fs;
|
|
|
- fs = o+1 + funcproto(fn)->framesize;
|
|
|
- if (fs > ntop) ntop = fs; /* Update top for newly added frames. */
|
|
|
- if (s != 0) newbase = o+1;
|
|
|
+ frame = L->base-1;
|
|
|
+ for (n = 0; n < nent; n++) {
|
|
|
+ IRRef ref = snap_ref(map[n]);
|
|
|
+ BCReg s = snap_slot(map[n]);
|
|
|
+ TValue *o = &frame[s]; /* Stack slots are relative to start frame. */
|
|
|
+ IRIns *ir = &T->ir[ref];
|
|
|
+ if (irref_isk(ref)) { /* Restore constant slot. */
|
|
|
+ lj_ir_kvalue(L, o, ir);
|
|
|
+ } else {
|
|
|
+ IRType1 t = ir->t;
|
|
|
+ RegSP rs = ir->prev;
|
|
|
+ if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
|
|
|
+ rs = snap_renameref(T, snapno, ref, rs);
|
|
|
+ if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
|
|
|
+ int32_t *sps = &ex->spill[regsp_spill(rs)];
|
|
|
+ if (irt_isinteger(t)) {
|
|
|
+ setintV(o, *sps);
|
|
|
+ } else if (irt_isnum(t)) {
|
|
|
+ o->u64 = *(uint64_t *)sps;
|
|
|
+ } else {
|
|
|
+ lua_assert(!irt_ispri(t)); /* PRI refs never have a spill slot. */
|
|
|
+ setgcrefi(o->gcr, *sps);
|
|
|
+ setitype(o, irt_toitype(t));
|
|
|
+ }
|
|
|
+ } else if (ra_hasreg(regsp_reg(rs))) { /* Restore from register. */
|
|
|
+ Reg r = regsp_reg(rs);
|
|
|
+ if (irt_isinteger(t)) {
|
|
|
+ setintV(o, ex->gpr[r-RID_MIN_GPR]);
|
|
|
+ } else if (irt_isnum(t)) {
|
|
|
+ setnumV(o, ex->fpr[r-RID_MIN_FPR]);
|
|
|
+ } else {
|
|
|
+ if (!irt_ispri(t))
|
|
|
+ setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
|
|
|
+ setitype(o, irt_toitype(t));
|
|
|
+ }
|
|
|
+ } else { /* Restore frame slot. */
|
|
|
+ lua_assert(ir->o == IR_FRAME);
|
|
|
+ /* This works for both PTR and FUNC IR_FRAME. */
|
|
|
+ setgcrefp(o->fr.func, mref(T->ir[ir->op2].ptr, void));
|
|
|
+ if (s != 0) /* Do not overwrite link to previous frame. */
|
|
|
+ o->fr.tp.ftsz = (int32_t)*--flinks;
|
|
|
+ if (irt_isfunc(ir->t)) {
|
|
|
+ GCfunc *fn = gco2func(gcref(T->ir[ir->op2].gcr));
|
|
|
+ if (isluafunc(fn)) {
|
|
|
+ MSize framesize = funcproto(fn)->framesize;
|
|
|
+ TValue *fs;
|
|
|
+ L->base = ++o;
|
|
|
+ if (LJ_UNLIKELY(o + framesize > L->maxstack)) { /* Grow again? */
|
|
|
+ ptrdiff_t fsave = savestack(L, frame);
|
|
|
+ L->top = o;
|
|
|
+ lj_state_growstack(L, framesize);
|
|
|
+ frame = restorestack(L, fsave);
|
|
|
+ o = L->top;
|
|
|
}
|
|
|
+ fs = o + framesize;
|
|
|
+ if (s == 0) /* Only partially clear tail call frame at #0. */
|
|
|
+ o = &frame[nslots];
|
|
|
+ while (o < fs) /* Clear slots of newly added frames. */
|
|
|
+ setnilV(o++);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
- } else {
|
|
|
- lua_assert(!newbase);
|
|
|
}
|
|
|
}
|
|
|
- if (newbase) L->base = newbase;
|
|
|
- if (ntop >= L->maxstack) { /* Need to grow the stack again. */
|
|
|
- MSize need = (MSize)(ntop - o);
|
|
|
- L->top = o;
|
|
|
- lj_state_growstack(L, need);
|
|
|
- o = L->top;
|
|
|
- ntop = o + need;
|
|
|
- }
|
|
|
L->top = curr_topL(L);
|
|
|
- for (; o < ntop; o++) /* Clear remainder of newly added frames. */
|
|
|
- setnilV(o);
|
|
|
- lua_assert(map + nslots == flinks-1);
|
|
|
- J->pc = (const BCIns *)(uintptr_t)(*--flinks);
|
|
|
+ J->pc = snap_pc(*--flinks);
|
|
|
+ lua_assert(map + nent == flinks);
|
|
|
}
|
|
|
|
|
|
#undef IR
|
Mike Pall