Add SPLIT pass to split 64 bit IR instructions for 32 bit CPUs.

Add generic HIOP instruction for extra backend functionality.
Add support for HIOP to x86 backend.
Use POWI for 64 bit integer x^k, too.
POWI is lowered to a call by SPLIT or the x64 backend.

src/Makefile

@@ -331,7 +331,7 @@ LJCORE_O= lj_gc.o lj_err.o lj_char.o lj_bc.o lj_obj.o \
 	  lj_state.o lj_dispatch.o lj_vmevent.o lj_api.o \
 	  lj_lex.o lj_parse.o \
 	  lj_ir.o lj_opt_mem.o lj_opt_fold.o lj_opt_narrow.o \
-	  lj_opt_dce.o lj_opt_loop.o \
+	  lj_opt_dce.o lj_opt_loop.o lj_opt_split.o \
 	  lj_mcode.o lj_snap.o lj_record.o lj_crecord.o lj_ffrecord.o \
 	  lj_asm.o lj_trace.o lj_gdbjit.o \
 	  lj_ctype.o lj_cdata.o lj_cconv.o lj_ccall.o lj_carith.o lj_clib.o \

src/Makefile.dep

@@ -128,6 +128,8 @@ lj_opt_mem.o: lj_opt_mem.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
 lj_opt_narrow.o: lj_opt_narrow.c lj_obj.h lua.h luaconf.h lj_def.h \
  lj_arch.h lj_str.h lj_bc.h lj_ir.h lj_jit.h lj_iropt.h lj_trace.h \
  lj_dispatch.h lj_traceerr.h
+lj_opt_split.o: lj_opt_split.c lj_obj.h lua.h luaconf.h lj_def.h \
+ lj_arch.h
 lj_parse.o: lj_parse.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
  lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_func.h lj_state.h \
  lj_bc.h lj_ctype.h lj_lex.h lj_parse.h lj_vm.h lj_vmevent.h
@@ -167,10 +169,11 @@ ljamalg.o: ljamalg.c lua.h luaconf.h lauxlib.h lj_gc.c lj_obj.h lj_def.h \
  lj_cconv.h lj_cconv.c lj_ccall.c lj_ccall.h lj_carith.c lj_carith.h \
  lj_clib.c lj_clib.h lj_cparse.c lj_cparse.h lj_lib.c lj_lib.h lj_ir.c \
  lj_iropt.h lj_opt_mem.c lj_opt_fold.c lj_folddef.h lj_opt_narrow.c \
- lj_opt_dce.c lj_opt_loop.c lj_snap.h lj_mcode.c lj_mcode.h lj_snap.c \
- lj_target.h lj_target_*.h lj_record.c lj_record.h lj_ffrecord.h \
- lj_crecord.c lj_crecord.h lj_ffrecord.c lj_recdef.h lj_asm.c lj_asm.h \
- lj_trace.c lj_gdbjit.h lj_gdbjit.c lj_alloc.c lib_aux.c lib_base.c \
- lj_libdef.h lib_math.c lib_string.c lib_table.c lib_io.c lib_os.c \
- lib_package.c lib_debug.c lib_bit.c lib_jit.c lib_ffi.c lib_init.c
+ lj_opt_dce.c lj_opt_loop.c lj_snap.h lj_opt_split.c lj_mcode.c \
+ lj_mcode.h lj_snap.c lj_target.h lj_target_*.h lj_record.c lj_record.h \
+ lj_ffrecord.h lj_crecord.c lj_crecord.h lj_ffrecord.c lj_recdef.h \
+ lj_asm.c lj_asm.h lj_trace.c lj_gdbjit.h lj_gdbjit.c lj_alloc.c \
+ lib_aux.c lib_base.c lj_libdef.h lib_math.c lib_string.c lib_table.c \
+ lib_io.c lib_os.c lib_package.c lib_debug.c lib_bit.c lib_jit.c \
+ lib_ffi.c lib_init.c
 luajit.o: luajit.c lua.h luaconf.h lauxlib.h lualib.h luajit.h lj_arch.h

src/lj_asm.c

@@ -347,6 +347,20 @@ static void emit_addptr(ASMState *as, Reg r, int32_t ofs)
   }
 }
 
+/* op rm/mrm, i */
+static void emit_gmrmi(ASMState *as, x86Group xg, Reg rb, int32_t i)
+{
+  x86Op xo;
+  if (checki8(i)) {
+    emit_i8(as, i);
+    xo = XG_TOXOi8(xg);
+  } else {
+    emit_i32(as, i);
+    xo = XG_TOXOi(xg);
+  }
+  emit_mrm(as, xo, (Reg)(xg & 7) | (rb & REX_64), (rb & ~REX_64));
+}
+
 /* -- Emit moves ---------------------------------------------------------- */
 
 /* mov [base+ofs], i */
@@ -371,7 +385,10 @@ static void emit_movmroi(ASMState *as, Reg base, int32_t ofs, int32_t i)
 /* mov r, i / xor r, r */
 static void emit_loadi(ASMState *as, Reg r, int32_t i)
 {
-  if (i == 0) {
+  /* XOR r,r is shorter, but modifies the flags. This is bad for HIOP. */
+  if (i == 0 && !(LJ_32 && (IR(as->curins)->o == IR_HIOP ||
+			    (as->curins+1 < as->T->nins &&
+			     IR(as->curins+1)->o == IR_HIOP)))) {
     emit_rr(as, XO_ARITH(XOg_XOR), r, r);
   } else {
     MCode *p = as->mcp;
@@ -422,6 +439,19 @@ static void emit_loadn(ASMState *as, Reg r, cTValue *tv)
 /* Label for short jumps. */
 typedef MCode *MCLabel;
 
+#if LJ_32 && LJ_HASFFI
+/* jmp short target */
+static void emit_sjmp(ASMState *as, MCLabel target)
+{
+  MCode *p = as->mcp;
+  ptrdiff_t delta = target - p;
+  lua_assert(delta == (int8_t)delta);
+  p[-1] = (MCode)(int8_t)delta;
+  p[-2] = XI_JMPs;
+  as->mcp = p - 2;
+}
+#endif
+
 /* jcc short target */
 static void emit_sjcc(ASMState *as, int cc, MCLabel target)
 {
@@ -630,7 +660,7 @@ static Reg ra_rematk(ASMState *as, IRIns *ir)
   } else if (ir->o == IR_KPRI) {  /* REF_NIL stores ASMREF_L register. */
     lua_assert(irt_isnil(ir->t));
     emit_getgl(as, r, jit_L);
-#if LJ_64  /* NYI: 32 bit register pairs. */
+#if LJ_64
   } else if (ir->o == IR_KINT64) {
     emit_loadu64(as, r, ir_kint64(ir)->u64);
 #endif
@@ -681,8 +711,7 @@ static Reg ra_releasetmp(ASMState *as, IRRef ref)
 #if LJ_64
 #define REX_64IR(ir, r)		((r) + (irt_is64((ir)->t) ? REX_64 : 0))
 #else
-/* NYI: 32 bit register pairs. */
-#define REX_64IR(ir, r)		check_exp(!irt_is64((ir)->t), (r))
+#define REX_64IR(ir, r)		(r)
 #endif
 
 /* Generic move between two regs. */
@@ -939,7 +968,7 @@ static void ra_left(ASMState *as, Reg dest, IRRef lref)
 	  emit_loadn(as, dest, tv);
 	  return;
 	}
-#if LJ_64  /* NYI: 32 bit register pairs. */
+#if LJ_64
       } else if (ir->o == IR_KINT64) {
 	emit_loadu64(as, dest, ir_kint64(ir)->u64);
 	return;
@@ -1463,7 +1492,7 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
 #endif
     if (r) {  /* Argument is in a register. */
       if (r < RID_MAX_GPR && ref < ASMREF_TMP1) {
-#if LJ_64  /* NYI: 32 bit register pairs. */
+#if LJ_64
 	if (ir->o == IR_KINT64)
 	  emit_loadu64(as, r, ir_kint64(ir)->u64);
 	else
@@ -1519,7 +1548,7 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
   ra_evictset(as, drop);  /* Evictions must be performed first. */
   if (ra_used(ir)) {
     if (irt_isfp(ir->t)) {
-      int32_t ofs = sps_scale(ir->s); /* Use spill slot or temp slots. */
+      int32_t ofs = sps_scale(ir->s);  /* Use spill slot or temp slots. */
 #if LJ_64
       if ((ci->flags & CCI_CASTU64)) {
 	Reg dest = ir->r;
@@ -1632,19 +1661,24 @@ static void asm_conv(ASMState *as, IRIns *ir)
   int stfp = (st == IRT_NUM || st == IRT_FLOAT);
   IRRef lref = ir->op1;
   lua_assert(irt_type(ir->t) != st);
+  lua_assert(!(LJ_32 && (irt_isint64(ir->t) || st64)));  /* Handled by SPLIT. */
   if (irt_isfp(ir->t)) {
     Reg dest = ra_dest(as, ir, RSET_FPR);
     if (stfp) {  /* FP to FP conversion. */
       Reg left = asm_fuseload(as, lref, RSET_FPR);
       emit_mrm(as, st == IRT_NUM ? XO_CVTSD2SS : XO_CVTSS2SD, dest, left);
       if (left == dest) return;  /* Avoid the XO_XORPS. */
-#if LJ_32
-    } else if (st >= IRT_U32) {
-      /* NYI: 64 bit integer or uint32_t to number conversion. */
-      setintV(&as->J->errinfo, ir->o);
-      lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
+    } else if (LJ_32 && st == IRT_U32) {  /* U32 to FP conversion on x86. */
+      /* number = (2^52+2^51 .. u32) - (2^52+2^51) */
+      cTValue *k = lj_ir_k64_find(as->J, U64x(43380000,00000000));
+      Reg bias = ra_scratch(as, rset_exclude(RSET_FPR, dest));
+      if (irt_isfloat(ir->t))
+	emit_rr(as, XO_CVTSD2SS, dest, dest);
+      emit_rr(as, XO_SUBSD, dest, bias);  /* Subtract 2^52+2^51 bias. */
+      emit_rr(as, XO_XORPS, dest, bias);  /* Merge bias and integer. */
+      emit_loadn(as, bias, k);
+      emit_mrm(as, XO_MOVD, dest, asm_fuseload(as, lref, RSET_GPR));
       return;
-#endif
     } else {  /* Integer to FP conversion. */
       Reg left = (LJ_64 && (st == IRT_U32 || st == IRT_U64)) ?
 		 ra_alloc1(as, lref, RSET_GPR) :
@@ -1663,41 +1697,47 @@ static void asm_conv(ASMState *as, IRIns *ir)
       emit_rr(as, XO_XORPS, dest, dest);  /* Avoid partial register stall. */
   } else if (stfp) {  /* FP to integer conversion. */
     if (irt_isguard(ir->t)) {
-      lua_assert(!irt_is64(ir->t));  /* No support for checked 64 bit conv. */
+      /* Checked conversions are only supported from number to int. */
+      lua_assert(irt_isint(ir->t) && st == IRT_NUM);
       asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
-#if LJ_32
-    } else if (irt_isi64(ir->t) || irt_isu64(ir->t) || irt_isu32(ir->t)) {
-      /* NYI: number to 64 bit integer or uint32_t conversion. */
-      setintV(&as->J->errinfo, ir->o);
-      lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
-#endif
     } else {
       Reg dest = ra_dest(as, ir, RSET_GPR);
       x86Op op = st == IRT_NUM ?
 		 ((ir->op2 & IRCONV_TRUNC) ? XO_CVTTSD2SI : XO_CVTSD2SI) :
 		 ((ir->op2 & IRCONV_TRUNC) ? XO_CVTTSS2SI : XO_CVTSS2SI);
-      if (LJ_64 && irt_isu64(ir->t)) {
-	const void *k = lj_ir_k64_find(as->J, U64x(c3f00000,00000000));
-	MCLabel l_end = emit_label(as);
-	Reg left = IR(lref)->r;
+      if (LJ_32 && irt_isu32(ir->t)) {  /* FP to U32 conversion on x86. */
+	/* u32 = (int32_t)(number - 2^31) + 2^31 */
+	Reg tmp = ra_noreg(IR(lref)->r) ? ra_alloc1(as, lref, RSET_FPR) :
+					  ra_scratch(as, RSET_FPR);
+	emit_gri(as, XG_ARITHi(XOg_ADD), dest, (int32_t)0x80000000);
+	emit_rr(as, op, dest, tmp);
+	if (st == IRT_NUM)
+	  emit_rma(as, XO_ADDSD, tmp,
+		   lj_ir_k64_find(as->J, U64x(c1e00000,00000000)));
+	else
+	  emit_rma(as, XO_ADDSS, tmp,
+		   lj_ir_k64_find(as->J, U64x(00000000,cf000000)));
+	ra_left(as, tmp, lref);
+      } else if (LJ_64 && irt_isu64(ir->t)) {
 	/* For inputs in [2^63,2^64-1] add -2^64 and convert again. */
-	if (ra_hasreg(left)) {
-	  Reg tmpn = ra_scratch(as, rset_exclude(RSET_FPR, left));
-	  emit_rr(as, op, dest|REX_64, tmpn);
-	  emit_rr(as, st == IRT_NUM ? XO_ADDSD : XO_ADDSS, tmpn, left);
-	  emit_rma(as, st == IRT_NUM ? XMM_MOVRM(as) : XO_MOVSS, tmpn, k);
-	} else {
-	  left = ra_allocref(as, lref, RSET_FPR);
-	  emit_rr(as, op, dest|REX_64, left);
-	  emit_rma(as, st == IRT_NUM ? XO_ADDSD : XO_ADDSS, left, k);
-	}
+	Reg tmp = ra_noreg(IR(lref)->r) ? ra_alloc1(as, lref, RSET_FPR) :
+					  ra_scratch(as, RSET_FPR);
+	MCLabel l_end = emit_label(as);
+	emit_rr(as, op, dest|REX_64, tmp);
+	if (st == IRT_NUM)
+	  emit_rma(as, XO_ADDSD, tmp,
+		   lj_ir_k64_find(as->J, U64x(c3f00000,00000000)));
+	else
+	  emit_rma(as, XO_ADDSS, tmp,
+		   lj_ir_k64_find(as->J, U64x(00000000,df800000)));
 	emit_sjcc(as, CC_NS, l_end);
 	emit_rr(as, XO_TEST, dest|REX_64, dest);  /* Check if dest < 2^63. */
-	emit_rr(as, op, dest|REX_64, left);
+	emit_rr(as, op, dest|REX_64, tmp);
+	ra_left(as, tmp, lref);
       } else {
 	Reg left = asm_fuseload(as, lref, RSET_FPR);
 	if (LJ_64 && irt_isu32(ir->t))
-	  emit_rr(as, XO_MOV, dest, dest);  /* Zero upper 32 bits. */
+	  emit_rr(as, XO_MOV, dest, dest);  /* Zero hiword. */
 	emit_mrm(as, op,
 		 dest|((LJ_64 &&
 			(irt_is64(ir->t) || irt_isu32(ir->t))) ? REX_64 : 0),
@@ -1728,12 +1768,10 @@ static void asm_conv(ASMState *as, IRIns *ir)
       emit_mrm(as, op, dest, left);
     }
   } else {  /* 32/64 bit integer conversions. */
-    if (irt_is64(ir->t)) {
-#if LJ_32
-      /* NYI: conversion to 64 bit integers. */
-      setintV(&as->J->errinfo, ir->o);
-      lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
-#else
+    if (LJ_32) {  /* Only need to handle 32/32 bit no-op (cast) on x86. */
+      Reg dest = ra_dest(as, ir, RSET_GPR);
+      ra_left(as, dest, lref);  /* Do nothing, but may need to move regs. */
+    } else if (irt_is64(ir->t)) {
       Reg dest = ra_dest(as, ir, RSET_GPR);
       if (st64 || !(ir->op2 & IRCONV_SEXT)) {
 	/* 64/64 bit no-op (cast) or 32 to 64 bit zero extension. */
@@ -1742,21 +1780,14 @@ static void asm_conv(ASMState *as, IRIns *ir)
 	Reg left = asm_fuseload(as, lref, RSET_GPR);
 	emit_mrm(as, XO_MOVSXd, dest|REX_64, left);
       }
-#endif
     } else {
       Reg dest = ra_dest(as, ir, RSET_GPR);
       if (st64) {
-#if LJ_32
-	/* NYI: conversion from 64 bit integers. */
-	setintV(&as->J->errinfo, ir->o);
-	lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
-#else
 	Reg left = asm_fuseload(as, lref, RSET_GPR);
-	/* This is either a 32 bit reg/reg mov which zeroes the hi-32 bits
-	** or a load of the lower 32 bits from a 64 bit address.
+	/* This is either a 32 bit reg/reg mov which zeroes the hiword
+	** or a load of the loword from a 64 bit address.
 	*/
 	emit_mrm(as, XO_MOV, dest, left);
-#endif
       } else {  /* 32/32 bit no-op (cast). */
 	ra_left(as, dest, lref);  /* Do nothing, but may need to move regs. */
       }
@@ -1764,6 +1795,93 @@ static void asm_conv(ASMState *as, IRIns *ir)
   }
 }
 
+#if LJ_32 && LJ_HASFFI
+/* No SSE conversions to/from 64 bit on x86, so resort to ugly x87 code. */
+
+/* 64 bit integer to FP conversion in 32 bit mode. */
+static void asm_conv_fp_int64(ASMState *as, IRIns *ir)
+{
+  Reg hi = ra_alloc1(as, ir->op1, RSET_GPR);
+  Reg lo = ra_alloc1(as, (ir-1)->op1, rset_exclude(RSET_GPR, hi));
+  int32_t ofs = sps_scale(ir->s);  /* Use spill slot or temp slots. */
+  Reg dest = ir->r;
+  if (ra_hasreg(dest)) {
+    ra_free(as, dest);
+    ra_modified(as, dest);
+    emit_rmro(as, irt_isnum(ir->t) ? XMM_MOVRM(as) : XO_MOVSS,
+	      dest, RID_ESP, ofs);
+  }
+  emit_rmro(as, irt_isnum(ir->t) ? XO_FSTPq : XO_FSTPd,
+	    irt_isnum(ir->t) ? XOg_FSTPq : XOg_FSTPd, RID_ESP, ofs);
+  if (((ir-1)->op2 & IRCONV_SRCMASK) == IRT_U64) {
+    /* For inputs in [2^63,2^64-1] add 2^64 to compensate. */
+    MCLabel l_end = emit_label(as);
+    emit_rma(as, XO_FADDq, XOg_FADDq,
+	     lj_ir_k64_find(as->J, U64x(43f00000,00000000)));
+    emit_sjcc(as, CC_NS, l_end);
+    emit_rr(as, XO_TEST, hi, hi);  /* Check if u64 >= 2^63. */
+  } else {
+    lua_assert(((ir-1)->op2 & IRCONV_SRCMASK) == IRT_I64);
+  }
+  emit_rmro(as, XO_FILDq, XOg_FILDq, RID_ESP, 0);
+  /* NYI: Avoid narrow-to-wide store-to-load forwarding stall. */
+  emit_rmro(as, XO_MOVto, hi, RID_ESP, 4);
+  emit_rmro(as, XO_MOVto, lo, RID_ESP, 0);
+}
+
+/* FP to 64 bit integer conversion in 32 bit mode. */
+static void asm_conv_int64_fp(ASMState *as, IRIns *ir)
+{
+  IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
+  IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
+  Reg lo, hi;
+  lua_assert(st == IRT_NUM || st == IRT_FLOAT);
+  lua_assert(dt == IRT_I64 || dt == IRT_U64);
+  lua_assert(((ir-1)->op2 & IRCONV_TRUNC));
+  hi = ra_dest(as, ir, RSET_GPR);
+  lo = ra_dest(as, ir-1, rset_exclude(RSET_GPR, hi));
+  if (ra_used(ir-1)) emit_rmro(as, XO_MOV, lo, RID_ESP, 0);
+  /* NYI: Avoid wide-to-narrow store-to-load forwarding stall. */
+  if (!(as->flags & JIT_F_SSE3)) {  /* Set FPU rounding mode to default. */
+    emit_rmro(as, XO_FLDCW, XOg_FLDCW, RID_ESP, 4);
+    emit_rmro(as, XO_MOVto, lo, RID_ESP, 4);
+    emit_gri(as, XG_ARITHi(XOg_AND), lo, 0xf3ff);
+  }
+  if (dt == IRT_U64) {
+    /* For inputs in [2^63,2^64-1] add -2^64 and convert again. */
+    MCLabel l_pop, l_end = emit_label(as);
+    emit_x87op(as, XI_FPOP);
+    l_pop = emit_label(as);
+    emit_sjmp(as, l_end);
+    emit_rmro(as, XO_MOV, hi, RID_ESP, 4);
+    if ((as->flags & JIT_F_SSE3))
+      emit_rmro(as, XO_FISTTPq, XOg_FISTTPq, RID_ESP, 0);
+    else
+      emit_rmro(as, XO_FISTPq, XOg_FISTPq, RID_ESP, 0);
+    emit_rma(as, XO_FADDq, XOg_FADDq,
+	     lj_ir_k64_find(as->J, U64x(c3f00000,00000000)));
+    emit_sjcc(as, CC_NS, l_pop);
+    emit_rr(as, XO_TEST, hi, hi);  /* Check if out-of-range (2^63). */
+  }
+  emit_rmro(as, XO_MOV, hi, RID_ESP, 4);
+  if ((as->flags & JIT_F_SSE3)) {  /* Truncation is easy with SSE3. */
+    emit_rmro(as, XO_FISTTPq, XOg_FISTTPq, RID_ESP, 0);
+  } else {  /* Otherwise set FPU rounding mode to truncate before the store. */
+    emit_rmro(as, XO_FISTPq, XOg_FISTPq, RID_ESP, 0);
+    emit_rmro(as, XO_FLDCW, XOg_FLDCW, RID_ESP, 0);
+    emit_rmro(as, XO_MOVtow, lo, RID_ESP, 0);
+    emit_rmro(as, XO_ARITHw(XOg_OR), lo, RID_ESP, 0);
+    emit_loadi(as, lo, 0xc00);
+    emit_rmro(as, XO_FNSTCW, XOg_FNSTCW, RID_ESP, 0);
+  }
+  if (dt == IRT_U64)
+    emit_x87op(as, XI_FDUP);
+  emit_mrm(as, st == IRT_NUM ? XO_FLDq : XO_FLDd,
+	   st == IRT_NUM ? XOg_FLDq: XOg_FLDd,
+	   asm_fuseload(as, ir->op1, RSET_EMPTY));
+}
+#endif
+
 static void asm_strto(ASMState *as, IRIns *ir)
 {
   /* Force a spill slot for the destination register (if any). */
@@ -2644,6 +2762,18 @@ static void asm_powi(ASMState *as, IRIns *ir)
   ra_left(as, RID_EAX, ir->op2);
 }
 
+#if LJ_64 && LJ_HASFFI
+static void asm_arith64(ASMState *as, IRIns *ir, IRCallID id)
+{
+  const CCallInfo *ci = &lj_ir_callinfo[id];
+  IRRef args[2];
+  args[0] = ir->op1;
+  args[1] = ir->op2;
+  asm_setupresult(as, ir, ci);
+  asm_gencall(as, ci, args);
+}
+#endif
+
 /* Find out whether swapping operands might be beneficial. */
 static int swapops(ASMState *as, IRIns *ir)
 {
@@ -2877,12 +3007,30 @@ static void asm_bitshift(ASMState *as, IRIns *ir, x86Shift xs)
 /* -- Comparisons --------------------------------------------------------- */
 
 /* Virtual flags for unordered FP comparisons. */
-#define VCC_U	0x100		/* Unordered. */
-#define VCC_P	0x200		/* Needs extra CC_P branch. */
-#define VCC_S	0x400		/* Swap avoids CC_P branch. */
+#define VCC_U	0x1000		/* Unordered. */
+#define VCC_P	0x2000		/* Needs extra CC_P branch. */
+#define VCC_S	0x4000		/* Swap avoids CC_P branch. */
 #define VCC_PS	(VCC_P|VCC_S)
 
-static void asm_comp_(ASMState *as, IRIns *ir, int cc)
+/* Map of comparisons to flags. ORDER IR. */
+#define COMPFLAGS(ci, cin, cu, cf)	((ci)+((cu)<<4)+((cin)<<8)+(cf))
+static const uint16_t asm_compmap[IR_ABC+1] = {
+  /*                 signed non-eq unsigned flags */
+  /* LT  */ COMPFLAGS(CC_GE, CC_G,  CC_AE, VCC_PS),
+  /* GE  */ COMPFLAGS(CC_L,  CC_L,  CC_B,  0),
+  /* LE  */ COMPFLAGS(CC_G,  CC_G,  CC_A,  VCC_PS),
+  /* GT  */ COMPFLAGS(CC_LE, CC_L,  CC_BE, 0),
+  /* ULT */ COMPFLAGS(CC_AE, CC_A,  CC_AE, VCC_U),
+  /* UGE */ COMPFLAGS(CC_B,  CC_B,  CC_B,  VCC_U|VCC_PS),
+  /* ULE */ COMPFLAGS(CC_A,  CC_A,  CC_A,  VCC_U),
+  /* UGT */ COMPFLAGS(CC_BE, CC_B,  CC_BE, VCC_U|VCC_PS),
+  /* EQ  */ COMPFLAGS(CC_NE, CC_NE, CC_NE, VCC_P),
+  /* NE  */ COMPFLAGS(CC_E,  CC_E,  CC_E,  VCC_U|VCC_P),
+  /* ABC */ COMPFLAGS(CC_BE, CC_B,  CC_BE, VCC_U|VCC_PS)  /* Same as UGT. */
+};
+
+/* FP and integer comparisons. */
+static void asm_comp(ASMState *as, IRIns *ir, uint32_t cc)
 {
   if (irt_isnum(ir->t)) {
     IRRef lref = ir->op1;
@@ -3008,15 +3156,7 @@ static void asm_comp_(ASMState *as, IRIns *ir, int cc)
 	  if (irl+1 == ir)  /* Referencing previous ins? */
 	    as->testmcp = as->mcp;  /* Set flag to drop test r,r if possible. */
 	} else {
-	  x86Op xo;
-	  if (checki8(imm)) {
-	    emit_i8(as, imm);
-	    xo = XO_ARITHi8;
-	  } else {
-	    emit_i32(as, imm);
-	    xo = XO_ARITHi;
-	  }
-	  emit_mrm(as, xo, r64 + XOg_CMP, left);
+	  emit_gmrmi(as, XG_ARITHi(XOg_CMP), r64 + left, imm);
 	}
       }
     } else {
@@ -3028,8 +3168,133 @@ static void asm_comp_(ASMState *as, IRIns *ir, int cc)
   }
 }
 
-#define asm_comp(as, ir, ci, cf, cu) \
-  asm_comp_(as, ir, (ci)+((cf)<<4)+(cu))
+#if LJ_32 && LJ_HASFFI
+/* 64 bit integer comparisons in 32 bit mode. */
+static void asm_comp_int64(ASMState *as, IRIns *ir)
+{
+  uint32_t cc = asm_compmap[(ir-1)->o];
+  RegSet allow = RSET_GPR;
+  Reg lefthi = RID_NONE, leftlo = RID_NONE;
+  Reg righthi = RID_NONE, rightlo = RID_NONE;
+  MCLabel l_around;
+  x86ModRM mrm;
+
+  as->curins--;  /* Skip loword ins. Avoids failing in noconflict(), too. */
+
+  /* Allocate/fuse hiword operands. */
+  if (irref_isk(ir->op2)) {
+    lefthi = asm_fuseload(as, ir->op1, allow);
+  } else {
+    lefthi = ra_alloc1(as, ir->op1, allow);
+    righthi = asm_fuseload(as, ir->op2, allow);
+    if (righthi == RID_MRM) {
+      if (as->mrm.base != RID_NONE) rset_clear(allow, as->mrm.base);
+      if (as->mrm.idx != RID_NONE) rset_clear(allow, as->mrm.idx);
+    } else {
+      rset_clear(allow, righthi);
+    }
+  }
+  mrm = as->mrm;  /* Save state for hiword instruction. */
+
+  /* Allocate/fuse loword operands. */
+  if (irref_isk((ir-1)->op2)) {
+    leftlo = asm_fuseload(as, (ir-1)->op1, allow);
+  } else {
+    leftlo = ra_alloc1(as, (ir-1)->op1, allow);
+    rightlo = asm_fuseload(as, (ir-1)->op2, allow);
+    if (rightlo == RID_MRM) {
+      if (as->mrm.base != RID_NONE) rset_clear(allow, as->mrm.base);
+      if (as->mrm.idx != RID_NONE) rset_clear(allow, as->mrm.idx);
+    } else {
+      rset_clear(allow, rightlo);
+    }
+  }
+
+  /* All register allocations must be performed _before_ this point. */
+  l_around = emit_label(as);
+  as->invmcp = as->testmcp = NULL;  /* Cannot use these optimizations. */
+
+  /* Loword comparison and branch. */
+  asm_guardcc(as, cc >> 4);  /* Always use unsigned compare for loword. */
+  if (ra_noreg(rightlo)) {
+    int32_t imm = IR((ir-1)->op2)->i;
+    if (imm == 0 && ((cc >> 4) & 0xa) != 0x2 && leftlo != RID_MRM)
+      emit_rr(as, XO_TEST, leftlo, leftlo);
+    else
+      emit_gmrmi(as, XG_ARITHi(XOg_CMP), leftlo, imm);
+  } else {
+    emit_mrm(as, XO_CMP, leftlo, rightlo);
+  }
+
+  /* Hiword comparison and branches. */
+  if ((cc & 15) != CC_NE)
+    emit_sjcc(as, CC_NE, l_around);  /* Hiword unequal: skip loword compare. */
+  if ((cc & 15) != CC_E)
+    asm_guardcc(as, cc >> 8);  /* Hiword compare without equality check. */
+  as->mrm = mrm;  /* Restore state. */
+  if (ra_noreg(righthi)) {
+    int32_t imm = IR(ir->op2)->i;
+    if (imm == 0 && (cc & 0xa) != 0x2 && lefthi != RID_MRM)
+      emit_rr(as, XO_TEST, lefthi, lefthi);
+    else
+      emit_gmrmi(as, XG_ARITHi(XOg_CMP), lefthi, imm);
+  } else {
+    emit_mrm(as, XO_CMP, lefthi, righthi);
+  }
+}
+#endif
+
+/* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
+
+/* Hiword op of a split 64 bit op. Previous op must be the loword op. */
+static void asm_hiop(ASMState *as, IRIns *ir)
+{
+#if LJ_32 && LJ_HASFFI
+  /* HIOP is marked as a store because it needs its own DCE logic. */
+  int uselo = ra_used(ir-1), usehi = ra_used(ir);  /* Loword/hiword used? */
+  if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
+  if ((ir-1)->o == IR_CONV) {  /* Conversions to/from 64 bit. */
+    if (usehi || uselo) {
+      if (irt_isfp(ir->t))
+	asm_conv_fp_int64(as, ir);
+      else
+	asm_conv_int64_fp(as, ir);
+    }
+    as->curins--;  /* Always skip the CONV. */
+    return;
+  } else if ((ir-1)->o <= IR_NE) {  /* 64 bit integer comparisons. ORDER IR. */
+    asm_comp_int64(as, ir);
+    return;
+  }
+  if (!usehi) return;  /* Skip unused hiword op for all remaining ops. */
+  switch ((ir-1)->o) {
+  case IR_ADD:
+    asm_intarith(as, ir, uselo ? XOg_ADC : XOg_ADD);
+    break;
+  case IR_SUB:
+    asm_intarith(as, ir, uselo ? XOg_SBB : XOg_SUB);
+    break;
+  case IR_NEG: {
+    Reg dest = ra_dest(as, ir, RSET_GPR);
+    emit_rr(as, XO_GROUP3, XOg_NEG, dest);
+    if (uselo) {
+      emit_i8(as, 0);
+      emit_rr(as, XO_ARITHi8, XOg_ADC, dest);
+    }
+    ra_left(as, dest, ir->op1);
+    break;
+    }
+  case IR_CALLN:
+    ra_destreg(as, ir, RID_RETHI);
+    if (!uselo)
+      ra_allocref(as, ir->op1, RID2RSET(RID_RET));  /* Mark call as used. */
+    break;
+  default: lua_assert(0); break;
+  }
+#else
+  UNUSED(as); UNUSED(ir); lua_assert(0);  /* Unused on x64 or without FFI. */
+#endif
+}
 
 /* -- Stack handling ------------------------------------------------------ */
 
@@ -3682,21 +3947,16 @@ static void asm_ir(ASMState *as, IRIns *ir)
   switch ((IROp)ir->o) {
   /* Miscellaneous ops. */
   case IR_LOOP: asm_loop(as); break;
-  case IR_NOP: break;
+  case IR_NOP: lua_assert(!ra_used(ir)); break;
   case IR_PHI: asm_phi(as, ir); break;
+  case IR_HIOP: asm_hiop(as, ir); break;
 
   /* Guarded assertions. */
-  case IR_LT:  asm_comp(as, ir, CC_GE, CC_AE, VCC_PS); break;
-  case IR_GE:  asm_comp(as, ir, CC_L,  CC_B,  0); break;
-  case IR_LE:  asm_comp(as, ir, CC_G,  CC_A,  VCC_PS); break;
-  case IR_GT:  asm_comp(as, ir, CC_LE, CC_BE, 0); break;
-  case IR_ULT: asm_comp(as, ir, CC_AE, CC_AE, VCC_U); break;
-  case IR_UGE: asm_comp(as, ir, CC_B,  CC_B,  VCC_U|VCC_PS); break;
-  case IR_ULE: asm_comp(as, ir, CC_A,  CC_A,  VCC_U); break;
-  case IR_ABC:
-  case IR_UGT: asm_comp(as, ir, CC_BE, CC_BE, VCC_U|VCC_PS); break;
-  case IR_EQ:  asm_comp(as, ir, CC_NE, CC_NE, VCC_P); break;
-  case IR_NE:  asm_comp(as, ir, CC_E,  CC_E,  VCC_U|VCC_P); break;
+  case IR_LT: case IR_GE: case IR_LE: case IR_GT:
+  case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT:
+  case IR_EQ: case IR_NE: case IR_ABC:
+    asm_comp(as, ir, asm_compmap[ir->o]);
+    break;
 
   case IR_RETF: asm_retf(as, ir); break;
 
@@ -3744,7 +4004,15 @@ static void asm_ir(ASMState *as, IRIns *ir)
   case IR_FPMATH: case IR_ATAN2: case IR_LDEXP:
     asm_fpmath(as, ir);
     break;
-  case IR_POWI: asm_powi(as, ir); break;
+  case IR_POWI:
+#if LJ_64 && LJ_HASFFI
+    if (!irt_isnum(ir->t))
+      asm_arith64(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
+					     IRCALL_lj_carith_powu64);
+    else
+#endif
+      asm_powi(as, ir);
+    break;
 
   /* Overflow-checking arithmetic ops. Note: don't use LEA here! */
   case IR_ADDOV: asm_intarith(as, ir, XOg_ADD); break;
@@ -3801,6 +4069,7 @@ static void asm_trace(ASMState *as)
 {
   for (as->curins--; as->curins > as->stopins; as->curins--) {
     IRIns *ir = IR(as->curins);
+    lua_assert(!(LJ_32 && irt_isint64(ir->t)));  /* Handled by SPLIT. */
     if (!ra_used(ir) && !ir_sideeff(ir) && (as->flags & JIT_F_OPT_DCE))
       continue;  /* Dead-code elimination can be soooo easy. */
     if (irt_isguard(ir->t))
@@ -3864,11 +4133,10 @@ static void asm_setup_regsp(ASMState *as, GCtrace *T)
     case IR_CALLN: case IR_CALLL: case IR_CALLS: {
       const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
 #if LJ_64
-      /* NYI: add stack slots for x64 calls with many args. */
       lua_assert(CCI_NARGS(ci) <= (LJ_ABI_WIN ? 4 : 6));
       ir->prev = REGSP_HINT(irt_isnum(ir->t) ? RID_FPRET : RID_RET);
 #else
-      /* NYI: not fastcall-aware, but doesn't matter (yet). */
+      lua_assert(!(ci->flags & CCI_FASTCALL) || CCI_NARGS(ci) <= 2);
       if (CCI_NARGS(ci) > (uint32_t)as->evenspill)  /* Leave room for args. */
 	as->evenspill = (int32_t)CCI_NARGS(ci);
       ir->prev = REGSP_HINT(RID_RET);
@@ -3878,6 +4146,12 @@ static void asm_setup_regsp(ASMState *as, GCtrace *T)
 		      (RSET_SCRATCH & ~RSET_FPR) : RSET_SCRATCH;
       continue;
       }
+#if LJ_32 && LJ_HASFFI
+    case IR_HIOP:
+      if ((ir-1)->o == IR_CALLN)
+	ir->prev = REGSP_HINT(RID_RETHI);
+      break;
+#endif
     /* C calls evict all scratch regs and return results in RID_RET. */
     case IR_SNEW: case IR_NEWREF:
 #if !LJ_64
@@ -3894,6 +4168,14 @@ static void asm_setup_regsp(ASMState *as, GCtrace *T)
 	as->modset = RSET_SCRATCH;
       break;
     case IR_POWI:
+#if LJ_64 && LJ_HASFFI
+      if (!irt_isnum(ir->t)) {
+	ir->prev = REGSP_HINT(RID_RET);
+	if (inloop)
+	  as->modset |= (RSET_SCRATCH & RSET_GPR);
+	continue;
+      }
+#endif
       ir->prev = REGSP_HINT(RID_XMM0);
       if (inloop)
 	as->modset |= RSET_RANGE(RID_XMM0, RID_XMM1+1)|RID2RSET(RID_EAX);

src/lj_carith.c

@@ -230,6 +230,14 @@ int lj_carith_op(lua_State *L, MMS mm)
 
 /* -- 64 bit integer arithmetic helpers ----------------------------------- */
 
+#if LJ_32
+/* Signed/unsigned 64 bit multiply. */
+int64_t lj_carith_mul64(int64_t a, int64_t b)
+{
+  return a * b;
+}
+#endif
+
 /* Unsigned 64 bit x^k. */
 uint64_t lj_carith_powu64(uint64_t x, uint64_t k)
 {

src/lj_carith.h

@@ -12,6 +12,9 @@
 
 LJ_FUNC int lj_carith_op(lua_State *L, MMS mm);
 
+#if LJ_32
+LJ_FUNC int64_t lj_carith_mul64(int64_t x, int64_t k);
+#endif
 LJ_FUNC uint64_t lj_carith_powu64(uint64_t x, uint64_t k);
 LJ_FUNC int64_t lj_carith_powi64(int64_t x, int64_t k);
 

src/lj_crecord.c

@@ -189,6 +189,7 @@ static void crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
       sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, 0);
 #endif
   xstore:
+    if (dt == IRT_I64 || dt == IRT_U64) lj_needsplit(J);
     emitir(IRT(IR_XSTORE, dt), dp, sp);
     break;
   case CCX(I, C):
@@ -311,6 +312,7 @@ static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp)
       TRef ptr = emitir(IRT(IR_ADD, IRT_PTR), dp,
 			lj_ir_kintp(J, sizeof(GCcdata)));
       emitir(IRT(IR_XSTORE, t), ptr, tr);
+      lj_needsplit(J);
       return dp;
     } else if ((sinfo & CTF_BOOL)) {
       /* Assume not equal to zero. Fixup and emit pending guard later. */
@@ -406,7 +408,10 @@ static void crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, TValue *sval)
     if (ctype_isenum(s->info)) s = ctype_child(cts, s);
     if (ctype_isnum(s->info)) {  /* Load number value. */
       IRType t = crec_ct2irt(s);
-      if (t != IRT_CDATA) sp = emitir(IRT(IR_XLOAD, t), sp, 0);
+      if (t != IRT_CDATA) {
+	sp = emitir(IRT(IR_XLOAD, t), sp, 0);
+	if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
+      }
     }
     goto doconv;
   }
@@ -499,8 +504,10 @@ void LJ_FASTCALL recff_cdata_index(jit_State *J, RecordFFData *rd)
     if (ctype_isinteger(ctk->info) && (t = crec_ct2irt(ctk)) != IRT_CDATA) {
       idx = emitir(IRT(IR_ADD, IRT_PTR), idx, lj_ir_kintp(J, sizeof(GCcdata)));
       idx = emitir(IRT(IR_XLOAD, t), idx, 0);
-      if (!LJ_64 && (t == IRT_I64 || t == IRT_U64))
+      if (!LJ_64 && (t == IRT_I64 || t == IRT_U64)) {
 	idx = emitconv(idx, IRT_INT, t, 0);
+	lj_needsplit(J);
+      }
       goto integer_key;
     }
   } else if (tref_isstr(idx)) {
@@ -664,6 +671,7 @@ static TRef crec_arith_int64(jit_State *J, TRef *sp, CType **s, MMS mm)
     CTypeID id;
     TRef tr, dp, ptr;
     MSize i;
+    lj_needsplit(J);
     if (((s[0]->info & CTF_UNSIGNED) && s[0]->size == 8) ||
 	((s[1]->info & CTF_UNSIGNED) && s[1]->size == 8)) {
       dt = IRT_U64; id = CTID_UINT64;
@@ -691,9 +699,6 @@ static TRef crec_arith_int64(jit_State *J, TRef *sp, CType **s, MMS mm)
       lj_ir_set(J, IRTG(op, dt), sp[0], sp[1]);
       J->postproc = LJ_POST_FIXGUARD;
       return TREF_TRUE;
-    } else if (mm == MM_pow) {
-      tr = lj_ir_call(J, dt == IRT_I64 ? IRCALL_lj_carith_powi64 :
-					 IRCALL_lj_carith_powu64, sp[0], sp[1]);
     } else {
       if (mm == MM_div || mm == MM_mod)
 	return 0;  /* NYI: integer div, mod. */
@@ -754,10 +759,11 @@ static TRef crec_arith_ptr(jit_State *J, TRef *sp, CType **s, MMS mm)
       tr = emitconv(tr, IRT_INTP, IRT_INT,
 		    ((t - IRT_I8) & 1) ? 0 : IRCONV_SEXT);
 #else
-    if (!tref_typerange(sp[1], IRT_I8, IRT_U32))
+    if (!tref_typerange(sp[1], IRT_I8, IRT_U32)) {
       tr = emitconv(tr, IRT_INTP, t,
 		    (t == IRT_NUM || t == IRT_FLOAT) ?
 		    IRCONV_TRUNC|IRCONV_ANY : 0);
+    }
 #endif
     tr = emitir(IRT(IR_MUL, IRT_INTP), tr, lj_ir_kintp(J, sz));
     tr = emitir(IRT(IR_ADD, IRT_PTR), sp[0], tr);
@@ -790,6 +796,7 @@ void LJ_FASTCALL recff_cdata_arith(jit_State *J, RecordFFData *rd)
       if (ctype_isnum(ct->info)) {
 	IRType t = crec_ct2irt(ct);
 	if (t == IRT_CDATA) goto err_type;
+	if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
 	tr = emitir(IRT(IR_XLOAD, t), tr, 0);
       } else if (!(ctype_isptr(ct->info) || ctype_isrefarray(ct->info))) {
 	goto err_type;
@@ -842,6 +849,7 @@ void LJ_FASTCALL lj_crecord_tonumber(jit_State *J, RecordFFData *rd)
     IRType t = crec_ct2irt(s);
     if (t != IRT_CDATA) {
       TRef tr = emitir(IRT(IR_XLOAD, t), sp, 0);  /* Load number value. */
+      if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
       if (t == IRT_FLOAT || t == IRT_U32 || t == IRT_I64 || t == IRT_U64)
 	tr = emitconv(tr, IRT_NUM, t, 0);
       J->base[0] = tr;

src/lj_ir.h

@@ -33,6 +33,7 @@
   /* Miscellaneous ops. */ \
   _(NOP,	N , ___, ___) \
   _(BASE,	N , lit, lit) \
+  _(HIOP,	S , ref, ref) \
   _(LOOP,	S , ___, ___) \
   _(PHI,	S , ref, ref) \
   _(RENAME,	S , ref, lit) \
@@ -212,8 +213,9 @@ IRFLDEF(FLENUM)
 /* CONV mode, stored in op2. */
 #define IRCONV_SRCMASK		0x001f	/* Source IRType. */
 #define IRCONV_DSTMASK		0x03e0	/* Dest. IRType (also in ir->t). */
-#define IRCONV_NUM_INT		((IRT_NUM<<5)|IRT_INT)
-#define IRCONV_INT_NUM		((IRT_INT<<5)|IRT_NUM)
+#define IRCONV_DSH		5
+#define IRCONV_NUM_INT		((IRT_NUM<<IRCONV_DSH)|IRT_INT)
+#define IRCONV_INT_NUM		((IRT_INT<<IRCONV_DSH)|IRT_NUM)
 #define IRCONV_TRUNC		0x0400	/* Truncate number to integer. */
 #define IRCONV_SEXT		0x0800	/* Sign-extend integer to integer. */
 #define IRCONV_MODEMASK		0x0fff
@@ -251,13 +253,21 @@ typedef struct CCallInfo {
 #define CCI_CASTU64		0x0200	/* Cast u64 result to number. */
 #define CCI_NOFPRCLOBBER	0x0400	/* Does not clobber any FPRs. */
 #define CCI_FASTCALL		0x0800	/* Fastcall convention. */
-#define CCI_STACK64		0x1000	/* Needs 64 bits per argument. */
 
 /* Function definitions for CALL* instructions. */
 #if LJ_HASFFI
+#if LJ_32
+#define ARG2_64		4	/* Treat as 4 32 bit arguments. */
+#define IRCALLDEF_FFI32(_) \
+  _(lj_carith_mul64,	ARG2_64,   N, I64, CCI_NOFPRCLOBBER)
+#else
+#define ARG2_64		2
+#define IRCALLDEF_FFI32(_)
+#endif
 #define IRCALLDEF_FFI(_) \
-  _(lj_carith_powi64,	2,   N, I64, CCI_STACK64|CCI_NOFPRCLOBBER) \
-  _(lj_carith_powu64,	2,   N, U64, CCI_STACK64|CCI_NOFPRCLOBBER)
+  IRCALLDEF_FFI32(_) \
+  _(lj_carith_powi64,	ARG2_64,   N, I64, CCI_NOFPRCLOBBER) \
+  _(lj_carith_powu64,	ARG2_64,   N, U64, CCI_NOFPRCLOBBER)
 #else
 #define IRCALLDEF_FFI(_)
 #endif
@@ -402,6 +412,7 @@ typedef struct IRType1 { uint8_t irt; } IRType1;
 #define irt_isinteger(t)	(irt_typerange((t), IRT_I8, IRT_INT))
 #define irt_isgcv(t)		(irt_typerange((t), IRT_STR, IRT_UDATA))
 #define irt_isaddr(t)		(irt_typerange((t), IRT_LIGHTUD, IRT_UDATA))
+#define irt_isint64(t)		(irt_typerange((t), IRT_I64, IRT_U64))
 
 #if LJ_64
 #define IRT_IS64 \

src/lj_iropt.h

@@ -141,6 +141,12 @@ LJ_FUNC IRType lj_opt_narrow_forl(cTValue *forbase);
 /* Optimization passes. */
 LJ_FUNC void lj_opt_dce(jit_State *J);
 LJ_FUNC int lj_opt_loop(jit_State *J);
+#if LJ_HASFFI && LJ_32
+LJ_FUNC void lj_opt_split(jit_State *J);
+#else
+#define lj_opt_split(J)		UNUSED(J)
+#endif
+
 #endif
 
 #endif

src/lj_jit.h

@@ -240,6 +240,15 @@ enum {
 #define LJ_KSIMD(J, n) \
   ((TValue *)(((intptr_t)&J->ksimd[2*(n)] + 15) & ~(intptr_t)15))
 
+/* Set/reset flag to activate the SPLIT pass for the current trace. */
+#if LJ_32 && LJ_HASFFI
+#define lj_needsplit(J)		(J->needsplit = 1)
+#define lj_resetsplit(J)	(J->needsplit = 0)
+#else
+#define lj_needsplit(J)		UNUSED(J)
+#define lj_resetsplit(J)	UNUSED(J)
+#endif
+
 /* Fold state is used to fold instructions on-the-fly. */
 typedef struct FoldState {
   IRIns ins;		/* Currently emitted instruction. */
@@ -293,6 +302,9 @@ typedef struct jit_State {
   MSize sizesnapmap;	/* Size of temp. snapshot map buffer. */
 
   PostProc postproc;	/* Required post-processing after execution. */
+#if LJ_32 && LJ_HASFFI
+  int needsplit;	/* Need SPLIT pass. */
+#endif
 
   GCRef *trace;		/* Array of traces. */
   TraceNo freetrace;	/* Start of scan for next free trace. */

src/lj_opt_fold.c

@@ -538,6 +538,13 @@ LJFOLDF(kfold_conv_knum_int_num)
   }
 }
 
+LJFOLD(CONV KNUM IRCONV_U32_NUM)
+LJFOLDF(kfold_conv_knum_u32_num)
+{
+  lua_assert((fins->op2 & IRCONV_TRUNC));
+  return INTFOLD((int32_t)(uint32_t)knumleft);
+}
+
 LJFOLD(CONV KNUM IRCONV_I64_NUM)
 LJFOLDF(kfold_conv_knum_i64_num)
 {
@@ -805,6 +812,7 @@ LJFOLDF(simplify_conv_u32_num)
 }
 
 LJFOLD(CONV CONV IRCONV_I64_NUM)  /* _INT or _U32*/
+LJFOLD(CONV CONV IRCONV_U64_NUM)  /* _INT or _U32*/
 LJFOLDF(simplify_conv_i64_num)
 {
   PHIBARRIER(fleft);
@@ -826,23 +834,6 @@ LJFOLDF(simplify_conv_i64_num)
   return NEXTFOLD;
 }
 
-LJFOLD(CONV CONV IRCONV_U64_NUM)  /* _U32*/
-LJFOLDF(simplify_conv_u64_num)
-{
-  PHIBARRIER(fleft);
-  if ((fleft->op2 & IRCONV_SRCMASK) == IRT_U32) {
-#if LJ_TARGET_X64
-    return fleft->op1;
-#else
-    /* Reduce to a zero-extension. */
-    fins->op1 = fleft->op1;
-    fins->op2 = (IRT_U64<<5)|IRT_U32;
-    return RETRYFOLD;
-#endif
-  }
-  return NEXTFOLD;
-}
-
 /* Shortcut TOBIT + IRT_NUM <- IRT_INT/IRT_U32 conversion. */
 LJFOLD(TOBIT CONV KNUM)
 LJFOLDF(simplify_tobit_conv)

src/lj_opt_split.c

@@ -0,0 +1,343 @@
+/*
+** SPLIT: Split 64 bit IR instructions into 32 bit IR instructions.
+** Copyright (C) 2005-2011 Mike Pall. See Copyright Notice in luajit.h
+*/
+
+#define lj_opt_split_c
+#define LUA_CORE
+
+#include "lj_obj.h"
+
+#if LJ_HASJIT && LJ_HASFFI && LJ_32
+
+#include "lj_err.h"
+#include "lj_str.h"
+#include "lj_ir.h"
+#include "lj_jit.h"
+#include "lj_iropt.h"
+#include "lj_vm.h"
+
+/* SPLIT pass:
+**
+** This pass splits up 64 bit IR instructions into multiple 32 bit IR
+** instructions. It's only active for 32 bit CPUs which lack native 64 bit
+** operations. The FFI is currently the only emitter for 64 bit
+** instructions, so this pass is disabled if the FFI is disabled.
+**
+** Splitting the IR in a separate pass keeps each 32 bit IR assembler
+** backend simple. Only a small amount of extra functionality needs to be
+** implemented. This is much easier than adding support for allocating
+** register pairs to each backend (believe me, I tried). A few simple, but
+** important optimizations can be performed by the SPLIT pass, which would
+** be tedious to do in the backend.
+**
+** The basic idea is to replace each 64 bit IR instruction with its 32 bit
+** equivalent plus an extra HIOP instruction. The splitted IR is not passed
+** through FOLD or any other optimizations, so each HIOP is guaranteed to
+** immediately follow it's counterpart. The actual functionality of HIOP is
+** inferred from the previous instruction.
+**
+** The operands of HIOP hold the hiword input references. The output of HIOP
+** is the hiword output reference, which is also used to hold the hiword
+** register or spill slot information. The register allocator treats this
+** instruction independent of any other instruction, which improves code
+** quality compared to using fixed register pairs.
+**
+** It's easier to split up some instructions into two regular 32 bit
+** instructions. E.g. XLOAD is split up into two XLOADs with two different
+** addresses. Obviously 64 bit constants need to be split up into two 32 bit
+** constants, too. Some hiword instructions can be entirely omitted, e.g.
+** when zero-extending a 32 bit value to 64 bits.
+**
+** Here's the IR and x64 machine code for 'x.b = x.a + 1' for a struct with
+** two int64_t fields:
+**
+** 0100    p32 ADD    base  +8
+** 0101    i64 XLOAD  0100
+** 0102    i64 ADD    0101  +1
+** 0103    p32 ADD    base  +16
+** 0104    i64 XSTORE 0103  0102
+**
+**         mov rax, [esi+0x8]
+**         add rax, +0x01
+**         mov [esi+0x10], rax
+**
+** Here's the transformed IR and the x86 machine code after the SPLIT pass:
+**
+** 0100    p32 ADD    base  +8
+** 0101    int XLOAD  0100
+** 0102    p32 ADD    base  +12
+** 0103    int XLOAD  0102
+** 0104    int ADD    0101  +1
+** 0105    int HIOP   0103  +0
+** 0106    p32 ADD    base  +16
+** 0107    int XSTORE 0106  0104
+** 0108    p32 ADD    base  +20
+** 0109    int XSTORE 0108  0105
+**
+**         mov eax, [esi+0x8]
+**         mov ecx, [esi+0xc]
+**         add eax, +0x01
+**         adc ecx, +0x00
+**         mov [esi+0x10], eax
+**         mov [esi+0x14], ecx
+**
+** You may notice the reassociated hiword address computation, which is
+** later fused into the mov operands by the assembler.
+*/
+
+/* Some local macros to save typing. Undef'd at the end. */
+#define IR(ref)		(&J->cur.ir[(ref)])
+
+/* Directly emit the transformed IR without updating chains etc. */
+static IRRef split_emit(jit_State *J, uint16_t ot, IRRef1 op1, IRRef1 op2)
+{
+  IRRef nref = lj_ir_nextins(J);
+  IRIns *ir = IR(nref);
+  ir->ot = ot;
+  ir->op1 = op1;
+  ir->op2 = op2;
+  return nref;
+}
+
+/* Emit a CALLN with two split 64 bit arguments. */
+static IRRef split_call64(jit_State *J, IRRef1 *hisubst, IRIns *oir,
+			  IRIns *ir, IRCallID id)
+{
+  IRRef tmp, op1 = ir->op1, op2 = ir->op2;
+  J->cur.nins--;
+#if LJ_LE
+  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), oir[op1].prev, hisubst[op1]);
+  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, oir[op2].prev);
+  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, hisubst[op2]);
+#else
+  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
+  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, hisubst[op2]);
+  tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, oir[op2].prev);
+#endif
+  ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
+  return split_emit(J, IRTI(IR_HIOP), tmp, tmp);
+}
+
+/* Get a pointer to the other 32 bit word (LE: hiword, BE: loword). */
+static IRRef split_ptr(jit_State *J, IRRef ref)
+{
+  IRIns *ir = IR(ref);
+  int32_t ofs = 4;
+  if (ir->o == IR_ADD && irref_isk(ir->op2)) {  /* Reassociate address. */
+    ofs += IR(ir->op2)->i;
+    ref = ir->op1;
+    if (ofs == 0) return ref;
+  }
+  return split_emit(J, IRTI(IR_ADD), ref, lj_ir_kint(J, ofs));
+}
+
+/* Transform the old IR to the new IR. */
+static void split_ir(jit_State *J)
+{
+  IRRef nins = J->cur.nins, nk = J->cur.nk;
+  MSize irlen = nins - nk;
+  MSize need = (irlen+1)*(sizeof(IRIns) + sizeof(IRRef1));
+  IRIns *oir = (IRIns *)lj_str_needbuf(J->L, &G(J->L)->tmpbuf, need);
+  IRRef1 *hisubst;
+  IRRef ref;
+
+  /* Copy old IR to buffer. */
+  memcpy(oir, IR(nk), irlen*sizeof(IRIns));
+  /* Bias hiword substitution table and old IR. Loword kept in field prev. */
+  hisubst = (IRRef1 *)&oir[irlen] - nk;
+  oir -= nk;
+
+  /* Remove all IR instructions, but retain IR constants. */
+  J->cur.nins = REF_FIRST;
+
+  /* Process constants and fixed references. */
+  for (ref = nk; ref <= REF_BASE; ref++) {
+    IRIns *ir = &oir[ref];
+    if (ir->o == IR_KINT64) {  /* Split up 64 bit constant. */
+      TValue tv = *ir_k64(ir);
+      ir->prev = lj_ir_kint(J, (int32_t)tv.u32.lo);
+      hisubst[ref] = lj_ir_kint(J, (int32_t)tv.u32.hi);
+    } else {
+      ir->prev = (IRRef1)ref;  /* Identity substitution for loword. */
+    }
+  }
+
+  /* Process old IR instructions. */
+  for (ref = REF_FIRST; ref < nins; ref++) {
+    IRIns *ir = &oir[ref];
+    IRRef nref = lj_ir_nextins(J);
+    IRIns *nir = IR(nref);
+
+    /* Copy-substitute old instruction to new instruction. */
+    nir->op1 = ir->op1 < nk ? ir->op1 : oir[ir->op1].prev;
+    nir->op2 = ir->op2 < nk ? ir->op2 : oir[ir->op2].prev;
+    ir->prev = nref;  /* Loword substitution. */
+    nir->o = ir->o;
+    nir->t.irt = ir->t.irt & ~(IRT_MARK|IRT_ISPHI);
+
+    /* Split 64 bit instructions. */
+    if (irt_isint64(ir->t)) {
+      IRRef hi = hisubst[ir->op1];
+      nir->t.irt = IRT_INT | (nir->t.irt & IRT_GUARD);  /* Turn into INT op. */
+      switch (ir->o) {
+      case IR_ADD:
+      case IR_SUB:
+	/* Use plain op for hiword if loword cannot produce a carry/borrow. */
+	if (irref_isk(nir->op2) && IR(nir->op2)->i == 0) {
+	  ir->prev = nir->op1;  /* Pass through loword. */
+	  nir->op1 = hi; nir->op2 = hisubst[ir->op2];
+	  hi = nref;
+	  break;
+	}
+	/* fallthrough */
+      case IR_NEG:
+	hi = split_emit(J, IRTI(IR_HIOP), hi, hisubst[ir->op2]);
+	break;
+      case IR_MUL:
+	hi = split_call64(J, hisubst, oir, ir, IRCALL_lj_carith_mul64);
+	break;
+      case IR_POWI:
+	hi = split_call64(J, hisubst, oir, ir,
+			  irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
+					     IRCALL_lj_carith_powu64);
+	break;
+      case IR_XLOAD:
+	hi = split_emit(J, IRTI(IR_XLOAD), split_ptr(J, nir->op1), ir->op2);
+#if LJ_BE
+	ir->prev = hi; hi = nref;
+#endif
+	break;
+      case IR_XSTORE:
+#if LJ_LE
+	hi = hisubst[ir->op2];
+#else
+	hi = nir->op2; nir->op2 = hisubst[ir->op2];
+#endif
+	split_emit(J, IRTI(IR_XSTORE), split_ptr(J, nir->op1), hi);
+	continue;
+      case IR_CONV: {  /* Conversion to 64 bit integer. Others handled below. */
+	IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
+	if (st == IRT_NUM || st == IRT_FLOAT) {  /* FP to 64 bit int conv. */
+	  hi = split_emit(J, IRTI(IR_HIOP), nir->op1, nref);
+	} else if (st == IRT_I64 || st == IRT_U64) {  /* 64/64 bit cast. */
+	  /* Drop cast, since assembler doesn't care. */
+	  hisubst[ref] = hi;
+	  goto fwdlo;
+	} else if ((ir->op2 & IRCONV_SEXT)) {  /* Sign-extend to 64 bit. */
+	  IRRef k31 = lj_ir_kint(J, 31);
+	  nir = IR(nref);  /* May have been reallocated. */
+	  ir->prev = nir->op1;  /* Pass through loword. */
+	  nir->o = IR_BSAR;  /* hi = bsar(lo, 31). */
+	  nir->op2 = k31;
+	  hi = nref;
+	} else {  /* Zero-extend to 64 bit. */
+	  hisubst[ref] = lj_ir_kint(J, 0);
+	  goto fwdlo;
+	}
+	break;
+	}
+      case IR_PHI: {
+	IRRef hi2;
+	if ((irref_isk(nir->op1) && irref_isk(nir->op2)) ||
+	    nir->op1 == nir->op2)
+	  J->cur.nins--;  /* Drop useless PHIs. */
+	hi2 = hisubst[ir->op2];
+	if (!((irref_isk(hi) && irref_isk(hi2)) || hi == hi2))
+	  split_emit(J, IRTI(IR_PHI), hi, hi2);
+	continue;
+	}
+      default:
+	lua_assert(ir->o <= IR_NE);
+	split_emit(J, IRTGI(IR_HIOP), hi, hisubst[ir->op2]);  /* Comparisons. */
+	continue;
+      }
+      hisubst[ref] = hi;  /* Store hiword substitution. */
+    } else if (ir->o == IR_CONV) {  /* See above, too. */
+      IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
+      if (st == IRT_I64 || st == IRT_U64) {  /* Conversion from 64 bit int. */
+	if (irt_isfp(ir->t)) {  /* 64 bit integer to FP conversion. */
+	  ir->prev = split_emit(J, IRT(IR_HIOP, irt_type(ir->t)),
+				hisubst[ir->op1], nref);
+	} else {  /* Truncate to lower 32 bits. */
+	fwdlo:
+	  ir->prev = nir->op1;  /* Forward loword. */
+	  /* Replace with NOP to avoid messing up the snapshot logic. */
+	  nir->ot = IRT(IR_NOP, IRT_NIL);
+	  nir->op1 = nir->op2 = 0;
+	}
+      }
+    } else if (ir->o == IR_LOOP) {
+      J->loopref = nref;  /* Needed by assembler. */
+    }
+  }
+
+  /* Add PHI marks. */
+  for (ref = J->cur.nins-1; ref >= REF_FIRST; ref--) {
+    IRIns *ir = IR(ref);
+    if (ir->o != IR_PHI) break;
+    if (!irref_isk(ir->op1)) irt_setphi(IR(ir->op1)->t);
+    if (ir->op2 > J->loopref) irt_setphi(IR(ir->op2)->t);
+  }
+
+  /* Substitute snapshot maps. */
+  oir[nins].prev = J->cur.nins;  /* Substitution for last snapshot. */
+  {
+    SnapNo i, nsnap = J->cur.nsnap;
+    for (i = 0; i < nsnap; i++) {
+      SnapShot *snap = &J->cur.snap[i];
+      SnapEntry *map = &J->cur.snapmap[snap->mapofs];
+      MSize n, nent = snap->nent;
+      snap->ref = oir[snap->ref].prev;
+      for (n = 0; n < nent; n++) {
+	SnapEntry sn = map[n];
+	map[n] = ((sn & 0xffff0000) | oir[snap_ref(sn)].prev);
+      }
+    }
+  }
+}
+
+/* Protected callback for split pass. */
+static TValue *cpsplit(lua_State *L, lua_CFunction dummy, void *ud)
+{
+  jit_State *J = (jit_State *)ud;
+  split_ir(J);
+  UNUSED(L); UNUSED(dummy);
+  return NULL;
+}
+
+#ifdef LUA_USE_ASSERT
+/* Slow, but sure way to check whether a SPLIT pass is needed. */
+static int split_needsplit(jit_State *J)
+{
+  IRIns *ir, *irend;
+  IRRef ref;
+  for (ir = IR(REF_FIRST), irend = IR(J->cur.nins); ir < irend; ir++)
+    if (irt_isint64(ir->t))
+      return 1;
+  for (ref = J->chain[IR_CONV]; ref; ref = IR(ref)->prev)
+    if ((IR(ref)->op2 & IRCONV_SRCMASK) == IRT_I64 ||
+	(IR(ref)->op2 & IRCONV_SRCMASK) == IRT_U64)
+      return 1;
+  return 0;  /* Nope. */
+}
+#endif
+
+/* SPLIT pass. */
+void lj_opt_split(jit_State *J)
+{
+  lua_assert(J->needsplit >= split_needsplit(J));  /* Verify flag. */
+  if (J->needsplit) {
+    int errcode = lj_vm_cpcall(J->L, NULL, J, cpsplit);
+    if (errcode) {
+      /* Completely reset the trace to avoid inconsistent dump on abort. */
+      J->cur.nins = J->cur.nk = REF_BASE;
+      J->cur.nsnap = 0;
+      lj_err_throw(J->L, errcode);  /* Propagate errors. */
+    }
+  }
+}
+
+#undef IR
+
+#endif

src/lj_target_x86.h

@@ -193,6 +193,7 @@ typedef enum {
   XI_FLD1 =	0xe8d9,
   XI_FLDLG2 =	0xecd9,
   XI_FLDLN2 =	0xedd9,
+  XI_FDUP =	0xc0d9,  /* Really fld st0. */
   XI_FPOP =	0xd8dd,  /* Really fstp st0. */
   XI_FPOP1 =	0xd9dd,  /* Really fstp st1. */
   XI_FRNDINT =	0xfcd9,
@@ -263,10 +264,17 @@ typedef enum {
   XO_MOVD =	XO_660f(6e),
   XO_MOVDto =	XO_660f(7e),
 
+  XO_FLDd =	XO_(d9), XOg_FLDd = 0,
   XO_FLDq =	XO_(dd), XOg_FLDq = 0,
   XO_FILDd =	XO_(db), XOg_FILDd = 0,
+  XO_FILDq =	XO_(df), XOg_FILDq = 5,
+  XO_FSTPd =	XO_(d9), XOg_FSTPd = 3,
   XO_FSTPq =	XO_(dd), XOg_FSTPq = 3,
   XO_FISTPq =	XO_(df), XOg_FISTPq = 7,
+  XO_FISTTPq =	XO_(dd), XOg_FISTTPq = 1,
+  XO_FADDq =	XO_(dc), XOg_FADDq = 0,
+  XO_FLDCW =	XO_(d9), XOg_FLDCW = 5,
+  XO_FNSTCW =	XO_(d9), XOg_FNSTCW = 7
 } x86Op;
 
 /* x86 opcode groups. */
@@ -278,6 +286,7 @@ typedef uint32_t x86Group;
 #define XG_TOXOi8(xg)	((x86Op)(0x000000fe + (((xg)<<8) & 0xff000000)))
 
 #define XO_ARITH(a)	((x86Op)(0x030000fe + ((a)<<27)))
+#define XO_ARITHw(a)	((x86Op)(0x036600fd + ((a)<<27)))
 
 typedef enum {
   XOg_ADD, XOg_OR, XOg_ADC, XOg_SBB, XOg_AND, XOg_SUB, XOg_XOR, XOg_CMP,

src/lj_trace.c

@@ -394,6 +394,7 @@ static void trace_start(jit_State *J)
   J->bcskip = 0;
   J->guardemit.irt = 0;
   J->postproc = LJ_POST_NONE;
+  lj_resetsplit(J);
   setgcref(J->cur.startpt, obj2gco(J->pt));
 
   L = J->L;
@@ -592,6 +593,7 @@ static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud)
 	}
 	J->loopref = J->chain[IR_LOOP];  /* Needed by assembler. */
       }
+      lj_opt_split(J);
       J->state = LJ_TRACE_ASM;
       break;
 

src/ljamalg.c

@@ -58,6 +58,7 @@
 #include "lj_opt_narrow.c"
 #include "lj_opt_dce.c"
 #include "lj_opt_loop.c"
+#include "lj_opt_split.c"
 #include "lj_mcode.c"
 #include "lj_snap.c"
 #include "lj_record.c"