FFI: Add 64 bit integer comparisons and pointer comparisons.

src/buildvm_x86.dasc

@@ -1042,6 +1042,19 @@ static void build_subroutines(BuildCtx *ctx, int cmov, int sse)
   |  call extern lj_meta_equal	// (lua_State *L, GCobj *o1, *o2, int ne)
   |  // 0/1 or TValue * (metamethod) returned in eax (RC).
   |  jmp <3
+  |
+  |->vmeta_equal_cd:
+#if LJ_HASFFI
+  |  sub PC, 4
+  |  mov L:RB, SAVE_L
+  |  mov L:RB->base, BASE
+  |  mov FCARG1, L:RB
+  |  mov FCARG2, dword [PC-4]
+  |  mov SAVE_PC, PC
+  |  call extern lj_meta_equal_cd@8	// (lua_State *L, BCIns op)
+  |  // 0/1 or TValue * (metamethod) returned in eax (RC).
+  |  jmp <3
+#endif
   |
   |//-- Arithmetic metamethods ---------------------------------------------
   |
@@ -3593,7 +3606,13 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
       |  movzx RD, PC_RD
       |  branchPC RD
       |2:				// NE: Fallthrough to next instruction.
+      if (!LJ_HASFFI) {
+	|3:
+      }
     } else {
+      if (!LJ_HASFFI) {
+	|3:
+      }
       |2:				// NE: Branch to the target.
       |  movzx RD, PC_RD
       |  branchPC RD
@@ -3603,6 +3622,10 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     |
     if (op == BC_ISEQV || op == BC_ISNEV) {
       |5:  // Either or both types are not numbers.
+      if (LJ_HASFFI) {
+	|  cmp RB, LJ_TCDATA; je ->vmeta_equal_cd
+	|  checktp RA, LJ_TCDATA; je ->vmeta_equal_cd
+      }
       |  checktp RA, RB			// Compare types.
       |  jne <2				// Not the same type?
       |  cmp RB, LJ_TISPRI
@@ -3629,13 +3652,18 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 	|  mov RB, 1			// ne = 1
       }
       |  jmp ->vmeta_equal		// Handle __eq metamethod.
+    } else if (LJ_HASFFI) {
+      |3:
+      |  cmp RB, LJ_TCDATA; jne <2
+      |  jmp ->vmeta_equal_cd
     }
     break;
   case BC_ISEQS: case BC_ISNES:
     vk = op == BC_ISEQS;
     |  ins_AND	// RA = src, RD = str const, JMP with RD = target
+    |  mov RB, [BASE+RA*8+4]
     |  add PC, 4
-    |  checkstr RA, >2
+    |  cmp RB, LJ_TSTR; jne >3
     |  mov RA, [BASE+RA*8]
     |  cmp RA, [KBASE+RD*4]
   iseqne_test:
@@ -3648,8 +3676,9 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
   case BC_ISEQN: case BC_ISNEN:
     vk = op == BC_ISEQN;
     |  ins_AD	// RA = src, RD = num const, JMP with RD = target
+    |  mov RB, [BASE+RA*8+4]
     |  add PC, 4
-    |  checknum RA, >2
+    |  cmp RB, LJ_TISNUM; ja >3
     if (sse) {
       |  movsd xmm0, qword [KBASE+RD*8]
       |  ucomisd xmm0, qword [BASE+RA*8]
@@ -3662,9 +3691,28 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
   case BC_ISEQP: case BC_ISNEP:
     vk = op == BC_ISEQP;
     |  ins_AND	// RA = src, RD = primitive type (~), JMP with RD = target
+    |  mov RB, [BASE+RA*8+4]
     |  add PC, 4
-    |  checktp RA, RD
-    goto iseqne_test;
+    |  cmp RB, RD
+    if (!LJ_HASFFI) goto iseqne_test;
+    if (vk) {
+      |  jne >3
+      |  movzx RD, PC_RD
+      |  branchPC RD
+      |2:
+      |  ins_next
+      |3:
+      |  cmp RB, LJ_TCDATA; jne <2
+      |  jmp ->vmeta_equal_cd
+    } else {
+      |  je >2
+      |  cmp RB, LJ_TCDATA; je ->vmeta_equal_cd
+      |  movzx RD, PC_RD
+      |  branchPC RD
+      |2:
+      |  ins_next
+    }
+    break;
 
   /* -- Unary test and copy ops ------------------------------------------- */
 

src/lib_ffi.c

@@ -85,9 +85,10 @@ typedef struct FFIArith {
 } FFIArith;
 
 /* Check arguments for arithmetic metamethods. */
-static void ffi_checkarith(lua_State *L, CTState *cts, FFIArith *fa)
+static int ffi_checkarith(lua_State *L, CTState *cts, FFIArith *fa)
 {
   TValue *o = L->base;
+  int ok = 1;
   MSize i;
   if (o+1 >= L->top)
     lj_err_argt(L, 1, LUA_TCDATA);
@@ -105,10 +106,16 @@ static void ffi_checkarith(lua_State *L, CTState *cts, FFIArith *fa)
     } else if (tvisnum(o)) {
       fa->ct[i] = ctype_get(cts, CTID_DOUBLE);
       fa->p[i] = (uint8_t *)&o->n;
+    } else if (tvisnil(o)) {
+      fa->ct[i] = ctype_get(cts, CTID_P_VOID);
+      fa->p[i] = (uint8_t *)0;
     } else {
-      lj_err_optype(L, o, LJ_ERR_OPARITH);
+      fa->ct[i] = NULL;
+      fa->p[i] = NULL;
+      ok = 0;
     }
   }
+  return ok;
 }
 
 /* Pointer arithmetic. */
@@ -120,26 +127,38 @@ static int ffi_arith_ptr(lua_State *L, CTState *cts, FFIArith *fa, MMS mm)
   CTSize sz;
   CTypeID id;
   GCcdata *cd;
-  if (!(mm == MM_add || mm == MM_sub))
-    return 0;
   if (ctype_isptr(ctp->info) || ctype_isrefarray(ctp->info)) {
-    if (mm == MM_sub &&
+    if ((mm == MM_sub || mm == MM_eq || mm == MM_lt || mm == MM_le) &&
 	(ctype_isptr(fa->ct[1]->info) || ctype_isrefarray(fa->ct[1]->info))) {
-      /* Pointer difference. */
-      intptr_t diff;
+      uint8_t *pp2 = fa->p[1];
+      if (mm == MM_eq) {  /* Pointer equality. Incompatible pointers are ok. */
+	setboolV(L->top-1, (pp == pp2));
+	return 1;
+      }
       if (!lj_cconv_compatptr(cts, ctp, fa->ct[1], CCF_IGNQUAL))
-	lj_err_caller(L, LJ_ERR_FFI_INVTYPE);
-      sz = lj_ctype_size(cts, ctype_cid(ctp->info));  /* Element size. */
-      if (sz == 0 || sz == CTSIZE_INVALID)
-	lj_err_caller(L, LJ_ERR_FFI_INVSIZE);
-      diff = ((intptr_t)pp - (intptr_t)fa->p[1]) / (int32_t)sz;
-      /* All valid pointer differences on x64 are in (-2^47, +2^47),
-      ** which fits into a double without loss of precision.
-      */
-      setnumV(L->top-1, (lua_Number)diff);
-      return 1;
+	return 0;
+      if (mm == MM_sub) {  /* Pointer difference. */
+	intptr_t diff;
+	sz = lj_ctype_size(cts, ctype_cid(ctp->info));  /* Element size. */
+	if (sz == 0 || sz == CTSIZE_INVALID)
+	  return 0;
+	diff = ((intptr_t)pp - (intptr_t)pp2) / (int32_t)sz;
+	/* All valid pointer differences on x64 are in (-2^47, +2^47),
+	** which fits into a double without loss of precision.
+	*/
+	setnumV(L->top-1, (lua_Number)diff);
+	return 1;
+      } else if (mm == MM_lt) {  /* Pointer comparison (unsigned). */
+	setboolV(L->top-1, ((uintptr_t)pp < (uintptr_t)pp2));
+	return 1;
+      } else {
+	lua_assert(mm == MM_le);
+	setboolV(L->top-1, ((uintptr_t)pp <= (uintptr_t)pp2));
+	return 1;
+      }
     }
-    if (!ctype_isnum(fa->ct[1]->info)) return 0;
+    if (!((mm == MM_add || mm == MM_sub) &&
+	  ctype_isnum(fa->ct[1]->info))) return 0;
     lj_cconv_ct_ct(cts, ctype_get(cts, CTID_INT_PSZ), fa->ct[1],
 		   (uint8_t *)&idx, fa->p[1], 0);
     if (mm == MM_sub) idx = -idx;
@@ -155,7 +174,7 @@ static int ffi_arith_ptr(lua_State *L, CTState *cts, FFIArith *fa, MMS mm)
   }
   sz = lj_ctype_size(cts, ctype_cid(ctp->info));  /* Element size. */
   if (sz == CTSIZE_INVALID)
-    lj_err_caller(L, LJ_ERR_FFI_INVSIZE);
+    return 0;
   pp += idx*(int32_t)sz;  /* Compute pointer + index. */
   id = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(ctp->info)),
 		       CTSIZE_PTR);
@@ -180,7 +199,19 @@ static int ffi_arith_int64(lua_State *L, CTState *cts, FFIArith *fa, MMS mm)
     lj_cconv_ct_ct(cts, ct, fa->ct[0], (uint8_t *)&u0, fa->p[0], 0);
     if (mm != MM_unm)
       lj_cconv_ct_ct(cts, ct, fa->ct[1], (uint8_t *)&u1, fa->p[1], 0);
-    if ((mm == MM_div || mm == MM_mod)) {
+    switch (mm) {
+    case MM_eq:
+      setboolV(L->top-1, (u0 == u1));
+      return 1;
+    case MM_lt:
+      setboolV(L->top-1,
+	       id == CTID_INT64 ? ((int64_t)u0 < (int64_t)u1) : (u0 < u1));
+      return 1;
+    case MM_le:
+      setboolV(L->top-1,
+	       id == CTID_INT64 ? ((int64_t)u0 <= (int64_t)u1) : (u0 <= u1));
+      return 1;
+    case MM_div: case MM_mod:
       if (u1 == 0) {  /* Division by zero. */
 	if (u0 == 0)
 	  setnanV(L->top-1);
@@ -194,6 +225,8 @@ static int ffi_arith_int64(lua_State *L, CTState *cts, FFIArith *fa, MMS mm)
 	if (mm == MM_div) id = CTID_UINT64; else u0 = 0;
 	mm = MM_unm;  /* Result is 0x8000000000000000ULL or 0LL. */
       }
+      break;
+    default: break;
     }
     cd = lj_cdata_new(cts, id, 8);
     up = (uint64_t *)cdataptr(cd);
@@ -229,17 +262,27 @@ static int ffi_arith(lua_State *L)
 {
   CTState *cts = ctype_cts(L);
   FFIArith fa;
-  MMS mm = (MMS)(curr_func(L)->c.ffid - (int)FF_ffi_meta___add + (int)MM_add);
-  ffi_checkarith(L, cts, &fa);
-  if (!ffi_arith_int64(L, cts, &fa, mm) &&
-      !ffi_arith_ptr(L, cts, &fa, mm)) {
+  MMS mm = (MMS)(curr_func(L)->c.ffid - (int)FF_ffi_meta___eq + (int)MM_eq);
+  if (ffi_checkarith(L, cts, &fa)) {
+    if (ffi_arith_int64(L, cts, &fa, mm) || ffi_arith_ptr(L, cts, &fa, mm))
+      return 1;
+  }
+  /* NYI: per-cdata metamethods. */
+  {
     const char *repr[2];
     int i;
-    for (i = 0; i < 2; i++)
-      repr[i] = strdata(lj_ctype_repr(L, ctype_typeid(cts, fa.ct[i]), NULL));
-    lj_err_callerv(L, LJ_ERR_FFI_BADARITH, repr[0], repr[1]);
+    for (i = 0; i < 2; i++) {
+      if (fa.ct[i])
+	repr[i] = strdata(lj_ctype_repr(L, ctype_typeid(cts, fa.ct[i]), NULL));
+      else
+	repr[i] = typename(&L->base[i]);
+    }
+    lj_err_callerv(L, mm == MM_len ? LJ_ERR_FFI_BADLEN :
+		      mm == MM_concat ? LJ_ERR_FFI_BADCONCAT :
+		      mm < MM_add ? LJ_ERR_FFI_BADCOMP : LJ_ERR_FFI_BADARITH,
+		   repr[0], repr[1]);
   }
-  return 1;
+  return 0;  /* unreachable */
 }
 
 /* -- C type metamethods -------------------------------------------------- */
@@ -275,6 +318,32 @@ LJLIB_CF(ffi_meta___newindex)	LJLIB_REC(cdata_index 1)
   return 0;
 }
 
+/* The following functions must be in contiguous ORDER MM. */
+LJLIB_CF(ffi_meta___eq)
+{
+  return ffi_arith(L);
+}
+
+LJLIB_CF(ffi_meta___len)
+{
+  return ffi_arith(L);
+}
+
+LJLIB_CF(ffi_meta___lt)
+{
+  return ffi_arith(L);
+}
+
+LJLIB_CF(ffi_meta___le)
+{
+  return ffi_arith(L);
+}
+
+LJLIB_CF(ffi_meta___concat)
+{
+  return ffi_arith(L);
+}
+
 /* Forward declaration. */
 static int lj_cf_ffi_new(lua_State *L);
 
@@ -324,6 +393,7 @@ LJLIB_CF(ffi_meta___unm)	LJLIB_REC(cdata_arith MM_unm)
 {
   return ffi_arith(L);
 }
+/* End of contiguous ORDER MM. */
 
 LJLIB_CF(ffi_meta___tostring)
 {

src/lj_crecord.h

@@ -19,6 +19,7 @@ LJ_FUNC void LJ_FASTCALL lj_crecord_tonumber(jit_State *J, RecordFFData *rd);
 #else
 #define recff_cdata_index	recff_nyi
 #define recff_cdata_call	recff_nyi
+#define recff_cdata_arith	recff_nyi
 #define recff_ffi_new		recff_nyi
 #endif
 

src/lj_errmsg.h

@@ -147,7 +147,10 @@ ERRDEF(FFI_BADTAG,	"undeclared or implicit tag " LUA_QS)
 ERRDEF(FFI_REDEF,	"attempt to redefine " LUA_QS)
 ERRDEF(FFI_INITOV,	"too many initializers for " LUA_QS)
 ERRDEF(FFI_BADCONV,	"cannot convert " LUA_QS " to " LUA_QS)
+ERRDEF(FFI_BADLEN,	"attempt to get length of " LUA_QS)
+ERRDEF(FFI_BADCONCAT,	"attempt to concatenate " LUA_QS " and " LUA_QS)
 ERRDEF(FFI_BADARITH,	"attempt to perform arithmetic on " LUA_QS " and " LUA_QS)
+ERRDEF(FFI_BADCOMP,	"attempt to compare " LUA_QS " with " LUA_QS)
 ERRDEF(FFI_BADCALL,	LUA_QS " is not callable")
 ERRDEF(FFI_NUMARG,	"wrong number of arguments for function call")
 ERRDEF(FFI_BADMEMBER,	LUA_QS " has no member named " LUA_QS)

src/lj_meta.c

@@ -302,10 +302,48 @@ TValue *lj_meta_equal(lua_State *L, GCobj *o1, GCobj *o2, int ne)
   return cast(TValue *, (intptr_t)ne);
 }
 
+#if LJ_HASFFI
+TValue * LJ_FASTCALL lj_meta_equal_cd(lua_State *L, BCIns ins)
+{
+  ASMFunction cont = (bc_op(ins) & 1) ? lj_cont_condf : lj_cont_condt;
+  int op = (int)bc_op(ins) & ~1;
+  TValue tv;
+  cTValue *mo, *o2, *o1 = &L->base[bc_a(ins)];
+  if (op == BC_ISEQV) {
+    cTValue *o = &L->base[bc_d(ins)];
+    if (tviscdata(o1)) {
+      o2 = o;
+    } else {
+      o2 = o1; o1 = o;
+    }
+  } else if (op == BC_ISEQS) {
+    setstrV(L, &tv, gco2str(proto_kgc(curr_proto(L), ~(ptrdiff_t)bc_d(ins))));
+    o2 = &tv;
+  } else if (op == BC_ISEQN) {
+    o2 = &mref(curr_proto(L)->k, cTValue)[bc_d(ins)];
+  } else {
+    lua_assert(op == BC_ISEQP);
+    setitype(&tv, ~bc_d(ins));
+    o2 = &tv;
+  }
+  mo = lj_meta_lookup(L, o1, MM_eq);
+  if (LJ_LIKELY(!tvisnil(mo)))
+    return mmcall(L, cont, mo, o1, o2);
+  else
+    return cast(TValue *, (intptr_t)(bc_op(ins) & 1));
+}
+#endif
+
 /* Helper for ordered comparisons. String compare, __lt/__le metamethods. */
 TValue *lj_meta_comp(lua_State *L, cTValue *o1, cTValue *o2, int op)
 {
-  if (itype(o1) == itype(o2)) {  /* Never called with two numbers. */
+  if (LJ_HASFFI && (tviscdata(o1) || tviscdata(o2))) {
+    ASMFunction cont = (op & 1) ? lj_cont_condf : lj_cont_condt;
+    MMS mm = (op & 2) ? MM_le : MM_lt;
+    cTValue *mo = lj_meta_lookup(L, tviscdata(o1) ? o1 : o2, mm);
+    if (LJ_UNLIKELY(tvisnil(mo))) goto err;
+    return mmcall(L, cont, mo, o1, o2);
+  } else if (itype(o1) == itype(o2)) {  /* Never called with two numbers. */
     if (tvisstr(o1) && tvisstr(o2)) {
       int32_t res = lj_str_cmp(strV(o1), strV(o2));
       return cast(TValue *, (intptr_t)(((op&2) ? res <= 0 : res < 0) ^ (op&1)));

src/lj_meta.h

@@ -26,6 +26,7 @@ LJ_FUNCA TValue *lj_meta_arith(lua_State *L, TValue *ra, cTValue *rb,
 LJ_FUNCA TValue *lj_meta_cat(lua_State *L, TValue *top, int left);
 LJ_FUNCA TValue * LJ_FASTCALL lj_meta_len(lua_State *L, cTValue *o);
 LJ_FUNCA TValue *lj_meta_equal(lua_State *L, GCobj *o1, GCobj *o2, int ne);
+LJ_FUNCA TValue * LJ_FASTCALL lj_meta_equal_cd(lua_State *L, BCIns ins);
 LJ_FUNCA TValue *lj_meta_comp(lua_State *L, cTValue *o1, cTValue *o2, int op);
 LJ_FUNCA void lj_meta_call(lua_State *L, TValue *func, TValue *top);
 LJ_FUNCA void LJ_FASTCALL lj_meta_for(lua_State *L, TValue *base);

src/lj_obj.h

@@ -434,7 +434,7 @@ enum {
 
 #define setvmstate(g, st)	((g)->vmstate = ~LJ_VMST_##st)
 
-/* Metamethods. */
+/* Metamethods. ORDER MM */
 #ifdef LUAJIT_ENABLE_LUA52COMPAT
 #define MMDEF_52(_) _(pairs) _(ipairs)
 #else

src/lj_record.c

@@ -812,6 +812,23 @@ static void rec_mm_comp(jit_State *J, RecordIndex *ix, int op)
   }
 }
 
+#if LJ_HASFFI
+/* Setup call to cdata comparison metamethod. */
+static void rec_mm_comp_cdata(jit_State *J, RecordIndex *ix, int op, MMS mm)
+{
+  if (tref_iscdata(ix->val)) {
+    ix->tab = ix->val;
+    copyTV(J->L, &ix->tabv, &ix->valv);
+  } else {
+    lua_assert(tref_iscdata(ix->key));
+    ix->tab = ix->key;
+    copyTV(J->L, &ix->tabv, &ix->keyv);
+  }
+  lj_record_mm_lookup(J, ix, mm);
+  rec_mm_callcomp(J, ix, op);
+}
+#endif
+
 /* -- Indexed access ------------------------------------------------------ */
 
 /* Record bounds-check. */
@@ -1410,6 +1427,12 @@ void lj_record_ins(jit_State *J)
   /* -- Comparison ops ---------------------------------------------------- */
 
   case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
+#if LJ_HASFFI
+    if (tref_iscdata(ra) || tref_iscdata(rc)) {
+      rec_mm_comp_cdata(J, &ix, op, ((int)op & 2) ? MM_le : MM_lt);
+      break;
+    }
+#endif
     /* Emit nothing for two numeric or string consts. */
     if (!(tref_isk2(ra,rc) && tref_isnumber_str(ra) && tref_isnumber_str(rc))) {
       IRType ta = tref_isinteger(ra) ? IRT_INT : tref_type(ra);
@@ -1452,6 +1475,12 @@ void lj_record_ins(jit_State *J)
   case BC_ISEQS: case BC_ISNES:
   case BC_ISEQN: case BC_ISNEN:
   case BC_ISEQP: case BC_ISNEP:
+#if LJ_HASFFI
+    if (tref_iscdata(ra) || tref_iscdata(rc)) {
+      rec_mm_comp_cdata(J, &ix, op, MM_eq);
+      break;
+    }
+#endif
     /* Emit nothing for two non-table, non-udata consts. */
     if (!(tref_isk2(ra, rc) && !(tref_istab(ra) || tref_isudata(ra)))) {
       int diff;