ARM64: Add build infrastructure and initial port of interpreter.

src/Makefile

@@ -48,6 +48,7 @@ CCOPT= -O2 -fomit-frame-pointer
 CCOPT_x86= -march=i686 -msse -msse2 -mfpmath=sse
 CCOPT_x64=
 CCOPT_arm=
+CCOPT_arm64=
 CCOPT_ppc=
 CCOPT_ppcspe=
 CCOPT_mips=
@@ -217,6 +218,9 @@ else
 ifneq (,$(findstring LJ_TARGET_ARM ,$(TARGET_TESTARCH)))
   TARGET_LJARCH= arm
 else
+ifneq (,$(findstring LJ_TARGET_ARM64 ,$(TARGET_TESTARCH)))
+  TARGET_LJARCH= arm64
+else
 ifneq (,$(findstring LJ_TARGET_PPC ,$(TARGET_TESTARCH)))
   TARGET_LJARCH= ppc
 else
@@ -236,6 +240,7 @@ endif
 endif
 endif
 endif
+endif
 
 ifneq (,$(findstring LJ_TARGET_PS3 1,$(TARGET_TESTARCH)))
   TARGET_SYS= PS3
@@ -402,7 +407,9 @@ ifeq (Windows,$(TARGET_SYS))
   DASM_AFLAGS+= -D WIN
 endif
 ifeq (x64,$(TARGET_LJARCH))
-  DASM_ARCH= x86
+  ifeq (,$(findstring LJ_FR2 1,$(TARGET_TESTARCH)))
+    DASM_ARCH= x86
+  endif
 else
 ifeq (arm,$(TARGET_LJARCH))
   ifeq (iOS,$(TARGET_SYS))

src/host/buildvm.c

@@ -59,6 +59,8 @@ static int collect_reloc(BuildCtx *ctx, uint8_t *addr, int idx, int type);
 #include "../dynasm/dasm_x86.h"
 #elif LJ_TARGET_ARM
 #include "../dynasm/dasm_arm.h"
+#elif LJ_TARGET_ARM64
+#include "../dynasm/dasm_arm64.h"
 #elif LJ_TARGET_PPC
 #include "../dynasm/dasm_ppc.h"
 #elif LJ_TARGET_PPCSPE

src/host/buildvm_asm.c

@@ -121,6 +121,15 @@ static void emit_asm_wordreloc(BuildCtx *ctx, uint8_t *p, int n,
 	    ins, sym);
     exit(1);
   }
+#elif LJ_TARGET_ARM64
+  if ((ins >> 26) == 0x25u) {
+    fprintf(ctx->fp, "\tbl %s\n", sym);
+  } else {
+    fprintf(stderr,
+	    "Error: unsupported opcode %08x for %s symbol relocation.\n",
+	    ins, sym);
+    exit(1);
+  }
 #elif LJ_TARGET_PPC || LJ_TARGET_PPCSPE
 #if LJ_TARGET_PS3
 #define TOCPREFIX "."

src/lib_jit.c

@@ -695,6 +695,8 @@ static uint32_t jit_cpudetect(lua_State *L)
 	   ver >= 60 ? JIT_F_ARMV6_ : 0;
   flags |= LJ_ARCH_HASFPU == 0 ? 0 : ver >= 70 ? JIT_F_VFPV3 : JIT_F_VFPV2;
 #endif
+#elif LJ_TARGET_ARM64
+  /* No optional CPU features to detect (for now). */
 #elif LJ_TARGET_PPC
 #if LJ_HASJIT
 #if LJ_ARCH_SQRT

src/lj_arch.h

@@ -19,12 +19,14 @@
 #define LUAJIT_ARCH_x64		2
 #define LUAJIT_ARCH_ARM		3
 #define LUAJIT_ARCH_arm		3
-#define LUAJIT_ARCH_PPC		4
-#define LUAJIT_ARCH_ppc		4
-#define LUAJIT_ARCH_PPCSPE	5
-#define LUAJIT_ARCH_ppcspe	5
-#define LUAJIT_ARCH_MIPS	6
-#define LUAJIT_ARCH_mips	6
+#define LUAJIT_ARCH_ARM64	4
+#define LUAJIT_ARCH_arm64	4
+#define LUAJIT_ARCH_PPC		5
+#define LUAJIT_ARCH_ppc		6
+#define LUAJIT_ARCH_PPCSPE	6
+#define LUAJIT_ARCH_ppcspe	6
+#define LUAJIT_ARCH_MIPS	7
+#define LUAJIT_ARCH_mips	7
 
 /* Target OS. */
 #define LUAJIT_OS_OTHER		0
@@ -43,6 +45,8 @@
 #define LUAJIT_TARGET	LUAJIT_ARCH_X64
 #elif defined(__arm__) || defined(__arm) || defined(__ARM__) || defined(__ARM)
 #define LUAJIT_TARGET	LUAJIT_ARCH_ARM
+#elif defined(__aarch64__)
+#define LUAJIT_TARGET	LUAJIT_ARCH_ARM64
 #elif defined(__ppc__) || defined(__ppc) || defined(__PPC__) || defined(__PPC) || defined(__powerpc__) || defined(__powerpc) || defined(__POWERPC__) || defined(__POWERPC) || defined(_M_PPC)
 #ifdef __NO_FPRS__
 #define LUAJIT_TARGET	LUAJIT_ARCH_PPCSPE
@@ -191,6 +195,24 @@
 #define LJ_ARCH_VERSION		50
 #endif
 
+#elif LUAJIT_TARGET == LUAJIT_ARCH_ARM64
+
+#define LJ_ARCH_NAME		"arm64"
+#define LJ_ARCH_BITS		64
+#define LJ_ARCH_ENDIAN		LUAJIT_LE
+#define LJ_TARGET_ARM64		1
+#define LJ_TARGET_EHRETREG	0
+#define LJ_TARGET_JUMPRANGE	27	/* +-2^27 = +-128MB */
+#define LJ_TARGET_MASKSHIFT	1
+#define LJ_TARGET_MASKROT	1
+#define LJ_TARGET_UNIFYROT	2	/* Want only IR_BROR. */
+#define LJ_TARGET_GC64		1
+#define LJ_ARCH_NUMMODE		LJ_NUMMODE_DUAL
+#define LJ_ARCH_NOFFI		1	/* NYI */
+#define LJ_ARCH_NOJIT		1	/* NYI */
+
+#define LJ_ARCH_VERSION		80
+
 #elif LUAJIT_TARGET == LUAJIT_ARCH_PPC
 
 #define LJ_ARCH_NAME		"ppc"
@@ -327,6 +349,13 @@
 #if !(__ARM_EABI__ || LJ_TARGET_IOS)
 #error "Only ARM EABI or iOS 3.0+ ABI is supported"
 #endif
+#elif LJ_TARGET_ARM64
+#if defined(__AARCH64EB__)
+#error "No support for big-endian ARM64"
+#endif
+#if defined(_ILP32)
+#error "No support for ILP32 model on ARM64"
+#endif
 #elif LJ_TARGET_PPC || LJ_TARGET_PPCSPE
 #if defined(_SOFT_FLOAT) || defined(_SOFT_DOUBLE)
 #error "No support for PowerPC CPUs without double-precision FPU"

src/lj_frame.h

@@ -159,6 +159,15 @@ enum { LJ_CONT_TAILCALL, LJ_CONT_FFI_CALLBACK };  /* Special continuations. */
 #define CFRAME_SIZE		64
 #endif
 #define CFRAME_SHIFT_MULTRES	3
+#elif LJ_TARGET_ARM64
+#define CFRAME_OFS_ERRF		196
+#define CFRAME_OFS_NRES		200
+#define CFRAME_OFS_PREV		160
+#define CFRAME_OFS_L		176
+#define CFRAME_OFS_PC		168
+#define CFRAME_OFS_MULTRES	192
+#define CFRAME_SIZE		208
+#define CFRAME_SHIFT_MULTRES	3
 #elif LJ_TARGET_PPC
 #if LJ_TARGET_XBOX360
 #define CFRAME_OFS_ERRF		424

src/lj_vm.h

@@ -50,7 +50,7 @@ LJ_ASMF void lj_vm_exit_handler(void);
 LJ_ASMF void lj_vm_exit_interp(void);
 
 /* Internal math helper functions. */
-#if LJ_TARGET_PPC
+#if LJ_TARGET_PPC || LJ_TARGET_ARM64
 #define lj_vm_floor	floor
 #define lj_vm_ceil	ceil
 #else
@@ -77,7 +77,7 @@ LJ_ASMF void lj_vm_powi_sse(void);
 #else
 LJ_ASMF double lj_vm_powi(double, int32_t);
 #endif
-#if LJ_TARGET_PPC
+#if LJ_TARGET_PPC || LJ_TARGET_ARM64
 #define lj_vm_trunc	trunc
 #else
 LJ_ASMF double lj_vm_trunc(double);

src/lj_vmmath.c

@@ -73,7 +73,7 @@ double lj_vm_exp2(double a)
 }
 #endif
 
-#if !(LJ_TARGET_ARM || LJ_TARGET_PPC)
+#if !(LJ_TARGET_ARM || LJ_TARGET_ARM64 || LJ_TARGET_PPC)
 int32_t LJ_FASTCALL lj_vm_modi(int32_t a, int32_t b)
 {
   uint32_t y, ua, ub;

src/vm_arm64.dasc

@@ -0,0 +1,3649 @@
+|// Low-level VM code for ARM64 CPUs.
+|// Bytecode interpreter, fast functions and helper functions.
+|// Copyright (C) 2005-2014 Mike Pall. See Copyright Notice in luajit.h
+|
+|.arch arm64
+|.section code_op, code_sub
+|
+|.actionlist build_actionlist
+|.globals GLOB_
+|.globalnames globnames
+|.externnames extnames
+|
+|// Note: The ragged indentation of the instructions is intentional.
+|//       The starting columns indicate data dependencies.
+|
+|//-----------------------------------------------------------------------
+|
+|// ARM64 registers and the AAPCS64 ABI 1.0 at a glance:
+|//
+|// x0-x17 temp, x19-x28 callee-saved, x29 fp, x30 lr
+|// x18 is reserved on most platforms. Don't use it, save it or restore it.
+|// x31 doesn't exist. Register number 31 either means xzr/wzr (zero) or sp,
+|// depending on the instruction.
+|// v0-v7 temp, v8-v15 callee-saved (only d8-d15 preserved), v16-v31 temp
+|//
+|// x0-x7/v0-v7 hold parameters and results.
+|
+|// Fixed register assignments for the interpreter.
+|
+|// The following must be C callee-save.
+|.define BASE,		x19	// Base of current Lua stack frame.
+|.define KBASE,		x20	// Constants of current Lua function.
+|.define PC,		x21	// Next PC.
+|.define GLREG,		x22	// Global state.
+|.define LREG,		x23	// Register holding lua_State (also in SAVE_L).
+|.define TISNUM,	x24	// Constant LJ_TISNUM << 47.
+|.define TISNUMhi,	x25	// Constant LJ_TISNUM << 15.
+|.define TISNIL,	x26	// Constant -1LL.
+|.define fp,		x29	// Yes, we have to maintain a frame pointer.
+|
+|.define ST_INTERP,	w26	// Constant -1.
+|
+|// The following temporaries are not saved across C calls, except for RA/RC.
+|.define RA,		x27
+|.define RC,		x28
+|.define RB,		x17
+|.define RAw,		w27
+|.define RCw,		w28
+|.define RBw,		w17
+|.define INS,		x16
+|.define INSw,		w16
+|.define ITYPE,		x15
+|.define TMP0,		x8
+|.define TMP1,		x9
+|.define TMP2,		x10
+|.define TMP3,		x11
+|.define TMP0w,		w8
+|.define TMP1w,		w9
+|.define TMP2w,		w10
+|.define TMP3w,		w11
+|
+|// Calling conventions. Also used as temporaries.
+|.define CARG1,		x0
+|.define CARG2,		x1
+|.define CARG3,		x2
+|.define CARG4,		x3
+|.define CARG5,		x4
+|.define CARG1w,	w0
+|.define CARG2w,	w1
+|.define CARG3w,	w2
+|.define CARG4w,	w3
+|.define CARG5w,	w4
+|
+|.define FARG1,		d0
+|.define FARG2,		d1
+|
+|.define CRET1,		x0
+|.define CRET1w,	w0
+|
+|// Stack layout while in interpreter. Must match with lj_frame.h.
+|
+|.define CFRAME_SPACE,	208
+|//----- 16 byte aligned, <-- sp entering interpreter
+|// Unused		[sp, #204]	// 32 bit values
+|.define SAVE_NRES,	[sp, #200]
+|.define SAVE_ERRF,	[sp, #196]
+|.define SAVE_MULTRES,	[sp, #192]
+|.define TMPD,		[sp, #184]	// 64 bit values
+|.define SAVE_L,	[sp, #176]
+|.define SAVE_PC,	[sp, #168]
+|.define SAVE_CFRAME,	[sp, #160]
+|.define SAVE_FPR_,	96		// 96+8*8: 64 bit FPR saves
+|.define SAVE_GPR_,	16		// 16+10*8: 64 bit GPR saves
+|.define SAVE_LR,	[sp, #8]
+|.define SAVE_FP,	[sp]
+|//----- 16 byte aligned, <-- sp while in interpreter.
+|
+|.define TMPDofs,	#184
+|
+|.macro save_, gpr1, gpr2, fpr1, fpr2
+|  stp d..fpr1, d..fpr2, [sp, # SAVE_FPR_+(fpr1-8)*8]
+|  stp x..gpr1, x..gpr2, [sp, # SAVE_GPR_+(gpr1-19)*8]
+|.endmacro
+|.macro rest_, gpr1, gpr2, fpr1, fpr2
+|  ldp d..fpr1, d..fpr2, [sp, # SAVE_FPR_+(fpr1-8)*8]
+|  ldp x..gpr1, x..gpr2, [sp, # SAVE_GPR_+(gpr1-19)*8]
+|.endmacro
+|
+|.macro saveregs
+|  stp fp, lr, [sp, #-CFRAME_SPACE]!
+|  add fp, sp, #0
+|  stp x19, x20, [sp, # SAVE_GPR_]
+|  save_ 21, 22, 8, 9
+|  save_ 23, 24, 10, 11
+|  save_ 25, 26, 12, 13
+|  save_ 27, 28, 14, 15
+|.endmacro
+|.macro restoreregs
+|  ldp x19, x20, [sp, # SAVE_GPR_]
+|  rest_ 21, 22, 8, 9
+|  rest_ 23, 24, 10, 11
+|  rest_ 25, 26, 12, 13
+|  rest_ 27, 28, 14, 15
+|  ldp fp, lr, [sp], # CFRAME_SPACE
+|.endmacro
+|
+|// Type definitions. Some of these are only used for documentation.
+|.type L,		lua_State,	LREG
+|.type GL,		global_State,	GLREG
+|.type TVALUE,		TValue
+|.type GCOBJ,		GCobj
+|.type STR,		GCstr
+|.type TAB,		GCtab
+|.type LFUNC,		GCfuncL
+|.type CFUNC,		GCfuncC
+|.type PROTO,		GCproto
+|.type UPVAL,		GCupval
+|.type NODE,		Node
+|.type NARGS8,		int
+|.type TRACE,		GCtrace
+|.type SBUF,		SBuf
+|
+|//-----------------------------------------------------------------------
+|
+|// Trap for not-yet-implemented parts.
+|.macro NYI; brk; .endmacro
+|
+|//-----------------------------------------------------------------------
+|
+|// Access to frame relative to BASE.
+|.define FRAME_FUNC,	#-16
+|.define FRAME_PC,	#-8
+|
+|.macro decode_RA, dst, ins; ubfx dst, ins, #8, #8; .endmacro
+|.macro decode_RB, dst, ins; ubfx dst, ins, #24, #8; .endmacro
+|.macro decode_RC, dst, ins; ubfx dst, ins, #16, #8; .endmacro
+|.macro decode_RD, dst, ins; ubfx dst, ins, #16, #16; .endmacro
+|.macro decode_RC8RD, dst, src; ubfiz dst, src, #3, #8; .endmacro
+|
+|// Instruction decode+dispatch.
+|.macro ins_NEXT
+|  ldr INSw, [PC], #4
+|  add TMP1, GL, INS, uxtb #3
+|   decode_RA RA, INS
+|  ldr TMP0, [TMP1, #GG_G2DISP]
+|   decode_RD RC, INS
+|  br TMP0
+|.endmacro
+|
+|// Instruction footer.
+|.if 1
+|  // Replicated dispatch. Less unpredictable branches, but higher I-Cache use.
+|  .define ins_next, ins_NEXT
+|  .define ins_next_, ins_NEXT
+|.else
+|  // Common dispatch. Lower I-Cache use, only one (very) unpredictable branch.
+|  // Affects only certain kinds of benchmarks (and only with -j off).
+|  .macro ins_next
+|    b ->ins_next
+|  .endmacro
+|  .macro ins_next_
+|  ->ins_next:
+|    ins_NEXT
+|  .endmacro
+|.endif
+|
+|// Call decode and dispatch.
+|.macro ins_callt
+|  // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
+|  ldr PC, LFUNC:CARG3->pc
+|  ldr INSw, [PC], #4
+|  add TMP1, GL, INS, uxtb #3
+|   decode_RA RA, INS
+|  ldr TMP0, [TMP1, #GG_G2DISP]
+|   add RA, BASE, RA, lsl #3
+|  br TMP0
+|.endmacro
+|
+|.macro ins_call
+|  // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, PC = caller PC
+|  str PC, [BASE, FRAME_PC]
+|  ins_callt
+|.endmacro
+|
+|//-----------------------------------------------------------------------
+|
+|// Macros to check the TValue type and extract the GCobj. Branch on failure.
+|.macro checktp, reg, tp, target
+|  asr ITYPE, reg, #47
+|  cmn ITYPE, #-tp
+|   and reg, reg, #LJ_GCVMASK
+|  bne target
+|.endmacro
+|.macro checktp, dst, reg, tp, target
+|  asr ITYPE, reg, #47
+|  cmn ITYPE, #-tp
+|   and dst, reg, #LJ_GCVMASK
+|  bne target
+|.endmacro
+|.macro checkstr, reg, target; checktp reg, LJ_TSTR, target; .endmacro
+|.macro checktab, reg, target; checktp reg, LJ_TTAB, target; .endmacro
+|.macro checkfunc, reg, target; checktp reg, LJ_TFUNC, target; .endmacro
+|.macro checkint, reg, target
+|  cmp TISNUMhi, reg, lsr #32
+|  bne target
+|.endmacro
+|.macro checknum, reg, target
+|  cmp TISNUMhi, reg, lsr #32
+|  bls target
+|.endmacro
+|.macro checknumber, reg, target
+|  cmp TISNUMhi, reg, lsr #32
+|  blo target
+|.endmacro
+|
+|.macro mov_false, reg; movn reg, #0x8000, lsl #32; .endmacro
+|.macro mov_true, reg; movn reg, #0x0001, lsl #48; .endmacro
+|
+#define GL_J(field)	(GG_OFS(J) + (int)offsetof(jit_State, field))
+|
+#define PC2PROTO(field)  ((int)offsetof(GCproto, field)-(int)sizeof(GCproto))
+|
+|.macro hotcheck, delta
+|  NYI
+|.endmacro
+|
+|.macro hotloop
+|  hotcheck HOTCOUNT_LOOP
+|  blo ->vm_hotloop
+|.endmacro
+|
+|.macro hotcall
+|  hotcheck HOTCOUNT_CALL
+|  blo ->vm_hotcall
+|.endmacro
+|
+|// Set current VM state.
+|.macro mv_vmstate, reg, st; movn reg, #LJ_VMST_..st; .endmacro
+|.macro st_vmstate, reg; str reg, GL->vmstate; .endmacro
+|
+|// Move table write barrier back. Overwrites mark and tmp.
+|.macro barrierback, tab, mark, tmp
+|  ldr tmp, GL->gc.grayagain
+|   and mark, mark, #~LJ_GC_BLACK	// black2gray(tab)
+|  str tab, GL->gc.grayagain
+|   strb mark, tab->marked
+|  str tmp, tab->gclist
+|.endmacro
+|
+|//-----------------------------------------------------------------------
+
+#if !LJ_DUALNUM
+#error "Only dual-number mode supported for ARM64 target"
+#endif
+
+/* Generate subroutines used by opcodes and other parts of the VM. */
+/* The .code_sub section should be last to help static branch prediction. */
+static void build_subroutines(BuildCtx *ctx)
+{
+  |.code_sub
+  |
+  |//-----------------------------------------------------------------------
+  |//-- Return handling ----------------------------------------------------
+  |//-----------------------------------------------------------------------
+  |
+  |->vm_returnp:
+  |  // See vm_return. Also: RB = previous base.
+  |  tbz PC, #2, ->cont_dispatch	// (PC & FRAME_P) == 0?
+  |
+  |  // Return from pcall or xpcall fast func.
+  |  ldr PC, [RB, FRAME_PC]		// Fetch PC of previous frame.
+  |   mov_true TMP0
+  |  mov BASE, RB
+  |  // Prepending may overwrite the pcall frame, so do it at the end.
+  |   str TMP0, [RA, #-8]!		// Prepend true to results.
+  |
+  |->vm_returnc:
+  |  adds RC, RC, #8			// RC = (nresults+1)*8.
+  |  mov CRET1, #LUA_YIELD
+  |  beq ->vm_unwind_c_eh
+  |  str RCw, SAVE_MULTRES
+  |  ands CARG1, PC, #FRAME_TYPE
+  |  beq ->BC_RET_Z			// Handle regular return to Lua.
+  |
+  |->vm_return:
+  |  // BASE = base, RA = resultptr, RC/MULTRES = (nresults+1)*8, PC = return
+  |  // CARG1 = PC & FRAME_TYPE
+  |  and RB, PC, #~FRAME_TYPEP
+  |   cmp CARG1, #FRAME_C
+  |  sub RB, BASE, RB			// RB = previous base.
+  |   bne ->vm_returnp
+  |
+  |  str RB, L->base
+  |   ldrsw CARG2, SAVE_NRES		// CARG2 = nresults+1.
+  |    mv_vmstate TMP0w, C
+  |   sub BASE, BASE, #16
+  |  subs TMP2, RC, #8
+  |    st_vmstate TMP0w
+  |  beq >2
+  |1:
+  |  subs TMP2, TMP2, #8
+  |   ldr TMP0, [RA], #8
+  |   str TMP0, [BASE], #8
+  |  bne <1
+  |2:
+  |  cmp RC, CARG2, lsl #3		// More/less results wanted?
+  |  bne >6
+  |3:
+  |  str BASE, L->top			// Store new top.
+  |
+  |->vm_leave_cp:
+  |  ldr RC, SAVE_CFRAME		// Restore previous C frame.
+  |   mov CRET1, #0			// Ok return status for vm_pcall.
+  |  str RC, L->cframe
+  |
+  |->vm_leave_unw:
+  |  restoreregs
+  |  ret
+  |
+  |6:
+  |  bgt >7				// Less results wanted?
+  |  // More results wanted. Check stack size and fill up results with nil.
+  |  ldr CARG3, L->maxstack
+  |  cmp BASE, CARG3
+  |  bhs >8
+  |   str TISNIL, [BASE], #8
+  |  add RC, RC, #8
+  |  b <2
+  |
+  |7:  // Less results wanted.
+  |  cbz CARG2, <3			// LUA_MULTRET+1 case?
+  |  sub CARG1, RC, CARG2, lsl #3
+  |  sub BASE, BASE, CARG1		// Shrink top.
+  |  b <3
+  |
+  |8:  // Corner case: need to grow stack for filling up results.
+  |  // This can happen if:
+  |  // - A C function grows the stack (a lot).
+  |  // - The GC shrinks the stack in between.
+  |  // - A return back from a lua_call() with (high) nresults adjustment.
+  |  str BASE, L->top			// Save current top held in BASE (yes).
+  |  mov CARG1, L
+  |  bl extern lj_state_growstack	// (lua_State *L, int n)
+  |  ldr BASE, L->top			// Need the (realloced) L->top in BASE.
+  |  ldrsw CARG2, SAVE_NRES
+  |  b <2
+  |
+  |->vm_unwind_c:			// Unwind C stack, return from vm_pcall.
+  |  // (void *cframe, int errcode)
+  |  mov sp, CARG1
+  |  mov CRET1, CARG2
+  |->vm_unwind_c_eh:			// Landing pad for external unwinder.
+  |  ldr L, SAVE_L
+  |   mv_vmstate TMP0w, C
+  |  ldr GL, L->glref
+  |   st_vmstate TMP0w
+  |  b ->vm_leave_unw
+  |
+  |->vm_unwind_ff:			// Unwind C stack, return from ff pcall.
+  |  // (void *cframe)
+  |  and sp, CARG1, #CFRAME_RAWMASK
+  |->vm_unwind_ff_eh:			// Landing pad for external unwinder.
+  |  ldr L, SAVE_L
+  |    movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
+  |    movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
+  |    movn TISNIL, #0
+  |    mov RC, #16			// 2 results: false + error message.
+  |  ldr BASE, L->base
+  |   ldr GL, L->glref			// Setup pointer to global state.
+  |    mov_false TMP0
+  |  sub RA, BASE, #8			// Results start at BASE-8.
+  |  ldr PC, [BASE, FRAME_PC]		// Fetch PC of previous frame.
+  |    str TMP0, [BASE, #-8]		// Prepend false to error message.
+  |   st_vmstate ST_INTERP
+  |  b ->vm_returnc
+  |
+  |//-----------------------------------------------------------------------
+  |//-- Grow stack for calls -----------------------------------------------
+  |//-----------------------------------------------------------------------
+  |
+  |->vm_growstack_c:			// Grow stack for C function.
+  |  // CARG1 = L
+  |  mov CARG2, #LUA_MINSTACK
+  |  b >2
+  |
+  |->vm_growstack_l:			// Grow stack for Lua function.
+  |  // BASE = new base, RA = BASE+framesize*8, RC = nargs*8, PC = first PC
+  |  add RC, BASE, RC
+  |   sub RA, RA, BASE
+  |    mov CARG1, L
+  |  stp BASE, RC, L->base
+  |   add PC, PC, #4			// Must point after first instruction.
+  |   lsr CARG2, RA, #3
+  |2:
+  |  // L->base = new base, L->top = top
+  |  str PC, SAVE_PC
+  |  bl extern lj_state_growstack	// (lua_State *L, int n)
+  |  ldp BASE, RC, L->base
+  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
+  |   sub NARGS8:RC, RC, BASE
+  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
+  |  // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
+  |  ins_callt				// Just retry the call.
+  |
+  |//-----------------------------------------------------------------------
+  |//-- Entry points into the assembler VM ---------------------------------
+  |//-----------------------------------------------------------------------
+  |
+  |->vm_resume:				// Setup C frame and resume thread.
+  |  // (lua_State *L, TValue *base, int nres1 = 0, ptrdiff_t ef = 0)
+  |  saveregs
+  |  mov L, CARG1
+  |    ldr GL, L->glref			// Setup pointer to global state.
+  |  mov BASE, CARG2
+  |   str L, SAVE_L
+  |  mov PC, #FRAME_CP
+  |   str wzr, SAVE_NRES
+  |    add TMP0, sp, #CFRAME_RESUME
+  |  ldrb TMP1w, L->status
+  |   str wzr, SAVE_ERRF
+  |   str L, SAVE_PC			// Any value outside of bytecode is ok.
+  |   str xzr, SAVE_CFRAME
+  |    str TMP0, L->cframe
+  |  cbz TMP1w, >3
+  |
+  |  // Resume after yield (like a return).
+  |  str L, GL->cur_L
+  |  mov RA, BASE
+  |   ldp BASE, CARG1, L->base
+  |    movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
+  |    movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
+  |  ldr PC, [BASE, FRAME_PC]
+  |     strb wzr, L->status
+  |    movn TISNIL, #0
+  |   sub RC, CARG1, BASE
+  |  ands CARG1, PC, #FRAME_TYPE
+  |   add RC, RC, #8
+  |     st_vmstate ST_INTERP
+  |   str RCw, SAVE_MULTRES
+  |  beq ->BC_RET_Z
+  |  b ->vm_return
+  |
+  |->vm_pcall:				// Setup protected C frame and enter VM.
+  |  // (lua_State *L, TValue *base, int nres1, ptrdiff_t ef)
+  |  saveregs
+  |  mov PC, #FRAME_CP
+  |  str CARG4w, SAVE_ERRF
+  |  b >1
+  |
+  |->vm_call:				// Setup C frame and enter VM.
+  |  // (lua_State *L, TValue *base, int nres1)
+  |  saveregs
+  |  mov PC, #FRAME_C
+  |
+  |1:  // Entry point for vm_pcall above (PC = ftype).
+  |  ldr RC, L:CARG1->cframe
+  |   str CARG3w, SAVE_NRES
+  |    mov L, CARG1
+  |   str CARG1, SAVE_L
+  |    ldr GL, L->glref			// Setup pointer to global state.
+  |     mov BASE, CARG2
+  |   str CARG1, SAVE_PC		// Any value outside of bytecode is ok.
+  |  str RC, SAVE_CFRAME
+  |  str fp, L->cframe			// Add our C frame to cframe chain.
+  |
+  |3:  // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype).
+  |  str L, GL->cur_L
+  |  ldp RB, CARG1, L->base		// RB = old base (for vmeta_call).
+  |    movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
+  |    movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
+  |  add PC, PC, BASE
+  |    movn TISNIL, #0
+  |  sub PC, PC, RB			// PC = frame delta + frame type
+  |   sub NARGS8:RC, CARG1, BASE
+  |    st_vmstate ST_INTERP
+  |
+  |->vm_call_dispatch:
+  |  // RB = old base, BASE = new base, RC = nargs*8, PC = caller PC
+  |  ldr CARG3, [BASE, FRAME_FUNC]
+  |  checkfunc CARG3, ->vmeta_call
+  |
+  |->vm_call_dispatch_f:
+  |  ins_call
+  |  // BASE = new base, CARG3 = func, RC = nargs*8, PC = caller PC
+  |
+  |->vm_cpcall:				// Setup protected C frame, call C.
+  |  // (lua_State *L, lua_CFunction func, void *ud, lua_CPFunction cp)
+  |  saveregs
+  |  mov L, CARG1
+  |   ldr RA, L:CARG1->stack
+  |  str CARG1, SAVE_L
+  |    ldr GL, L->glref			// Setup pointer to global state.
+  |   ldr RB, L->top
+  |  str CARG1, SAVE_PC			// Any value outside of bytecode is ok.
+  |  ldr RC, L->cframe
+  |   sub RA, RA, RB			// Compute -savestack(L, L->top).
+  |   str RAw, SAVE_NRES		// Neg. delta means cframe w/o frame.
+  |  str wzr, SAVE_ERRF			// No error function.
+  |  str RC, SAVE_CFRAME
+  |  str fp, L->cframe			// Add our C frame to cframe chain.
+  |    str L, GL->cur_L
+  |  blr CARG4			// (lua_State *L, lua_CFunction func, void *ud)
+  |  mov BASE, CRET1
+  |   mov PC, #FRAME_CP
+  |  cbnz BASE, <3			// Else continue with the call.
+  |  b ->vm_leave_cp			// No base? Just remove C frame.
+  |
+  |//-----------------------------------------------------------------------
+  |//-- Metamethod handling ------------------------------------------------
+  |//-----------------------------------------------------------------------
+  |
+  |//-- Continuation dispatch ----------------------------------------------
+  |
+  |->cont_dispatch:
+  |  // BASE = meta base, RA = resultptr, RC = (nresults+1)*8
+  |  ldr LFUNC:CARG3, [RB, FRAME_FUNC]
+  |    ldr CARG1, [BASE, #-32]		// Get continuation.
+  |   mov CARG4, BASE
+  |   mov BASE, RB			// Restore caller BASE.
+  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
+  |.if FFI
+  |    cmp CARG1, #1
+  |.endif
+  |   ldr PC, [CARG4, #-24]		// Restore PC from [cont|PC].
+  |  ldr CARG3, LFUNC:CARG3->pc
+  |    add TMP0, RA, RC
+  |    str TISNIL, [TMP0, #-8]		// Ensure one valid arg.
+  |.if FFI
+  |    bls >1
+  |.endif
+  |  ldr KBASE, [CARG3, #PC2PROTO(k)]
+  |  // BASE = base, RA = resultptr, CARG4 = meta base
+  |    br CARG1
+  |
+  |.if FFI
+  |1:
+  |  beq ->cont_ffi_callback		// cont = 1: return from FFI callback.
+  |  // cont = 0: tailcall from C function.
+  |   sub CARG4, CARG4, #32
+  |   sub RC, CARG4, BASE
+  |  b ->vm_call_tail
+  |.endif
+  |
+  |->cont_cat:				// RA = resultptr, CARG4 = meta base
+  |  ldr INSw, [PC, #-4]
+  |   sub CARG2, CARG4, #32
+  |   ldr TMP0, [RA]
+  |     str BASE, L->base
+  |  decode_RB RB, INS
+  |   decode_RA RA, INS
+  |  add TMP1, BASE, RB, lsl #3
+  |  subs TMP1, CARG2, TMP1
+  |  beq >1
+  |   str TMP0, [CARG2]
+  |  lsr CARG3, TMP1, #3
+  |  b ->BC_CAT_Z
+  |
+  |1:
+  |   str TMP0, [BASE, RA, lsl #3]
+  |  b ->cont_nop
+  |
+  |//-- Table indexing metamethods -----------------------------------------
+  |
+  |->vmeta_tgets1:
+  |  movn CARG4, #~LJ_TSTR
+  |   add CARG2, BASE, RB, lsl #3
+  |  add CARG4, STR:RC, CARG4, lsl #47
+  |  b >2
+  |
+  |->vmeta_tgets:
+  |  movk CARG2, #(LJ_TTAB>>1)&0xffff, lsl #48
+  |  str CARG2, GL->tmptv
+  |  add CARG2, GL, #offsetof(global_State, tmptv)
+  |2:
+  |   add CARG3, sp, TMPDofs
+  |  str CARG4, TMPD
+  |  b >1
+  |
+  |->vmeta_tgetb:			// RB = table, RC = index
+  |  add RC, RC, TISNUM
+  |   add CARG2, BASE, RB, lsl #3
+  |   add CARG3, sp, TMPDofs
+  |  str RC, TMPD
+  |  b >1
+  |
+  |->vmeta_tgetv:			// RB = table, RC = key
+  |  add CARG2, BASE, RB, lsl #3
+  |   add CARG3, BASE, RC, lsl #3
+  |1:
+  |   str BASE, L->base
+  |  mov CARG1, L
+  |   str PC, SAVE_PC
+  |  bl extern lj_meta_tget		// (lua_State *L, TValue *o, TValue *k)
+  |  // Returns TValue * (finished) or NULL (metamethod).
+  |  cbz CRET1, >3
+  |  ldr TMP0, [CRET1]
+  |  str TMP0, [BASE, RA, lsl #3]
+  |  ins_next
+  |
+  |3:  // Call __index metamethod.
+  |  // BASE = base, L->top = new base, stack = cont/func/t/k
+  |   sub TMP1, BASE, #FRAME_CONT
+  |  ldr BASE, L->top
+  |    mov NARGS8:RC, #16		// 2 args for func(t, k).
+  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Guaranteed to be a function here.
+  |    str PC, [BASE, #-24]		// [cont|PC]
+  |   sub PC, BASE, TMP1
+  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
+  |  b ->vm_call_dispatch_f
+  |
+  |->vmeta_tgetr:
+  |  sxtw CARG2, TMP1w
+  |  bl extern lj_tab_getinth		// (GCtab *t, int32_t key)
+  |  // Returns cTValue * or NULL.
+  |  mov TMP0, TISNIL
+  |  cbz CRET1, ->BC_TGETR_Z
+  |  ldr TMP0, [CRET1]
+  |  b ->BC_TGETR_Z
+  |
+  |//-----------------------------------------------------------------------
+  |
+  |->vmeta_tsets1:
+  |  movn CARG4, #~LJ_TSTR
+  |   add CARG2, BASE, RB, lsl #3
+  |  add CARG4, STR:RC, CARG4, lsl #47
+  |  b >2
+  |
+  |->vmeta_tsets:
+  |  movk CARG2, #(LJ_TTAB>>1)&0xffff, lsl #48
+  |  str CARG2, GL->tmptv
+  |  add CARG2, GL, #offsetof(global_State, tmptv)
+  |2:
+  |   add CARG3, sp, TMPDofs
+  |  str CARG4, TMPD
+  |  b >1
+  |
+  |->vmeta_tsetb:			// RB = table, RC = index
+  |  add RC, RC, TISNUM
+  |   add CARG2, BASE, RB, lsl #3
+  |   add CARG3, sp, TMPDofs
+  |  str RC, TMPD
+  |  b >1
+  |
+  |->vmeta_tsetv:
+  |  add CARG2, BASE, RB, lsl #3
+  |   add CARG3, BASE, RC, lsl #3
+  |1:
+  |   str BASE, L->base
+  |  mov CARG1, L
+  |   str PC, SAVE_PC
+  |  bl extern lj_meta_tset		// (lua_State *L, TValue *o, TValue *k)
+  |  // Returns TValue * (finished) or NULL (metamethod).
+  |   ldr TMP0, [BASE, RA, lsl #3]
+  |  cbz CRET1, >3
+  |  // NOBARRIER: lj_meta_tset ensures the table is not black.
+  |   str TMP0, [CRET1]
+  |  ins_next
+  |
+  |3:  // Call __newindex metamethod.
+  |  // BASE = base, L->top = new base, stack = cont/func/t/k/(v)
+  |   sub TMP1, BASE, #FRAME_CONT
+  |  ldr BASE, L->top
+  |    mov NARGS8:RC, #24		// 3 args for func(t, k, v).
+  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Guaranteed to be a function here.
+  |   str TMP0, [BASE, #16]		// Copy value to third argument.
+  |    str PC, [BASE, #-24]		// [cont|PC]
+  |   sub PC, BASE, TMP1
+  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
+  |  b ->vm_call_dispatch_f
+  |
+  |->vmeta_tsetr:
+  |  sxtw CARG3, TMP1w
+  |  str BASE, L->base
+  |  str PC, SAVE_PC
+  |  bl extern lj_tab_setinth  // (lua_State *L, GCtab *t, int32_t key)
+  |  // Returns TValue *.
+  |  b ->BC_TSETR_Z
+  |
+  |//-- Comparison metamethods ---------------------------------------------
+  |
+  |->vmeta_comp:
+  |  add CARG2, BASE, RA, lsl #3
+  |   sub PC, PC, #4
+  |  add CARG3, BASE, RC, lsl #3
+  |   str BASE, L->base
+  |  mov CARG1, L
+  |   str PC, SAVE_PC
+  |  uxtb CARG4w, INSw
+  |  bl extern lj_meta_comp  // (lua_State *L, TValue *o1, *o2, int op)
+  |  // Returns 0/1 or TValue * (metamethod).
+  |3:
+  |  cmp CRET1, #1
+  |  bhi ->vmeta_binop
+  |4:
+  |   ldrh RBw, [PC, #2]
+  |    add PC, PC, #4
+  |   add RB, PC, RB, lsl #2
+  |   sub RB, RB, #0x20000
+  |  csel PC, PC, RB, lo
+  |->cont_nop:
+  |  ins_next
+  |
+  |->cont_ra:				// RA = resultptr
+  |  ldr INSw, [PC, #-4]
+  |   ldr TMP0, [RA]
+  |  decode_RA TMP1, INS
+  |   str TMP0, [BASE, TMP1, lsl #3]
+  |  b ->cont_nop
+  |
+  |->cont_condt:			// RA = resultptr
+  |  ldr TMP0, [RA]
+  |   mov_true TMP1
+  |  cmp TMP1, TMP0			// Branch if result is true.
+  |  b <4
+  |
+  |->cont_condf:			// RA = resultptr
+  |  ldr TMP0, [RA]
+  |   mov_false TMP1
+  |  cmp TMP0, TMP1			// Branch if result is false.
+  |  b <4
+  |
+  |->vmeta_equal:
+  |  // CARG2, CARG3, CARG4 are already set by BC_ISEQV/BC_ISNEV.
+  |  and TAB:CARG3, CARG3, #LJ_GCVMASK
+  |  sub PC, PC, #4
+  |   str BASE, L->base
+  |   mov CARG1, L
+  |  str PC, SAVE_PC
+  |  bl extern lj_meta_equal  // (lua_State *L, GCobj *o1, *o2, int ne)
+  |  // Returns 0/1 or TValue * (metamethod).
+  |  b <3
+  |
+  |->vmeta_equal_cd:
+  |.if FFI
+  |  sub PC, PC, #4
+  |   str BASE, L->base
+  |   mov CARG1, L
+  |   mov CARG2, INS
+  |  str PC, SAVE_PC
+  |  bl extern lj_meta_equal_cd		// (lua_State *L, BCIns op)
+  |  // Returns 0/1 or TValue * (metamethod).
+  |  b <3
+  |.endif
+  |
+  |->vmeta_istype:
+  |  sub PC, PC, #4
+  |   str BASE, L->base
+  |   mov CARG1, L
+  |   mov CARG2, RA
+  |   mov CARG3, RC
+  |  str PC, SAVE_PC
+  |  bl extern lj_meta_istype  // (lua_State *L, BCReg ra, BCReg tp)
+  |  b ->cont_nop
+  |
+  |//-- Arithmetic metamethods ---------------------------------------------
+  |
+  |->vmeta_arith_vn:
+  |  add CARG3, BASE, RB, lsl #3
+  |   add CARG4, KBASE, RC, lsl #3
+  |  b >1
+  |
+  |->vmeta_arith_nv:
+  |  add CARG4, BASE, RB, lsl #3
+  |   add CARG3, KBASE, RC, lsl #3
+  |  b >1
+  |
+  |->vmeta_unm:
+  |  add CARG3, BASE, RC, lsl #3
+  |  mov CARG4, CARG3
+  |  b >1
+  |
+  |->vmeta_arith_vv:
+  |  add CARG3, BASE, RB, lsl #3
+  |   add CARG4, BASE, RC, lsl #3
+  |1:
+  |  uxtb CARG5w, INSw
+  |   add CARG2, BASE, RA, lsl #3
+  |    str BASE, L->base
+  |   mov CARG1, L
+  |    str PC, SAVE_PC
+  |  bl extern lj_meta_arith  // (lua_State *L, TValue *ra,*rb,*rc, BCReg op)
+  |  // Returns NULL (finished) or TValue * (metamethod).
+  |  cbz CRET1, ->cont_nop
+  |
+  |  // Call metamethod for binary op.
+  |->vmeta_binop:
+  |  // BASE = old base, CRET1 = new base, stack = cont/func/o1/o2
+  |  sub TMP1, CRET1, BASE
+  |   str PC, [CRET1, #-24]		// [cont|PC]
+  |  add PC, TMP1, #FRAME_CONT
+  |  mov BASE, CRET1
+  |   mov NARGS8:RC, #16		// 2 args for func(o1, o2).
+  |  b ->vm_call_dispatch
+  |
+  |->vmeta_len:
+  |  add CARG2, BASE, RC, lsl #3
+#if LJ_52
+  |  mov TAB:RC, TAB:CARG1		// Save table (ignored for other types).
+#endif
+  |   str BASE, L->base
+  |  mov CARG1, L
+  |   str PC, SAVE_PC
+  |  bl extern lj_meta_len		// (lua_State *L, TValue *o)
+  |  // Returns NULL (retry) or TValue * (metamethod base).
+#if LJ_52
+  |  cbnz CRET1, ->vmeta_binop		// Binop call for compatibility.
+  |  mov TAB:CARG1, TAB:RC
+  |  b ->BC_LEN_Z
+#else
+  |  b ->vmeta_binop			// Binop call for compatibility.
+#endif
+  |
+  |//-- Call metamethod ----------------------------------------------------
+  |
+  |->vmeta_call:			// Resolve and call __call metamethod.
+  |  // RB = old base, BASE = new base, RC = nargs*8
+  |  mov CARG1, L
+  |   str RB, L->base			// This is the callers base!
+  |  sub CARG2, BASE, #16
+  |   str PC, SAVE_PC
+  |  add CARG3, BASE, NARGS8:RC
+  |  bl extern lj_meta_call	// (lua_State *L, TValue *func, TValue *top)
+  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]  // Guaranteed to be a function here.
+  |   add NARGS8:RC, NARGS8:RC, #8	// Got one more argument now.
+  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
+  |  ins_call
+  |
+  |->vmeta_callt:			// Resolve __call for BC_CALLT.
+  |  // BASE = old base, RA = new base, RC = nargs*8
+  |  mov CARG1, L
+  |   str BASE, L->base
+  |  sub CARG2, RA, #16
+  |   str PC, SAVE_PC
+  |  add CARG3, RA, NARGS8:RC
+  |  bl extern lj_meta_call	// (lua_State *L, TValue *func, TValue *top)
+  |  ldp LFUNC:CARG3, PC, [RA, FRAME_FUNC]  // Guaranteed to be a function here.
+  |   add NARGS8:RC, NARGS8:RC, #8	// Got one more argument now.
+  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
+  |  b ->BC_CALLT2_Z
+  |
+  |//-- Argument coercion for 'for' statement ------------------------------
+  |
+  |->vmeta_for:
+  |  mov CARG1, L
+  |   str BASE, L->base
+  |  mov CARG2, RA
+  |   str PC, SAVE_PC
+  |  bl extern lj_meta_for	// (lua_State *L, TValue *base)
+  |  ldr INSw, [PC, #-4]
+  |.if JIT
+  |   uxtb TMP0, INS
+  |.endif
+  |  decode_RA RA, INS
+  |  decode_RD RC, INS
+  |.if JIT
+  |   cmp TMP0, #BC_JFORI
+  |   beq =>BC_JFORI
+  |.endif
+  |  b =>BC_FORI
+  |
+  |//-----------------------------------------------------------------------
+  |//-- Fast functions -----------------------------------------------------
+  |//-----------------------------------------------------------------------
+  |
+  |.macro .ffunc, name
+  |->ff_ .. name:
+  |.endmacro
+  |
+  |.macro .ffunc_1, name
+  |->ff_ .. name:
+  |  ldr CARG1, [BASE]
+  |   cmp NARGS8:RC, #8
+  |   blo ->fff_fallback
+  |.endmacro
+  |
+  |.macro .ffunc_2, name
+  |->ff_ .. name:
+  |  ldp CARG1, CARG2, [BASE]
+  |   cmp NARGS8:RC, #16
+  |   blo ->fff_fallback
+  |.endmacro
+  |
+  |.macro .ffunc_n, name
+  |  .ffunc name
+  |  ldr CARG1, [BASE]
+  |   cmp NARGS8:RC, #8
+  |  ldr FARG1, [BASE]
+  |   blo ->fff_fallback
+  |  checknum CARG1, ->fff_fallback
+  |.endmacro
+  |
+  |.macro .ffunc_nn, name
+  |  .ffunc name
+  |  ldp CARG1, CARG2, [BASE]
+  |   cmp NARGS8:RC, #16
+  |  ldp FARG1, FARG2, [BASE]
+  |   blo ->fff_fallback
+  |  checknum CARG1, ->fff_fallback
+  |  checknum CARG2, ->fff_fallback
+  |.endmacro
+  |
+  |// Inlined GC threshold check. Caveat: uses CARG1 and CARG2.
+  |.macro ffgccheck
+  |  ldp CARG1, CARG2, GL->gc.total	// Assumes threshold follows total.
+  |  cmp CARG1, CARG2
+  |  blt >1
+  |  bl ->fff_gcstep
+  |1:
+  |.endmacro
+  |
+  |//-- Base library: checks -----------------------------------------------
+  |
+  |.ffunc_1 assert
+  |   ldr PC, [BASE, FRAME_PC]
+  |  mov_false TMP1
+  |  cmp CARG1, TMP1
+  |  bhs ->fff_fallback
+  |  str CARG1, [BASE, #-16]
+  |  sub RB, BASE, #8
+  |  subs RA, NARGS8:RC, #8
+  |   add RC, NARGS8:RC, #8		// Compute (nresults+1)*8.
+  |  cbz RA, ->fff_res			// Done if exactly 1 argument.
+  |1:
+  |   ldr CARG1, [RB, #16]
+  |  sub RA, RA, #8
+  |   str CARG1, [RB], #8
+  |  cbnz RA, <1
+  |  b ->fff_res
+  |
+  |.ffunc_1 type
+  |  mov TMP0, #~LJ_TISNUM
+  |  asr ITYPE, CARG1, #47
+  |  cmn ITYPE, #~LJ_TISNUM
+  |  csinv TMP1, TMP0, ITYPE, lo
+  |  add TMP1, TMP1, #offsetof(GCfuncC, upvalue)/8
+  |  ldr CARG1, [CFUNC:CARG3, TMP1, lsl #3]
+  |  b ->fff_restv
+  |
+  |//-- Base library: getters and setters ---------------------------------
+  |
+  |.ffunc_1 getmetatable
+  |  asr ITYPE, CARG1, #47
+  |  cmn ITYPE, #-LJ_TTAB
+  |  ccmn ITYPE, #-LJ_TUDATA, #4, ne
+  |   and TAB:CARG1, CARG1, #LJ_GCVMASK
+  |  bne >6
+  |1:  // Field metatable must be at same offset for GCtab and GCudata!
+  |  ldr TAB:RB, TAB:CARG1->metatable
+  |2:
+  |   mov CARG1, TISNIL
+  |   ldr STR:RC, GL->gcroot[GCROOT_MMNAME+MM_metatable]
+  |  cbz TAB:RB, ->fff_restv
+  |  ldr TMP1w, TAB:RB->hmask
+  |   ldr TMP2w, STR:RC->hash
+  |    ldr NODE:CARG3, TAB:RB->node
+  |  and TMP1w, TMP1w, TMP2w		// idx = str->hash & tab->hmask
+  |  add TMP1, TMP1, TMP1, lsl #1
+  |  movn CARG4, #~LJ_TSTR
+  |    add NODE:CARG3, NODE:CARG3, TMP1, lsl #3  // node = tab->node + idx*3*8
+  |  add CARG4, STR:RC, CARG4, lsl #47	// Tagged key to look for.
+  |3:  // Rearranged logic, because we expect _not_ to find the key.
+  |  ldp CARG1, TMP0, NODE:CARG3->val
+  |   ldr NODE:CARG3, NODE:CARG3->next
+  |  cmp TMP0, CARG4
+  |  beq >5
+  |  cbnz NODE:CARG3, <3
+  |4:
+  |  mov CARG1, RB			// Use metatable as default result.
+  |  movk CARG1, #(LJ_TTAB>>1)&0xffff, lsl #48
+  |  b ->fff_restv
+  |5:
+  |  cmp TMP0, TISNIL
+  |  bne ->fff_restv
+  |  b <4
+  |
+  |6:
+  |  movn TMP0, #~LJ_TISNUM
+  |  cmp ITYPE, TMP0
+  |  csel ITYPE, ITYPE, TMP0, hs
+  |  sub TMP1, GL, ITYPE, lsl #3
+  |  ldr TAB:RB, [TMP1, #offsetof(global_State, gcroot[GCROOT_BASEMT])-8]
+  |  b <2
+  |
+  |.ffunc_2 setmetatable
+  |  // Fast path: no mt for table yet and not clearing the mt.
+  |  checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
+  |   ldr TAB:TMP0, TAB:TMP1->metatable
+  |  asr ITYPE, CARG2, #47
+  |   ldrb TMP2w, TAB:TMP1->marked
+  |  cmn ITYPE, #-LJ_TTAB
+  |    and TAB:CARG2, CARG2, #LJ_GCVMASK
+  |  ccmp TAB:TMP0, #0, #0, eq
+  |  bne ->fff_fallback
+  |    str TAB:CARG2, TAB:TMP1->metatable
+  |   tbz TMP2w, #2, ->fff_restv	// isblack(table)
+  |  barrierback TAB:TMP1, TMP2w, TMP0
+  |  b ->fff_restv
+  |
+  |.ffunc rawget
+  |  ldr CARG2, [BASE]
+  |   cmp NARGS8:RC, #16
+  |   blo ->fff_fallback
+  |  checktab CARG2, ->fff_fallback
+  |   mov CARG1, L
+  |   add CARG3, BASE, #8
+  |  bl extern lj_tab_get  // (lua_State *L, GCtab *t, cTValue *key)
+  |  // Returns cTValue *.
+  |  ldr CARG1, [CRET1]
+  |  b ->fff_restv
+  |
+  |//-- Base library: conversions ------------------------------------------
+  |
+  |.ffunc tonumber
+  |  // Only handles the number case inline (without a base argument).
+  |  ldr CARG1, [BASE]
+  |   cmp NARGS8:RC, #8
+  |   bne ->fff_fallback
+  |  checknumber CARG1, ->fff_fallback
+  |  b ->fff_restv
+  |
+  |.ffunc_1 tostring
+  |  // Only handles the string or number case inline.
+  |  asr ITYPE, CARG1, #47
+  |  cmn ITYPE, #-LJ_TSTR
+  |  // A __tostring method in the string base metatable is ignored.
+  |  beq ->fff_restv
+  |  // Handle numbers inline, unless a number base metatable is present.
+  |  ldr TMP1, GL->gcroot[GCROOT_BASEMT_NUM]
+  |   str BASE, L->base
+  |  cmn ITYPE, #-LJ_TISNUM
+  |  ccmp TMP1, #0, #0, ls
+  |   str PC, SAVE_PC			// Redundant (but a defined value).
+  |  bne ->fff_fallback
+  |  ffgccheck
+  |  mov CARG1, L
+  |  mov CARG2, BASE
+  |  bl extern lj_strfmt_number		// (lua_State *L, cTValue *o)
+  |  // Returns GCstr *.
+  |   movn TMP1, #~LJ_TSTR
+  |  ldr BASE, L->base
+  |   add CARG1, CARG1, TMP1, lsl #47
+  |  b ->fff_restv
+  |
+  |//-- Base library: iterators -------------------------------------------
+  |
+  |.ffunc_1 next
+  |  checktp CARG2, CARG1, LJ_TTAB, ->fff_fallback
+  |  str TISNIL, [BASE, NARGS8:RC]	// Set missing 2nd arg to nil.
+  |  ldr PC, [BASE, FRAME_PC]
+  |   stp BASE, BASE, L->base		// Add frame since C call can throw.
+  |  mov CARG1, L
+  |  add CARG3, BASE, #8
+  |   str PC, SAVE_PC
+  |  bl extern lj_tab_next	// (lua_State *L, GCtab *t, TValue *key)
+  |  // Returns 0 at end of traversal.
+  |  str TISNIL, [BASE, #-16]
+  |  cbz CRET1, ->fff_res1		// End of traversal: return nil.
+  |  ldp CARG1, CARG2, [BASE, #8]	// Copy key and value to results.
+  |    mov RC, #(2+1)*8
+  |  stp CARG1, CARG2, [BASE, #-16]
+  |  b ->fff_res
+  |
+  |.ffunc_1 pairs
+  |  checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
+#if LJ_52
+  |  ldr TAB:CARG2, TAB:TMP1->metatable
+#endif
+  |   ldr CFUNC:CARG4, CFUNC:CARG3->upvalue[0]
+  |    ldr PC, [BASE, FRAME_PC]
+#if LJ_52
+  |  cbnz TAB:CARG2, ->fff_fallback
+#endif
+  |  mov RC, #(3+1)*8
+  |  stp CARG1, TISNIL, [BASE, #-8]
+  |   str CFUNC:CARG4, [BASE, #-16]
+  |  b ->fff_res
+  |
+  |.ffunc_2 ipairs_aux
+  |  checktab CARG1, ->fff_fallback
+  |   checkint CARG2, ->fff_fallback
+  |  ldr TMP1w, TAB:CARG1->asize
+  |   ldr CARG3, TAB:CARG1->array
+  |    ldr TMP0w, TAB:CARG1->hmask
+  |  add CARG2w, CARG2w, #1
+  |  cmp CARG2w, TMP1w
+  |    ldr PC, [BASE, FRAME_PC]
+  |     add TMP2, CARG2, TISNUM
+  |   mov RC, #(0+1)*8
+  |     str TMP2, [BASE, #-16]
+  |  bhs >2				// Not in array part?
+  |  ldr TMP0, [CARG3, CARG2, lsl #3]
+  |1:
+  |   mov TMP1, #(2+1)*8
+  |   cmp TMP0, TISNIL
+  |  str TMP0, [BASE, #-8]
+  |   csel RC, RC, TMP1, eq
+  |  b ->fff_res
+  |2:  // Check for empty hash part first. Otherwise call C function.
+  |  cbz TMP0w, ->fff_res
+  |  bl extern lj_tab_getinth		// (GCtab *t, int32_t key)
+  |  // Returns cTValue * or NULL.
+  |  cbz CRET1, ->fff_res
+  |  ldr TMP0, [CRET1]
+  |  b <1
+  |
+  |.ffunc_1 ipairs
+  |  checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
+#if LJ_52
+  |  ldr TAB:CARG2, TAB:TMP1->metatable
+#endif
+  |   ldr CFUNC:CARG4, CFUNC:CARG3->upvalue[0]
+  |    ldr PC, [BASE, FRAME_PC]
+#if LJ_52
+  |  cbnz TAB:CARG2, ->fff_fallback
+#endif
+  |  mov RC, #(3+1)*8
+  |  stp CARG1, TISNUM, [BASE, #-8]
+  |   str CFUNC:CARG4, [BASE, #-16]
+  |  b ->fff_res
+  |
+  |//-- Base library: catch errors ----------------------------------------
+  |
+  |.ffunc pcall
+  |  ldrb TMP0w, GL->hookmask
+  |   subs NARGS8:RC, NARGS8:RC, #8
+  |   blo ->fff_fallback
+  |    mov RB, BASE
+  |    add BASE, BASE, #16
+  |  ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1
+  |  add PC, TMP0, #16+FRAME_PCALL
+  |   beq ->vm_call_dispatch
+  |1:
+  |   add TMP2, BASE, NARGS8:RC
+  |2:
+  |   ldr TMP0, [TMP2, #-16]
+  |   str TMP0, [TMP2, #-8]!
+  |  cmp TMP2, BASE
+  |  bne <2
+  |  b ->vm_call_dispatch
+  |
+  |.ffunc xpcall
+  |     ldp CARG1, CARG2, [BASE]
+  |  ldrb TMP0w, GL->hookmask
+  |   subs NARGS8:RC, NARGS8:RC, #16
+  |   blo ->fff_fallback
+  |    mov RB, BASE
+  |    add BASE, BASE, #24
+  |     asr ITYPE, CARG2, #47
+  |  ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1
+  |     cmn ITYPE, #-LJ_TFUNC
+  |  add PC, TMP0, #24+FRAME_PCALL
+  |     bne ->fff_fallback		// Traceback must be a function.
+  |     stp CARG2, CARG1, [RB]		// Swap function and traceback.
+  |   cbz NARGS8:RC, ->vm_call_dispatch
+  |  b <1
+  |
+  |//-- Coroutine library --------------------------------------------------
+  |
+  |.macro coroutine_resume_wrap, resume
+  |.if resume
+  |.ffunc_1 coroutine_resume
+  |  checktp CARG1, LJ_TTHREAD, ->fff_fallback
+  |.else
+  |.ffunc coroutine_wrap_aux
+  |  ldr L:CARG1, CFUNC:CARG3->upvalue[0].gcr
+  |  and L:CARG1, CARG1, #LJ_GCVMASK
+  |.endif
+  |   ldr PC, [BASE, FRAME_PC]
+  |     str BASE, L->base
+  |  ldp RB, CARG2, L:CARG1->base
+  |   ldrb TMP1w, L:CARG1->status
+  |  add TMP0, CARG2, TMP1
+  |   str PC, SAVE_PC
+  |  cmp TMP0, RB
+  |  beq ->fff_fallback
+  |   cmp TMP1, #LUA_YIELD
+  |    add TMP0, CARG2, #8
+  |   csel CARG2, CARG2, TMP0, hs
+  |   ldr CARG4, L:CARG1->maxstack
+  |   add CARG3, CARG2, NARGS8:RC
+  |    ldr RB, L:CARG1->cframe
+  |   ccmp CARG3, CARG4, #2, ls
+  |    ccmp RB, #0, #2, ls
+  |    bhi ->fff_fallback
+  |.if resume
+  |  sub CARG3, CARG3, #8		// Keep resumed thread in stack for GC.
+  |  add BASE, BASE, #8
+  |  sub NARGS8:RC, NARGS8:RC, #8
+  |.endif
+  |  str CARG3, L:CARG1->top
+  |  str BASE, L->top
+  |  cbz NARGS8:RC, >3
+  |2:  // Move args to coroutine.
+  |   ldr TMP0, [BASE, RB]
+  |  cmp RB, NARGS8:RC
+  |   str TMP0, [CARG2, RB]
+  |   add RB, RB, #8
+  |  bne <2
+  |3:
+  |  mov CARG3, #0
+  |   mov L:RA, L:CARG1
+  |  mov CARG4, #0
+  |  bl ->vm_resume			// (lua_State *L, TValue *base, 0, 0)
+  |  // Returns thread status.
+  |4:
+  |  ldp CARG3, CARG4, L:RA->base
+  |   cmp CRET1, #LUA_YIELD
+  |  ldr BASE, L->base
+  |    str L, GL->cur_L
+  |    st_vmstate ST_INTERP
+  |   bhi >8
+  |  sub RC, CARG4, CARG3
+  |   ldr CARG1, L->maxstack
+  |   add CARG2, BASE, RC
+  |  cbz RC, >6				// No results?
+  |  cmp CARG2, CARG1
+  |   mov RB, #0
+  |  bhi >9				// Need to grow stack?
+  |
+  |  sub CARG4, RC, #8
+  |   str CARG3, L:RA->top		// Clear coroutine stack.
+  |5:  // Move results from coroutine.
+  |   ldr TMP0, [CARG3, RB]
+  |  cmp RB, CARG4
+  |   str TMP0, [BASE, RB]
+  |   add RB, RB, #8
+  |  bne <5
+  |6:
+  |.if resume
+  |  mov_true TMP1
+  |   add RC, RC, #16
+  |7:
+  |  str TMP1, [BASE, #-8]		// Prepend true/false to results.
+  |   sub RA, BASE, #8
+  |.else
+  |   mov RA, BASE
+  |   add RC, RC, #8
+  |.endif
+  |  ands CARG1, PC, #FRAME_TYPE
+  |   str PC, SAVE_PC
+  |   str RCw, SAVE_MULTRES
+  |  beq ->BC_RET_Z
+  |  b ->vm_return
+  |
+  |8:  // Coroutine returned with error (at co->top-1).
+  |.if resume
+  |  ldr TMP0, [CARG4, #-8]!
+  |   mov_false TMP1
+  |    mov RC, #(2+1)*8
+  |  str CARG4, L:RA->top		// Remove error from coroutine stack.
+  |  str TMP0, [BASE]			// Copy error message.
+  |  b <7
+  |.else
+  |  mov CARG1, L
+  |  mov CARG2, L:RA
+  |  bl extern lj_ffh_coroutine_wrap_err  // (lua_State *L, lua_State *co)
+  |  // Never returns.
+  |.endif
+  |
+  |9:  // Handle stack expansion on return from yield.
+  |  mov CARG1, L
+  |  lsr CARG2, RC, #3
+  |  bl extern lj_state_growstack	// (lua_State *L, int n)
+  |  mov CRET1, #0
+  |  b <4
+  |.endmacro
+  |
+  |  coroutine_resume_wrap 1		// coroutine.resume
+  |  coroutine_resume_wrap 0		// coroutine.wrap
+  |
+  |.ffunc coroutine_yield
+  |  ldr TMP0, L->cframe
+  |   add TMP1, BASE, NARGS8:RC
+  |    mov CRET1, #LUA_YIELD
+  |   stp BASE, TMP1, L->base
+  |  tbz TMP0, #0, ->fff_fallback
+  |   str xzr, L->cframe
+  |    strb CRET1w, L->status
+  |  b ->vm_leave_unw
+  |
+  |//-- Math library -------------------------------------------------------
+  |
+  |.macro math_round, func, round
+  |  .ffunc math_ .. func
+  |  ldr CARG1, [BASE]
+  |   cmp NARGS8:RC, #8
+  |  ldr d0, [BASE]
+  |   blo ->fff_fallback
+  |  cmp TISNUMhi, CARG1, lsr #32
+  |  beq ->fff_restv
+  |  round d0, d0
+  |  b ->fff_resn
+  |.endmacro
+  |
+  |  math_round floor, frintm
+  |  math_round ceil, frintp
+  |
+  |.ffunc_1 math_abs
+  |  checknumber CARG1, ->fff_fallback
+  |  and CARG1, CARG1, #U64x(7fffffff,ffffffff)
+  |  bne ->fff_restv
+  |  eor CARG2w, CARG1w, CARG1w, asr #31
+  |   movz CARG3, #0x41e0, lsl #48	// 2^31.
+  |  subs CARG1w, CARG2w, CARG1w, asr #31
+  |   add CARG1, CARG1, TISNUM
+  |  csel CARG1, CARG1, CARG3, pl
+  |  // Fallthrough.
+  |
+  |->fff_restv:
+  |  // CARG1 = TValue result.
+  |  ldr PC, [BASE, FRAME_PC]
+  |  str CARG1, [BASE, #-16]
+  |->fff_res1:
+  |  // PC = return.
+  |  mov RC, #(1+1)*8
+  |->fff_res:
+  |  // RC = (nresults+1)*8, PC = return.
+  |  ands CARG1, PC, #FRAME_TYPE
+  |   str RCw, SAVE_MULTRES
+  |   sub RA, BASE, #16
+  |  bne ->vm_return
+  |  ldr INSw, [PC, #-4]
+  |  decode_RB RB, INS
+  |5:
+  |  cmp RC, RB, lsl #3			// More results expected?
+  |  blo >6
+  |  decode_RA TMP1, INS
+  |  // Adjust BASE. KBASE is assumed to be set for the calling frame.
+  |  sub BASE, RA, TMP1, lsl #3
+  |  ins_next
+  |
+  |6:  // Fill up results with nil.
+  |  add TMP1, RA, RC
+  |   add RC, RC, #8
+  |  str TISNIL, [TMP1, #-8]
+  |  b <5
+  |
+  |.macro math_extern, func
+  |  .ffunc_n math_ .. func
+  |  bl extern func
+  |  b ->fff_resn
+  |.endmacro
+  |
+  |.macro math_extern2, func
+  |  .ffunc_nn math_ .. func
+  |  bl extern func
+  |  b ->fff_resn
+  |.endmacro
+  |
+  |.ffunc_n math_sqrt
+  |  fsqrt d0, d0
+  |->fff_resn:
+  |  ldr PC, [BASE, FRAME_PC]
+  |  str d0, [BASE, #-16]
+  |  b ->fff_res1
+  |
+  |.ffunc math_log
+  |  ldr CARG1, [BASE]
+  |   cmp NARGS8:RC, #8
+  |  ldr FARG1, [BASE]
+  |   bne ->fff_fallback			// Need exactly 1 argument.
+  |  checknum CARG1, ->fff_fallback
+  |  bl extern log
+  |  b ->fff_resn
+  |
+  |  math_extern log10
+  |  math_extern exp
+  |  math_extern sin
+  |  math_extern cos
+  |  math_extern tan
+  |  math_extern asin
+  |  math_extern acos
+  |  math_extern atan
+  |  math_extern sinh
+  |  math_extern cosh
+  |  math_extern tanh
+  |  math_extern2 pow
+  |  math_extern2 atan2
+  |  math_extern2 fmod
+  |
+  |.ffunc_2 math_ldexp
+  |  ldr FARG1, [BASE]
+  |  checknum CARG1, ->fff_fallback
+  |  checkint CARG2, ->fff_fallback
+  |  sxtw CARG1, CARG2w
+  |  bl extern ldexp			// (double x, int exp)
+  |  b ->fff_resn
+  |
+  |.ffunc_n math_frexp
+  |  add CARG1, sp, TMPDofs
+  |  bl extern frexp
+  |   ldr CARG2w, TMPD
+  |    ldr PC, [BASE, FRAME_PC]
+  |  str d0, [BASE, #-16]
+  |    mov RC, #(2+1)*8
+  |   add CARG2, CARG2, TISNUM
+  |   str CARG2, [BASE, #-8]
+  |  b ->fff_res
+  |
+  |.ffunc_n math_modf
+  |  sub CARG1, BASE, #16
+  |   ldr PC, [BASE, FRAME_PC]
+  |  bl extern modf
+  |   mov RC, #(2+1)*8
+  |  str d0, [BASE, #-8]
+  |  b ->fff_res
+  |
+  |.macro math_minmax, name, cond, fcond
+  |  .ffunc_1 name
+  |   add RB, BASE, RC
+  |   add RA, BASE, #8
+  |  checkint CARG1, >4
+  |1:  // Handle integers.
+  |  ldr CARG2, [RA]
+  |   cmp RA, RB
+  |   bhs ->fff_restv
+  |  checkint CARG2, >3
+  |  cmp CARG1w, CARG2w
+  |   add RA, RA, #8
+  |  csel CARG1, CARG2, CARG1, cond
+  |  b <1
+  |3:  // Convert intermediate result to number and continue below.
+  |  scvtf d0, CARG1w
+  |  blo ->fff_fallback
+  |  ldr d1, [RA]
+  |  b >6
+  |
+  |4:
+  |  ldr d0, [BASE]
+  |  blo ->fff_fallback
+  |5:  // Handle numbers.
+  |  ldr CARG2, [RA]
+  |  ldr d1, [RA]
+  |   cmp RA, RB
+  |   bhs ->fff_resn
+  |  checknum CARG2, >7
+  |6:
+  |  fcmp d0, d1
+  |   add RA, RA, #8
+  |  fcsel d0, d1, d0, fcond
+  |  b <5
+  |7:  // Convert integer to number and continue above.
+  |  scvtf d1, CARG2w
+  |  blo ->fff_fallback
+  |  b <6
+  |.endmacro
+  |
+  |  math_minmax math_min, gt, hi
+  |  math_minmax math_max, lt, lo
+  |
+  |//-- String library -----------------------------------------------------
+  |
+  |.ffunc string_byte			// Only handle the 1-arg case here.
+  |  ldp PC, CARG1, [BASE, FRAME_PC]
+  |   cmp NARGS8:RC, #8
+  |  asr ITYPE, CARG1, #47
+  |  ccmn ITYPE, #-LJ_TSTR, #0, eq
+  |   and STR:CARG1, CARG1, #LJ_GCVMASK
+  |  bne ->fff_fallback
+  |  ldrb TMP0w, STR:CARG1[1]		// Access is always ok (NUL at end).
+  |   ldr CARG3w, STR:CARG1->len
+  |  add TMP0, TMP0, TISNUM
+  |  str TMP0, [BASE, #-16]
+  |  mov RC, #(0+1)*8
+  |   cbz CARG3, ->fff_res
+  |  b ->fff_res1
+  |
+  |.ffunc string_char			// Only handle the 1-arg case here.
+  |  ffgccheck
+  |  ldp PC, CARG1, [BASE, FRAME_PC]
+  |  cmp CARG1w, #255
+  |   ccmp NARGS8:RC, #8, #0, ls		// Need exactly 1 argument.
+  |  bne ->fff_fallback
+  |  checkint CARG1, ->fff_fallback
+  |  mov CARG3, #1
+  |  mov CARG2, BASE			// Points to stack. Little-endian.
+  |->fff_newstr:
+  |  // CARG2 = str, CARG3 = len.
+  |   str BASE, L->base
+  |  mov CARG1, L
+  |   str PC, SAVE_PC
+  |  bl extern lj_str_new		// (lua_State *L, char *str, size_t l)
+  |->fff_resstr:
+  |  // Returns GCstr *.
+  |  ldr BASE, L->base
+  |   movn TMP1, #~LJ_TSTR
+  |  add CARG1, CARG1, TMP1, lsl #47
+  |  b ->fff_restv
+  |
+  |.ffunc string_sub
+  |  ffgccheck
+  |  ldr CARG1, [BASE]
+  |    ldr CARG3, [BASE, #16]
+  |   cmp NARGS8:RC, #16
+  |    movn RB, #0
+  |   beq >1
+  |   blo ->fff_fallback
+  |    checkint CARG3, ->fff_fallback
+  |    sxtw RB, CARG3w
+  |1:
+  |  ldr CARG2, [BASE, #8]
+  |  checkstr CARG1, ->fff_fallback
+  |   ldr TMP1w, STR:CARG1->len
+  |  checkint CARG2, ->fff_fallback
+  |  sxtw CARG2, CARG2w
+  |  // CARG1 = str, TMP1 = str->len, CARG2 = start, RB = end
+  |   add TMP2, RB, TMP1
+  |   cmp RB, #0
+  |  add TMP0, CARG2, TMP1
+  |   csinc RB, RB, TMP2, ge		// if (end < 0) end += len+1
+  |  cmp CARG2, #0
+  |  csinc CARG2, CARG2, TMP0, ge	// if (start < 0) start += len+1
+  |   cmp RB, #0
+  |   csel RB, RB, xzr, ge		// if (end < 0) end = 0
+  |  cmp CARG2, #1
+  |  csinc CARG2, CARG2, xzr, ge	// if (start < 1) start = 1
+  |   cmp RB, TMP1
+  |   csel RB, RB, TMP1, le		// if (end > len) end = len
+  |  add CARG1, STR:CARG1, #sizeof(GCstr)-1
+  |   subs CARG3, RB, CARG2		// len = end - start
+  |  add CARG2, CARG1, CARG2
+  |   add CARG3, CARG3, #1		// len += 1
+  |   bge ->fff_newstr
+  |  add STR:CARG1, GL, #offsetof(global_State, strempty)
+  |   movn TMP1, #~LJ_TSTR
+  |  add CARG1, CARG1, TMP1, lsl #47
+  |  b ->fff_restv
+  |
+  |.macro ffstring_op, name
+  |  .ffunc string_ .. name
+  |  ffgccheck
+  |  ldr CARG2, [BASE]
+  |   cmp NARGS8:RC, #8
+  |  asr ITYPE, CARG2, #47
+  |  ccmn ITYPE, #-LJ_TSTR, #0, hs
+  |   and STR:CARG2, CARG2, #LJ_GCVMASK
+  |  bne ->fff_fallback
+  |  ldr TMP0, GL->tmpbuf.b
+  |   add SBUF:CARG1, GL, #offsetof(global_State, tmpbuf)
+  |   str BASE, L->base
+  |   str PC, SAVE_PC
+  |   str L, GL->tmpbuf.L
+  |  str TMP0, GL->tmpbuf.p
+  |  bl extern lj_buf_putstr_ .. name
+  |  bl extern lj_buf_tostr
+  |  b ->fff_resstr
+  |.endmacro
+  |
+  |ffstring_op reverse
+  |ffstring_op lower
+  |ffstring_op upper
+  |
+  |//-- Bit library --------------------------------------------------------
+  |
+  |// FP number to bit conversion for soft-float. Clobbers CARG1-CARG3
+  |->vm_tobit_fb:
+  |  bls ->fff_fallback
+  |  add CARG2, CARG1, CARG1
+  |  mov CARG3, #1076
+  |  sub CARG3, CARG3, CARG2, lsr #53
+  |  cmp CARG3, #53
+  |  bhi >1
+  |  and CARG2, CARG2, #U64x(001fffff,ffffffff)
+  |  orr CARG2, CARG2, #U64x(00200000,00000000)
+  |   cmp CARG1, #0
+  |  lsr CARG2, CARG2, CARG3
+  |   cneg CARG1w, CARG2w, mi
+  |  br lr
+  |1:
+  |  mov CARG1w, #0
+  |  br lr
+  |
+  |.macro .ffunc_bit, name
+  |  .ffunc_1 bit_..name
+  |  adr lr, >1
+  |  checkint CARG1, ->vm_tobit_fb
+  |1:
+  |.endmacro
+  |
+  |.macro .ffunc_bit_op, name, ins
+  |  .ffunc_bit name
+  |  mov RA, #8
+  |  mov TMP0w, CARG1w
+  |  adr lr, >2
+  |1:
+  |  ldr CARG1, [BASE, RA]
+  |   cmp RA, NARGS8:RC
+  |    add RA, RA, #8
+  |   bge >9
+  |  checkint CARG1, ->vm_tobit_fb
+  |2:
+  |  ins TMP0w, TMP0w, CARG1w
+  |  b <1
+  |.endmacro
+  |
+  |.ffunc_bit_op band, and
+  |.ffunc_bit_op bor, orr
+  |.ffunc_bit_op bxor, eor
+  |
+  |.ffunc_bit tobit
+  |  mov TMP0w, CARG1w
+  |9:  // Label reused by .ffunc_bit_op users.
+  |  add CARG1, TMP0, TISNUM
+  |  b ->fff_restv
+  |
+  |.ffunc_bit bswap
+  |  rev TMP0w, CARG1w
+  |  add CARG1, TMP0, TISNUM
+  |  b ->fff_restv
+  |
+  |.ffunc_bit bnot
+  |  mvn TMP0w, CARG1w
+  |  add CARG1, TMP0, TISNUM
+  |  b ->fff_restv
+  |
+  |.macro .ffunc_bit_sh, name, ins, shmod
+  |  .ffunc bit_..name
+  |  ldp TMP0, CARG1, [BASE]
+  |   cmp NARGS8:RC, #16
+  |   blo ->fff_fallback
+  |  adr lr, >1
+  |  checkint CARG1, ->vm_tobit_fb
+  |1:
+  |.if shmod == 0
+  |  mov TMP1, CARG1
+  |.else
+  |  neg TMP1, CARG1
+  |.endif
+  |  mov CARG1, TMP0
+  |  adr lr, >2
+  |  checkint CARG1, ->vm_tobit_fb
+  |2:
+  |  ins TMP0w, CARG1w, TMP1w
+  |  add CARG1, TMP0, TISNUM
+  |  b ->fff_restv
+  |.endmacro
+  |
+  |.ffunc_bit_sh lshift, lsl, 0
+  |.ffunc_bit_sh rshift, lsr, 0
+  |.ffunc_bit_sh arshift, asr, 0
+  |.ffunc_bit_sh rol, ror, 1
+  |.ffunc_bit_sh ror, ror, 0
+  |
+  |//-----------------------------------------------------------------------
+  |
+  |->fff_fallback:			// Call fast function fallback handler.
+  |  // BASE = new base, RC = nargs*8
+  |   ldp CFUNC:CARG3, PC, [BASE, FRAME_FUNC]	// Fallback may overwrite PC.
+  |  ldr TMP2, L->maxstack
+  |  add TMP1, BASE, NARGS8:RC
+  |  stp BASE, TMP1, L->base
+  |   and CFUNC:CARG3, CARG3, #LJ_GCVMASK
+  |  add TMP1, TMP1, #8*LUA_MINSTACK
+  |   ldr CARG3, CFUNC:CARG3->f
+  |    str PC, SAVE_PC			// Redundant (but a defined value).
+  |  cmp TMP1, TMP2
+  |   mov CARG1, L
+  |  bhi >5				// Need to grow stack.
+  |   blr CARG3				// (lua_State *L)
+  |  // Either throws an error, or recovers and returns -1, 0 or nresults+1.
+  |   ldr BASE, L->base
+  |  cmp CRET1w, #0
+  |   lsl RC, CRET1, #3
+  |   sub RA, BASE, #16
+  |  bgt ->fff_res			// Returned nresults+1?
+  |1:  // Returned 0 or -1: retry fast path.
+  |   ldr CARG1, L->top
+  |    ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
+  |   sub NARGS8:RC, CARG1, BASE
+  |  bne ->vm_call_tail			// Returned -1?
+  |    and CFUNC:CARG3, CARG3, #LJ_GCVMASK
+  |  ins_callt				// Returned 0: retry fast path.
+  |
+  |// Reconstruct previous base for vmeta_call during tailcall.
+  |->vm_call_tail:
+  |  ands TMP0, PC, #FRAME_TYPE
+  |   and TMP1, PC, #~FRAME_TYPEP
+  |  bne >3
+  |  ldrb RAw, [PC, #-3]
+  |  lsl RA, RA, #3
+  |  add TMP1, RA, #16
+  |3:
+  |  sub RB, BASE, TMP1
+  |  b ->vm_call_dispatch		// Resolve again for tailcall.
+  |
+  |5:  // Grow stack for fallback handler.
+  |  mov CARG2, #LUA_MINSTACK
+  |  bl extern lj_state_growstack	// (lua_State *L, int n)
+  |  ldr BASE, L->base
+  |  cmp CARG1, CARG1			// Set zero-flag to force retry.
+  |  b <1
+  |
+  |->fff_gcstep:			// Call GC step function.
+  |  // BASE = new base, RC = nargs*8
+  |   add CARG2, BASE, NARGS8:RC	// Calculate L->top.
+  |  mov RA, lr
+  |   stp BASE, CARG2, L->base
+  |   str PC, SAVE_PC			// Redundant (but a defined value).
+  |  mov CARG1, L
+  |  bl extern lj_gc_step		// (lua_State *L)
+  |  ldp BASE, CARG2, L->base
+  |   ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
+  |  mov lr, RA				// Help return address predictor.
+  |  sub NARGS8:RC, CARG2, BASE		// Calculate nargs*8.
+  |   and CFUNC:CARG3, CARG3, #LJ_GCVMASK
+  |  ret
+  |
+  |//-----------------------------------------------------------------------
+  |//-- Special dispatch targets -------------------------------------------
+  |//-----------------------------------------------------------------------
+  |
+  |->vm_record:				// Dispatch target for recording phase.
+  |  NYI
+  |
+  |->vm_rethook:			// Dispatch target for return hooks.
+  |  ldrb TMP2w, GL->hookmask
+  |  tbz TMP2w, #HOOK_ACTIVE_SHIFT, >1	// Hook already active?
+  |5:  // Re-dispatch to static ins.
+  |  ldr TMP0, [TMP1, #GG_G2DISP+GG_DISP2STATIC]
+  |  br TMP0
+  |
+  |->vm_inshook:			// Dispatch target for instr/line hooks.
+  |  ldrb TMP2w, GL->hookmask
+  |   ldr TMP3w, GL->hookcount
+  |  tbnz TMP2w, #HOOK_ACTIVE_SHIFT, <5	// Hook already active?
+  |  tst TMP2w, #LUA_MASKLINE|LUA_MASKCOUNT
+  |  beq <5
+  |   sub TMP3w, TMP3w, #1
+  |   str TMP3w, GL->hookcount
+  |   cbz TMP3w, >1
+  |  tbz TMP2w, #LUA_HOOKLINE, <5
+  |1:
+  |  mov CARG1, L
+  |   str BASE, L->base
+  |  mov CARG2, PC
+  |  // SAVE_PC must hold the _previous_ PC. The callee updates it with PC.
+  |  bl extern lj_dispatch_ins		// (lua_State *L, const BCIns *pc)
+  |3:
+  |  ldr BASE, L->base
+  |4:  // Re-dispatch to static ins.
+  |  ldr INSw, [PC, #-4]
+  |  add TMP1, GL, INS, uxtb #3
+  |   decode_RA RA, INS
+  |  ldr TMP0, [TMP1, #GG_G2DISP+GG_DISP2STATIC]
+  |   decode_RD RC, INS
+  |  br TMP0
+  |
+  |->cont_hook:				// Continue from hook yield.
+  |  ldr CARG1, [CARG4, #-40]
+  |   add PC, PC, #4
+  |  str CARG1w, SAVE_MULTRES		// Restore MULTRES for *M ins.
+  |  b <4
+  |
+  |->vm_hotloop:			// Hot loop counter underflow.
+  |  NYI
+  |
+  |->vm_callhook:			// Dispatch target for call hooks.
+  |  mov CARG2, PC
+  |.if JIT
+  |  b >1
+  |.endif
+  |
+  |->vm_hotcall:			// Hot call counter underflow.
+  |.if JIT
+  |  orr CARG2, PC, #1
+  |1:
+  |.endif
+  |  add TMP1, BASE, NARGS8:RC
+  |   str PC, SAVE_PC
+  |   mov CARG1, L
+  |   sub RA, RA, BASE
+  |  stp BASE, TMP1, L->base
+  |  bl extern lj_dispatch_call		// (lua_State *L, const BCIns *pc)
+  |  // Returns ASMFunction.
+  |  ldp BASE, TMP1, L->base
+  |   str xzr, SAVE_PC			// Invalidate for subsequent line hook.
+  |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
+  |  add RA, BASE, RA
+  |  sub NARGS8:RC, TMP1, BASE
+  |   ldr INSw, [PC, #-4]
+  |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
+  |  br CRET1
+  |
+  |->cont_stitch:			// Trace stitching.
+  |  NYI
+  |
+  |->vm_profhook:			// Dispatch target for profiler hook.
+#if LJ_HASPROFILE
+  |  mov CARG1, L
+  |   str BASE, L->base
+  |  mov CARG2, PC
+  |  bl extern lj_dispatch_profile	// (lua_State *L, const BCIns *pc)
+  |  // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
+  |  ldr BASE, L->base
+  |  sub PC, PC, #4
+  |  b ->cont_nop
+#endif
+  |
+  |//-----------------------------------------------------------------------
+  |//-- Trace exit handler -------------------------------------------------
+  |//-----------------------------------------------------------------------
+  |
+  |->vm_exit_handler:
+  |  NYI
+  |->vm_exit_interp:
+  |  NYI
+  |
+  |//-----------------------------------------------------------------------
+  |//-- Math helper functions ----------------------------------------------
+  |//-----------------------------------------------------------------------
+  |
+  |  // int lj_vm_modi(int dividend, int divisor);
+  |->vm_modi:
+  |    eor CARG4w, CARG1w, CARG2w
+  |    cmp CARG4w, #0
+  |  eor CARG3w, CARG1w, CARG1w, asr #31
+  |   eor CARG4w, CARG2w, CARG2w, asr #31
+  |  sub CARG3w, CARG3w, CARG1w, asr #31
+  |   sub CARG4w, CARG4w, CARG2w, asr #31
+  |  udiv CARG1w, CARG3w, CARG4w
+  |  msub CARG1w, CARG1w, CARG4w, CARG3w
+  |    ccmp CARG1w, #0, #4, mi
+  |    sub CARG3w, CARG1w, CARG4w
+  |    csel CARG1w, CARG1w, CARG3w, eq
+  |  eor CARG3w, CARG1w, CARG2w
+  |  cmp CARG3w, #0
+  |  cneg CARG1w, CARG1w, mi
+  |  ret
+  |
+  |//-----------------------------------------------------------------------
+  |//-- Miscellaneous functions --------------------------------------------
+  |//-----------------------------------------------------------------------
+  |
+  |//-----------------------------------------------------------------------
+  |//-- FFI helper functions -----------------------------------------------
+  |//-----------------------------------------------------------------------
+  |
+  |// Handler for callback functions.
+  |// Saveregs already performed. Callback slot number in [sp], g in r12.
+  |->vm_ffi_callback:
+  |.if FFI
+  |  NYI
+  |.endif
+  |
+  |->cont_ffi_callback:			// Return from FFI callback.
+  |.if FFI
+  |  NYI
+  |.endif
+  |
+  |->vm_ffi_call:			// Call C function via FFI.
+  |  // Caveat: needs special frame unwinding, see below.
+  |.if FFI
+  |  NYI
+  |.endif
+  |// Note: vm_ffi_call must be the last function in this object file!
+  |
+  |//-----------------------------------------------------------------------
+}
+
+/* Generate the code for a single instruction. */
+static void build_ins(BuildCtx *ctx, BCOp op, int defop)
+{
+  int vk = 0;
+  |=>defop:
+
+  switch (op) {
+
+  /* -- Comparison ops ---------------------------------------------------- */
+
+  /* Remember: all ops branch for a true comparison, fall through otherwise. */
+
+  case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
+    |  // RA = src1, RC = src2, JMP with RC = target
+    |  ldr CARG1, [BASE, RA, lsl #3]
+    |    ldrh RBw, [PC, #2]
+    |   ldr CARG2, [BASE, RC, lsl #3]
+    |    add PC, PC, #4
+    |    add RB, PC, RB, lsl #2
+    |    sub RB, RB, #0x20000
+    |  checkint CARG1, >3
+    |   checkint CARG2, >4
+    |  cmp CARG1w, CARG2w
+    if (op == BC_ISLT) {
+      |  csel PC, RB, PC, lt
+    } else if (op == BC_ISGE) {
+      |  csel PC, RB, PC, ge
+    } else if (op == BC_ISLE) {
+      |  csel PC, RB, PC, le
+    } else {
+      |  csel PC, RB, PC, gt
+    }
+    |1:
+    |  ins_next
+    |
+    |3:  // RA not int.
+    |    ldr FARG1, [BASE, RA, lsl #3]
+    |  blo ->vmeta_comp
+    |    ldr FARG2, [BASE, RC, lsl #3]
+    |   cmp TISNUMhi, CARG2, lsr #32
+    |   bhi >5
+    |   bne ->vmeta_comp
+    |  // RA number, RC int.
+    |  scvtf FARG2, CARG2w
+    |  b >5
+    |
+    |4:  // RA int, RC not int
+    |    ldr FARG2, [BASE, RC, lsl #3]
+    |   blo ->vmeta_comp
+    |  // RA int, RC number.
+    |  scvtf FARG1, CARG1w
+    |
+    |5:  // RA number, RC number
+    |  fcmp FARG1, FARG2
+    |  // To preserve NaN semantics GE/GT branch on unordered, but LT/LE don't.
+    if (op == BC_ISLT) {
+      |  csel PC, RB, PC, lo
+    } else if (op == BC_ISGE) {
+      |  csel PC, RB, PC, hs
+    } else if (op == BC_ISLE) {
+      |  csel PC, RB, PC, ls
+    } else {
+      |  csel PC, RB, PC, hi
+    }
+    |  b <1
+    break;
+
+  case BC_ISEQV: case BC_ISNEV:
+    vk = op == BC_ISEQV;
+    |  // RA = src1, RC = src2, JMP with RC = target
+    |  ldr CARG1, [BASE, RA, lsl #3]
+    |   add RC, BASE, RC, lsl #3
+    |    ldrh RBw, [PC, #2]
+    |   ldr CARG3, [RC]
+    |    add PC, PC, #4
+    |    add RB, PC, RB, lsl #2
+    |    sub RB, RB, #0x20000
+    |  asr ITYPE, CARG3, #47
+    |  cmn ITYPE, #-LJ_TISNUM
+    if (vk) {
+      |  bls ->BC_ISEQN_Z
+    } else {
+      |  bls ->BC_ISNEN_Z
+    }
+    |  // RC is not a number.
+    |   asr TMP0, CARG1, #47
+    |.if FFI
+    |  // Check if RC or RA is a cdata.
+    |  cmn ITYPE, #-LJ_TCDATA
+    |   ccmn TMP0, #-LJ_TCDATA, #4, ne
+    |  beq ->vmeta_equal_cd
+    |.endif
+    |  cmp CARG1, CARG3
+    |  bne >2
+    |  // Tag and value are equal.
+    if (vk) {
+      |->BC_ISEQV_Z:
+      |  mov PC, RB			// Perform branch.
+    }
+    |1:
+    |  ins_next
+    |
+    |2:  // Check if the tags are the same and it's a table or userdata.
+    |  cmp ITYPE, TMP0
+    |  ccmn ITYPE, #-LJ_TISTABUD, #2, eq
+    if (vk) {
+      |  bhi <1
+    } else {
+      |  bhi ->BC_ISEQV_Z		// Reuse code from opposite instruction.
+    }
+    |  // Different tables or userdatas. Need to check __eq metamethod.
+    |  // Field metatable must be at same offset for GCtab and GCudata!
+    |  and TAB:CARG2, CARG1, #LJ_GCVMASK
+    |  ldr TAB:TMP2, TAB:CARG2->metatable
+    if (vk) {
+      |  cbz TAB:TMP2, <1		// No metatable?
+      |  ldrb TMP1w, TAB:TMP2->nomm
+      |   mov CARG4, #0			// ne = 0
+      |  tbnz TMP1w, #MM_eq, <1		// 'no __eq' flag set: done.
+    } else {
+      |  cbz TAB:TMP2, ->BC_ISEQV_Z	// No metatable?
+      |  ldrb TMP1w, TAB:TMP2->nomm
+      |   mov CARG4, #1			// ne = 1.
+      |  tbnz TMP1w, #MM_eq, ->BC_ISEQV_Z	// 'no __eq' flag set: done.
+    }
+    |  b ->vmeta_equal
+    break;
+
+  case BC_ISEQS: case BC_ISNES:
+    vk = op == BC_ISEQS;
+    |  // RA = src, RC = str_const (~), JMP with RC = target
+    |  ldr CARG1, [BASE, RA, lsl #3]
+    |   mvn RC, RC
+    |    ldrh RBw, [PC, #2]
+    |   ldr CARG2, [KBASE, RC, lsl #3]
+    |    add PC, PC, #4
+    |   movn TMP0, #~LJ_TSTR
+    |.if FFI
+    |  asr ITYPE, CARG1, #47
+    |.endif
+    |    add RB, PC, RB, lsl #2
+    |   add CARG2, CARG2, TMP0, lsl #47
+    |    sub RB, RB, #0x20000
+    |.if FFI
+    |  cmn ITYPE, #-LJ_TCDATA
+    |  beq ->vmeta_equal_cd
+    |.endif
+    |  cmp CARG1, CARG2
+    if (vk) {
+      |  csel PC, RB, PC, eq
+    } else {
+      |  csel PC, RB, PC, ne
+    }
+    |  ins_next
+    break;
+
+  case BC_ISEQN: case BC_ISNEN:
+    vk = op == BC_ISEQN;
+    |  // RA = src, RC = num_const (~), JMP with RC = target
+    |  ldr CARG1, [BASE, RA, lsl #3]
+    |   add RC, KBASE, RC, lsl #3
+    |    ldrh RBw, [PC, #2]
+    |   ldr CARG3, [RC]
+    |    add PC, PC, #4
+    |    add RB, PC, RB, lsl #2
+    |    sub RB, RB, #0x20000
+    if (vk) {
+      |->BC_ISEQN_Z:
+    } else {
+      |->BC_ISNEN_Z:
+    }
+    |  checkint CARG1, >4
+    |   checkint CARG3, >6
+    |  cmp CARG1w, CARG3w
+    |1:
+    if (vk) {
+      |  csel PC, RB, PC, eq
+      |2:
+    } else {
+      |2:
+      |  csel PC, RB, PC, ne
+    }
+    |3:
+    |  ins_next
+    |
+    |4:  // RA not int.
+    |.if FFI
+    |  blo >7
+    |.else
+    |  blo <2
+    |.endif
+    |    ldr FARG1, [BASE, RA, lsl #3]
+    |    ldr FARG2, [RC]
+    |   cmp TISNUMhi, CARG3, lsr #32
+    |   bne >5
+    |  // RA number, RC int.
+    |  scvtf FARG2, CARG3w
+    |5:
+    |  // RA number, RC number.
+    |  fcmp FARG1, FARG2
+    |  b <1
+    |
+    |6:  // RA int, RC number
+    |  ldr FARG2, [RC]
+    |  scvtf FARG1, CARG1w
+    |  fcmp FARG1, FARG2
+    |  b <1
+    |
+    |.if FFI
+    |7:
+    |  asr ITYPE, TMP0, #47
+    |  cmn ITYPE, #-LJ_TCDATA
+    |  bne <2
+    |  b ->vmeta_equal_cd
+    |.endif
+    break;
+
+  case BC_ISEQP: case BC_ISNEP:
+    vk = op == BC_ISEQP;
+    |  // RA = src, RC = primitive_type (~), JMP with RC = target
+    |  ldr TMP0, [BASE, RA, lsl #3]
+    |   ldrh RBw, [PC, #2]
+    |   add PC, PC, #4
+    |  add RC, RC, #1
+    |   add RB, PC, RB, lsl #2
+    |.if FFI
+    |  asr ITYPE, TMP0, #47
+    |  cmn ITYPE, #-LJ_TCDATA
+    |  beq ->vmeta_equal_cd
+    |  cmn RC, ITYPE
+    |.else
+    |  cmn RC, TMP0, asr #47
+    |.endif
+    |   sub RB, RB, #0x20000
+    if (vk) {
+      |  csel PC, RB, PC, eq
+    } else {
+      |  csel PC, RB, PC, ne
+    }
+    |  ins_next
+    break;
+
+  /* -- Unary test and copy ops ------------------------------------------- */
+
+  case BC_ISTC: case BC_ISFC: case BC_IST: case BC_ISF:
+    |  // RA = dst or unused, RC = src, JMP with RC = target
+    |   ldrh RBw, [PC, #2]
+    |  ldr TMP0, [BASE, RC, lsl #3]
+    |   add PC, PC, #4
+    |  mov_false TMP1
+    |   add RB, PC, RB, lsl #2
+    |  cmp TMP0, TMP1
+    |   sub RB, RB, #0x20000
+    if (op == BC_ISTC || op == BC_IST) {
+      if (op == BC_ISTC) {
+	|  csel RA, RA, RC, lo
+      }
+      |  csel PC, RB, PC, lo
+    } else {
+      if (op == BC_ISFC) {
+	|  csel RA, RA, RC, hs
+      }
+      |  csel PC, RB, PC, hs
+    }
+    if (op == BC_ISTC || op == BC_ISFC) {
+      |  str TMP0, [BASE, RA, lsl #3]
+    }
+    |  ins_next
+    break;
+
+  case BC_ISTYPE:
+    |  // RA = src, RC = -type
+    |  ldr TMP0, [BASE, RA, lsl #3]
+    |  cmn RC, TMP0, asr #47
+    |  bne ->vmeta_istype
+    |  ins_next
+    break;
+  case BC_ISNUM:
+    |  // RA = src, RC = -(TISNUM-1)
+    |  ldr TMP0, [BASE, RA]
+    |  checknum TMP0, ->vmeta_istype
+    |  ins_next
+    break;
+
+  /* -- Unary ops --------------------------------------------------------- */
+
+  case BC_MOV:
+    |  // RA = dst, RC = src
+    |  ldr TMP0, [BASE, RC, lsl #3]
+    |  str TMP0, [BASE, RA, lsl #3]
+    |  ins_next
+    break;
+  case BC_NOT:
+    |  // RA = dst, RC = src
+    |  ldr TMP0, [BASE, RC, lsl #3]
+    |   mov_false TMP1
+    |   mov_true TMP2
+    |  cmp TMP0, TMP1
+    |  csel TMP0, TMP1, TMP2, lo
+    |  str TMP0, [BASE, RA, lsl #3]
+    |  ins_next
+    break;
+  case BC_UNM:
+    |  // RA = dst, RC = src
+    |  ldr TMP0, [BASE, RC, lsl #3]
+    |  asr ITYPE, TMP0, #47
+    |  cmn ITYPE, #-LJ_TISNUM
+    |  bhi ->vmeta_unm
+    |  eor TMP0, TMP0, #U64x(80000000,00000000)
+    |  bne >5
+    |  negs TMP0w, TMP0w
+    |   movz CARG3, #0x41e0, lsl #48	// 2^31.
+    |   add TMP0, TMP0, TISNUM
+    |  csel TMP0, TMP0, CARG3, vc
+    |5:
+    |  str TMP0, [BASE, RA, lsl #3]
+    |  ins_next
+    break;
+  case BC_LEN:
+    |  // RA = dst, RC = src
+    |  ldr CARG1, [BASE, RC, lsl #3]
+    |  asr ITYPE, CARG1, #47
+    |  cmn ITYPE, #-LJ_TSTR
+    |   and CARG1, CARG1, #LJ_GCVMASK
+    |  bne >2
+    |  ldr CARG1w, STR:CARG1->len
+    |1:
+    |  add CARG1, CARG1, TISNUM
+    |  str CARG1, [BASE, RA, lsl #3]
+    |  ins_next
+    |
+    |2:
+    |  cmn ITYPE, #-LJ_TTAB
+    |  bne ->vmeta_len
+#if LJ_52
+    |  ldr TAB:CARG2, TAB:CARG1->metatable
+    |  cbnz TAB:CARG2, >9
+    |3:
+#endif
+    |->BC_LEN_Z:
+    |  bl extern lj_tab_len		// (GCtab *t)
+    |  // Returns uint32_t (but less than 2^31).
+    |  b <1
+    |
+#if LJ_52
+    |9:
+    |  ldrb TMP1w, TAB:CARG2->nomm
+    |  tbnz TMP1w, #MM_len, <3		// 'no __len' flag set: done.
+    |  b ->vmeta_len
+#endif
+    break;
+
+  /* -- Binary ops -------------------------------------------------------- */
+
+    |.macro ins_arithcheck_int, target
+    |  checkint CARG1, target
+    |  checkint CARG2, target
+    |.endmacro
+    |
+    |.macro ins_arithcheck_num, target
+    |  checknum CARG1, target
+    |  checknum CARG2, target
+    |.endmacro
+    |
+    |.macro ins_arithcheck_nzdiv, target
+    |  cbz CARG2w, target
+    |.endmacro
+    |
+    |.macro ins_arithhead
+    ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
+    ||if (vk == 1) {
+    |   and RC, RC, #255
+    |    decode_RB RB, INS
+    ||} else {
+    |   decode_RB RB, INS
+    |    and RC, RC, #255
+    ||}
+    |.endmacro
+    |
+    |.macro ins_arithload, reg1, reg2
+    |  // RA = dst, RB = src1, RC = src2 | num_const
+    ||switch (vk) {
+    ||case 0:
+    |   ldr reg1, [BASE, RB, lsl #3]
+    |    ldr reg2, [KBASE, RC, lsl #3]
+    ||  break;
+    ||case 1:
+    |   ldr reg1, [KBASE, RC, lsl #3]
+    |    ldr reg2, [BASE, RB, lsl #3]
+    ||  break;
+    ||default:
+    |   ldr reg1, [BASE, RB, lsl #3]
+    |    ldr reg2, [BASE, RC, lsl #3]
+    ||  break;
+    ||}
+    |.endmacro
+    |
+    |.macro ins_arithfallback, ins
+    ||switch (vk) {
+    ||case 0:
+    |   ins ->vmeta_arith_vn
+    ||  break;
+    ||case 1:
+    |   ins ->vmeta_arith_nv
+    ||  break;
+    ||default:
+    |   ins ->vmeta_arith_vv
+    ||  break;
+    ||}
+    |.endmacro
+    |
+    |.macro ins_arithmod, res, reg1, reg2
+    |  fdiv d2, reg1, reg2
+    |  frintm d2, d2
+    |  fmsub res, d2, reg2, reg1
+    |.endmacro
+    |
+    |.macro ins_arithdn, intins, fpins
+    |  ins_arithhead
+    |  ins_arithload CARG1, CARG2
+    |  ins_arithcheck_int >5
+    |.if "intins" == "smull"
+    |  smull CARG1, CARG1w, CARG2w
+    |  cmp CARG1, CARG1, sxtw
+    |   mov CARG1w, CARG1w
+    |  ins_arithfallback bne
+    |.elif "intins" == "ins_arithmodi"
+    |  ins_arithfallback ins_arithcheck_nzdiv
+    |  bl ->vm_modi
+    |.else
+    |  intins CARG1w, CARG1w, CARG2w
+    |  ins_arithfallback bvs
+    |.endif
+    |  add CARG1, CARG1, TISNUM
+    |  str CARG1, [BASE, RA, lsl #3]
+    |4:
+    |  ins_next
+    |
+    |5:  // FP variant.
+    |  ins_arithload FARG1, FARG2
+    |  ins_arithfallback ins_arithcheck_num
+    |  fpins FARG1, FARG1, FARG2
+    |  str FARG1, [BASE, RA, lsl #3]
+    |  b <4
+    |.endmacro
+    |
+    |.macro ins_arithfp, fpins
+    |  ins_arithhead
+    |  ins_arithload CARG1, CARG2
+    |  ins_arithload FARG1, FARG2
+    |  ins_arithfallback ins_arithcheck_num
+    |.if "fpins" == "fpow"
+    |  bl extern pow
+    |.else
+    |  fpins FARG1, FARG1, FARG2
+    |.endif
+    |  str FARG1, [BASE, RA, lsl #3]
+    |  ins_next
+    |.endmacro
+
+  case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
+    |  ins_arithdn adds, fadd
+    break;
+  case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
+    |  ins_arithdn subs, fsub
+    break;
+  case BC_MULVN: case BC_MULNV: case BC_MULVV:
+    |  ins_arithdn smull, fmul
+    break;
+  case BC_DIVVN: case BC_DIVNV: case BC_DIVVV:
+    |  ins_arithfp fdiv
+    break;
+  case BC_MODVN: case BC_MODNV: case BC_MODVV:
+    |  ins_arithdn ins_arithmodi, ins_arithmod
+    break;
+  case BC_POW:
+    |  // NYI: (partial) integer arithmetic.
+    |  ins_arithfp fpow
+    break;
+
+  case BC_CAT:
+    |  decode_RB RB, INS
+    |   and RC, RC, #255
+    |  // RA = dst, RB = src_start, RC = src_end
+    |   str BASE, L->base
+    |  sub CARG3, RC, RB
+    |  add CARG2, BASE, RC, lsl #3
+    |->BC_CAT_Z:
+    |  // RA = dst, CARG2 = top-1, CARG3 = left
+    |  mov CARG1, L
+    |   str PC, SAVE_PC
+    |  bl extern lj_meta_cat		// (lua_State *L, TValue *top, int left)
+    |  // Returns NULL (finished) or TValue * (metamethod).
+    |  ldrb RBw, [PC, #-1]
+    |   ldr BASE, L->base
+    |   cbnz CRET1, ->vmeta_binop
+    |  ldr TMP0, [BASE, RB, lsl #3]
+    |  str TMP0, [BASE, RA, lsl #3]	// Copy result to RA.
+    |  ins_next
+    break;
+
+  /* -- Constant ops ------------------------------------------------------ */
+
+  case BC_KSTR:
+    |  // RA = dst, RC = str_const (~)
+    |  mvn RC, RC
+    |  ldr TMP0, [KBASE, RC, lsl #3]
+    |   movn TMP1, #~LJ_TSTR
+    |  add TMP0, TMP0, TMP1, lsl #47
+    |  str TMP0, [BASE, RA, lsl #3]
+    |  ins_next
+    break;
+  case BC_KCDATA:
+    |.if FFI
+    |  // RA = dst, RC = cdata_const (~)
+    |  mvn RC, RC
+    |  ldr TMP0, [KBASE, RC, lsl #3]
+    |   movn TMP1, #~LJ_TCDATA
+    |  add TMP0, TMP0, TMP1, lsl #47
+    |  str TMP0, [BASE, RA, lsl #3]
+    |  ins_next
+    |.endif
+    break;
+  case BC_KSHORT:
+    |  // RA = dst, RC = int16_literal
+    |  sxth RCw, RCw
+    |  add TMP0, RC, TISNUM
+    |  str TMP0, [BASE, RA, lsl #3]
+    |  ins_next
+    break;
+  case BC_KNUM:
+    |  // RA = dst, RC = num_const
+    |  ldr TMP0, [KBASE, RC, lsl #3]
+    |  str TMP0, [BASE, RA, lsl #3]
+    |  ins_next
+    break;
+  case BC_KPRI:
+    |  // RA = dst, RC = primitive_type (~)
+    |  mvn TMP0, RC, lsl #47
+    |  str TMP0, [BASE, RA, lsl #3]
+    |  ins_next
+    break;
+  case BC_KNIL:
+    |  // RA = base, RC = end
+    |  add RA, BASE, RA, lsl #3
+    |   add RC, BASE, RC, lsl #3
+    |  str TISNIL, [RA], #8
+    |1:
+    |   cmp RA, RC
+    |  str TISNIL, [RA], #8
+    |   blt <1
+    |  ins_next_
+    break;
+
+  /* -- Upvalue and function ops ------------------------------------------ */
+
+  case BC_UGET:
+    |  // RA = dst, RC = uvnum
+    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
+    |   add RC, RC, #offsetof(GCfuncL, uvptr)/8
+    |  and LFUNC:CARG2, CARG2, #LJ_GCVMASK
+    |  ldr UPVAL:CARG2, [LFUNC:CARG2, RC, lsl #3]
+    |  ldr CARG2, UPVAL:CARG2->v
+    |  ldr TMP0, [CARG2]
+    |  str TMP0, [BASE, RA, lsl #3]
+    |  ins_next
+    break;
+  case BC_USETV:
+    |  // RA = uvnum, RC = src
+    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
+    |   add RA, RA, #offsetof(GCfuncL, uvptr)/8
+    |  and LFUNC:CARG2, CARG2, #LJ_GCVMASK
+    |  ldr UPVAL:CARG1, [LFUNC:CARG2, RA, lsl #3]
+    |   ldr CARG3, [BASE, RC, lsl #3]
+    |    ldr CARG2, UPVAL:CARG1->v
+    |  ldrb TMP2w, UPVAL:CARG1->marked
+    |  ldrb TMP0w, UPVAL:CARG1->closed
+    |    asr ITYPE, CARG3, #47
+    |   str CARG3, [CARG2]
+    |    add ITYPE, ITYPE, #-LJ_TISGCV
+    |  tst TMP2w, #LJ_GC_BLACK		// isblack(uv)
+    |  ccmp TMP0w, #0, #4, ne		// && uv->closed
+    |    ccmn ITYPE, #-(LJ_TNUMX - LJ_TISGCV), #0, ne	// && tvisgcv(v)
+    |  bhi >2
+    |1:
+    |  ins_next
+    |
+    |2:  // Check if new value is white.
+    |  and GCOBJ:CARG3, CARG3, #LJ_GCVMASK
+    |  ldrb TMP1w, GCOBJ:CARG3->gch.marked
+    |  tst TMP1w, #LJ_GC_WHITES		// iswhite(str)
+    |  beq <1
+    |  // Crossed a write barrier. Move the barrier forward.
+    |  mov CARG1, GL
+    |  bl extern lj_gc_barrieruv	// (global_State *g, TValue *tv)
+    |  b <1
+    break;
+  case BC_USETS:
+    |  // RA = uvnum, RC = str_const (~)
+    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
+    |   add RA, RA, #offsetof(GCfuncL, uvptr)/8
+    |    mvn RC, RC
+    |  and LFUNC:CARG2, CARG2, #LJ_GCVMASK
+    |  ldr UPVAL:CARG1, [LFUNC:CARG2, RA, lsl #3]
+    |   ldr STR:CARG3, [KBASE, RC, lsl #3]
+    |   movn TMP0, #~LJ_TSTR
+    |    ldr CARG2, UPVAL:CARG1->v
+    |  ldrb TMP2w, UPVAL:CARG1->marked
+    |   add TMP0, STR:CARG3, TMP0, lsl #47
+    |    ldrb TMP1w, STR:CARG3->marked
+    |   str TMP0, [CARG2]
+    |  tbnz TMP2w, #2, >2		// isblack(uv)
+    |1:
+    |  ins_next
+    |
+    |2:  // Check if string is white and ensure upvalue is closed.
+    |  ldrb TMP0w, UPVAL:CARG1->closed
+    |    tst TMP1w, #LJ_GC_WHITES	// iswhite(str)
+    |  ccmp TMP0w, #0, #0, ne
+    |  beq <1
+    |  // Crossed a write barrier. Move the barrier forward.
+    |  mov CARG1, GL
+    |  bl extern lj_gc_barrieruv	// (global_State *g, TValue *tv)
+    |  b <1
+    break;
+  case BC_USETN:
+    |  // RA = uvnum, RC = num_const
+    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
+    |   add RA, RA, #offsetof(GCfuncL, uvptr)/8
+    |  and LFUNC:CARG2, CARG2, #LJ_GCVMASK
+    |  ldr UPVAL:CARG2, [LFUNC:CARG2, RA, lsl #3]
+    |   ldr TMP0, [KBASE, RC, lsl #3]
+    |  ldr CARG2, UPVAL:CARG2->v
+    |   str TMP0, [CARG2]
+    |  ins_next
+    break;
+  case BC_USETP:
+    |  // RA = uvnum, RC = primitive_type (~)
+    |  ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
+    |   add RA, RA, #offsetof(GCfuncL, uvptr)/8
+    |  and LFUNC:CARG2, CARG2, #LJ_GCVMASK
+    |  ldr UPVAL:CARG2, [LFUNC:CARG2, RA, lsl #3]
+    |   mvn TMP0, RC, lsl #47
+    |  ldr CARG2, UPVAL:CARG2->v
+    |   str TMP0, [CARG2]
+    |  ins_next
+    break;
+
+  case BC_UCLO:
+    |  // RA = level, RC = target
+    |  ldr CARG3, L->openupval
+    |   add RC, PC, RC, lsl #2
+    |    str BASE, L->base
+    |   sub PC, RC, #0x20000
+    |  cbz CARG3, >1
+    |  mov CARG1, L
+    |  add CARG2, BASE, RA, lsl #3
+    |  bl extern lj_func_closeuv	// (lua_State *L, TValue *level)
+    |  ldr BASE, L->base
+    |1:
+    |  ins_next
+    break;
+
+  case BC_FNEW:
+    |  // RA = dst, RC = proto_const (~) (holding function prototype)
+    |  mvn RC, RC
+    |   str BASE, L->base
+    |  ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
+    |    str PC, SAVE_PC
+    |   ldr CARG2, [KBASE, RC, lsl #3]
+    |    mov CARG1, L
+    |  and LFUNC:CARG3, CARG3, #LJ_GCVMASK
+    |  // (lua_State *L, GCproto *pt, GCfuncL *parent)
+    |  bl extern lj_func_newL_gc
+    |  // Returns GCfuncL *.
+    |  ldr BASE, L->base
+    |   movn TMP0, #~LJ_TFUNC
+    |   add CRET1, CRET1, TMP0, lsl #47
+    |  str CRET1, [BASE, RA, lsl #3]
+    |  ins_next
+    break;
+
+  /* -- Table ops --------------------------------------------------------- */
+
+  case BC_TNEW:
+  case BC_TDUP:
+    |  // RA = dst, RC = (hbits|asize) | tab_const (~)
+    |  ldp CARG3, CARG4, GL->gc.total	// Assumes threshold follows total.
+    |   str BASE, L->base
+    |   str PC, SAVE_PC
+    |   mov CARG1, L
+    |  cmp CARG3, CARG4
+    |  bhs >5
+    |1:
+    if (op == BC_TNEW) {
+      |  and CARG2, RC, #0x7ff
+      |   lsr CARG3, RC, #11
+      |  cmp CARG2, #0x7ff
+      |  mov TMP0, #0x801
+      |  csel CARG2, CARG2, TMP0, ne
+      |  bl extern lj_tab_new  // (lua_State *L, int32_t asize, uint32_t hbits)
+      |  // Returns GCtab *.
+    } else {
+      |  mvn RC, RC
+      |  ldr CARG2, [KBASE, RC, lsl #3]
+      |  bl extern lj_tab_dup  // (lua_State *L, Table *kt)
+      |  // Returns GCtab *.
+    }
+    |  ldr BASE, L->base
+    |   movk CRET1, #(LJ_TTAB>>1)&0xffff, lsl #48
+    |  str CRET1, [BASE, RA, lsl #3]
+    |  ins_next
+    |
+    |5:
+    |  bl extern lj_gc_step_fixtop  // (lua_State *L)
+    |  mov CARG1, L
+    |  b <1
+    break;
+
+  case BC_GGET:
+    |  // RA = dst, RC = str_const (~)
+  case BC_GSET:
+    |  // RA = dst, RC = str_const (~)
+    |  ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
+    |   mvn RC, RC
+    |  and LFUNC:CARG1, CARG1, #LJ_GCVMASK
+    |  ldr TAB:CARG2, LFUNC:CARG1->env
+    |   ldr STR:RC, [KBASE, RC, lsl #3]
+    if (op == BC_GGET) {
+      |  b ->BC_TGETS_Z
+    } else {
+      |  b ->BC_TSETS_Z
+    }
+    break;
+
+  case BC_TGETV:
+    |  decode_RB RB, INS
+    |   and RC, RC, #255
+    |  // RA = dst, RB = table, RC = key
+    |  ldr CARG2, [BASE, RB, lsl #3]
+    |   ldr TMP1, [BASE, RC, lsl #3]
+    |  checktab CARG2, ->vmeta_tgetv
+    |  checkint TMP1, >9		// Integer key?
+    |  ldr CARG3, TAB:CARG2->array
+    |   ldr CARG1w, TAB:CARG2->asize
+    |  add CARG3, CARG3, TMP1, uxtw #3
+    |   cmp TMP1w, CARG1w		// In array part?
+    |   bhs ->vmeta_tgetv
+    |  ldr TMP0, [CARG3]
+    |  cmp TMP0, TISNIL
+    |  beq >5
+    |1:
+    |  str TMP0, [BASE, RA, lsl #3]
+    |  ins_next
+    |
+    |5:  // Check for __index if table value is nil.
+    |  ldr TAB:CARG1, TAB:CARG2->metatable
+    |  cbz TAB:CARG1, <1		// No metatable: done.
+    |  ldrb TMP1w, TAB:CARG1->nomm
+    |  tbnz TMP1w, #MM_index, <1	// 'no __index' flag set: done.
+    |  b ->vmeta_tgetv
+    |
+    |9:
+    |  asr ITYPE, TMP1, #47
+    |  cmn ITYPE, #-LJ_TSTR		// String key?
+    |  bne ->vmeta_tgetv
+    |   and STR:RC, TMP1, #LJ_GCVMASK
+    |  b ->BC_TGETS_Z
+    break;
+  case BC_TGETS:
+    |  decode_RB RB, INS
+    |   and RC, RC, #255
+    |  // RA = dst, RB = table, RC = str_const (~)
+    |  ldr CARG2, [BASE, RB, lsl #3]
+    |   mvn RC, RC
+    |   ldr STR:RC, [KBASE, RC, lsl #3]
+    |  checktab CARG2, ->vmeta_tgets1
+    |->BC_TGETS_Z:
+    |  // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = dst
+    |  ldr TMP1w, TAB:CARG2->hmask
+    |   ldr TMP2w, STR:RC->hash
+    |    ldr NODE:CARG3, TAB:CARG2->node
+    |  and TMP1w, TMP1w, TMP2w		// idx = str->hash & tab->hmask
+    |  add TMP1, TMP1, TMP1, lsl #1
+    |  movn CARG4, #~LJ_TSTR
+    |    add NODE:CARG3, NODE:CARG3, TMP1, lsl #3  // node = tab->node + idx*3*8
+    |  add CARG4, STR:RC, CARG4, lsl #47	// Tagged key to look for.
+    |1:
+    |  ldp TMP0, CARG1, NODE:CARG3->val
+    |   ldr NODE:CARG3, NODE:CARG3->next
+    |  cmp CARG1, CARG4
+    |  bne >4
+    |  cmp TMP0, TISNIL
+    |  beq >5
+    |3:
+    |  str TMP0, [BASE, RA, lsl #3]
+    |  ins_next
+    |
+    |4:  // Follow hash chain.
+    |  cbnz NODE:CARG3, <1
+    |  // End of hash chain: key not found, nil result.
+    |   mov TMP0, TISNIL
+    |
+    |5:  // Check for __index if table value is nil.
+    |  ldr TAB:CARG1, TAB:CARG2->metatable
+    |  cbz TAB:CARG1, <3		// No metatable: done.
+    |  ldrb TMP1w, TAB:CARG1->nomm
+    |  tbnz TMP1w, #MM_index, <3	// 'no __index' flag set: done.
+    |  b ->vmeta_tgets
+    break;
+  case BC_TGETB:
+    |  decode_RB RB, INS
+    |   and RC, RC, #255
+    |  // RA = dst, RB = table, RC = index
+    |  ldr CARG2, [BASE, RB, lsl #3]
+    |  checktab CARG2, ->vmeta_tgetb
+    |  ldr CARG3, TAB:CARG2->array
+    |   ldr CARG1w, TAB:CARG2->asize
+    |  add CARG3, CARG3, RC, lsl #3
+    |   cmp RCw, CARG1w			// In array part?
+    |   bhs ->vmeta_tgetb
+    |  ldr TMP0, [CARG3]
+    |  cmp TMP0, TISNIL
+    |  beq >5
+    |1:
+    |  str TMP0, [BASE, RA, lsl #3]
+    |  ins_next
+    |
+    |5:  // Check for __index if table value is nil.
+    |  ldr TAB:CARG1, TAB:CARG2->metatable
+    |  cbz TAB:CARG1, <1		// No metatable: done.
+    |  ldrb TMP1w, TAB:CARG1->nomm
+    |  tbnz TMP1w, #MM_index, <1	// 'no __index' flag set: done.
+    |  b ->vmeta_tgetb
+    break;
+  case BC_TGETR:
+    |  decode_RB RB, INS
+    |   and RC, RC, #255
+    |  // RA = dst, RB = table, RC = key
+    |  ldr CARG1, [BASE, RB, lsl #3]
+    |   ldr TMP1, [BASE, RC, lsl #3]
+    |  and TAB:CARG1, CARG1, #LJ_GCVMASK
+    |  ldr CARG3, TAB:CARG1->array
+    |   ldr TMP2w, TAB:CARG1->asize
+    |  add CARG3, CARG3, TMP1w, uxtw #3
+    |   cmp TMP1w, TMP2w		// In array part?
+    |   bhs ->vmeta_tgetr
+    |  ldr TMP0, [CARG3]
+    |->BC_TGETR_Z:
+    |  str TMP0, [BASE, RA, lsl #3]
+    |  ins_next
+    break;
+
+  case BC_TSETV:
+    |  decode_RB RB, INS
+    |   and RC, RC, #255
+    |  // RA = src, RB = table, RC = key
+    |  ldr CARG2, [BASE, RB, lsl #3]
+    |   ldr TMP1, [BASE, RC, lsl #3]
+    |  checktab CARG2, ->vmeta_tsetv
+    |  checkint TMP1, >9		// Integer key?
+    |  ldr CARG3, TAB:CARG2->array
+    |   ldr CARG1w, TAB:CARG2->asize
+    |  add CARG3, CARG3, TMP1, uxtw #3
+    |   cmp TMP1w, CARG1w		// In array part?
+    |   bhs ->vmeta_tsetv
+    |  ldr TMP1, [CARG3]
+    |   ldr TMP0, [BASE, RA, lsl #3]
+    |    ldrb TMP2w, TAB:CARG2->marked
+    |  cmp TMP1, TISNIL			// Previous value is nil?
+    |  beq >5
+    |1:
+    |   str TMP0, [CARG3]
+    |    tbnz TMP2w, #2, >7		// isblack(table)
+    |2:
+    |   ins_next
+    |
+    |5:  // Check for __newindex if previous value is nil.
+    |  ldr TAB:CARG1, TAB:CARG2->metatable
+    |  cbz TAB:CARG1, <1		// No metatable: done.
+    |  ldrb TMP1w, TAB:CARG1->nomm
+    |  tbnz TMP1w, #MM_newindex, <1	// 'no __newindex' flag set: done.
+    |  b ->vmeta_tsetv
+    |
+    |7:  // Possible table write barrier for the value. Skip valiswhite check.
+    |  barrierback TAB:CARG2, TMP2w, TMP1
+    |  b <2
+    |
+    |9:
+    |  asr ITYPE, TMP1, #47
+    |  cmn ITYPE, #-LJ_TSTR		// String key?
+    |  bne ->vmeta_tsetv
+    |   and STR:RC, TMP1, #LJ_GCVMASK
+    |  b ->BC_TSETS_Z
+    break;
+  case BC_TSETS:
+    |  decode_RB RB, INS
+    |   and RC, RC, #255
+    |  // RA = dst, RB = table, RC = str_const (~)
+    |  ldr CARG2, [BASE, RB, lsl #3]
+    |   mvn RC, RC
+    |   ldr STR:RC, [KBASE, RC, lsl #3]
+    |  checktab CARG2, ->vmeta_tsets1
+    |->BC_TSETS_Z:
+    |  // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = src
+    |  ldr TMP1w, TAB:CARG2->hmask
+    |   ldr TMP2w, STR:RC->hash
+    |    ldr NODE:CARG3, TAB:CARG2->node
+    |  and TMP1w, TMP1w, TMP2w		// idx = str->hash & tab->hmask
+    |  add TMP1, TMP1, TMP1, lsl #1
+    |  movn CARG4, #~LJ_TSTR
+    |    add NODE:CARG3, NODE:CARG3, TMP1, lsl #3  // node = tab->node + idx*3*8
+    |  add CARG4, STR:RC, CARG4, lsl #47	// Tagged key to look for.
+    |   strb wzr, TAB:CARG2->nomm	// Clear metamethod cache.
+    |1:
+    |  ldp TMP1, CARG1, NODE:CARG3->val
+    |   ldr NODE:TMP3, NODE:CARG3->next
+    |    ldrb TMP2w, TAB:CARG2->marked
+    |  cmp CARG1, CARG4
+    |  bne >5
+    |   ldr TMP0, [BASE, RA, lsl #3]
+    |  cmp TMP1, TISNIL			// Previous value is nil?
+    |  beq >4
+    |2:
+    |   str TMP0, NODE:CARG3->val
+    |    tbnz TMP2w, #2, >7		// isblack(table)
+    |3:
+    |  ins_next
+    |
+    |4:  // Check for __newindex if previous value is nil.
+    |  ldr TAB:CARG1, TAB:CARG2->metatable
+    |  cbz TAB:CARG1, <2		// No metatable: done.
+    |  ldrb TMP1w, TAB:CARG1->nomm
+    |  tbnz TMP1w, #MM_newindex, <2	// 'no __newindex' flag set: done.
+    |  b ->vmeta_tsets
+    |
+    |5:  // Follow hash chain.
+    |  mov NODE:CARG3, NODE:TMP3
+    |  cbnz NODE:TMP3, <1
+    |  // End of hash chain: key not found, add a new one.
+    |
+    |  // But check for __newindex first.
+    |  ldr TAB:CARG1, TAB:CARG2->metatable
+    |  cbz TAB:CARG1, >6		// No metatable: continue.
+    |  ldrb TMP1w, TAB:CARG1->nomm
+    |  // 'no __newindex' flag NOT set: check.
+    |  tbz TMP1w, #MM_newindex, ->vmeta_tsets
+    |6:
+    |  movn TMP1, #~LJ_TSTR
+    |   str PC, SAVE_PC
+    |  add TMP0, STR:RC, TMP1, lsl #47
+    |   str BASE, L->base
+    |   mov CARG1, L
+    |  str TMP0, TMPD
+    |   add CARG3, sp, TMPDofs
+    |  bl extern lj_tab_newkey		// (lua_State *L, GCtab *t, TValue *k)
+    |  // Returns TValue *.
+    |  ldr BASE, L->base
+    |  ldr TMP0, [BASE, RA, lsl #3]
+    |  str TMP0, [CRET1]
+    |  b <3				// No 2nd write barrier needed.
+    |
+    |7:  // Possible table write barrier for the value. Skip valiswhite check.
+    |  barrierback TAB:CARG2, TMP2w, TMP1
+    |  b <3
+    break;
+  case BC_TSETB:
+    |  decode_RB RB, INS
+    |   and RC, RC, #255
+    |  // RA = src, RB = table, RC = index
+    |  ldr CARG2, [BASE, RB, lsl #3]
+    |  checktab CARG2, ->vmeta_tsetb
+    |  ldr CARG3, TAB:CARG2->array
+    |   ldr CARG1w, TAB:CARG2->asize
+    |  add CARG3, CARG3, RC, lsl #3
+    |   cmp RCw, CARG1w			// In array part?
+    |   bhs ->vmeta_tsetb
+    |  ldr TMP1, [CARG3]
+    |   ldr TMP0, [BASE, RA, lsl #3]
+    |    ldrb TMP2w, TAB:CARG2->marked
+    |  cmp TMP1, TISNIL			// Previous value is nil?
+    |  beq >5
+    |1:
+    |   str TMP0, [CARG3]
+    |    tbnz TMP2w, #2, >7		// isblack(table)
+    |2:
+    |   ins_next
+    |
+    |5:  // Check for __newindex if previous value is nil.
+    |  ldr TAB:CARG1, TAB:CARG2->metatable
+    |  cbz TAB:CARG1, <1		// No metatable: done.
+    |  ldrb TMP1w, TAB:CARG1->nomm
+    |  tbnz TMP1w, #MM_newindex, <1	// 'no __newindex' flag set: done.
+    |  b ->vmeta_tsetb
+    |
+    |7:  // Possible table write barrier for the value. Skip valiswhite check.
+    |  barrierback TAB:CARG2, TMP2w, TMP1
+    |  b <2
+    break;
+  case BC_TSETR:
+    |  decode_RB RB, INS
+    |   and RC, RC, #255
+    |  // RA = src, RB = table, RC = key
+    |  ldr CARG2, [BASE, RB, lsl #3]
+    |   ldr TMP1, [BASE, RC, lsl #3]
+    |  and TAB:CARG2, CARG2, #LJ_GCVMASK
+    |  ldr CARG1, TAB:CARG2->array
+    |    ldrb TMP2w, TAB:CARG2->marked
+    |   ldr CARG4w, TAB:CARG2->asize
+    |  add CARG1, CARG1, TMP1, uxtw #3
+    |    tbnz TMP2w, #2, >7		// isblack(table)
+    |2:
+    |   cmp TMP1w, CARG4w		// In array part?
+    |   bhs ->vmeta_tsetr
+    |->BC_TSETR_Z:
+    |   ldr TMP0, [BASE, RA, lsl #3]
+    |   str TMP0, [CARG1]
+    |   ins_next
+    |
+    |7:  // Possible table write barrier for the value. Skip valiswhite check.
+    |  barrierback TAB:CARG2, TMP2w, TMP0
+    |  b <2
+    break;
+
+  case BC_TSETM:
+    |  // RA = base (table at base-1), RC = num_const (start index)
+    |  add RA, BASE, RA, lsl #3
+    |1:
+    |   ldr RBw, SAVE_MULTRES
+    |  ldr TAB:CARG2, [RA, #-8]		// Guaranteed to be a table.
+    |   ldr TMP1, [KBASE, RC, lsl #3]	// Integer constant is in lo-word.
+    |    sub RB, RB, #8
+    |    cbz RB, >4			// Nothing to copy?
+    |  and TAB:CARG2, CARG2, #LJ_GCVMASK
+    |  ldr CARG1w, TAB:CARG2->asize
+    |   add CARG3w, TMP1w, RBw, lsr #3
+    |   ldr CARG4, TAB:CARG2->array
+    |  cmp CARG3, CARG1
+    |    add RB, RA, RB
+    |  bhi >5
+    |   add TMP1, CARG4, TMP1w, uxtw #3
+    |    ldrb TMP2w, TAB:CARG2->marked
+    |3:  // Copy result slots to table.
+    |   ldr TMP0, [RA], #8
+    |   str TMP0, [TMP1], #8
+    |  cmp RA, RB
+    |  blo <3
+    |    tbnz TMP2w, #2, >7		// isblack(table)
+    |4:
+    |  ins_next
+    |
+    |5:  // Need to resize array part.
+    |   str BASE, L->base
+    |  mov CARG1, L
+    |   str PC, SAVE_PC
+    |  bl extern lj_tab_reasize		// (lua_State *L, GCtab *t, int nasize)
+    |  // Must not reallocate the stack.
+    |  b <1
+    |
+    |7:  // Possible table write barrier for any value. Skip valiswhite check.
+    |  barrierback TAB:CARG2, TMP2w, TMP1
+    |  b <4
+    break;
+
+  /* -- Calls and vararg handling ----------------------------------------- */
+
+  case BC_CALLM:
+    |  // RA = base, (RB = nresults+1,) RC = extra_nargs
+    |  ldr TMP0w, SAVE_MULTRES
+    |  decode_RC8RD NARGS8:RC, RC
+    |  add NARGS8:RC, NARGS8:RC, TMP0
+    |  b ->BC_CALL_Z
+    break;
+  case BC_CALL:
+    |  decode_RC8RD NARGS8:RC, RC
+    |  // RA = base, (RB = nresults+1,) RC = (nargs+1)*8
+    |->BC_CALL_Z:
+    |  mov RB, BASE			// Save old BASE for vmeta_call.
+    |  add BASE, BASE, RA, lsl #3
+    |  ldr CARG3, [BASE]
+    |   sub NARGS8:RC, NARGS8:RC, #8
+    |   add BASE, BASE, #16
+    |  checkfunc CARG3, ->vmeta_call
+    |  ins_call
+    break;
+
+  case BC_CALLMT:
+    |  // RA = base, (RB = 0,) RC = extra_nargs
+    |  ldr TMP0w, SAVE_MULTRES
+    |  add NARGS8:RC, TMP0, RC, lsl #3
+    |  b ->BC_CALLT1_Z
+    break;
+  case BC_CALLT:
+    |  lsl NARGS8:RC, RC, #3
+    |  // RA = base, (RB = 0,) RC = (nargs+1)*8
+    |->BC_CALLT1_Z:
+    |  add RA, BASE, RA, lsl #3
+    |  ldr TMP1, [RA]
+    |   sub NARGS8:RC, NARGS8:RC, #8
+    |   add RA, RA, #16
+    |  checktp CARG3, TMP1, LJ_TFUNC, ->vmeta_callt
+    |  ldr PC, [BASE, FRAME_PC]
+    |->BC_CALLT2_Z:
+    |   mov RB, #0
+    |   ldrb TMP2w, LFUNC:CARG3->ffid
+    |  tst PC, #FRAME_TYPE
+    |  bne >7
+    |1:
+    |  str TMP1, [BASE, FRAME_FUNC]	// Copy function down, but keep PC.
+    |  cbz NARGS8:RC, >3
+    |2:
+    |  ldr TMP0, [RA, RB]
+    |   add TMP1, RB, #8
+    |   cmp TMP1, NARGS8:RC
+    |  str TMP0, [BASE, RB]
+    |    mov RB, TMP1
+    |   bne <2
+    |3:
+    |  cmp TMP2, #1			// (> FF_C) Calling a fast function?
+    |  bhi >5
+    |4:
+    |  ins_callt
+    |
+    |5:  // Tailcall to a fast function with a Lua frame below.
+    |  ldrb RAw, [PC, #-3]
+    |  sub CARG1, BASE, RA, lsl #3
+    |  ldr LFUNC:CARG1, [CARG1, #-32]
+    |  and LFUNC:CARG1, CARG1, #LJ_GCVMASK
+    |  ldr CARG1, LFUNC:CARG1->pc
+    |  ldr KBASE, [CARG1, #PC2PROTO(k)]
+    |  b <4
+    |
+    |7:  // Tailcall from a vararg function.
+    |  eor PC, PC, #FRAME_VARG
+    |  tst PC, #FRAME_TYPEP		// Vararg frame below?
+    |  csel TMP2, RB, TMP2, ne		// Clear ffid if no Lua function below.
+    |  bne <1
+    |  sub BASE, BASE, PC
+    |  ldr PC, [BASE, FRAME_PC]
+    |  tst PC, #FRAME_TYPE
+    |  csel TMP2, RB, TMP2, ne		// Clear ffid if no Lua function below.
+    |  b <1
+    break;
+
+  case BC_ITERC:
+    |  // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
+    |  add RA, BASE, RA, lsl #3
+    |  ldr CARG3, [RA, #-24]
+    |    mov RB, BASE			// Save old BASE for vmeta_call.
+    |   ldp CARG1, CARG2, [RA, #-16]
+    |    add BASE, RA, #16
+    |    mov NARGS8:RC, #16		// Iterators get 2 arguments.
+    |  str CARG3, [RA]			// Copy callable.
+    |   stp CARG1, CARG2, [RA, #16]	// Copy state and control var.
+    |  checkfunc CARG3, ->vmeta_call
+    |  ins_call
+    break;
+
+  case BC_ITERN:
+    |  // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
+    |.if JIT
+    |  // NYI: add hotloop, record BC_ITERN.
+    |.endif
+    |  add RA, BASE, RA, lsl #3
+    |  ldr TAB:RB, [RA, #-16]
+    |    ldrh TMP3w, [PC, #2]
+    |  ldr CARG1w, [RA, #-8]		// Get index from control var.
+    |    add PC, PC, #4
+    |    add TMP3, PC, TMP3, lsl #2
+    |  and TAB:RB, RB, #LJ_GCVMASK
+    |    sub TMP3, TMP3, #0x20000
+    |  ldr TMP1w, TAB:RB->asize
+    |   ldr CARG2, TAB:RB->array
+    |1:  // Traverse array part.
+    |  subs RC, CARG1, TMP1
+    |   add CARG3, CARG2, CARG1, lsl #3
+    |  bhs >5				// Index points after array part?
+    |   ldr TMP0, [CARG3]
+    |   cmp TMP0, TISNIL
+    |   cinc CARG1, CARG1, eq		// Skip holes in array part.
+    |   beq <1
+    |   add CARG1, CARG1, TISNUM
+    |   stp CARG1, TMP0, [RA]
+    |    add CARG1, CARG1, #1
+    |3:
+    |    str CARG1w, [RA, #-8]		// Update control var.
+    |  mov PC, TMP3
+    |4:
+    |  ins_next
+    |
+    |5:  // Traverse hash part.
+    |  ldr TMP2w, TAB:RB->hmask
+    |   ldr NODE:RB, TAB:RB->node
+    |6:
+    |   add CARG1, RC, RC, lsl #1
+    |  cmp RC, TMP2			// End of iteration? Branch to ITERN+1.
+    |   add NODE:CARG3, NODE:RB, CARG1, lsl #3  // node = tab->node + idx*3*8
+    |  bhi <4
+    |  ldp TMP0, CARG1, NODE:CARG3->val
+    |  cmp TMP0, TISNIL
+    |   add RC, RC, #1
+    |  beq <6				// Skip holes in hash part.
+    |  stp CARG1, TMP0, [RA]
+    |  add CARG1, RC, TMP1
+    |  b <3
+    break;
+
+  case BC_ISNEXT:
+    |  // RA = base, RC = target (points to ITERN)
+    |  add RA, BASE, RA, lsl #3
+    |  ldr CFUNC:CARG1, [RA, #-24]
+    |     add RC, PC, RC, lsl #2
+    |   ldp TAB:CARG3, CARG4, [RA, #-16]
+    |     sub RC, RC, #0x20000
+    |  checkfunc CFUNC:CARG1, >5
+    |   asr TMP0, TAB:CARG3, #47
+    |  ldrb TMP1w, CFUNC:CARG1->ffid
+    |   cmn TMP0, #-LJ_TTAB
+    |   ccmp CARG4, TISNIL, #0, eq
+    |  ccmp TMP1w, #FF_next_N, #0, eq
+    |  bne >5
+    |  mov TMP0w, #0xfffe7fff
+    |  lsl TMP0, TMP0, #32
+    |  str TMP0, [RA, #-8]		// Initialize control var.
+    |1:
+    |     mov PC, RC
+    |  ins_next
+    |
+    |5:  // Despecialize bytecode if any of the checks fail.
+    |  mov TMP0, #BC_JMP
+    |   mov TMP1, #BC_ITERC
+    |  strb TMP0w, [PC, #-4]
+    |   strb TMP1w, [RC]
+    |  b <1
+    break;
+
+  case BC_VARG:
+    |  decode_RB RB, INS
+    |   and RC, RC, #255
+    |  // RA = base, RB = (nresults+1), RC = numparams
+    |  ldr TMP1, [BASE, FRAME_PC]
+    |  add RC, BASE, RC, lsl #3
+    |   add RA, BASE, RA, lsl #3
+    |  add RC, RC, #FRAME_VARG
+    |   add TMP2, RA, RB, lsl #3
+    |  sub RC, RC, TMP1			// RC = vbase
+    |  // Note: RC may now be even _above_ BASE if nargs was < numparams.
+    |   sub TMP3, BASE, #16		// TMP3 = vtop
+    |  cbz RB, >5
+    |   sub TMP2, TMP2, #16
+    |1:  // Copy vararg slots to destination slots.
+    |  cmp RC, TMP3
+    |  ldr TMP0, [RC], #8
+    |  csel TMP0, TMP0, TISNIL, lo
+    |   cmp RA, TMP2
+    |  str TMP0, [RA], #8
+    |   blo <1
+    |2:
+    |  ins_next
+    |
+    |5:  // Copy all varargs.
+    |  ldr TMP0, L->maxstack
+    |   subs TMP2, TMP3, RC
+    |   csel RB, xzr, TMP2, le		// MULTRES = (max(vtop-vbase,0)+1)*8
+    |   add RB, RB, #8
+    |  add TMP1, RA, TMP2
+    |   str RBw, SAVE_MULTRES
+    |   ble <2				// Nothing to copy.
+    |  cmp TMP1, TMP0
+    |  bhi >7
+    |6:
+    |  ldr TMP0, [RC], #8
+    |  str TMP0, [RA], #8
+    |  cmp RC, TMP3
+    |  blo <6
+    |  b <2
+    |
+    |7:  // Grow stack for varargs.
+    |  lsr CARG2, TMP2, #3
+    |   stp BASE, RA, L->base
+    |  mov CARG1, L
+    |  sub RC, RC, BASE			// Need delta, because BASE may change.
+    |   str PC, SAVE_PC
+    |  bl extern lj_state_growstack	// (lua_State *L, int n)
+    |  ldp BASE, RA, L->base
+    |  add RC, BASE, RC
+    |  sub TMP3, BASE, #16
+    |  b <6
+    break;
+
+  /* -- Returns ----------------------------------------------------------- */
+
+  case BC_RETM:
+    |  // RA = results, RC = extra results
+    |  ldr TMP0w, SAVE_MULTRES
+    |   ldr PC, [BASE, FRAME_PC]
+    |    add RA, BASE, RA, lsl #3
+    |  add RC, TMP0, RC, lsl #3
+    |  b ->BC_RETM_Z
+    break;
+
+  case BC_RET:
+    |  // RA = results, RC = nresults+1
+    |  ldr PC, [BASE, FRAME_PC]
+    |   lsl RC, RC, #3
+    |    add RA, BASE, RA, lsl #3
+    |->BC_RETM_Z:
+    |   str RCw, SAVE_MULTRES
+    |1:
+    |  ands CARG1, PC, #FRAME_TYPE
+    |   eor CARG2, PC, #FRAME_VARG
+    |  bne ->BC_RETV2_Z
+    |
+    |->BC_RET_Z:
+    |  // BASE = base, RA = resultptr, RC = (nresults+1)*8, PC = return
+    |  ldr INSw, [PC, #-4]
+    |  subs TMP1, RC, #8
+    |   sub CARG3, BASE, #16
+    |  beq >3
+    |2:
+    |  ldr TMP0, [RA], #8
+    |   add BASE, BASE, #8
+    |   sub TMP1, TMP1, #8
+    |  str TMP0, [BASE, #-24]
+    |   cbnz TMP1, <2
+    |3:
+    |  decode_RA RA, INS
+    |  sub CARG4, CARG3, RA, lsl #3
+    |   decode_RB RB, INS
+    |  ldr LFUNC:CARG1, [CARG4, FRAME_FUNC]
+    |5:
+    |  cmp RC, RB, lsl #3		// More results expected?
+    |  blo >6
+    |  and LFUNC:CARG1, CARG1, #LJ_GCVMASK
+    |  mov BASE, CARG4
+    |  ldr CARG2, LFUNC:CARG1->pc
+    |  ldr KBASE, [CARG2, #PC2PROTO(k)]
+    |   ins_next
+    |
+    |6:  // Fill up results with nil.
+    |  add BASE, BASE, #8
+    |   add RC, RC, #8
+    |  str TISNIL, [BASE, #-24]
+    |  b <5
+    |
+    |->BC_RETV1_Z:  // Non-standard return case.
+    |  add RA, BASE, RA, lsl #3
+    |->BC_RETV2_Z:
+    |  tst CARG2, #FRAME_TYPEP
+    |  bne ->vm_return
+    |  // Return from vararg function: relocate BASE down.
+    |  sub BASE, BASE, CARG2
+    |  ldr PC, [BASE, FRAME_PC]
+    |  b <1
+    break;
+
+  case BC_RET0: case BC_RET1:
+    |  // RA = results, RC = nresults+1
+    |  ldr PC, [BASE, FRAME_PC]
+    |   lsl RC, RC, #3
+    |   str RCw, SAVE_MULTRES
+    |  ands CARG1, PC, #FRAME_TYPE
+    |   eor CARG2, PC, #FRAME_VARG
+    |  bne ->BC_RETV1_Z
+    |   ldr INSw, [PC, #-4]
+    if (op == BC_RET1) {
+      |  ldr TMP0, [BASE, RA, lsl #3]
+    }
+    |  sub CARG4, BASE, #16
+    |   decode_RA RA, INS
+    |  sub BASE, CARG4, RA, lsl #3
+    if (op == BC_RET1) {
+      |  str TMP0, [CARG4], #8
+    }
+    |   decode_RB RB, INS
+    |  ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
+    |5:
+    |  cmp RC, RB, lsl #3
+    |  blo >6
+    |  and LFUNC:CARG1, CARG1, #LJ_GCVMASK
+    |  ldr CARG2, LFUNC:CARG1->pc
+    |  ldr KBASE, [CARG2, #PC2PROTO(k)]
+    |  ins_next
+    |
+    |6:  // Fill up results with nil.
+    |  add RC, RC, #8
+    |  str TISNIL, [CARG4], #8
+    |  b <5
+    break;
+
+  /* -- Loops and branches ------------------------------------------------ */
+
+  |.define FOR_IDX,  [RA];      .define FOR_TIDX,  [RA, #4]
+  |.define FOR_STOP, [RA, #8];  .define FOR_TSTOP, [RA, #12]
+  |.define FOR_STEP, [RA, #16]; .define FOR_TSTEP, [RA, #20]
+  |.define FOR_EXT,  [RA, #24]; .define FOR_TEXT,  [RA, #28]
+
+  case BC_FORL:
+    |.if JIT
+    |  hotloop
+    |.endif
+    |  // Fall through. Assumes BC_IFORL follows.
+    break;
+
+  case BC_JFORI:
+  case BC_JFORL:
+#if !LJ_HASJIT
+    break;
+#endif
+  case BC_FORI:
+  case BC_IFORL:
+    |  // RA = base, RC = target (after end of loop or start of loop)
+    vk = (op == BC_IFORL || op == BC_JFORL);
+    |  add RA, BASE, RA, lsl #3
+    |  ldp CARG1, CARG2, FOR_IDX		// CARG1 = IDX, CARG2 = STOP
+    |   ldr CARG3, FOR_STEP			// CARG3 = STEP
+    if (op != BC_JFORL) {
+      |   add RC, PC, RC, lsl #2
+      |   sub RC, RC, #0x20000
+    }
+    |  checkint CARG1, >5
+    if (!vk) {
+      |  checkint CARG2, ->vmeta_for
+      |   checkint CARG3, ->vmeta_for
+      |  tbnz CARG3w, #31, >4
+      |  cmp CARG1w, CARG2w
+    } else {
+      |  adds CARG1w, CARG1w, CARG3w
+      |  bvs >2
+      |   add TMP0, CARG1, TISNUM
+      |  tbnz CARG3w, #31, >4
+      |  cmp CARG1w, CARG2w
+    }
+    |1:
+    if (op == BC_FORI) {
+      |  csel PC, RC, PC, gt
+    } else if (op == BC_JFORI) {
+      |  ldrh RCw, [RC, #-2]
+    } else if (op == BC_IFORL) {
+      |  csel PC, RC, PC, le
+    }
+    if (vk) {
+      |   str TMP0, FOR_IDX
+      |   str TMP0, FOR_EXT
+    } else {
+      |  str CARG1, FOR_EXT
+    }
+    if (op == BC_JFORI || op == BC_JFORL) {
+      |  ble =>BC_JLOOP
+    }
+    |2:
+    |   ins_next
+    |
+    |4:  // Invert check for negative step.
+    |  cmp CARG2w, CARG1w
+    |  b <1
+    |
+    |5:  // FP loop.
+    |  ldp d0, d1, FOR_IDX
+    |  blo ->vmeta_for
+    if (!vk) {
+      |  checknum CARG2, ->vmeta_for
+      |   checknum CARG3, ->vmeta_for
+      |  str d0, FOR_EXT
+    } else {
+      |  ldr d2, FOR_STEP
+      |  fadd d0, d0, d2
+    }
+    |  tbnz CARG3, #63, >7
+    |  fcmp d0, d1
+    |6:
+    if (vk) {
+      |  str d0, FOR_IDX
+      |  str d0, FOR_EXT
+    }
+    if (op == BC_FORI) {
+      |  csel PC, RC, PC, hi
+    } else if (op == BC_JFORI) {
+      |  ldrh RCw, [RC, #-2]
+      |  bls =>BC_JLOOP
+    } else if (op == BC_IFORL) {
+      |  csel PC, RC, PC, ls
+    } else {
+      |  bls =>BC_JLOOP
+    }
+    |  b <2
+    |
+    |7:  // Invert check for negative step.
+    |  fcmp d1, d0
+    |  b <6
+    break;
+
+  case BC_ITERL:
+    |.if JIT
+    |  hotloop
+    |.endif
+    |  // Fall through. Assumes BC_IITERL follows.
+    break;
+
+  case BC_JITERL:
+#if !LJ_HASJIT
+    break;
+#endif
+  case BC_IITERL:
+    |  // RA = base, RC = target
+    |  ldr CARG1, [BASE, RA, lsl #3]
+    |   add TMP1, BASE, RA, lsl #3
+    |  cmp CARG1, TISNIL
+    |  beq >1				// Stop if iterator returned nil.
+    if (op == BC_JITERL) {
+      |  str CARG1, [TMP1, #-8]
+      |  b =>BC_JLOOP
+    } else {
+      |  add TMP0, PC, RC, lsl #2	// Otherwise save control var + branch.
+      |  sub PC, TMP0, #0x20000
+      |  str CARG1, [TMP1, #-8]
+    }
+    |1:
+    |  ins_next
+    break;
+
+  case BC_LOOP:
+    |  // RA = base, RC = target (loop extent)
+    |  // Note: RA/RC is only used by trace recorder to determine scope/extent
+    |  // This opcode does NOT jump, it's only purpose is to detect a hot loop.
+    |.if JIT
+    |  hotloop
+    |.endif
+    |  // Fall through. Assumes BC_ILOOP follows.
+    break;
+
+  case BC_ILOOP:
+    |  // RA = base, RC = target (loop extent)
+    |  ins_next
+    break;
+
+  case BC_JLOOP:
+    |.if JIT
+    |  NYI
+    |.endif
+    break;
+
+  case BC_JMP:
+    |  // RA = base (only used by trace recorder), RC = target
+    |  add RC, PC, RC, lsl #2
+    |  sub PC, RC, #0x20000
+    |  ins_next
+    break;
+
+  /* -- Function headers -------------------------------------------------- */
+
+  case BC_FUNCF:
+    |.if JIT
+    |  hotcall
+    |.endif
+  case BC_FUNCV:  /* NYI: compiled vararg functions. */
+    |  // Fall through. Assumes BC_IFUNCF/BC_IFUNCV follow.
+    break;
+
+  case BC_JFUNCF:
+#if !LJ_HASJIT
+    break;
+#endif
+  case BC_IFUNCF:
+    |  // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
+    |  ldr CARG1, L->maxstack
+    |   ldrb TMP1w, [PC, #-4+PC2PROTO(numparams)]
+    |    ldr KBASE, [PC, #-4+PC2PROTO(k)]
+    |  cmp RA, CARG1
+    |  bhi ->vm_growstack_l
+    |2:
+    |  cmp NARGS8:RC, TMP1, lsl #3	// Check for missing parameters.
+    |  blo >3
+    if (op == BC_JFUNCF) {
+      |  decode_RD RC, INS
+      |  b =>BC_JLOOP
+    } else {
+      |  ins_next
+    }
+    |
+    |3:  // Clear missing parameters.
+    |  str TISNIL, [BASE, NARGS8:RC]
+    |  add NARGS8:RC, NARGS8:RC, #8
+    |  b <2
+    break;
+
+  case BC_JFUNCV:
+#if !LJ_HASJIT
+    break;
+#endif
+    |  NYI  // NYI: compiled vararg functions
+    break;  /* NYI: compiled vararg functions. */
+
+  case BC_IFUNCV:
+    |  // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
+    |  ldr CARG1, L->maxstack
+    |   add TMP2, BASE, RC
+    |  add RA, RA, RC
+    |   add TMP0, RC, #16+FRAME_VARG
+    |   str LFUNC:CARG3, [TMP2], #8	// Store (untagged) copy of LFUNC.
+    |    ldr KBASE, [PC, #-4+PC2PROTO(k)]
+    |  cmp RA, CARG1
+    |   str TMP0, [TMP2], #8		// Store delta + FRAME_VARG.
+    |  bhs ->vm_growstack_l
+    |   sub RC, TMP2, #16
+    |  ldrb TMP1w, [PC, #-4+PC2PROTO(numparams)]
+    |   mov RA, BASE
+    |   mov BASE, TMP2
+    |  cbz TMP1, >2
+    |1:
+    |  cmp RA, RC			// Less args than parameters?
+    |  bhs >3
+    |   ldr TMP0, [RA]
+    |  sub TMP1, TMP1, #1
+    |    str TISNIL, [RA], #8		// Clear old fixarg slot (help the GC).
+    |   str TMP0, [TMP2], #8
+    |  cbnz TMP1, <1
+    |2:
+    |  ins_next
+    |
+    |3:
+    |  sub TMP1, TMP1, #1
+    |   str TISNIL, [TMP2], #8
+    |  cbz TMP1, <2
+    |  b <3
+    break;
+
+  case BC_FUNCC:
+  case BC_FUNCCW:
+    |  // BASE = new base, RA = BASE+framesize*8, CARG3 = CFUNC, RC = nargs*8
+    if (op == BC_FUNCC) {
+      |  ldr CARG4, CFUNC:CARG3->f
+    } else {
+      |  ldr CARG4, GL->wrapf
+    }
+    |   add CARG2, RA, NARGS8:RC
+    |   ldr CARG1, L->maxstack
+    |  add RC, BASE, NARGS8:RC
+    |   cmp CARG2, CARG1
+    |  stp BASE, RC, L->base
+    if (op == BC_FUNCCW) {
+      |  ldr CARG2, CFUNC:CARG3->f
+    }
+    |    mv_vmstate TMP0w, C
+    |  mov CARG1, L
+    |   bhi ->vm_growstack_c		// Need to grow stack.
+    |    st_vmstate TMP0w
+    |  blr CARG4			// (lua_State *L [, lua_CFunction f])
+    |  // Returns nresults.
+    |  ldp BASE, TMP1, L->base
+    |    str L, GL->cur_L
+    |   sbfiz RC, CRET1, #3, #32
+    |    st_vmstate ST_INTERP
+    |  ldr PC, [BASE, FRAME_PC]
+    |   sub RA, TMP1, RC		// RA = L->top - nresults*8
+    |  b ->vm_returnc
+    break;
+
+  /* ---------------------------------------------------------------------- */
+
+  default:
+    fprintf(stderr, "Error: undefined opcode BC_%s\n", bc_names[op]);
+    exit(2);
+    break;
+  }
+}
+
+static int build_backend(BuildCtx *ctx)
+{
+  int op;
+
+  dasm_growpc(Dst, BC__MAX);
+
+  build_subroutines(ctx);
+
+  |.code_op
+  for (op = 0; op < BC__MAX; op++)
+    build_ins(ctx, (BCOp)op, op);
+
+  return BC__MAX;
+}
+
+/* Emit pseudo frame-info for all assembler functions. */
+static void emit_asm_debug(BuildCtx *ctx)
+{
+  int fcofs = (int)((uint8_t *)ctx->glob[GLOB_vm_ffi_call] - ctx->code);
+  int i, cf = CFRAME_SIZE >> 3;
+  switch (ctx->mode) {
+  case BUILD_elfasm:
+    fprintf(ctx->fp, "\t.section .debug_frame,\"\",%%progbits\n");
+    fprintf(ctx->fp,
+	".Lframe0:\n"
+	"\t.long .LECIE0-.LSCIE0\n"
+	".LSCIE0:\n"
+	"\t.long 0xffffffff\n"
+	"\t.byte 0x1\n"
+	"\t.string \"\"\n"
+	"\t.uleb128 0x1\n"
+	"\t.sleb128 -8\n"
+	"\t.byte 30\n"				/* Return address is in lr. */
+	"\t.byte 0xc\n\t.uleb128 31\n\t.uleb128 0\n"	/* def_cfa sp */
+	"\t.align 3\n"
+	".LECIE0:\n\n");
+    fprintf(ctx->fp,
+	".LSFDE0:\n"
+	"\t.long .LEFDE0-.LASFDE0\n"
+	".LASFDE0:\n"
+	"\t.long .Lframe0\n"
+	"\t.quad .Lbegin\n"
+	"\t.quad %d\n"
+	"\t.byte 0xe\n\t.uleb128 %d\n"		/* def_cfa_offset */
+	"\t.byte 0x9d\n\t.uleb128 %d\n"		/* offset fp */
+	"\t.byte 0x9e\n\t.uleb128 %d\n",	/* offset lr */
+	fcofs, CFRAME_SIZE, cf, cf-1);
+    for (i = 19; i <= 28; i++)  /* offset x19-x28 */
+      fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, cf-i+17);
+    for (i = 8; i <= 15; i++)  /* offset d8-d15 */
+      fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x, %d\n", 64+i, cf-i-4);
+    fprintf(ctx->fp,
+	"\t.align 3\n"
+	".LEFDE0:\n\n");
+#if LJ_HASFFI
+#error "NYI"
+#endif
+    fprintf(ctx->fp, "\t.section .eh_frame,\"a\",%%progbits\n");
+    fprintf(ctx->fp,
+	".Lframe1:\n"
+	"\t.long .LECIE1-.LSCIE1\n"
+	".LSCIE1:\n"
+	"\t.long 0\n"
+	"\t.byte 0x1\n"
+	"\t.string \"zPR\"\n"
+	"\t.uleb128 0x1\n"
+	"\t.sleb128 -8\n"
+	"\t.byte 30\n"				/* Return address is in lr. */
+	"\t.uleb128 6\n"			/* augmentation length */
+	"\t.byte 0x1b\n"			/* pcrel|sdata4 */
+        "\t.long lj_err_unwind_dwarf-.\n"
+	"\t.byte 0x1b\n"			/* pcrel|sdata4 */
+	"\t.byte 0xc\n\t.uleb128 31\n\t.uleb128 0\n"	/* def_cfa sp */
+	"\t.align 3\n"
+	".LECIE1:\n\n");
+    fprintf(ctx->fp,
+	".LSFDE2:\n"
+	"\t.long .LEFDE2-.LASFDE2\n"
+	".LASFDE2:\n"
+	"\t.long .LASFDE2-.Lframe1\n"
+	"\t.long .Lbegin-.\n"
+	"\t.long %d\n"
+        "\t.uleb128 0\n"			/* augmentation length */
+	"\t.byte 0xe\n\t.uleb128 %d\n"		/* def_cfa_offset */
+	"\t.byte 0x9d\n\t.uleb128 %d\n"		/* offset fp */
+	"\t.byte 0x9e\n\t.uleb128 %d\n",	/* offset lr */
+	fcofs, CFRAME_SIZE, cf, cf-1);
+    for (i = 19; i <= 28; i++)  /* offset x19-x28 */
+      fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, cf-i+17);
+    for (i = 8; i <= 15; i++)  /* offset d8-d15 */
+      fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x, %d\n", 64+i, cf-i-4);
+    fprintf(ctx->fp,
+	"\t.align 3\n"
+	".LEFDE2:\n\n");
+#if LJ_HASFFI
+#error "NYI"
+#endif
+    break;
+  default:
+    break;
+  }
+}
+