Add cross-32/64 bit and deterministic bytecode generation.

Contributed by Peter Cawley. #993 #1008

doc/extensions.html

@@ -160,13 +160,33 @@ passes any arguments after the error function to the function
 which is called in a protected context.
 </p>
 
-<h3 id="load"><tt>loadfile()</tt> etc. handle UTF-8 source code</h3>
+<h3 id="load"><tt>load*()</tt> handle UTF-8 source code</h3>
 <p>
 Non-ASCII characters are handled transparently by the Lua source code parser.
 This allows the use of UTF-8 characters in identifiers and strings.
 A UTF-8 BOM is skipped at the start of the source code.
 </p>
 
+<h3 id="load_mode"><tt>load*()</tt> add a mode parameter</h3>
+<p>
+As an extension from Lua 5.2, the functions <tt>loadstring()</tt>,
+<tt>loadfile()</tt> and (new) <tt>load()</tt> add an optional
+<tt>mode</tt> parameter.
+</p>
+<p>
+The default mode string is <tt>"bt"</tt>, which allows loading of both
+source code and bytecode. Use <tt>"t"</tt> to allow only source code
+or <tt>"b"</tt> to allow only bytecode to be loaded.
+</p>
+<p>
+By default, the <tt>load*</tt> functions generate the native bytecode format.
+For cross-compilation purposes, add <tt>W</tt> to the mode string to
+force the 32 bit format and <tt>X</tt> to force the 64 bit format.
+Add both to force the opposite format. Note that non-native bytecode
+generated by <tt>load*</tt> cannot be run, but can still be passed
+to <tt>string.dump</tt>.
+</p>
+
 <h3 id="tostring"><tt>tostring()</tt> etc. canonicalize NaN and &plusmn;Inf</h3>
 <p>
 All number-to-string conversions consistently convert non-finite numbers
@@ -186,26 +206,33 @@ works independently of the current locale and it supports hex floating-point
 numbers (e.g. <tt>0x1.5p-3</tt>).
 </p>
 
-<h3 id="string_dump"><tt>string.dump(f [,strip])</tt> generates portable bytecode</h3>
+<h3 id="string_dump"><tt>string.dump(f [,mode])</tt> generates portable bytecode</h3>
 <p>
 An extra argument has been added to <tt>string.dump()</tt>. If set to
-<tt>true</tt>, 'stripped' bytecode without debug information is
-generated. This speeds up later bytecode loading and reduces memory
-usage. See also the
+<tt>true</tt> or to a string which contains the character <tt>s</tt>,
+'stripped' bytecode without debug information is generated. This speeds
+up later bytecode loading and reduces memory usage. See also the
 <a href="running.html#opt_b"><tt>-b</tt> command line option</a>.
 </p>
 <p>
 The generated bytecode is portable and can be loaded on any architecture
-that LuaJIT supports, independent of word size or endianess. However, the
-bytecode compatibility versions must match. Bytecode stays compatible
-for dot releases (x.y.0 &rarr; x.y.1), but may change with major or
-minor releases (2.0 &rarr; 2.1) or between any beta release. Foreign
-bytecode (e.g. from Lua 5.1) is incompatible and cannot be loaded.
+that LuaJIT supports. However, the bytecode compatibility versions must
+match. Bytecode only stays compatible within a major+minor version
+(x.y.aaa &rarr; x.y.bbb), except for development branches. Foreign bytecode
+(e.g. from Lua 5.1) is incompatible and cannot be loaded.
 </p>
 <p>
 Note: <tt>LJ_GC64</tt> mode requires a different frame layout, which implies
-a different, incompatible bytecode format for all 64 bit ports. This may be
-rectified in the future.
+a different, incompatible bytecode format between 32 bit and 64 bit ports.
+This may be rectified in the future. In the meantime, use the <tt>W</tt>
+and </tt>X</tt> <a href="#load_mode">modes of the <tt>load*</tt> functions</a>
+for cross-compilation purposes.
+</p>
+<p>
+Due to VM hardening, bytecode is not deterministic. Add <tt>d</tt> to the
+mode string to dump it in a deterministic manner: identical source code
+always gives a byte-for-byte identical bytecode dump. This feature is
+mainly useful for reproducible builds.
 </p>
 
 <h3 id="table_new"><tt>table.new(narray, nhash)</tt> allocates a pre-sized table</h3>

doc/running.html

@@ -106,6 +106,9 @@ are accepted:
 <li><tt>-l</tt> &mdash; Only list bytecode.</li>
 <li><tt>-s</tt> &mdash; Strip debug info (this is the default).</li>
 <li><tt>-g</tt> &mdash; Keep debug info.</li>
+<li><tt>-W</tt> &mdash; Generate 32 bit (non-GC64) bytecode.</li>
+<li><tt>-X</tt> &mdash; Generate 64 bit (GC64) bytecode.</li>
+<li><tt>-d</tt> &mdash; Generate bytecode in deterministic manner.</li>
 <li><tt>-n name</tt> &mdash; Set module name (default: auto-detect from input name)</li>
 <li><tt>-t type</tt> &mdash; Set output file type (default: auto-detect from output name).</li>
 <li><tt>-a arch</tt> &mdash; Override architecture for object files (default: native).</li>

src/host/genlibbc.lua

@@ -138,65 +138,73 @@ local function fixup_dump(dump, fixup)
   return { dump = ndump, startbc = startbc, sizebc = sizebc }
 end
 
-local function find_defs(src)
+local function find_defs(src, mode)
   local defs = {}
   for name, code in string.gmatch(src, "LJLIB_LUA%(([^)]*)%)%s*/%*(.-)%*/") do
-    local env = {}
     local tcode, fixup = transform_lua(code)
-    local func = assert(load(tcode, "", nil, env))()
-    defs[name] = fixup_dump(string.dump(func, true), fixup)
+    local func = assert(load(tcode, "", mode))
+    defs[name] = fixup_dump(string.dump(func, mode), fixup)
     defs[#defs+1] = name
   end
   return defs
 end
 
-local function gen_header(defs)
+local function gen_header(defs32, defs64)
   local t = {}
   local function w(x) t[#t+1] = x end
   w("/* This is a generated file. DO NOT EDIT! */\n\n")
   w("static const int libbc_endian = ") w(isbe and 1 or 0) w(";\n\n")
-  local s, sb = "", ""
-  for i,name in ipairs(defs) do
-    local d = defs[name]
-    s = s .. d.dump
-    sb = sb .. string.char(i) .. ("\0"):rep(d.startbc - 1)
-	    .. (isbe and "\0\0\0\255" or "\255\0\0\0"):rep(d.sizebc)
-	    .. ("\0"):rep(#d.dump - d.startbc - d.sizebc*4)
-  end
-  w("static const uint8_t libbc_code[] = {\n")
-  local n = 0
-  for i=1,#s do
-    local x = string.byte(s, i)
-    local xb = string.byte(sb, i)
-    if xb == 255 then
-      local name = BCN[x]
-      local m = #name + 4
-      if n + m > 78 then n = 0; w("\n") end
-      n = n + m
-      w("BC_"); w(name)
+  for j,defs in ipairs{defs64, defs32} do
+    local s, sb = "", ""
+    for i,name in ipairs(defs) do
+      local d = defs[name]
+      s = s .. d.dump
+      sb = sb .. string.char(i) .. ("\0"):rep(d.startbc - 1)
+	      .. (isbe and "\0\0\0\255" or "\255\0\0\0"):rep(d.sizebc)
+	      .. ("\0"):rep(#d.dump - d.startbc - d.sizebc*4)
+    end
+    if j == 1 then
+      w("static const uint8_t libbc_code[] = {\n#if LJ_FR2\n")
     else
-      local m = x < 10 and 2 or (x < 100 and 3 or 4)
-      if xb == 0 then
+      w("\n#else\n")
+    end
+    local n = 0
+    for i=1,#s do
+      local x = string.byte(s, i)
+      local xb = string.byte(sb, i)
+      if xb == 255 then
+	local name = BCN[x]
+	local m = #name + 4
 	if n + m > 78 then n = 0; w("\n") end
+	n = n + m
+	w("BC_"); w(name)
       else
-	local name = defs[xb]:gsub("_", ".")
-	if n ~= 0 then w("\n") end
-	w("/* "); w(name); w(" */ ")
-	n = #name + 7
+	local m = x < 10 and 2 or (x < 100 and 3 or 4)
+	if xb == 0 then
+	  if n + m > 78 then n = 0; w("\n") end
+	else
+	  local name = defs[xb]:gsub("_", ".")
+	  if n ~= 0 then w("\n") end
+	  w("/* "); w(name); w(" */ ")
+	  n = #name + 7
+	end
+	n = n + m
+	w(x)
       end
-      n = n + m
-      w(x)
+      w(",")
     end
-    w(",")
   end
-  w("\n0\n};\n\n")
+  w("\n#endif\n0\n};\n\n")
   w("static const struct { const char *name; int ofs; } libbc_map[] = {\n")
-  local m = 0
-  for _,name in ipairs(defs) do
-    w('{"'); w(name); w('",'); w(m) w('},\n')
-    m = m + #defs[name].dump
+  local m32, m64 = 0, 0
+  for i,name in ipairs(defs32) do
+    assert(name == defs64[i])
+    w('{"'); w(name); w('",'); w(m32) w('},\n')
+    m32 = m32 + #defs32[name].dump
+    m64 = m64 + #defs64[name].dump
+    assert(m32 == m64)
   end
-  w("{NULL,"); w(m); w("}\n};\n\n")
+  w("{NULL,"); w(m32); w("}\n};\n\n")
   return table.concat(t)
 end
 
@@ -219,7 +227,8 @@ end
 
 local outfile = parse_arg(arg)
 local src = read_files(arg)
-local defs = find_defs(src)
-local hdr = gen_header(defs)
+local defs32 = find_defs(src, "Wdts")
+local defs64 = find_defs(src, "Xdts")
+local hdr = gen_header(defs32, defs64)
 write_file(outfile, hdr)
 

src/jit/bcsave.lua

@@ -29,6 +29,9 @@ Save LuaJIT bytecode: luajit -b[options] input output
   -l        Only list bytecode.
   -s        Strip debug info (default).
   -g        Keep debug info.
+  -W        Generate 32 bit (non-GC64) bytecode.
+  -X        Generate 64 bit (GC64) bytecode.
+  -d        Generate bytecode in deterministic manner.
   -n name   Set module name (default: auto-detect from input name).
   -t type   Set output file type (default: auto-detect from output name).
   -a arch   Override architecture for object files (default: native).
@@ -51,8 +54,9 @@ local function check(ok, ...)
 end
 
 local function readfile(ctx, input)
-  if type(input) == "function" then return input end
-  if ctx.filename then
+  if ctx.string then
+    return check(loadstring(input, nil, ctx.mode))
+  elseif ctx.filename then
     local data
     if input == "-" then
       data = io.stdin:read("*a")
@@ -61,10 +65,10 @@ local function readfile(ctx, input)
       data = assert(fp:read("*a"))
       assert(fp:close())
     end
-    return check(load(data, ctx.filename))
+    return check(load(data, ctx.filename, ctx.mode))
   else
     if input == "-" then input = nil end
-    return check(loadfile(input))
+    return check(loadfile(input, ctx.mode))
   end
 end
 
@@ -624,7 +628,7 @@ end
 
 local function bcsave(ctx, input, output)
   local f = readfile(ctx, input)
-  local s = string.dump(f, ctx.strip)
+  local s = string.dump(f, ctx.mode)
   local t = ctx.type
   if not t then
     t = detecttype(output)
@@ -647,9 +651,11 @@ local function docmd(...)
   local n = 1
   local list = false
   local ctx = {
-    strip = true, arch = jit.arch, os = jit.os:lower(),
-    type = false, modname = false,
+    mode = "bt", arch = jit.arch, os = jit.os:lower(),
+    type = false, modname = false, string = false,
   }
+  local strip = "s"
+  local gc64 = ""
   while n <= #arg do
     local a = arg[n]
     if type(a) == "string" and a:sub(1, 1) == "-" and a ~= "-" then
@@ -660,14 +666,18 @@ local function docmd(...)
 	if opt == "l" then
 	  list = true
 	elseif opt == "s" then
-	  ctx.strip = true
+	  strip = "s"
 	elseif opt == "g" then
-	  ctx.strip = false
+	  strip = ""
+	elseif opt == "W" or opt == "X" then
+	  gc64 = opt
+	elseif opt == "d" then
+	  ctx.mode = ctx.mode .. opt
 	else
 	  if arg[n] == nil or m ~= #a then usage() end
 	  if opt == "e" then
 	    if n ~= 1 then usage() end
-	    arg[1] = check(loadstring(arg[1]))
+	    ctx.string = true
 	  elseif opt == "n" then
 	    ctx.modname = checkmodname(tremove(arg, n))
 	  elseif opt == "t" then
@@ -687,6 +697,7 @@ local function docmd(...)
       n = n + 1
     end
   end
+  ctx.mode = ctx.mode .. strip .. gc64
   if list then
     if #arg == 0 or #arg > 2 then usage() end
     bclist(ctx, arg[1], arg[2] or "-")

src/lib_base.c

@@ -360,7 +360,11 @@ LJLIB_ASM_(xpcall)		LJLIB_REC(.)
 static int load_aux(lua_State *L, int status, int envarg)
 {
   if (status == LUA_OK) {
-    if (tvistab(L->base+envarg-1)) {
+    /*
+    ** Set environment table for top-level function.
+    ** Don't do this for non-native bytecode, which returns a prototype.
+    */
+    if (tvistab(L->base+envarg-1) && tvisfunc(L->top-1)) {
       GCfunc *fn = funcV(L->top-1);
       GCtab *t = tabV(L->base+envarg-1);
       setgcref(fn->c.env, obj2gco(t));

src/lib_jit.c

@@ -161,24 +161,6 @@ LJLIB_PUSH(top-2) LJLIB_SET(version)
 
 /* -- Reflection API for Lua functions ------------------------------------ */
 
-/* Return prototype of first argument (Lua function or prototype object) */
-static GCproto *check_Lproto(lua_State *L, int nolua)
-{
-  TValue *o = L->base;
-  if (L->top > o) {
-    if (tvisproto(o)) {
-      return protoV(o);
-    } else if (tvisfunc(o)) {
-      if (isluafunc(funcV(o)))
-	return funcproto(funcV(o));
-      else if (nolua)
-	return NULL;
-    }
-  }
-  lj_err_argt(L, 1, LUA_TFUNCTION);
-  return NULL;  /* unreachable */
-}
-
 static void setintfield(lua_State *L, GCtab *t, const char *name, int32_t val)
 {
   setintV(lj_tab_setstr(L, t, lj_str_newz(L, name)), val);
@@ -187,7 +169,7 @@ static void setintfield(lua_State *L, GCtab *t, const char *name, int32_t val)
 /* local info = jit.util.funcinfo(func [,pc]) */
 LJLIB_CF(jit_util_funcinfo)
 {
-  GCproto *pt = check_Lproto(L, 1);
+  GCproto *pt = lj_lib_checkLproto(L, 1, 1);
   if (pt) {
     BCPos pc = (BCPos)lj_lib_optint(L, 2, 0);
     GCtab *t;
@@ -229,7 +211,7 @@ LJLIB_CF(jit_util_funcinfo)
 /* local ins, m = jit.util.funcbc(func, pc) */
 LJLIB_CF(jit_util_funcbc)
 {
-  GCproto *pt = check_Lproto(L, 0);
+  GCproto *pt = lj_lib_checkLproto(L, 1, 0);
   BCPos pc = (BCPos)lj_lib_checkint(L, 2);
   if (pc < pt->sizebc) {
     BCIns ins = proto_bc(pt)[pc];
@@ -246,7 +228,7 @@ LJLIB_CF(jit_util_funcbc)
 /* local k = jit.util.funck(func, idx) */
 LJLIB_CF(jit_util_funck)
 {
-  GCproto *pt = check_Lproto(L, 0);
+  GCproto *pt = lj_lib_checkLproto(L, 1, 0);
   ptrdiff_t idx = (ptrdiff_t)lj_lib_checkint(L, 2);
   if (idx >= 0) {
     if (idx < (ptrdiff_t)pt->sizekn) {
@@ -266,7 +248,7 @@ LJLIB_CF(jit_util_funck)
 /* local name = jit.util.funcuvname(func, idx) */
 LJLIB_CF(jit_util_funcuvname)
 {
-  GCproto *pt = check_Lproto(L, 0);
+  GCproto *pt = lj_lib_checkLproto(L, 1, 0);
   uint32_t idx = (uint32_t)lj_lib_checkint(L, 2);
   if (idx < pt->sizeuv) {
     setstrV(L, L->top-1, lj_str_newz(L, lj_debug_uvname(pt, idx)));

src/lib_string.c

@@ -122,11 +122,25 @@ static int writer_buf(lua_State *L, const void *p, size_t size, void *sb)
 
 LJLIB_CF(string_dump)
 {
-  GCfunc *fn = lj_lib_checkfunc(L, 1);
-  int strip = L->base+1 < L->top && tvistruecond(L->base+1);
-  SBuf *sb = lj_buf_tmp_(L);  /* Assumes lj_bcwrite() doesn't use tmpbuf. */
+  GCproto *pt = lj_lib_checkLproto(L, 1, 1);
+  uint32_t flags = 0;
+  SBuf *sb;
+  TValue *o = L->base+1;
+  if (o < L->top) {
+    if (tvisstr(o)) {
+      const char *mode = strVdata(o);
+      char c;
+      while ((c = *mode++)) {
+	if (c == 's') flags |= BCDUMP_F_STRIP;
+	if (c == 'd') flags |= BCDUMP_F_DETERMINISTIC;
+      }
+    } else if (tvistruecond(o)) {
+      flags |= BCDUMP_F_STRIP;
+    }
+  }
+  sb = lj_buf_tmp_(L);  /* Assumes lj_bcwrite() doesn't use tmpbuf. */
   L->top = L->base+1;
-  if (!isluafunc(fn) || lj_bcwrite(L, funcproto(fn), writer_buf, sb, strip))
+  if (!pt || lj_bcwrite(L, pt, writer_buf, sb, flags))
     lj_err_caller(L, LJ_ERR_STRDUMP);
   setstrV(L, L->top-1, lj_buf_str(L, sb));
   lj_gc_check(L);

src/lj_bcdump.h

@@ -46,6 +46,8 @@
 
 #define BCDUMP_F_KNOWN		(BCDUMP_F_FR2*2-1)
 
+#define BCDUMP_F_DETERMINISTIC	0x80000000
+
 /* Type codes for the GC constants of a prototype. Plus length for strings. */
 enum {
   BCDUMP_KGC_CHILD, BCDUMP_KGC_TAB, BCDUMP_KGC_I64, BCDUMP_KGC_U64,
@@ -61,7 +63,7 @@ enum {
 /* -- Bytecode reader/writer ---------------------------------------------- */
 
 LJ_FUNC int lj_bcwrite(lua_State *L, GCproto *pt, lua_Writer writer,
-		       void *data, int strip);
+		       void *data, uint32_t flags);
 LJ_FUNC GCproto *lj_bcread_proto(LexState *ls);
 LJ_FUNC GCproto *lj_bcread(LexState *ls);
 

src/lj_bcread.c

@@ -281,8 +281,11 @@ static void bcread_knum(LexState *ls, GCproto *pt, MSize sizekn)
 static void bcread_bytecode(LexState *ls, GCproto *pt, MSize sizebc)
 {
   BCIns *bc = proto_bc(pt);
-  bc[0] = BCINS_AD((pt->flags & PROTO_VARARG) ? BC_FUNCV : BC_FUNCF,
-		   pt->framesize, 0);
+  BCIns op;
+  if (ls->fr2 != LJ_FR2) op = BC_NOT;  /* Mark non-native prototype. */
+  else if ((pt->flags & PROTO_VARARG)) op = BC_FUNCV;
+  else op = BC_FUNCF;
+  bc[0] = BCINS_AD(op, pt->framesize, 0);
   bcread_block(ls, bc+1, (sizebc-1)*(MSize)sizeof(BCIns));
   /* Swap bytecode instructions if the endianess differs. */
   if (bcread_swap(ls)) {
@@ -395,7 +398,7 @@ static int bcread_header(LexState *ls)
       bcread_byte(ls) != BCDUMP_VERSION) return 0;
   bcread_flags(ls) = flags = bcread_uleb128(ls);
   if ((flags & ~(BCDUMP_F_KNOWN)) != 0) return 0;
-  if ((flags & BCDUMP_F_FR2) != LJ_FR2*BCDUMP_F_FR2) return 0;
+  if ((flags & BCDUMP_F_FR2) != (uint32_t)ls->fr2*BCDUMP_F_FR2) return 0;
   if ((flags & BCDUMP_F_FFI)) {
 #if LJ_HASFFI
     lua_State *L = ls->L;

src/lj_bcwrite.c

@@ -27,7 +27,9 @@ typedef struct BCWriteCtx {
   GCproto *pt;			/* Root prototype. */
   lua_Writer wfunc;		/* Writer callback. */
   void *wdata;			/* Writer callback data. */
-  int strip;			/* Strip debug info. */
+  TValue **heap;		/* Heap used for deterministic sorting. */
+  uint32_t heapsz;		/* Size of heap. */
+  uint32_t flags;		/* BCDUMP_F_* flags. */
   int status;			/* Status from writer callback. */
 #ifdef LUA_USE_ASSERT
   global_State *g;
@@ -76,6 +78,75 @@ static void bcwrite_ktabk(BCWriteCtx *ctx, cTValue *o, int narrow)
   ctx->sb.w = p;
 }
 
+/* Compare two template table keys. */
+static LJ_AINLINE int bcwrite_ktabk_lt(TValue *a, TValue *b)
+{
+  uint32_t at = itype(a), bt = itype(b);
+  if (at != bt) {  /* This also handles false and true keys. */
+    return at < bt;
+  } else if (at == LJ_TSTR) {
+    return lj_str_cmp(strV(a), strV(b)) < 0;
+  } else {
+    return a->u64 < b->u64;  /* This works for numbers and integers. */
+  }
+}
+
+/* Insert key into a sorted heap. */
+static void bcwrite_ktabk_heap_insert(TValue **heap, MSize idx, MSize end,
+				      TValue *key)
+{
+  MSize child;
+  while ((child = idx * 2 + 1) < end) {
+    /* Find lower of the two children. */
+    TValue *c0 = heap[child];
+    if (child + 1 < end) {
+      TValue *c1 = heap[child + 1];
+      if (bcwrite_ktabk_lt(c1, c0)) {
+	c0 = c1;
+	child++;
+      }
+    }
+    if (bcwrite_ktabk_lt(key, c0)) break;  /* Key lower? Found our position. */
+    heap[idx] = c0;  /* Move lower child up. */
+    idx = child;  /* Descend. */
+  }
+  heap[idx] = key;  /* Insert key here. */
+}
+
+/* Resize heap, dropping content. */
+static void bcwrite_heap_resize(BCWriteCtx *ctx, uint32_t nsz)
+{
+  lua_State *L = sbufL(&ctx->sb);
+  if (ctx->heapsz) {
+    lj_mem_freevec(G(L), ctx->heap, ctx->heapsz, TValue *);
+    ctx->heapsz = 0;
+  }
+  if (nsz) {
+    ctx->heap = lj_mem_newvec(L, nsz, TValue *);
+    ctx->heapsz = nsz;
+  }
+}
+
+/* Write hash part of template table in sorted order. */
+static void bcwrite_ktab_sorted_hash(BCWriteCtx *ctx, Node *node, MSize nhash)
+{
+  TValue **heap = ctx->heap;
+  MSize i = nhash;
+  for (;; node--) {  /* Build heap. */
+    if (!tvisnil(&node->val)) {
+      bcwrite_ktabk_heap_insert(heap, --i, nhash, &node->key);
+      if (i == 0) break;
+    }
+  }
+  do {  /* Drain heap. */
+    TValue *key = heap[0];  /* Output lowest key from top. */
+    bcwrite_ktabk(ctx, key, 0);
+    bcwrite_ktabk(ctx, (TValue *)((char *)key - offsetof(Node, key)), 1);
+    key = heap[--nhash];  /* Remove last key. */
+    bcwrite_ktabk_heap_insert(heap, 0, nhash, key);  /* Re-insert. */
+  } while (nhash);
+}
+
 /* Write a template table. */
 static void bcwrite_ktab(BCWriteCtx *ctx, char *p, const GCtab *t)
 {
@@ -105,14 +176,20 @@ static void bcwrite_ktab(BCWriteCtx *ctx, char *p, const GCtab *t)
       bcwrite_ktabk(ctx, o, 1);
   }
   if (nhash) {  /* Write hash entries. */
-    MSize i = nhash;
     Node *node = noderef(t->node) + t->hmask;
-    for (;; node--)
-      if (!tvisnil(&node->val)) {
-	bcwrite_ktabk(ctx, &node->key, 0);
-	bcwrite_ktabk(ctx, &node->val, 1);
-	if (--i == 0) break;
-      }
+    if ((ctx->flags & BCDUMP_F_DETERMINISTIC) && nhash > 1) {
+      if (ctx->heapsz < nhash)
+	bcwrite_heap_resize(ctx, t->hmask + 1);
+      bcwrite_ktab_sorted_hash(ctx, node, nhash);
+    } else {
+      MSize i = nhash;
+      for (;; node--)
+	if (!tvisnil(&node->val)) {
+	  bcwrite_ktabk(ctx, &node->key, 0);
+	  bcwrite_ktabk(ctx, &node->val, 1);
+	  if (--i == 0) break;
+	}
+    }
   }
 }
 
@@ -269,7 +346,7 @@ static void bcwrite_proto(BCWriteCtx *ctx, GCproto *pt)
   p = lj_strfmt_wuleb128(p, pt->sizekgc);
   p = lj_strfmt_wuleb128(p, pt->sizekn);
   p = lj_strfmt_wuleb128(p, pt->sizebc-1);
-  if (!ctx->strip) {
+  if (!(ctx->flags & BCDUMP_F_STRIP)) {
     if (proto_lineinfo(pt))
       sizedbg = pt->sizept - (MSize)((char *)proto_lineinfo(pt) - (char *)pt);
     p = lj_strfmt_wuleb128(p, sizedbg);
@@ -317,11 +394,10 @@ static void bcwrite_header(BCWriteCtx *ctx)
   *p++ = BCDUMP_HEAD2;
   *p++ = BCDUMP_HEAD3;
   *p++ = BCDUMP_VERSION;
-  *p++ = (ctx->strip ? BCDUMP_F_STRIP : 0) +
+  *p++ = (ctx->flags & (BCDUMP_F_STRIP | BCDUMP_F_FR2)) +
 	 LJ_BE*BCDUMP_F_BE +
-	 ((ctx->pt->flags & PROTO_FFI) ? BCDUMP_F_FFI : 0) +
-	 LJ_FR2*BCDUMP_F_FR2;
-  if (!ctx->strip) {
+	 ((ctx->pt->flags & PROTO_FFI) ? BCDUMP_F_FFI : 0);
+  if (!(ctx->flags & BCDUMP_F_STRIP)) {
     p = lj_strfmt_wuleb128(p, len);
     p = lj_buf_wmem(p, name, len);
   }
@@ -352,14 +428,16 @@ static TValue *cpwriter(lua_State *L, lua_CFunction dummy, void *ud)
 
 /* Write bytecode for a prototype. */
 int lj_bcwrite(lua_State *L, GCproto *pt, lua_Writer writer, void *data,
-	      int strip)
+	      uint32_t flags)
 {
   BCWriteCtx ctx;
   int status;
   ctx.pt = pt;
   ctx.wfunc = writer;
   ctx.wdata = data;
-  ctx.strip = strip;
+  ctx.heapsz = 0;
+  if ((bc_op(proto_bc(pt)[0]) != BC_NOT) == LJ_FR2) flags |= BCDUMP_F_FR2;
+  ctx.flags = flags;
   ctx.status = 0;
 #ifdef LUA_USE_ASSERT
   ctx.g = G(L);
@@ -368,6 +446,7 @@ int lj_bcwrite(lua_State *L, GCproto *pt, lua_Writer writer, void *data,
   status = lj_vm_cpcall(L, NULL, &ctx, cpwriter);
   if (status == 0) status = ctx.status;
   lj_buf_free(G(sbufL(&ctx.sb)), &ctx.sb);
+  bcwrite_heap_resize(&ctx, 0);
   return status;
 }
 

src/lj_lex.c

@@ -411,6 +411,7 @@ int lj_lex_setup(lua_State *L, LexState *ls)
   ls->linenumber = 1;
   ls->lastline = 1;
   ls->endmark = 0;
+  ls->fr2 = LJ_FR2;  /* Generate native bytecode by default. */
   lex_next(ls);  /* Read-ahead first char. */
   if (ls->c == 0xef && ls->p + 2 <= ls->pe && (uint8_t)ls->p[0] == 0xbb &&
       (uint8_t)ls->p[1] == 0xbf) {  /* Skip UTF-8 BOM (if buffered). */

src/lj_lex.h

@@ -74,6 +74,7 @@ typedef struct LexState {
   MSize sizebcstack;	/* Size of bytecode stack. */
   uint32_t level;	/* Syntactical nesting level. */
   int endmark;		/* Trust bytecode end marker, even if not at EOF. */
+  int fr2;		/* Generate bytecode for LJ_FR2 mode. */
 } LexState;
 
 LJ_FUNC int lj_lex_setup(lua_State *L, LexState *ls);

src/lj_lib.c

@@ -62,6 +62,7 @@ static const uint8_t *lib_read_lfunc(lua_State *L, const uint8_t *p, GCtab *tab)
   ls.pe = (const char *)~(uintptr_t)0;
   ls.c = -1;
   ls.level = (BCDUMP_F_STRIP|(LJ_BE*BCDUMP_F_BE));
+  ls.fr2 = LJ_FR2;
   ls.chunkname = name;
   pt = lj_bcread_proto(&ls);
   pt->firstline = ~(BCLine)0;
@@ -266,6 +267,23 @@ GCfunc *lj_lib_checkfunc(lua_State *L, int narg)
   return funcV(o);
 }
 
+GCproto *lj_lib_checkLproto(lua_State *L, int narg, int nolua)
+{
+  TValue *o = L->base + narg-1;
+  if (L->top > o) {
+    if (tvisproto(o)) {
+      return protoV(o);
+    } else if (tvisfunc(o)) {
+      if (isluafunc(funcV(o)))
+	return funcproto(funcV(o));
+      else if (nolua)
+	return NULL;
+    }
+  }
+  lj_err_argt(L, narg, LUA_TFUNCTION);
+  return NULL;  /* unreachable */
+}
+
 GCtab *lj_lib_checktab(lua_State *L, int narg)
 {
   TValue *o = L->base + narg-1;

src/lj_lib.h

@@ -42,6 +42,7 @@ LJ_FUNC lua_Number lj_lib_checknum(lua_State *L, int narg);
 LJ_FUNC int32_t lj_lib_checkint(lua_State *L, int narg);
 LJ_FUNC int32_t lj_lib_optint(lua_State *L, int narg, int32_t def);
 LJ_FUNC GCfunc *lj_lib_checkfunc(lua_State *L, int narg);
+LJ_FUNC GCproto *lj_lib_checkLproto(lua_State *L, int narg, int nolua);
 LJ_FUNC GCtab *lj_lib_checktab(lua_State *L, int narg);
 LJ_FUNC GCtab *lj_lib_checktabornil(lua_State *L, int narg);
 LJ_FUNC int lj_lib_checkopt(lua_State *L, int narg, int def, const char *lst);

src/lj_load.c

@@ -34,14 +34,28 @@ static TValue *cpparser(lua_State *L, lua_CFunction dummy, void *ud)
   UNUSED(dummy);
   cframe_errfunc(L->cframe) = -1;  /* Inherit error function. */
   bc = lj_lex_setup(L, ls);
-  if (ls->mode && !strchr(ls->mode, bc ? 'b' : 't')) {
-    setstrV(L, L->top++, lj_err_str(L, LJ_ERR_XMODE));
-    lj_err_throw(L, LUA_ERRSYNTAX);
+  if (ls->mode) {
+    int xmode = 1;
+    const char *mode = ls->mode;
+    char c;
+    while ((c = *mode++)) {
+      if (c == (bc ? 'b' : 't')) xmode = 0;
+      if (c == (LJ_FR2 ? 'W' : 'X')) ls->fr2 = !LJ_FR2;
+    }
+    if (xmode) {
+      setstrV(L, L->top++, lj_err_str(L, LJ_ERR_XMODE));
+      lj_err_throw(L, LUA_ERRSYNTAX);
+    }
   }
   pt = bc ? lj_bcread(ls) : lj_parse(ls);
-  fn = lj_func_newL_empty(L, pt, tabref(L->env));
-  /* Don't combine above/below into one statement. */
-  setfuncV(L, L->top++, fn);
+  if (ls->fr2 == LJ_FR2) {
+    fn = lj_func_newL_empty(L, pt, tabref(L->env));
+    /* Don't combine above/below into one statement. */
+    setfuncV(L, L->top++, fn);
+  } else {
+    /* Non-native generation returns a dumpable, but non-runnable prototype. */
+    setprotoV(L, L->top++, pt);
+  }
   return NULL;
 }
 
@@ -159,9 +173,10 @@ LUALIB_API int luaL_loadstring(lua_State *L, const char *s)
 LUA_API int lua_dump(lua_State *L, lua_Writer writer, void *data)
 {
   cTValue *o = L->top-1;
+  uint32_t flags = LJ_FR2*BCDUMP_F_FR2;  /* Default mode for legacy C API. */
   lj_checkapi(L->top > L->base, "top slot empty");
   if (tvisfunc(o) && isluafunc(funcV(o)))
-    return lj_bcwrite(L, funcproto(funcV(o)), writer, data, 0);
+    return lj_bcwrite(L, funcproto(funcV(o)), writer, data, flags);
   else
     return 1;
 }

src/lj_parse.c

@@ -667,19 +667,20 @@ static void bcemit_store(FuncState *fs, ExpDesc *var, ExpDesc *e)
 /* Emit method lookup expression. */
 static void bcemit_method(FuncState *fs, ExpDesc *e, ExpDesc *key)
 {
-  BCReg idx, func, obj = expr_toanyreg(fs, e);
+  BCReg idx, func, fr2, obj = expr_toanyreg(fs, e);
   expr_free(fs, e);
   func = fs->freereg;
-  bcemit_AD(fs, BC_MOV, func+1+LJ_FR2, obj);  /* Copy object to 1st argument. */
+  fr2 = fs->ls->fr2;
+  bcemit_AD(fs, BC_MOV, func+1+fr2, obj);  /* Copy object to 1st argument. */
   lj_assertFS(expr_isstrk(key), "bad usage");
   idx = const_str(fs, key);
   if (idx <= BCMAX_C) {
-    bcreg_reserve(fs, 2+LJ_FR2);
+    bcreg_reserve(fs, 2+fr2);
     bcemit_ABC(fs, BC_TGETS, func, obj, idx);
   } else {
-    bcreg_reserve(fs, 3+LJ_FR2);
-    bcemit_AD(fs, BC_KSTR, func+2+LJ_FR2, idx);
-    bcemit_ABC(fs, BC_TGETV, func, obj, func+2+LJ_FR2);
+    bcreg_reserve(fs, 3+fr2);
+    bcemit_AD(fs, BC_KSTR, func+2+fr2, idx);
+    bcemit_ABC(fs, BC_TGETV, func, obj, func+2+fr2);
     fs->freereg--;
   }
   e->u.s.info = func;
@@ -1326,9 +1327,12 @@ static void fs_fixup_bc(FuncState *fs, GCproto *pt, BCIns *bc, MSize n)
 {
   BCInsLine *base = fs->bcbase;
   MSize i;
+  BCIns op;
   pt->sizebc = n;
-  bc[0] = BCINS_AD((fs->flags & PROTO_VARARG) ? BC_FUNCV : BC_FUNCF,
-		   fs->framesize, 0);
+  if (fs->ls->fr2 != LJ_FR2) op = BC_NOT;  /* Mark non-native prototype. */
+  else if ((fs->flags & PROTO_VARARG)) op = BC_FUNCV;
+  else op = BC_FUNCF;
+  bc[0] = BCINS_AD(op, fs->framesize, 0);
   for (i = 1; i < n; i++)
     bc[i] = base[i].ins;
 }
@@ -1936,11 +1940,11 @@ static void parse_args(LexState *ls, ExpDesc *e)
   lj_assertFS(e->k == VNONRELOC, "bad expr type %d", e->k);
   base = e->u.s.info;  /* Base register for call. */
   if (args.k == VCALL) {
-    ins = BCINS_ABC(BC_CALLM, base, 2, args.u.s.aux - base - 1 - LJ_FR2);
+    ins = BCINS_ABC(BC_CALLM, base, 2, args.u.s.aux - base - 1 - ls->fr2);
   } else {
     if (args.k != VVOID)
       expr_tonextreg(fs, &args);
-    ins = BCINS_ABC(BC_CALL, base, 2, fs->freereg - base - LJ_FR2);
+    ins = BCINS_ABC(BC_CALL, base, 2, fs->freereg - base - ls->fr2);
   }
   expr_init(e, VCALL, bcemit_INS(fs, ins));
   e->u.s.aux = base;
@@ -1980,7 +1984,7 @@ static void expr_primary(LexState *ls, ExpDesc *v)
       parse_args(ls, v);
     } else if (ls->tok == '(' || ls->tok == TK_string || ls->tok == '{') {
       expr_tonextreg(fs, v);
-      if (LJ_FR2) bcreg_reserve(fs, 1);
+      if (ls->fr2) bcreg_reserve(fs, 1);
       parse_args(ls, v);
     } else {
       break;
@@ -2565,7 +2569,7 @@ static void parse_for_iter(LexState *ls, GCstr *indexname)
   line = ls->linenumber;
   assign_adjust(ls, 3, expr_list(ls, &e), &e);
   /* The iterator needs another 3 [4] slots (func [pc] | state ctl). */
-  bcreg_bump(fs, 3+LJ_FR2);
+  bcreg_bump(fs, 3+ls->fr2);
   isnext = (nvars <= 5 && predict_next(ls, fs, exprpc));
   var_add(ls, 3);  /* Hidden control variables. */
   lex_check(ls, TK_do);