Update README and compilation instructions

README

@@ -1,22 +1,32 @@
-# Lua-AOT 5.4
+# Lua-AOT 5.4.3
 
-This is a modified version of Lua 5.4.0 (beta) that allows Lua functions to be compiled ahead-of-time to more efficient code. It also includes a compiler called `luaot` that converts a Lua module to an equivalent one written in C.
+This is a modified version of Lua 5.4.3that allows Lua functions to be compiled ahead-of-time to more efficient code. It also includes a compiler called `luaot` that converts a Lua module to an equivalent one written in C.
 
-The luaot compiler is a simplistic compiler that compiles Lua bytecode to C. 
+The luaot compiler is a simplistic compiler that compiles Lua bytecode to C.
 One may see it as a partial evaluation of the reference Lua interpreter.
 The main optimizations are that Lua jumps are converted into C gotos, and that there is no overhead of interpreting bytecodes at run time.
 Additionally, the C compiler will be able to optimize some kinds of things, specially constant folding and dead-code elimination for the opcodes that include immediate parameters.
 
 The performance gains depend on the kind of program being run. For numerical benchmarks it can be twice as fast as regular Lua. But overall, it isn't super impressive, and is not as fast as LuaJIT or Pallene.
 
-#Usage
+# Compilation and instalation
 
+This is a modified version of the Lua interpreter so the process is the same you would use for compiling Lua. For full instructions, consult and license details, see doc/readme.html.
+
+    make linux -j4
+
+# Usage
+
+Our compiler generates a `.c` file. You can compile that into a `.so` module and then require it from Lua.
 
     ./src/luaot test.lua testcompiled.c
     gcc -shared -fPIC -O2 -I./src testcompiled.c -o testcompiled.so
     ./src/lua -l testcompiled
 
+# Experiments
+
+If you are interested in reproducing the experiments from our paper, please consult the documentation in the `experiments` and `scripts` directory. Note that to be inside the experiments directory when you run the scripts:
 
-For installation instructions, license details, and
-further information about Lua, see doc/readme.html.
+   ../scripts/compile binarytrees.lua
+   ../script/run binarytrees_fast 10
 

UPDATING

@@ -0,0 +1,29 @@
+Updating this compiler to a new release of Lua is a manual process.
+This document describes how to get started on that.
+
+Updating the code
+=================
+
+1. Back up the luaot.c, luaot_header.c and luaot_footer.c
+2. Replace the src folder with a clean copy of Lua.
+3. Reapply the modifications to the Makefile and the interpreter
+   - These are described in the paper
+   - I put some ".patch" files in the "script" folder.
+     I don't guarantee that they will apply cleanly on a future version; may require manual tweaks.
+4. Update the luaot files
+   - In the luaot.c, update the parts copied from luac.c.
+     The most common source of change is if Lua introduces a new opcode, or changes an existing one.
+   - In the luaot.c, update the parts copied from lvm.c
+     Parts that mess with the program counter or that do gotos require manual intervention
+   - Recreate the luaot_header.c. Go through the lvm.c and check each macro one by one.
+     The macros that do jumps or mess with the program counter need to be modified accordingly.
+
+Testing
+=======
+
+Since we're abusing the Lua internals, it is easy to break things.
+I highly recommend turning on the lua_assert assertions by assing the LUAI_ASSERT compiler flag
+
+    MYCFLAGS=-DLUAI_ASSERT
+
+In Lua ≤ 5.4.3, luac had a bug and did not work correctly with the assertions. This includes the luac code that we copied in the luaot.c. If this isn't fixed yet, you may need to fix the luac.c (see commit 1a1fefac) or just disable the assertions in that file with a `#undef LUAI_ASSERT` at the top of the file.

scripts/0001-Patch-the-makefile.patch

@@ -0,0 +1,40 @@
+From b0425c637bffba2f6db5f55ff6532716c151895e Mon Sep 17 00:00:00 2001
+From: Hugo Musso Gualandi <[email protected]>
+Date: Fri, 16 Apr 2021 15:50:29 -0300
+Subject: [PATCH 1/2] Patch the makefile
+
+---
+ src/Makefile | 10 ++++++++--
+ 1 file changed, 8 insertions(+), 2 deletions(-)
+
+diff --git a/src/Makefile b/src/Makefile
+index f78c0b8..ca46c5d 100644
+--- a/src/Makefile
++++ b/src/Makefile
+@@ -43,8 +43,11 @@ LUA_O=	lua.o
+ LUAC_T=	luac
+ LUAC_O=	luac.o
+ 
+-ALL_O= $(BASE_O) $(LUA_O) $(LUAC_O)
+-ALL_T= $(LUA_A) $(LUA_T) $(LUAC_T)
++AOT_T=	luaot
++AOT_O=	luaot.o
++
++ALL_O= $(BASE_O) $(LUA_O) $(LUAC_O) $(AOT_O)
++ALL_T= $(LUA_A) $(LUA_T) $(LUAC_T) $(AOT_T)
+ ALL_A= $(LUA_A)
+ 
+ # Targets start here.
+@@ -66,6 +69,9 @@ $(LUA_T): $(LUA_O) $(LUA_A)
+ $(LUAC_T): $(LUAC_O) $(LUA_A)
+ 	$(CC) -o $@ $(LDFLAGS) $(LUAC_O) $(LUA_A) $(LIBS)
+ 
++$(AOT_T): $(AOT_O) $(LUA_A)
++	$(CC) -o $@ $(LDFLAGS) $(AOT_O) $(LUA_A) $(LIBS)
++
+ test:
+ 	./$(LUA_T) -v
+ 
+-- 
+2.30.2
+

scripts/0002-Changes-to-the-core-Lua-interpreter.patch

@@ -0,0 +1,212 @@
+From aac8fb43a394f2fcf709ec0240a6a04c7d6d8860 Mon Sep 17 00:00:00 2001
+From: Hugo Musso Gualandi <[email protected]>
+Date: Mon, 26 Apr 2021 13:25:53 -0300
+Subject: [PATCH 2/2] Changes to the core Lua interpreter
+
+---
+ src/lfunc.c   |  1 +
+ src/lobject.h |  7 +++++
+ src/luaconf.h |  6 +---
+ src/lvm.c     | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++-
+ 4 files changed, 86 insertions(+), 6 deletions(-)
+
+diff --git a/src/lfunc.c b/src/lfunc.c
+index f5889a2..8948099 100644
+--- a/src/lfunc.c
++++ b/src/lfunc.c
+@@ -260,6 +260,7 @@ Proto *luaF_newproto (lua_State *L) {
+   f->linedefined = 0;
+   f->lastlinedefined = 0;
+   f->source = NULL;
++  f->aot_implementation = NULL;
+   return f;
+ }
+ 
+diff --git a/src/lobject.h b/src/lobject.h
+index 950bebb..06319c8 100644
+--- a/src/lobject.h
++++ b/src/lobject.h
+@@ -533,6 +533,12 @@ typedef struct AbsLineInfo {
+   int line;
+ } AbsLineInfo;
+ 
++
++/*
++** AOT implementation
++*/
++typedef void (*AotCompiledFunction) (lua_State *L, struct CallInfo *ci);
++
+ /*
+ ** Function Prototypes
+ */
+@@ -559,6 +565,7 @@ typedef struct Proto {
+   LocVar *locvars;  /* information about local variables (debug information) */
+   TString  *source;  /* used for debug information */
+   GCObject *gclist;
++  AotCompiledFunction aot_implementation;
+ } Proto;
+ 
+ /* }================================================================== */
+diff --git a/src/luaconf.h b/src/luaconf.h
+index e64d2ee..0f8340a 100644
+--- a/src/luaconf.h
++++ b/src/luaconf.h
+@@ -305,12 +305,8 @@
+ ** give a warning about it. To avoid these warnings, change to the
+ ** default definition.
+ */
+-#if defined(__GNUC__) && ((__GNUC__*100 + __GNUC_MINOR__) >= 302) && \
+-    defined(__ELF__)		/* { */
+-#define LUAI_FUNC	__attribute__((visibility("internal"))) extern
+-#else				/* }{ */
++/* AOT: export all internal APIs */
+ #define LUAI_FUNC	extern
+-#endif				/* } */
+ 
+ #define LUAI_DDEC(dec)	LUAI_FUNC dec
+ #define LUAI_DDEF	/* empty */
+diff --git a/src/lvm.c b/src/lvm.c
+index c9729bc..9a9a6dc 100644
+--- a/src/lvm.c
++++ b/src/lvm.c
+@@ -30,6 +30,11 @@
+ #include "ltm.h"
+ #include "lvm.h"
+ 
++/*
++** We use ifdefs to clearly mark the parts that we had to change
++*/
++#define LUAOT 1
++
+ 
+ /*
+ ** By default, use jump tables in the main interpreter loop on gcc
+@@ -1125,7 +1130,6 @@ void luaV_finishOp (lua_State *L) {
+ #define vmcase(l)	case l:
+ #define vmbreak		break
+ 
+-
+ void luaV_execute (lua_State *L, CallInfo *ci) {
+   LClosure *cl;
+   TValue *k;
+@@ -1135,10 +1139,19 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
+ #if LUA_USE_JUMPTABLE
+ #include "ljumptab.h"
+ #endif
++#if LUAOT
++  trap = L->hookmask;
++  cl = clLvalue(s2v(ci->func));
++  lua_assert(ci->callstatus & CIST_FRESH);
++  if (cl->p->aot_implementation) {
++      return cl->p->aot_implementation(L, ci);
++  }
++#else
+  startfunc:
+   trap = L->hookmask;
+  returning:  /* trap already set */
+   cl = clLvalue(s2v(ci->func));
++#endif
+   k = cl->p->k;
+   pc = ci->u.l.savedpc;
+   if (l_unlikely(trap)) {
+@@ -1613,6 +1626,24 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
+         }
+         vmbreak;
+       }
++#if LUAOT
++      vmcase(OP_CALL) {
++        CallInfo *newci;
++        int b = GETARG_B(i);
++        int nresults = GETARG_C(i) - 1;
++        if (b != 0)  /* fixed number of arguments? */
++          L->top = ra + b;  /* top signals number of arguments */
++        /* else previous instruction set top */
++        savepc(L);  /* in case of errors */
++        if ((newci = luaD_precall(L, ra, nresults)) == NULL)
++          updatetrap(ci);  /* C call; nothing else to be done */
++        else {  /* Lua call: run function in this same C frame */
++          newci->callstatus = CIST_FRESH;
++          Protect(luaV_execute(L, newci));
++        }
++        vmbreak;
++      }
++#else
+       vmcase(OP_CALL) {
+         CallInfo *newci;
+         int b = GETARG_B(i);
+@@ -1630,6 +1661,44 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
+         }
+         vmbreak;
+       }
++#endif
++
++#if LUAOT
++      vmcase(OP_TAILCALL) {
++        int b = GETARG_B(i);  /* number of arguments + 1 (function) */
++        int nparams1 = GETARG_C(i);
++        /* delta is virtual 'func' - real 'func' (vararg functions) */
++        int delta = (nparams1) ? ci->u.l.nextraargs + nparams1 : 0;
++        if (b != 0)
++          L->top = ra + b;
++        else  /* previous instruction set top */
++          b = cast_int(L->top - ra);
++        savepc(ci);  /* several calls here can raise errors */
++        if (TESTARG_k(i)) {
++          luaF_closeupval(L, base);  /* close upvalues from current call */
++          lua_assert(L->tbclist < base);  /* no pending tbc variables */
++          lua_assert(base == ci->func + 1);
++        }
++        while (!ttisfunction(s2v(ra))) {  /* not a function? */
++          luaD_tryfuncTM(L, ra);  /* try '__call' metamethod */
++          b++;  /* there is now one extra argument */
++          checkstackGCp(L, 1, ra);
++        }
++        if (!ttisLclosure(s2v(ra))) {  /* C function? */
++          luaD_precall(L, ra, LUA_MULTRET);  /* call it */
++          updatetrap(ci);
++          updatestack(ci);  /* stack may have been relocated */
++          ci->func -= delta;  /* restore 'func' (if vararg) */
++          luaD_poscall(L, ci, cast_int(L->top - ra));  /* finish caller */
++          updatetrap(ci);  /* 'luaD_poscall' can change hooks */
++          goto ret;  /* caller returns after the tail call */
++        }
++        ci->func -= delta;  /* restore 'func' (if vararg) */
++        luaD_pretailcall(L, ci, ra, b);  /* prepare call frame */
++        return luaV_execute(L, ci); /* execute the callee */
++        vmbreak;
++      }
++#else
+       vmcase(OP_TAILCALL) {
+         int b = GETARG_B(i);  /* number of arguments + 1 (function) */
+         int nparams1 = GETARG_C(i);
+@@ -1662,7 +1731,9 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
+         ci->func -= delta;  /* restore 'func' (if vararg) */
+         luaD_pretailcall(L, ci, ra, b);  /* prepare call frame */
+         goto startfunc;  /* execute the callee */
++        vmbreak;
+       }
++#endif
+       vmcase(OP_RETURN) {
+         int n = GETARG_B(i) - 1;  /* number of results */
+         int nparams1 = GETARG_C(i);
+@@ -1719,12 +1790,17 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
+           }
+         }
+        ret:  /* return from a Lua function */
++#if LUAOT
++        lua_assert(ci->callstatus & CIST_FRESH);
++        return;
++#else
+         if (ci->callstatus & CIST_FRESH)
+           return;  /* end this frame */
+         else {
+           ci = ci->previous;
+           goto returning;  /* continue running caller in this frame */
+         }
++#endif
+       }
+       vmcase(OP_FORLOOP) {
+         if (ttisinteger(s2v(ra + 2))) {  /* integer loop? */
+-- 
+2.30.2
+

scripts/compile

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+lua_file=$1
+base=${lua_file%.lua}
+c_file="$base"_fast.c
+so_file="$base"_fast.so
+
+echo "Generating $c_file ..."
+../src/luaot "$lua_file" "$c_file" || exit 1
+
+echo "Compiling $so_file ..."
+gcc -shared -fPIC -O2 -g -I../src "$c_file" -o "$so_file" || exit 1
+
+echo "Done!"
+exit 0

scripts/run

@@ -0,0 +1,2 @@
+#!/bin/sh
+../src/lua -l "$1" -e "$1($2)"