Check for failure to mark memory as executable (restricted kernels).

src/Makefile.dep

@@ -17,8 +17,8 @@ lib_ffi.o: lib_ffi.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
  lj_ccallback.h lj_clib.h lj_ff.h lj_ffdef.h lj_lib.h lj_libdef.h
 lib_init.o: lib_init.c lua.h luaconf.h lauxlib.h lualib.h lj_arch.h
 lib_io.o: lib_io.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
- lj_arch.h lj_err.h lj_errmsg.h lj_str.h lj_state.h lj_ff.h lj_ffdef.h \
- lj_lib.h lj_libdef.h
+ lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_state.h lj_ff.h \
+ lj_ffdef.h lj_lib.h lj_libdef.h
 lib_jit.o: lib_jit.c lua.h luaconf.h lauxlib.h lualib.h lj_arch.h \
  lj_obj.h lj_def.h lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_tab.h \
  lj_bc.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h lj_target.h \
@@ -129,8 +129,8 @@ lj_load.o: lj_load.c lua.h luaconf.h lauxlib.h lj_obj.h lj_def.h \
  lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_func.h lj_frame.h \
  lj_bc.h lj_vm.h lj_lex.h lj_bcdump.h lj_parse.h
 lj_mcode.o: lj_mcode.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_gc.h lj_jit.h lj_ir.h lj_mcode.h lj_trace.h lj_dispatch.h lj_bc.h \
- lj_traceerr.h lj_vm.h
+ lj_gc.h lj_err.h lj_errmsg.h lj_jit.h lj_ir.h lj_mcode.h lj_trace.h \
+ lj_dispatch.h lj_bc.h lj_traceerr.h lj_vm.h
 lj_meta.o: lj_meta.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
  lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_meta.h lj_frame.h lj_bc.h \
  lj_vm.h lj_strscan.h

src/lj_errmsg.h

@@ -102,6 +102,7 @@ ERRDEF(STRFMTW,	"invalid format (width or precision too long)")
 ERRDEF(STRGSRV,	"invalid replacement value (a %s)")
 ERRDEF(BADMODN,	"name conflict for module " LUA_QS)
 #if LJ_HASJIT
+ERRDEF(JITPROT,	"runtime code generation failed, restricted kernel?")
 #if LJ_TARGET_X86ORX64
 ERRDEF(NOJIT,	"JIT compiler disabled, CPU does not support SSE2")
 #else

src/lj_mcode.c

@@ -9,6 +9,7 @@
 #include "lj_obj.h"
 #if LJ_HASJIT
 #include "lj_gc.h"
+#include "lj_err.h"
 #include "lj_jit.h"
 #include "lj_mcode.h"
 #include "lj_trace.h"
@@ -78,10 +79,10 @@ static void mcode_free(jit_State *J, void *p, size_t sz)
   VirtualFree(p, 0, MEM_RELEASE);
 }
 
-static void mcode_setprot(void *p, size_t sz, DWORD prot)
+static int mcode_setprot(void *p, size_t sz, DWORD prot)
 {
   DWORD oprot;
-  VirtualProtect(p, sz, prot, &oprot);
+  return !VirtualProtect(p, sz, prot, &oprot);
 }
 
 #elif LJ_TARGET_POSIX
@@ -112,9 +113,9 @@ static void mcode_free(jit_State *J, void *p, size_t sz)
   munmap(p, sz);
 }
 
-static void mcode_setprot(void *p, size_t sz, int prot)
+static int mcode_setprot(void *p, size_t sz, int prot)
 {
-  mprotect(p, sz, prot);
+  return mprotect(p, sz, prot);
 }
 
 #elif LJ_64
@@ -140,8 +141,6 @@ static void mcode_free(jit_State *J, void *p, size_t sz)
   lj_mem_free(J2G(J), p, sz);
 }
 
-#define mcode_setprot(p, sz, prot)	UNUSED(p)
-
 #endif
 
 /* -- MCode area protection ----------------------------------------------- */
@@ -180,11 +179,23 @@ static void mcode_protect(jit_State *J, int prot)
 #define MCPROT_GEN	MCPROT_RW
 #define MCPROT_RUN	MCPROT_RX
 
+/* Protection twiddling failed. Probably due to kernel security. */
+static LJ_NOINLINE void mcode_protfail(jit_State *J)
+{
+  lua_CFunction panic = J2G(J)->panic;
+  if (panic) {
+    lua_State *L = J->L;
+    setstrV(L, L->top++, lj_err_str(L, LJ_ERR_JITPROT));
+    panic(L);
+  }
+}
+
 /* Change protection of MCode area. */
 static void mcode_protect(jit_State *J, int prot)
 {
   if (J->mcprot != prot) {
-    mcode_setprot(J->mcarea, J->szmcarea, prot);
+    if (LJ_UNLIKELY(mcode_setprot(J->mcarea, J->szmcarea, prot)))
+      mcode_protfail(J);
     J->mcprot = prot;
   }
 }
@@ -305,7 +316,8 @@ void lj_mcode_commit(jit_State *J, MCode *top)
 /* Abort the reservation. */
 void lj_mcode_abort(jit_State *J)
 {
-  mcode_protect(J, MCPROT_RUN);
+  if (J->mcarea)
+    mcode_protect(J, MCPROT_RUN);
 }
 
 /* Set/reset protection to allow patching of MCode areas. */
@@ -318,8 +330,8 @@ MCode *lj_mcode_patch(jit_State *J, MCode *ptr, int finish)
   if (finish) {
     if (J->mcarea == ptr)
       mcode_protect(J, MCPROT_RUN);
-    else
-      mcode_setprot(ptr, ((MCLink *)ptr)->size, MCPROT_RUN);
+    else if (LJ_UNLIKELY(mcode_setprot(ptr, ((MCLink *)ptr)->size, MCPROT_RUN)))
+      mcode_protfail(J);
     return NULL;
   } else {
     MCode *mc = J->mcarea;
@@ -333,7 +345,8 @@ MCode *lj_mcode_patch(jit_State *J, MCode *ptr, int finish)
       mc = ((MCLink *)mc)->next;
       lua_assert(mc != NULL);
       if (ptr >= mc && ptr < (MCode *)((char *)mc + ((MCLink *)mc)->size)) {
-	mcode_setprot(mc, ((MCLink *)mc)->size, MCPROT_GEN);
+	if (LJ_UNLIKELY(mcode_setprot(mc, ((MCLink *)mc)->size, MCPROT_GEN)))
+	  mcode_protfail(J);
 	return mc;
       }
     }