Protect g->hookmask with lock when using profiler thread.

src/Makefile.dep

@@ -24,9 +24,9 @@ lib_io.o: lib_io.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
  lj_strfmt.h lj_ff.h lj_ffdef.h lj_lib.h lj_libdef.h
 lib_jit.o: lib_jit.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
  lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_str.h lj_tab.h \
- lj_state.h lj_bc.h lj_ctype.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h \
- lj_target.h lj_target_*.h lj_dispatch.h lj_vm.h lj_vmevent.h lj_lib.h \
- luajit.h lj_libdef.h
+ lj_state.h lj_bc.h lj_ctype.h lj_ir.h lj_jit.h lj_trace.h lj_dispatch.h \
+ lj_traceerr.h lj_ircall.h lj_iropt.h lj_target.h lj_target_*.h lj_vm.h \
+ lj_vmevent.h lj_lib.h luajit.h lj_libdef.h
 lib_math.o: lib_math.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
  lj_def.h lj_arch.h lj_lib.h lj_vm.h lj_libdef.h
 lib_os.o: lib_os.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \

src/lib_jit.c

@@ -24,6 +24,7 @@
 #if LJ_HASJIT
 #include "lj_ir.h"
 #include "lj_jit.h"
+#include "lj_trace.h"
 #include "lj_ircall.h"
 #include "lj_iropt.h"
 #include "lj_target.h"
@@ -560,6 +561,7 @@ static void jit_profile_callback(lua_State *L2, lua_State *L, int samples,
       if (G(L2)->panic) G(L2)->panic(L2);
       exit(EXIT_FAILURE);
     }
+    lj_trace_abort(G(L2));
   }
 }
 

src/lj_dispatch.c

@@ -508,28 +508,18 @@ out:
 void LJ_FASTCALL lj_dispatch_profile(lua_State *L, const BCIns *pc)
 {
   ERRNO_SAVE
-  global_State *g = G(L);
-  uint8_t mask = g->hookmask;
-  g->hookmask = (mask & ~HOOK_PROFILE);
-  lj_dispatch_update(g);
-  if (!(mask & HOOK_VMEVENT)) {
-    GCfunc *fn = curr_func(L);
-    GCproto *pt = funcproto(fn);
-    void *cf = cframe_raw(L->cframe);
-    const BCIns *oldpc = cframe_pc(cf);
-    uint8_t oldh = hook_save(g);
-    BCReg slots;
-    hook_vmevent(g);
-    setcframe_pc(cf, pc);
-    slots = cur_topslot(pt, pc, cframe_multres_n(cf));
-    L->top = L->base + slots;  /* Fix top. */
-    lj_profile_interpreter(L);
-    setgcref(g->cur_L, obj2gco(L));
-    setcframe_pc(cf, oldpc);
-    hook_restore(g, oldh);
-    lj_trace_abort(g);
-    setvmstate(g, INTERP);
-  }
+  GCfunc *fn = curr_func(L);
+  GCproto *pt = funcproto(fn);
+  void *cf = cframe_raw(L->cframe);
+  const BCIns *oldpc = cframe_pc(cf);
+  global_State *g;
+  setcframe_pc(cf, pc);
+  L->top = L->base + cur_topslot(pt, pc, cframe_multres_n(cf));
+  lj_profile_interpreter(L);
+  setcframe_pc(cf, oldpc);
+  g = G(L);
+  setgcref(g->cur_L, obj2gco(L));
+  setvmstate(g, INTERP);
   ERRNO_RESTORE
 }
 #endif

src/lj_profile.c

@@ -26,6 +26,8 @@
 
 #include <sys/time.h>
 #include <signal.h>
+#define profile_lock(ps)	UNUSED(ps)
+#define profile_unlock(ps)	UNUSED(ps)
 
 #elif LJ_PROFILE_PTHREAD
 
@@ -34,12 +36,16 @@
 #if LJ_TARGET_PS3
 #include <sys/timer.h>
 #endif
+#define profile_lock(ps)	pthread_mutex_lock(&ps->lock)
+#define profile_unlock(ps)	pthread_mutex_unlock(&ps->lock)
 
 #elif LJ_PROFILE_WTHREAD
 
 #define WIN32_LEAN_AND_MEAN
 #include <windows.h>
 typedef unsigned int (WINAPI *WMM_TPFUNC)(unsigned int);
+#define profile_lock(ps)	EnterCriticalSection(&ps->lock)
+#define profile_unlock(ps)	LeaveCriticalSection(&ps->lock)
 
 #endif
 
@@ -55,6 +61,7 @@ typedef struct ProfileState {
 #if LJ_PROFILE_SIGPROF
   struct sigaction oldsa;	/* Previous SIGPROF state. */
 #elif LJ_PROFILE_PTHREAD
+  pthread_mutex_t lock;		/* g->hookmask update lock. */
   pthread_t thread;		/* Timer thread. */
   int abort;			/* Abort timer thread. */
 #elif LJ_PROFILE_WTHREAD
@@ -63,6 +70,7 @@ typedef struct ProfileState {
   WMM_TPFUNC wmm_tbp;		/* WinMM timeBeginPeriod function. */
   WMM_TPFUNC wmm_tep;		/* WinMM timeEndPeriod function. */
 #endif
+  CRITICAL_SECTION lock;	/* g->hookmask update lock. */
   HANDLE thread;		/* Timer thread. */
   int abort;			/* Abort timer thread. */
 #endif
@@ -85,9 +93,23 @@ static ProfileState profile_state;
 void LJ_FASTCALL lj_profile_interpreter(lua_State *L)
 {
   ProfileState *ps = &profile_state;
-  int samples = ps->samples;
-  ps->samples = 0;
-  ps->cb(ps->data, L, samples, ps->vmstate);  /* Invoke user callback. */
+  global_State *g = G(L);
+  uint8_t mask;
+  profile_lock(ps);
+  mask = (g->hookmask & ~HOOK_PROFILE);
+  if (!(mask & HOOK_VMEVENT)) {
+    int samples = ps->samples;
+    ps->samples = 0;
+    g->hookmask = HOOK_VMEVENT;
+    lj_dispatch_update(g);
+    profile_unlock(ps);
+    ps->cb(ps->data, L, samples, ps->vmstate);  /* Invoke user callback. */
+    profile_lock(ps);
+    mask |= (g->hookmask & HOOK_PROFILE);
+  }
+  g->hookmask = mask;
+  lj_dispatch_update(g);
+  profile_unlock(ps);
 }
 
 /* Trigger profile hook. Asynchronous call from OS-specific profile timer. */
@@ -95,9 +117,10 @@ static void profile_trigger(ProfileState *ps)
 {
   global_State *g = ps->g;
   uint8_t mask;
+  profile_lock(ps);
   ps->samples++;  /* Always increment number of samples. */
   mask = g->hookmask;
-  if (!(mask & HOOK_PROFILE)) {  /* Set profile hook, unless already set. */
+  if (!(mask & (HOOK_PROFILE|HOOK_VMEVENT))) {  /* Set profile hook. */
     int st = g->vmstate;
     ps->vmstate = st >= 0 ? 'N' :
 		  st == ~LJ_VMST_INTERP ? 'I' :
@@ -106,6 +129,7 @@ static void profile_trigger(ProfileState *ps)
     g->hookmask = (mask | HOOK_PROFILE);
     lj_dispatch_update(g);
   }
+  profile_unlock(ps);
 }
 
 /* -- OS-specific profile timer handling ---------------------------------- */
@@ -170,6 +194,7 @@ static void *profile_thread(ProfileState *ps)
 /* Start profiling timer thread. */
 static void profile_timer_start(ProfileState *ps)
 {
+  pthread_mutex_init(&ps->lock, 0);
   ps->abort = 0;
   pthread_create(&ps->thread, NULL, (void *(*)(void *))profile_thread, ps);
 }
@@ -179,6 +204,7 @@ static void profile_timer_stop(ProfileState *ps)
 {
   ps->abort = 1;
   pthread_join(ps->thread, NULL);
+  pthread_mutex_destroy(&ps->lock);
 }
 
 #elif LJ_PROFILE_WTHREAD
@@ -218,6 +244,7 @@ static void profile_timer_start(ProfileState *ps)
     }
   }
 #endif
+  InitializeCriticalSection(&ps->lock);
   ps->abort = 0;
   ps->thread = CreateThread(NULL, 0, profile_thread, ps, 0, NULL);
 }
@@ -227,6 +254,7 @@ static void profile_timer_stop(ProfileState *ps)
 {
   ps->abort = 1;
   WaitForSingleObject(ps->thread, INFINITE);
+  DeleteCriticalSection(&ps->lock);
 }
 
 #endif