avoid overflows in computation of step size

lgc.c

@@ -1,5 +1,5 @@
 /*
-** $Id: lgc.c,v 2.228 2017/05/04 13:32:01 roberto Exp roberto $
+** $Id: lgc.c,v 2.229 2017/05/26 19:14:29 roberto Exp roberto $
 ** Garbage Collector
 ** See Copyright Notice in lua.h
 */
@@ -1486,7 +1486,9 @@ void luaC_runtilstate (lua_State *L, int statesmask) {
 static void incstep (lua_State *L, global_State *g) {
   int stepmul = (g->gcstepmul | 1);  /* avoid division by 0 */
   l_mem debt = (g->GCdebt / WORK2MEM) * stepmul;
-  l_mem stepsize = cast(l_mem, 1) << g->gcstepsize;
+  l_mem stepsize = (g->gcstepsize <= log2maxs(l_mem))
+                   ? cast(l_mem, 1) << g->gcstepsize
+                   : MAX_LMEM;
   stepsize = -((stepsize / WORK2MEM) * stepmul);
   do {  /* repeat until pause or enough "credit" (negative debt) */
     lu_mem work = singlestep(L);  /* perform one single step */

llimits.h

@@ -1,5 +1,5 @@
 /*
-** $Id: llimits.h,v 1.141 2015/11/19 19:16:22 roberto Exp roberto $
+** $Id: llimits.h,v 1.142 2017/04/24 18:06:12 roberto Exp roberto $
 ** Limits, basic types, and some other 'installation-dependent' definitions
 ** See Copyright Notice in lua.h
 */
@@ -51,6 +51,13 @@ typedef unsigned char lu_byte;
 #define MAX_INT		INT_MAX  /* maximum value of an int */
 
 
+/*
+** floor of the log2 of the maximum signed value for integral type 't'.
+** (That is, maximum 'n' such that '2^n' fits in the given signed type.)
+*/
+#define log2maxs(t)	(sizeof(t) * 8 - 2)
+
+
 /*
 ** conversion of pointer to unsigned integer:
 ** this is for hashing only; there is no problem if the integer