Avoid overflows when incrementing parameters in C

Any C function can receive maxinteger as an integer argument, and
therefore cannot increment it without some care (e.g., doing unsigned
arithmetic as the core does).

lauxlib.h

@@ -154,6 +154,14 @@ LUALIB_API void (luaL_requiref) (lua_State *L, const char *modname,
 #define luaL_loadbuffer(L,s,sz,n)	luaL_loadbufferx(L,s,sz,n,NULL)
 
 
+/*
+** Perform arithmetic operations on lua_Integer values with wrap-around
+** semantics, as the Lua core does.
+*/
+#define luaL_intop(op,v1,v2)  \
+	((lua_Integer)((lua_Unsigned)(v1) op (lua_Unsigned)(v2)))
+
+
 /* push the value used to represent failure/error */
 #define luaL_pushfail(L)	lua_pushnil(L)
 

lbaselib.c

@@ -285,7 +285,8 @@ static int luaB_pairs (lua_State *L) {
 ** Traversal function for 'ipairs'
 */
 static int ipairsaux (lua_State *L) {
-  lua_Integer i = luaL_checkinteger(L, 2) + 1;
+  lua_Integer i = luaL_checkinteger(L, 2);
+  i = luaL_intop(+, i, 1);
   lua_pushinteger(L, i);
   return (lua_geti(L, 1, i) == LUA_TNIL) ? 1 : 2;
 }

ltablib.c

@@ -59,8 +59,9 @@ static void checktab (lua_State *L, int arg, int what) {
 
 
 static int tinsert (lua_State *L) {
-  lua_Integer e = aux_getn(L, 1, TAB_RW) + 1;  /* first empty element */
   lua_Integer pos;  /* where to insert new element */
+  lua_Integer e = aux_getn(L, 1, TAB_RW);
+  e = luaL_intop(+, e, 1);  /* first empty element */
   switch (lua_gettop(L)) {
     case 2: {  /* called with only 2 arguments */
       pos = e;  /* insert new element at the end */

lutf8lib.c

@@ -224,14 +224,11 @@ static int byteoffset (lua_State *L) {
 static int iter_aux (lua_State *L, int strict) {
   size_t len;
   const char *s = luaL_checklstring(L, 1, &len);
-  lua_Integer n = lua_tointeger(L, 2) - 1;
-  if (n < 0)  /* first iteration? */
-    n = 0;  /* start from here */
-  else if (n < (lua_Integer)len) {
-    n++;  /* skip current byte */
-    while (iscont(s + n)) n++;  /* and its continuations */
+  lua_Unsigned n = (lua_Unsigned)lua_tointeger(L, 2);
+  if (n < len) {
+    while (iscont(s + n)) n++;  /* skip continuation bytes */
   }
-  if (n >= (lua_Integer)len)
+  if (n >= len)  /* (also handles original 'n' being negative) */
     return 0;  /* no more codepoints */
   else {
     utfint code;

testes/nextvar.lua

@@ -43,6 +43,14 @@ assert(i == 4)
 assert(type(ipairs{}) == 'function' and ipairs{} == ipairs{})
 
 
+do   -- overflow (must wrap-around)
+  local f = ipairs{}
+  local k, v = f({[math.mininteger] = 10}, math.maxinteger)
+  assert(k == math.mininteger and v == 10)
+  k, v = f({[math.mininteger] = 10}, k)
+  assert(k == nil)
+end
+
 if not T then
   (Message or print)
     ('\n >>> testC not active: skipping tests for table sizes <<<\n')
@@ -499,6 +507,15 @@ do   -- testing table library with metamethods
 end
 
 
+do   -- testing overflow in table.insert (must wrap-around)
+
+  local t = setmetatable({},
+            {__len = function () return math.maxinteger end})
+  table.insert(t, 20)
+  local k, v = next(t)
+  assert(k == math.mininteger and v == 20)
+end
+
 if not T then
   (Message or print)
     ('\n >>> testC not active: skipping tests for table library on non-tables <<<\n')

testes/utf8.lua

@@ -112,6 +112,12 @@ do
   end
   errorcodes("ab\xff")
   errorcodes("\u{110000}")
+
+  -- calling interation function with invalid arguments
+  local f = utf8.codes("")
+  assert(f("", 2) == nil)
+  assert(f("", -1) == nil)
+  assert(f("", math.mininteger) == nil)
 end
 
 -- error in initial position for offset