correct way to check stack space for vararg functions

lcode.c

@@ -1,5 +1,5 @@
 /*
-** $Id: lcode.c,v 2.153 2018/02/09 15:16:06 roberto Exp roberto $
+** $Id: lcode.c,v 2.154 2018/02/15 15:34:29 roberto Exp roberto $
 ** Code generator for Lua
 ** See Copyright Notice in lua.h
 */
@@ -31,7 +31,7 @@
 
 
 /* Maximum number of registers in a Lua function (must fit in 8 bits) */
-#define MAXREGS		254
+#define MAXREGS		255
 
 
 #define hasjumps(e)	((e)->t != (e)->f)

lparser.c

@@ -1,5 +1,5 @@
 /*
-** $Id: lparser.c,v 2.176 2018/02/07 15:18:04 roberto Exp roberto $
+** $Id: lparser.c,v 2.177 2018/02/09 15:16:06 roberto Exp roberto $
 ** Lua Parser
 ** See Copyright Notice in lua.h
 */
@@ -578,8 +578,6 @@ static void close_func (LexState *ls) {
   luaM_shrinkvector(L, f->p, f->sizep, fs->np, Proto *);
   luaM_shrinkvector(L, f->locvars, f->sizelocvars, fs->nlocvars, LocVar);
   luaM_shrinkvector(L, f->upvalues, f->sizeupvalues, fs->nups, Upvaldesc);
-  if (f->is_vararg)
-    f->maxstacksize++;  /* ensure space to copy the function */
   ls->fs = fs->prev;
   luaC_checkGC(L);
 }

ltm.c

@@ -1,5 +1,5 @@
 /*
-** $Id: ltm.c,v 2.60 2018/02/09 15:16:06 roberto Exp roberto $
+** $Id: ltm.c,v 2.61 2018/02/15 15:34:29 roberto Exp roberto $
 ** Tag methods
 ** See Copyright Notice in lua.h
 */
@@ -216,12 +216,13 @@ int luaT_callorderiTM (lua_State *L, const TValue *p1, int v2,
 }
 
 
-void luaT_adjustvarargs (lua_State *L, int nfixparams, CallInfo *ci) {
+void luaT_adjustvarargs (lua_State *L, int nfixparams, CallInfo *ci,
+                         Proto *p) {
   int i;
   int actual = cast_int(L->top - ci->func) - 1;  /* number of arguments */
   int nextra = actual - nfixparams;  /* number of extra arguments */
   ci->u.l.nextraargs = nextra;
-  checkstackGC(L, nfixparams + 1);
+  checkstackGC(L, p->maxstacksize + 1);
   /* copy function to the top of the stack */
   setobjs2s(L, L->top++, ci->func);
   /* move fixed parameters to the top of the stack */
@@ -231,6 +232,7 @@ void luaT_adjustvarargs (lua_State *L, int nfixparams, CallInfo *ci) {
   }
   ci->func += actual + 1;
   ci->top += actual + 1;
+  lua_assert(L->top <= ci->top && ci->top <= L->stack_last);
 }
 
 

ltm.h

@@ -1,5 +1,5 @@
 /*
-** $Id: ltm.h,v 2.30 2018/02/07 15:18:04 roberto Exp roberto $
+** $Id: ltm.h,v 2.31 2018/02/09 15:16:06 roberto Exp roberto $
 ** Tag methods
 ** See Copyright Notice in lua.h
 */
@@ -79,7 +79,7 @@ LUAI_FUNC int luaT_callorderiTM (lua_State *L, const TValue *p1, int v2,
                                  int inv, TMS event);
 
 LUAI_FUNC void luaT_adjustvarargs (lua_State *L, int nfixparams,
-                                                 struct CallInfo *ci);
+                                   struct CallInfo *ci, Proto *p);
 LUAI_FUNC void luaT_getvarargs (lua_State *L, struct CallInfo *ci,
                                               StkId where, int wanted);
 

lvm.c

@@ -1,5 +1,5 @@
 /*
-** $Id: lvm.c,v 2.339 2018/02/09 15:16:06 roberto Exp roberto $
+** $Id: lvm.c,v 2.340 2018/02/15 15:34:29 roberto Exp roberto $
 ** Lua virtual machine
 ** See Copyright Notice in lua.h
 */
@@ -1713,13 +1713,13 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
         vmbreak;
       }
       vmcase(OP_PREPVARARG) {
-        luaT_adjustvarargs(L, GETARG_A(i), ci);
+        luaT_adjustvarargs(L, GETARG_A(i), ci, cl->p);
         updatetrap(ci);
         if (trap) {
           luaD_hookcall(L, ci);
-          L->oldpc = pc + 1;  /* next opcode will be seen as a new line */
+          L->oldpc = pc + 1;  /* next opcode will be seen as a "new" line */
         }
-        updatebase(ci);
+        updatebase(ci);  /* function has new base after adjustment */
         vmbreak;
       }
       vmcase(OP_EXTRAARG) {