Specialize bytecode for pairs()/next() iterator. Speedup: 3.5x.

Parser predict pairs/next and emits specialized bytecode.
Bytecode is descpecialized at runtime if the prediction was wrong.
Store slot index in hidden control var to avoid key lookups.

src/buildvm_ppc.dasc

@@ -2450,6 +2450,14 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  ins_call
     break;
 
+  case BC_ITERN:
+    |  NYI
+    break;
+
+  case BC_ISNEXT:
+    |  NYI
+    break;
+
   case BC_VARG:
     |  // RA = base*8, RB = (nresults+1)*8, RC = numparams*8
     |  lwz TMP0, FRAME_PC(BASE)

src/buildvm_x64.h

@@ -12,7 +12,7 @@
 #define DASM_SECTION_CODE_OP	0
 #define DASM_SECTION_CODE_SUB	1
 #define DASM_MAXSECTION		2
-static const unsigned char build_actionlist[13946] = {
+static const unsigned char build_actionlist[14263] = {
   254,1,248,10,252,247,195,237,15,132,244,11,131,227,252,248,41,218,72,141,
   76,25,252,248,139,90,252,252,199,68,10,4,237,248,12,131,192,1,137,68,36,4,
   252,247,195,237,15,132,244,13,248,14,129,252,243,239,252,247,195,237,15,133,
@@ -636,58 +636,73 @@ static const unsigned char build_actionlist[13946] = {
   105,232,72,139,65,252,240,72,137,41,72,137,65,8,139,105,224,139,65,228,137,
   105,252,248,137,65,252,252,129,252,248,239,184,237,15,133,244,28,137,202,
   137,90,252,252,139,157,233,139,11,15,182,252,233,15,182,205,131,195,4,65,
-  252,255,36,252,238,255,15,182,252,236,15,182,192,68,137,60,36,68,141,188,
-  253,194,233,141,12,202,68,43,122,252,252,133,252,237,15,132,244,251,141,108,
-  252,233,252,248,65,57,215,15,131,244,248,248,1,73,139,71,252,248,65,131,199,
-  8,72,137,1,131,193,8,57,252,233,15,131,244,249,65,57,215,15,130,244,1,248,
-  2,199,65,4,237,131,193,8,57,252,233,15,130,244,2,248,3,68,139,60,36,139,3,
-  15,182,204,15,182,232,131,195,4,193,232,16,65,252,255,36,252,238,248,5,199,
-  68,36,4,1,0,0,0,137,208,68,41,252,248,15,134,244,3,137,197,193,252,237,3,
-  131,197,1,137,108,36,4,139,108,36,24,1,200,59,133,233,15,135,244,253,248,
-  6,255,73,139,71,252,248,65,131,199,8,72,137,1,131,193,8,65,57,215,15,130,
-  244,6,252,233,244,3,248,7,137,149,233,137,141,233,137,92,36,28,65,41,215,
-  139,116,36,4,131,252,238,1,137,252,239,232,251,1,0,139,149,233,139,141,233,
-  65,1,215,252,233,244,6,255,193,225,3,255,248,1,139,90,252,252,137,68,36,4,
-  252,247,195,237,15,133,244,253,255,248,13,65,137,215,131,232,1,15,132,244,
-  249,248,2,73,139,44,15,73,137,111,252,248,65,131,199,8,131,232,1,15,133,244,
-  2,248,3,139,68,36,4,15,182,107,252,255,248,5,57,197,15,135,244,252,255,72,
-  139,44,10,72,137,106,252,248,255,248,5,56,67,252,255,15,135,244,252,255,15,
-  182,75,252,253,72,252,247,209,141,20,202,68,139,122,252,248,69,139,191,233,
-  69,139,191,233,139,3,15,182,204,15,182,232,131,195,4,193,232,16,65,252,255,
-  36,252,238,248,6,255,65,199,71,252,252,237,65,131,199,8,255,199,68,194,252,
-  244,237,255,131,192,1,252,233,244,5,248,7,141,171,233,252,247,197,237,15,
-  133,244,14,41,252,234,255,1,252,233,255,137,221,209,252,237,129,229,239,102,
-  65,131,172,253,46,233,1,15,132,244,140,255,141,12,202,255,129,121,253,4,239,
-  15,135,244,53,129,121,253,12,239,15,135,244,53,255,139,105,20,255,129,252,
-  253,239,15,135,244,53,255,252,242,15,16,1,252,242,15,16,73,8,255,252,242,
-  15,88,65,16,252,242,15,17,1,133,252,237,15,136,244,249,255,15,140,244,249,
-  255,102,15,46,200,248,1,252,242,15,17,65,24,255,221,65,8,221,1,255,220,65,
-  16,221,17,221,81,24,133,252,237,15,136,244,247,255,221,81,24,15,140,244,247,
-  255,217,201,248,1,255,15,183,67,252,254,255,15,131,244,248,141,156,253,131,
-  233,255,141,156,253,131,233,15,183,67,252,254,15,131,245,255,15,130,244,248,
-  141,156,253,131,233,255,248,3,102,15,46,193,252,233,244,1,255,141,12,202,
-  139,105,4,129,252,253,239,15,132,244,247,255,137,105,252,252,139,41,137,105,
-  252,248,252,233,245,255,141,156,253,131,233,139,1,137,105,252,252,137,65,
-  252,248,255,65,139,142,233,139,4,129,72,139,128,233,139,108,36,24,65,137,
-  150,233,65,137,174,233,76,137,36,36,76,137,108,36,8,72,131,252,236,16,252,
-  255,224,255,141,156,253,131,233,139,3,15,182,204,15,182,232,131,195,4,193,
-  232,16,65,252,255,36,252,238,255,137,221,209,252,237,129,229,239,102,65,131,
-  172,253,46,233,1,15,132,244,142,255,68,139,187,233,139,108,36,24,141,12,202,
-  59,141,233,15,135,244,23,15,182,139,233,57,200,15,134,244,249,248,2,255,15,
-  183,67,252,254,252,233,245,255,248,3,199,68,194,252,252,237,131,192,1,57,
-  200,15,134,244,3,252,233,244,2,255,141,44,197,237,141,4,194,68,139,122,252,
-  248,137,104,252,252,68,137,120,252,248,139,108,36,24,141,12,200,59,141,233,
-  15,135,244,22,137,209,137,194,15,182,171,233,133,252,237,15,132,244,248,248,
-  1,131,193,8,57,209,15,131,244,249,68,139,121,252,248,68,137,56,68,139,121,
-  252,252,68,137,120,4,131,192,8,199,65,252,252,237,131,252,237,1,15,133,244,
-  1,248,2,255,68,139,187,233,139,3,15,182,204,15,182,232,131,195,4,193,232,
-  16,65,252,255,36,252,238,255,248,3,199,64,4,237,131,192,8,131,252,237,1,15,
-  133,244,3,252,233,244,2,255,139,106,252,248,76,139,189,233,139,108,36,24,
-  141,68,194,252,248,137,149,233,141,136,233,59,141,233,137,133,233,255,137,
-  252,239,255,76,137,252,254,137,252,239,255,15,135,244,21,65,199,134,233,237,
-  255,65,252,255,215,255,65,252,255,150,233,255,65,199,134,233,237,139,149,
-  233,141,12,194,252,247,217,3,141,233,139,90,252,252,252,233,244,12,255,254,
-  0
+  252,255,36,252,238,255,68,137,60,36,68,137,116,36,4,139,108,202,252,240,139,
+  68,202,252,248,68,139,181,233,131,195,4,68,139,189,233,248,1,68,57,252,240,
+  15,131,244,251,65,129,124,253,199,4,239,15,132,244,250,255,252,242,15,42,
+  192,255,219,68,202,252,248,255,73,139,44,199,72,137,108,202,8,131,192,1,255,
+  137,68,202,252,248,248,2,15,183,67,252,254,141,156,253,131,233,248,3,68,139,
+  116,36,4,68,139,60,36,139,3,15,182,204,15,182,232,131,195,4,193,232,16,65,
+  252,255,36,252,238,248,4,131,192,1,255,137,68,202,252,248,255,252,233,244,
+  1,248,5,68,41,252,240,248,6,59,133,233,15,135,244,3,68,105,252,248,239,68,
+  3,189,233,65,129,191,233,239,15,132,244,253,70,141,116,48,1,73,139,175,233,
+  73,139,135,233,72,137,44,202,72,137,68,202,8,68,137,116,202,252,248,252,233,
+  244,2,248,7,131,192,1,252,233,244,6,255,129,124,253,202,252,236,239,15,133,
+  244,251,139,108,202,232,129,124,253,202,252,244,239,15,133,244,251,129,124,
+  253,202,252,252,239,15,133,244,251,128,189,233,235,15,133,244,251,141,156,
+  253,131,233,199,68,202,252,248,0,0,0,0,248,1,139,3,15,182,204,15,182,232,
+  131,195,4,193,232,16,65,252,255,36,252,238,248,5,198,67,252,252,235,141,156,
+  253,131,233,198,3,235,252,233,244,1,255,15,182,252,236,15,182,192,68,137,
+  60,36,68,141,188,253,194,233,141,12,202,68,43,122,252,252,133,252,237,15,
+  132,244,251,141,108,252,233,252,248,65,57,215,15,131,244,248,248,1,73,139,
+  71,252,248,65,131,199,8,72,137,1,131,193,8,57,252,233,15,131,244,249,65,57,
+  215,15,130,244,1,248,2,199,65,4,237,131,193,8,57,252,233,15,130,244,2,248,
+  3,68,139,60,36,139,3,15,182,204,15,182,232,131,195,4,193,232,16,65,252,255,
+  36,252,238,248,5,199,68,36,4,1,0,0,0,137,208,68,41,252,248,15,134,244,3,137,
+  197,193,252,237,3,131,197,1,137,108,36,4,139,108,36,24,1,200,59,133,233,15,
+  135,244,253,248,6,255,73,139,71,252,248,65,131,199,8,72,137,1,131,193,8,65,
+  57,215,15,130,244,6,252,233,244,3,248,7,137,149,233,137,141,233,137,92,36,
+  28,65,41,215,139,116,36,4,131,252,238,1,137,252,239,232,251,1,0,139,149,233,
+  139,141,233,65,1,215,252,233,244,6,255,193,225,3,255,248,1,139,90,252,252,
+  137,68,36,4,252,247,195,237,15,133,244,253,255,248,13,65,137,215,131,232,
+  1,15,132,244,249,248,2,73,139,44,15,73,137,111,252,248,65,131,199,8,131,232,
+  1,15,133,244,2,248,3,139,68,36,4,15,182,107,252,255,248,5,57,197,15,135,244,
+  252,255,72,139,44,10,72,137,106,252,248,255,248,5,56,67,252,255,15,135,244,
+  252,255,15,182,75,252,253,72,252,247,209,141,20,202,68,139,122,252,248,69,
+  139,191,233,69,139,191,233,139,3,15,182,204,15,182,232,131,195,4,193,232,
+  16,65,252,255,36,252,238,248,6,255,65,199,71,252,252,237,65,131,199,8,255,
+  199,68,194,252,244,237,255,131,192,1,252,233,244,5,248,7,141,171,233,252,
+  247,197,237,15,133,244,14,41,252,234,255,1,252,233,255,137,221,209,252,237,
+  129,229,239,102,65,131,172,253,46,233,1,15,132,244,140,255,141,12,202,255,
+  129,121,253,4,239,15,135,244,53,129,121,253,12,239,15,135,244,53,255,139,
+  105,20,255,129,252,253,239,15,135,244,53,255,252,242,15,16,1,252,242,15,16,
+  73,8,255,252,242,15,88,65,16,252,242,15,17,1,133,252,237,15,136,244,249,255,
+  15,140,244,249,255,102,15,46,200,248,1,252,242,15,17,65,24,255,221,65,8,221,
+  1,255,220,65,16,221,17,221,81,24,133,252,237,15,136,244,247,255,221,81,24,
+  15,140,244,247,255,217,201,248,1,255,15,183,67,252,254,255,15,131,244,248,
+  141,156,253,131,233,255,141,156,253,131,233,15,183,67,252,254,15,131,245,
+  255,15,130,244,248,141,156,253,131,233,255,248,3,102,15,46,193,252,233,244,
+  1,255,141,12,202,139,105,4,129,252,253,239,15,132,244,247,255,137,105,252,
+  252,139,41,137,105,252,248,252,233,245,255,141,156,253,131,233,139,1,137,
+  105,252,252,137,65,252,248,255,65,139,142,233,139,4,129,72,139,128,233,139,
+  108,36,24,65,137,150,233,65,137,174,233,76,137,36,36,76,137,108,36,8,72,131,
+  252,236,16,252,255,224,255,141,156,253,131,233,139,3,15,182,204,15,182,232,
+  131,195,4,193,232,16,65,252,255,36,252,238,255,137,221,209,252,237,129,229,
+  239,102,65,131,172,253,46,233,1,15,132,244,142,255,68,139,187,233,139,108,
+  36,24,141,12,202,59,141,233,15,135,244,23,15,182,139,233,57,200,15,134,244,
+  249,248,2,255,15,183,67,252,254,252,233,245,255,248,3,199,68,194,252,252,
+  237,131,192,1,57,200,15,134,244,3,252,233,244,2,255,141,44,197,237,141,4,
+  194,68,139,122,252,248,137,104,252,252,68,137,120,252,248,139,108,36,24,141,
+  12,200,59,141,233,15,135,244,22,137,209,137,194,15,182,171,233,133,252,237,
+  15,132,244,248,248,1,131,193,8,57,209,15,131,244,249,68,139,121,252,248,68,
+  137,56,68,139,121,252,252,68,137,120,4,131,192,8,199,65,252,252,237,131,252,
+  237,1,15,133,244,1,248,2,255,68,139,187,233,139,3,15,182,204,15,182,232,131,
+  195,4,193,232,16,65,252,255,36,252,238,255,248,3,199,64,4,237,131,192,8,131,
+  252,237,1,15,133,244,3,252,233,244,2,255,139,106,252,248,76,139,189,233,139,
+  108,36,24,141,68,194,252,248,137,149,233,141,136,233,59,141,233,137,133,233,
+  255,137,252,239,255,76,137,252,254,137,252,239,255,15,135,244,21,65,199,134,
+  233,237,255,65,252,255,215,255,65,252,255,150,233,255,65,199,134,233,237,
+  139,149,233,141,12,194,252,247,217,3,141,233,139,90,252,252,252,233,244,12,
+  255,254,0
 };
 
 enum {
@@ -1982,9 +1997,35 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     dasm_put(Dst, 12823, LJ_TFUNC, 2+1, Dt7(->pc));
     break;
 
+  case BC_ITERN:
+#if LJ_HASJIT
+#endif
+    dasm_put(Dst, 12895, Dt6(->asize), Dt6(->array), LJ_TNIL);
+    if (sse) {
+      dasm_put(Dst, 12947);
+    } else {
+      dasm_put(Dst, 12953);
+    }
+    dasm_put(Dst, 12959);
+    if (sse) {
+      dasm_put(Dst, 9917);
+    } else {
+      dasm_put(Dst, 9929);
+    }
+    dasm_put(Dst, 12972, -BCBIAS_J*4);
+    if (!sse) {
+      dasm_put(Dst, 13026);
+    }
+    dasm_put(Dst, 13032, Dt6(->hmask), sizeof(Node), Dt6(->node), DtB(->val.it), LJ_TNIL, DtB(->key), DtB(->val));
+    break;
+
+  case BC_ISNEXT:
+    dasm_put(Dst, 13111, LJ_TFUNC, LJ_TTAB, LJ_TNIL, Dt8(->ffid), FF_next_N, -BCBIAS_J*4, BC_JMP, -BCBIAS_J*4, BC_ITERC);
+    break;
+
   case BC_VARG:
-    dasm_put(Dst, 12895, (8+FRAME_VARG), LJ_TNIL, Dt1(->maxstack));
-    dasm_put(Dst, 13062, Dt1(->base), Dt1(->top), Dt1(->base), Dt1(->top));
+    dasm_put(Dst, 13212, (8+FRAME_VARG), LJ_TNIL, Dt1(->maxstack));
+    dasm_put(Dst, 13379, Dt1(->base), Dt1(->top), Dt1(->base), Dt1(->top));
     break;
 
   /* -- Returns ----------------------------------------------------------- */
@@ -1995,30 +2036,30 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_RET: case BC_RET0: case BC_RET1:
     if (op != BC_RET0) {
-      dasm_put(Dst, 13132);
+      dasm_put(Dst, 13449);
     }
-    dasm_put(Dst, 13136, FRAME_TYPE);
+    dasm_put(Dst, 13453, FRAME_TYPE);
     switch (op) {
     case BC_RET:
-      dasm_put(Dst, 13155);
+      dasm_put(Dst, 13472);
       break;
     case BC_RET1:
-      dasm_put(Dst, 13209);
+      dasm_put(Dst, 13526);
       /* fallthrough */
     case BC_RET0:
-      dasm_put(Dst, 13219);
+      dasm_put(Dst, 13536);
     default:
       break;
     }
-    dasm_put(Dst, 13230, Dt7(->pc), PC2PROTO(k));
+    dasm_put(Dst, 13547, Dt7(->pc), PC2PROTO(k));
     if (op == BC_RET) {
-      dasm_put(Dst, 13278, LJ_TNIL);
+      dasm_put(Dst, 13595, LJ_TNIL);
     } else {
-      dasm_put(Dst, 13289, LJ_TNIL);
+      dasm_put(Dst, 13606, LJ_TNIL);
     }
-    dasm_put(Dst, 13296, -FRAME_VARG, FRAME_TYPEP);
+    dasm_put(Dst, 13613, -FRAME_VARG, FRAME_TYPEP);
     if (op != BC_RET0) {
-      dasm_put(Dst, 13320);
+      dasm_put(Dst, 13637);
     }
     dasm_put(Dst, 4531);
     break;
@@ -2028,7 +2069,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_FORL:
 #if LJ_HASJIT
-    dasm_put(Dst, 13324, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 13641, HOTCOUNT_PCMASK, GG_DISP2HOT);
 #endif
     break;
 
@@ -2040,57 +2081,57 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
   case BC_FORI:
   case BC_IFORL:
     vk = (op == BC_IFORL || op == BC_JFORL);
-    dasm_put(Dst, 13345);
+    dasm_put(Dst, 13662);
     if (!vk) {
-      dasm_put(Dst, 13349, LJ_TISNUM, LJ_TISNUM);
+      dasm_put(Dst, 13666, LJ_TISNUM, LJ_TISNUM);
     }
-    dasm_put(Dst, 13368);
+    dasm_put(Dst, 13685);
     if (!vk) {
-      dasm_put(Dst, 13372, LJ_TISNUM);
+      dasm_put(Dst, 13689, LJ_TISNUM);
     }
     if (sse) {
-      dasm_put(Dst, 13381);
+      dasm_put(Dst, 13698);
       if (vk) {
-	dasm_put(Dst, 13393);
+	dasm_put(Dst, 13710);
       } else {
-	dasm_put(Dst, 13412);
+	dasm_put(Dst, 13729);
       }
-      dasm_put(Dst, 13417);
+      dasm_put(Dst, 13734);
     } else {
-      dasm_put(Dst, 13430);
+      dasm_put(Dst, 13747);
       if (vk) {
-	dasm_put(Dst, 13436);
+	dasm_put(Dst, 13753);
       } else {
-	dasm_put(Dst, 13452);
+	dasm_put(Dst, 13769);
       }
-      dasm_put(Dst, 13460);
+      dasm_put(Dst, 13777);
       if (cmov) {
       dasm_put(Dst, 9460);
       } else {
       dasm_put(Dst, 9466);
       }
       if (!cmov) {
-	dasm_put(Dst, 13465);
+	dasm_put(Dst, 13782);
       }
     }
     if (op == BC_FORI) {
-      dasm_put(Dst, 13471, -BCBIAS_J*4);
+      dasm_put(Dst, 13788, -BCBIAS_J*4);
     } else if (op == BC_JFORI) {
-      dasm_put(Dst, 13481, -BCBIAS_J*4, BC_JLOOP);
+      dasm_put(Dst, 13798, -BCBIAS_J*4, BC_JLOOP);
     } else if (op == BC_IFORL) {
-      dasm_put(Dst, 13495, -BCBIAS_J*4);
+      dasm_put(Dst, 13812, -BCBIAS_J*4);
     } else {
-      dasm_put(Dst, 13491, BC_JLOOP);
+      dasm_put(Dst, 13808, BC_JLOOP);
     }
     dasm_put(Dst, 9495);
     if (sse) {
-      dasm_put(Dst, 13505);
+      dasm_put(Dst, 13822);
     }
     break;
 
   case BC_ITERL:
 #if LJ_HASJIT
-    dasm_put(Dst, 13324, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 13641, HOTCOUNT_PCMASK, GG_DISP2HOT);
 #endif
     break;
 
@@ -2099,18 +2140,18 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     break;
 #endif
   case BC_IITERL:
-    dasm_put(Dst, 13516, LJ_TNIL);
+    dasm_put(Dst, 13833, LJ_TNIL);
     if (op == BC_JITERL) {
-      dasm_put(Dst, 13531, BC_JLOOP);
+      dasm_put(Dst, 13848, BC_JLOOP);
     } else {
-      dasm_put(Dst, 13545, -BCBIAS_J*4);
+      dasm_put(Dst, 13862, -BCBIAS_J*4);
     }
     dasm_put(Dst, 9799);
     break;
 
   case BC_LOOP:
 #if LJ_HASJIT
-    dasm_put(Dst, 13324, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 13641, HOTCOUNT_PCMASK, GG_DISP2HOT);
 #endif
     break;
 
@@ -2120,12 +2161,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_JLOOP:
 #if LJ_HASJIT
-    dasm_put(Dst, 13561, DISPATCH_J(trace), DtD(->mcode), DISPATCH_GL(jit_base), DISPATCH_GL(jit_L));
+    dasm_put(Dst, 13878, DISPATCH_J(trace), DtD(->mcode), DISPATCH_GL(jit_base), DISPATCH_GL(jit_L));
 #endif
     break;
 
   case BC_JMP:
-    dasm_put(Dst, 13602, -BCBIAS_J*4);
+    dasm_put(Dst, 13919, -BCBIAS_J*4);
     break;
 
   /* -- Function headers -------------------------------------------------- */
@@ -2139,7 +2180,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_FUNCF:
 #if LJ_HASJIT
-    dasm_put(Dst, 13628, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 13945, HOTCOUNT_PCMASK, GG_DISP2HOT);
 #endif
   case BC_FUNCV:  /* NYI: compiled vararg functions. */
     break;
@@ -2149,13 +2190,13 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     break;
 #endif
   case BC_IFUNCF:
-    dasm_put(Dst, 13649, -4+PC2PROTO(k), Dt1(->maxstack), -4+PC2PROTO(numparams));
+    dasm_put(Dst, 13966, -4+PC2PROTO(k), Dt1(->maxstack), -4+PC2PROTO(numparams));
     if (op == BC_JFUNCF) {
-      dasm_put(Dst, 13680, BC_JLOOP);
+      dasm_put(Dst, 13997, BC_JLOOP);
     } else {
       dasm_put(Dst, 9497);
     }
-    dasm_put(Dst, 13689, LJ_TNIL);
+    dasm_put(Dst, 14006, LJ_TNIL);
     break;
 
   case BC_JFUNCV:
@@ -2166,30 +2207,30 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     break;  /* NYI: compiled vararg functions. */
 
   case BC_IFUNCV:
-    dasm_put(Dst, 13711, FRAME_VARG, Dt1(->maxstack), -4+PC2PROTO(numparams), LJ_TNIL);
+    dasm_put(Dst, 14028, FRAME_VARG, Dt1(->maxstack), -4+PC2PROTO(numparams), LJ_TNIL);
     if (op == BC_JFUNCV) {
-      dasm_put(Dst, 13680, BC_JLOOP);
+      dasm_put(Dst, 13997, BC_JLOOP);
     } else {
-      dasm_put(Dst, 13808, -4+PC2PROTO(k));
+      dasm_put(Dst, 14125, -4+PC2PROTO(k));
     }
-    dasm_put(Dst, 13833, LJ_TNIL);
+    dasm_put(Dst, 14150, LJ_TNIL);
     break;
 
   case BC_FUNCC:
   case BC_FUNCCW:
-    dasm_put(Dst, 13855, Dt8(->f), Dt1(->base), 8*LUA_MINSTACK, Dt1(->maxstack), Dt1(->top));
+    dasm_put(Dst, 14172, Dt8(->f), Dt1(->base), 8*LUA_MINSTACK, Dt1(->maxstack), Dt1(->top));
     if (op == BC_FUNCC) {
-      dasm_put(Dst, 13885);
+      dasm_put(Dst, 14202);
     } else {
-      dasm_put(Dst, 13889);
+      dasm_put(Dst, 14206);
     }
-    dasm_put(Dst, 13897, DISPATCH_GL(vmstate), ~LJ_VMST_C);
+    dasm_put(Dst, 14214, DISPATCH_GL(vmstate), ~LJ_VMST_C);
     if (op == BC_FUNCC) {
-      dasm_put(Dst, 13907);
+      dasm_put(Dst, 14224);
     } else {
-      dasm_put(Dst, 13912, DISPATCH_GL(wrapf));
+      dasm_put(Dst, 14229, DISPATCH_GL(wrapf));
     }
-    dasm_put(Dst, 13918, DISPATCH_GL(vmstate), ~LJ_VMST_INTERP, Dt1(->base), Dt1(->top));
+    dasm_put(Dst, 14235, DISPATCH_GL(vmstate), ~LJ_VMST_INTERP, Dt1(->base), Dt1(->top));
     break;
 
   /* ---------------------------------------------------------------------- */
@@ -2217,7 +2258,7 @@ static int build_backend(BuildCtx *ctx)
 
   build_subroutines(ctx, cmov, sse);
 
-  dasm_put(Dst, 13944);
+  dasm_put(Dst, 14261);
   for (op = 0; op < BC__MAX; op++)
     build_ins(ctx, (BCOp)op, op, cmov, sse);
 

src/buildvm_x64win.h

@@ -12,7 +12,7 @@
 #define DASM_SECTION_CODE_OP	0
 #define DASM_SECTION_CODE_SUB	1
 #define DASM_MAXSECTION		2
-static const unsigned char build_actionlist[13907] = {
+static const unsigned char build_actionlist[14208] = {
   254,1,248,10,252,247,198,237,15,132,244,11,131,230,252,248,41,252,242,72,
   141,76,49,252,248,139,114,252,252,199,68,10,4,237,248,12,131,192,1,137,68,
   36,84,252,247,198,237,15,132,244,13,248,14,129,252,246,239,252,247,198,237,
@@ -635,60 +635,75 @@ static const unsigned char build_actionlist[13907] = {
   141,76,202,8,72,139,105,232,72,139,65,252,240,72,137,41,72,137,65,8,139,105,
   224,139,65,228,137,105,252,248,137,65,252,252,129,252,248,239,184,237,15,
   133,244,28,137,202,137,114,252,252,139,181,233,139,14,15,182,252,233,15,182,
-  205,131,198,4,252,255,36,252,235,255,15,182,252,236,15,182,192,137,124,36,
-  80,141,188,253,194,233,141,12,202,43,122,252,252,133,252,237,15,132,244,251,
-  141,108,252,233,252,248,57,215,15,131,244,248,248,1,72,139,71,252,248,131,
-  199,8,72,137,1,131,193,8,57,252,233,15,131,244,249,57,215,15,130,244,1,248,
-  2,199,65,4,237,131,193,8,57,252,233,15,130,244,2,248,3,139,124,36,80,139,
-  6,15,182,204,15,182,232,131,198,4,193,232,16,252,255,36,252,235,248,5,199,
-  68,36,84,1,0,0,0,137,208,41,252,248,15,134,244,3,137,197,193,252,237,3,131,
-  197,1,137,108,36,84,139,108,36,96,1,200,59,133,233,15,135,244,253,248,6,255,
-  72,139,71,252,248,131,199,8,72,137,1,131,193,8,57,215,15,130,244,6,252,233,
-  244,3,248,7,137,149,233,137,141,233,137,116,36,100,41,215,139,84,36,84,131,
-  252,234,1,137,252,233,232,251,1,0,139,149,233,139,141,233,1,215,252,233,244,
-  6,255,193,225,3,255,248,1,139,114,252,252,137,68,36,84,252,247,198,237,15,
-  133,244,253,255,248,13,137,215,131,232,1,15,132,244,249,248,2,72,139,44,15,
-  72,137,111,252,248,131,199,8,131,232,1,15,133,244,2,248,3,139,68,36,84,15,
-  182,110,252,255,248,5,57,197,15,135,244,252,255,72,139,44,10,72,137,106,252,
-  248,255,248,5,56,70,252,255,15,135,244,252,255,15,182,78,252,253,72,252,247,
-  209,141,20,202,139,122,252,248,139,191,233,139,191,233,139,6,15,182,204,15,
-  182,232,131,198,4,193,232,16,252,255,36,252,235,248,6,255,199,71,252,252,
-  237,131,199,8,255,199,68,194,252,244,237,255,131,192,1,252,233,244,5,248,
-  7,141,174,233,252,247,197,237,15,133,244,14,41,252,234,255,1,252,233,255,
-  137,252,245,209,252,237,129,229,239,102,131,172,253,43,233,1,15,132,244,140,
-  255,141,12,202,255,129,121,253,4,239,15,135,244,53,129,121,253,12,239,15,
-  135,244,53,255,139,105,20,255,129,252,253,239,15,135,244,53,255,252,242,15,
-  16,1,252,242,15,16,73,8,255,252,242,15,88,65,16,252,242,15,17,1,133,252,237,
-  15,136,244,249,255,15,140,244,249,255,102,15,46,200,248,1,252,242,15,17,65,
-  24,255,221,65,8,221,1,255,220,65,16,221,17,221,81,24,133,252,237,15,136,244,
-  247,255,221,81,24,15,140,244,247,255,217,201,248,1,255,15,183,70,252,254,
-  255,15,131,244,248,141,180,253,134,233,255,141,180,253,134,233,15,183,70,
-  252,254,15,131,245,255,15,130,244,248,141,180,253,134,233,255,248,3,102,15,
-  46,193,252,233,244,1,255,141,12,202,139,105,4,129,252,253,239,15,132,244,
-  247,255,137,105,252,252,139,41,137,105,252,248,252,233,245,255,141,180,253,
-  134,233,139,1,137,105,252,252,137,65,252,248,255,139,139,233,139,4,129,72,
-  139,128,233,139,108,36,96,137,147,233,137,171,233,76,137,100,36,80,76,137,
-  108,36,32,76,137,116,36,24,76,137,124,36,16,72,137,225,72,129,252,236,239,
-  102,15,127,49,102,15,127,185,233,102,68,15,127,129,233,102,68,15,127,137,
-  233,102,68,15,127,145,233,102,68,15,127,153,233,102,68,15,127,161,233,102,
-  68,15,127,169,233,102,68,15,127,177,233,102,68,15,127,185,233,252,255,224,
-  255,141,180,253,134,233,139,6,15,182,204,15,182,232,131,198,4,193,232,16,
-  252,255,36,252,235,255,137,252,245,209,252,237,129,229,239,102,131,172,253,
-  43,233,1,15,132,244,142,255,139,190,233,139,108,36,96,141,12,202,59,141,233,
-  15,135,244,23,15,182,142,233,57,200,15,134,244,249,248,2,255,15,183,70,252,
-  254,252,233,245,255,248,3,199,68,194,252,252,237,131,192,1,57,200,15,134,
-  244,3,252,233,244,2,255,141,44,197,237,141,4,194,139,122,252,248,137,104,
-  252,252,137,120,252,248,139,108,36,96,141,12,200,59,141,233,15,135,244,22,
-  137,209,137,194,15,182,174,233,133,252,237,15,132,244,248,248,1,131,193,8,
-  57,209,15,131,244,249,139,121,252,248,137,56,139,121,252,252,137,120,4,131,
-  192,8,199,65,252,252,237,131,252,237,1,15,133,244,1,248,2,255,139,190,233,
-  139,6,15,182,204,15,182,232,131,198,4,193,232,16,252,255,36,252,235,255,248,
-  3,199,64,4,237,131,192,8,131,252,237,1,15,133,244,3,252,233,244,2,255,139,
-  106,252,248,72,139,189,233,139,108,36,96,141,68,194,252,248,137,149,233,141,
-  136,233,59,141,233,137,133,233,255,137,252,233,255,72,137,252,250,137,252,
-  233,255,15,135,244,21,199,131,233,237,255,252,255,215,255,252,255,147,233,
-  255,199,131,233,237,139,149,233,141,12,194,252,247,217,3,141,233,139,114,
-  252,252,252,233,244,12,255,254,0
+  205,131,198,4,252,255,36,252,235,255,137,124,36,80,137,92,36,84,139,108,202,
+  252,240,139,68,202,252,248,139,157,233,131,198,4,139,189,233,248,1,57,216,
+  15,131,244,251,129,124,253,199,4,239,15,132,244,250,255,252,242,15,42,192,
+  255,219,68,202,252,248,255,72,139,44,199,72,137,108,202,8,131,192,1,255,137,
+  68,202,252,248,248,2,15,183,70,252,254,141,180,253,134,233,248,3,139,92,36,
+  84,139,124,36,80,139,6,15,182,204,15,182,232,131,198,4,193,232,16,252,255,
+  36,252,235,248,4,131,192,1,255,137,68,202,252,248,255,252,233,244,1,248,5,
+  41,216,248,6,59,133,233,15,135,244,3,105,252,248,239,3,189,233,129,191,233,
+  239,15,132,244,253,141,92,24,1,72,139,175,233,72,139,135,233,72,137,44,202,
+  72,137,68,202,8,137,92,202,252,248,252,233,244,2,248,7,131,192,1,252,233,
+  244,6,255,129,124,253,202,252,236,239,15,133,244,251,139,108,202,232,129,
+  124,253,202,252,244,239,15,133,244,251,129,124,253,202,252,252,239,15,133,
+  244,251,128,189,233,235,15,133,244,251,141,180,253,134,233,199,68,202,252,
+  248,0,0,0,0,248,1,139,6,15,182,204,15,182,232,131,198,4,193,232,16,252,255,
+  36,252,235,248,5,198,70,252,252,235,141,180,253,134,233,198,6,235,252,233,
+  244,1,255,15,182,252,236,15,182,192,137,124,36,80,141,188,253,194,233,141,
+  12,202,43,122,252,252,133,252,237,15,132,244,251,141,108,252,233,252,248,
+  57,215,15,131,244,248,248,1,72,139,71,252,248,131,199,8,72,137,1,131,193,
+  8,57,252,233,15,131,244,249,57,215,15,130,244,1,248,2,199,65,4,237,131,193,
+  8,57,252,233,15,130,244,2,248,3,139,124,36,80,139,6,15,182,204,15,182,232,
+  131,198,4,193,232,16,252,255,36,252,235,248,5,199,68,36,84,1,0,0,0,137,208,
+  41,252,248,15,134,244,3,137,197,193,252,237,3,131,197,1,137,108,36,84,139,
+  108,36,96,1,200,59,133,233,15,135,244,253,248,6,255,72,139,71,252,248,131,
+  199,8,72,137,1,131,193,8,57,215,15,130,244,6,252,233,244,3,248,7,137,149,
+  233,137,141,233,137,116,36,100,41,215,139,84,36,84,131,252,234,1,137,252,
+  233,232,251,1,0,139,149,233,139,141,233,1,215,252,233,244,6,255,193,225,3,
+  255,248,1,139,114,252,252,137,68,36,84,252,247,198,237,15,133,244,253,255,
+  248,13,137,215,131,232,1,15,132,244,249,248,2,72,139,44,15,72,137,111,252,
+  248,131,199,8,131,232,1,15,133,244,2,248,3,139,68,36,84,15,182,110,252,255,
+  248,5,57,197,15,135,244,252,255,72,139,44,10,72,137,106,252,248,255,248,5,
+  56,70,252,255,15,135,244,252,255,15,182,78,252,253,72,252,247,209,141,20,
+  202,139,122,252,248,139,191,233,139,191,233,139,6,15,182,204,15,182,232,131,
+  198,4,193,232,16,252,255,36,252,235,248,6,255,199,71,252,252,237,131,199,
+  8,255,199,68,194,252,244,237,255,131,192,1,252,233,244,5,248,7,141,174,233,
+  252,247,197,237,15,133,244,14,41,252,234,255,1,252,233,255,137,252,245,209,
+  252,237,129,229,239,102,131,172,253,43,233,1,15,132,244,140,255,141,12,202,
+  255,129,121,253,4,239,15,135,244,53,129,121,253,12,239,15,135,244,53,255,
+  139,105,20,255,129,252,253,239,15,135,244,53,255,252,242,15,16,1,252,242,
+  15,16,73,8,255,252,242,15,88,65,16,252,242,15,17,1,133,252,237,15,136,244,
+  249,255,15,140,244,249,255,102,15,46,200,248,1,252,242,15,17,65,24,255,221,
+  65,8,221,1,255,220,65,16,221,17,221,81,24,133,252,237,15,136,244,247,255,
+  221,81,24,15,140,244,247,255,217,201,248,1,255,15,183,70,252,254,255,15,131,
+  244,248,141,180,253,134,233,255,141,180,253,134,233,15,183,70,252,254,15,
+  131,245,255,15,130,244,248,141,180,253,134,233,255,248,3,102,15,46,193,252,
+  233,244,1,255,141,12,202,139,105,4,129,252,253,239,15,132,244,247,255,137,
+  105,252,252,139,41,137,105,252,248,252,233,245,255,141,180,253,134,233,139,
+  1,137,105,252,252,137,65,252,248,255,139,139,233,139,4,129,72,139,128,233,
+  139,108,36,96,137,147,233,137,171,233,76,137,100,36,80,76,137,108,36,32,76,
+  137,116,36,24,76,137,124,36,16,72,137,225,72,129,252,236,239,102,15,127,49,
+  102,15,127,185,233,102,68,15,127,129,233,102,68,15,127,137,233,102,68,15,
+  127,145,233,102,68,15,127,153,233,102,68,15,127,161,233,102,68,15,127,169,
+  233,102,68,15,127,177,233,102,68,15,127,185,233,252,255,224,255,141,180,253,
+  134,233,139,6,15,182,204,15,182,232,131,198,4,193,232,16,252,255,36,252,235,
+  255,137,252,245,209,252,237,129,229,239,102,131,172,253,43,233,1,15,132,244,
+  142,255,139,190,233,139,108,36,96,141,12,202,59,141,233,15,135,244,23,15,
+  182,142,233,57,200,15,134,244,249,248,2,255,15,183,70,252,254,252,233,245,
+  255,248,3,199,68,194,252,252,237,131,192,1,57,200,15,134,244,3,252,233,244,
+  2,255,141,44,197,237,141,4,194,139,122,252,248,137,104,252,252,137,120,252,
+  248,139,108,36,96,141,12,200,59,141,233,15,135,244,22,137,209,137,194,15,
+  182,174,233,133,252,237,15,132,244,248,248,1,131,193,8,57,209,15,131,244,
+  249,139,121,252,248,137,56,139,121,252,252,137,120,4,131,192,8,199,65,252,
+  252,237,131,252,237,1,15,133,244,1,248,2,255,139,190,233,139,6,15,182,204,
+  15,182,232,131,198,4,193,232,16,252,255,36,252,235,255,248,3,199,64,4,237,
+  131,192,8,131,252,237,1,15,133,244,3,252,233,244,2,255,139,106,252,248,72,
+  139,189,233,139,108,36,96,141,68,194,252,248,137,149,233,141,136,233,59,141,
+  233,137,133,233,255,137,252,233,255,72,137,252,250,137,252,233,255,15,135,
+  244,21,199,131,233,237,255,252,255,215,255,252,255,147,233,255,199,131,233,
+  237,139,149,233,141,12,194,252,247,217,3,141,233,139,114,252,252,252,233,
+  244,12,255,254,0
 };
 
 enum {
@@ -1984,9 +1999,35 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     dasm_put(Dst, 12750, LJ_TFUNC, 2+1, Dt7(->pc));
     break;
 
+  case BC_ITERN:
+#if LJ_HASJIT
+#endif
+    dasm_put(Dst, 12821, Dt6(->asize), Dt6(->array), LJ_TNIL);
+    if (sse) {
+      dasm_put(Dst, 12867);
+    } else {
+      dasm_put(Dst, 12873);
+    }
+    dasm_put(Dst, 12879);
+    if (sse) {
+      dasm_put(Dst, 9920);
+    } else {
+      dasm_put(Dst, 9932);
+    }
+    dasm_put(Dst, 12892, -BCBIAS_J*4);
+    if (!sse) {
+      dasm_put(Dst, 12944);
+    }
+    dasm_put(Dst, 12950, Dt6(->hmask), sizeof(Node), Dt6(->node), DtB(->val.it), LJ_TNIL, DtB(->key), DtB(->val));
+    break;
+
+  case BC_ISNEXT:
+    dasm_put(Dst, 13022, LJ_TFUNC, LJ_TTAB, LJ_TNIL, Dt8(->ffid), FF_next_N, -BCBIAS_J*4, BC_JMP, -BCBIAS_J*4, BC_ITERC);
+    break;
+
   case BC_VARG:
-    dasm_put(Dst, 12821, (8+FRAME_VARG), LJ_TNIL, Dt1(->maxstack));
-    dasm_put(Dst, 12981, Dt1(->base), Dt1(->top), Dt1(->base), Dt1(->top));
+    dasm_put(Dst, 13122, (8+FRAME_VARG), LJ_TNIL, Dt1(->maxstack));
+    dasm_put(Dst, 13282, Dt1(->base), Dt1(->top), Dt1(->base), Dt1(->top));
     break;
 
   /* -- Returns ----------------------------------------------------------- */
@@ -1997,30 +2038,30 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_RET: case BC_RET0: case BC_RET1:
     if (op != BC_RET0) {
-      dasm_put(Dst, 13047);
+      dasm_put(Dst, 13348);
     }
-    dasm_put(Dst, 13051, FRAME_TYPE);
+    dasm_put(Dst, 13352, FRAME_TYPE);
     switch (op) {
     case BC_RET:
-      dasm_put(Dst, 13070);
+      dasm_put(Dst, 13371);
       break;
     case BC_RET1:
-      dasm_put(Dst, 13122);
+      dasm_put(Dst, 13423);
       /* fallthrough */
     case BC_RET0:
-      dasm_put(Dst, 13132);
+      dasm_put(Dst, 13433);
     default:
       break;
     }
-    dasm_put(Dst, 13143, Dt7(->pc), PC2PROTO(k));
+    dasm_put(Dst, 13444, Dt7(->pc), PC2PROTO(k));
     if (op == BC_RET) {
-      dasm_put(Dst, 13187, LJ_TNIL);
+      dasm_put(Dst, 13488, LJ_TNIL);
     } else {
-      dasm_put(Dst, 13196, LJ_TNIL);
+      dasm_put(Dst, 13497, LJ_TNIL);
     }
-    dasm_put(Dst, 13203, -FRAME_VARG, FRAME_TYPEP);
+    dasm_put(Dst, 13504, -FRAME_VARG, FRAME_TYPEP);
     if (op != BC_RET0) {
-      dasm_put(Dst, 13227);
+      dasm_put(Dst, 13528);
     }
     dasm_put(Dst, 4507);
     break;
@@ -2030,7 +2071,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_FORL:
 #if LJ_HASJIT
-    dasm_put(Dst, 13231, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 13532, HOTCOUNT_PCMASK, GG_DISP2HOT);
 #endif
     break;
 
@@ -2042,57 +2083,57 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
   case BC_FORI:
   case BC_IFORL:
     vk = (op == BC_IFORL || op == BC_JFORL);
-    dasm_put(Dst, 13252);
+    dasm_put(Dst, 13553);
     if (!vk) {
-      dasm_put(Dst, 13256, LJ_TISNUM, LJ_TISNUM);
+      dasm_put(Dst, 13557, LJ_TISNUM, LJ_TISNUM);
     }
-    dasm_put(Dst, 13275);
+    dasm_put(Dst, 13576);
     if (!vk) {
-      dasm_put(Dst, 13279, LJ_TISNUM);
+      dasm_put(Dst, 13580, LJ_TISNUM);
     }
     if (sse) {
-      dasm_put(Dst, 13288);
+      dasm_put(Dst, 13589);
       if (vk) {
-	dasm_put(Dst, 13300);
+	dasm_put(Dst, 13601);
       } else {
-	dasm_put(Dst, 13319);
+	dasm_put(Dst, 13620);
       }
-      dasm_put(Dst, 13324);
+      dasm_put(Dst, 13625);
     } else {
-      dasm_put(Dst, 13337);
+      dasm_put(Dst, 13638);
       if (vk) {
-	dasm_put(Dst, 13343);
+	dasm_put(Dst, 13644);
       } else {
-	dasm_put(Dst, 13359);
+	dasm_put(Dst, 13660);
       }
-      dasm_put(Dst, 13367);
+      dasm_put(Dst, 13668);
       if (cmov) {
       dasm_put(Dst, 9470);
       } else {
       dasm_put(Dst, 9476);
       }
       if (!cmov) {
-	dasm_put(Dst, 13372);
+	dasm_put(Dst, 13673);
       }
     }
     if (op == BC_FORI) {
-      dasm_put(Dst, 13378, -BCBIAS_J*4);
+      dasm_put(Dst, 13679, -BCBIAS_J*4);
     } else if (op == BC_JFORI) {
-      dasm_put(Dst, 13388, -BCBIAS_J*4, BC_JLOOP);
+      dasm_put(Dst, 13689, -BCBIAS_J*4, BC_JLOOP);
     } else if (op == BC_IFORL) {
-      dasm_put(Dst, 13402, -BCBIAS_J*4);
+      dasm_put(Dst, 13703, -BCBIAS_J*4);
     } else {
-      dasm_put(Dst, 13398, BC_JLOOP);
+      dasm_put(Dst, 13699, BC_JLOOP);
     }
     dasm_put(Dst, 9505);
     if (sse) {
-      dasm_put(Dst, 13412);
+      dasm_put(Dst, 13713);
     }
     break;
 
   case BC_ITERL:
 #if LJ_HASJIT
-    dasm_put(Dst, 13231, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 13532, HOTCOUNT_PCMASK, GG_DISP2HOT);
 #endif
     break;
 
@@ -2101,18 +2142,18 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     break;
 #endif
   case BC_IITERL:
-    dasm_put(Dst, 13423, LJ_TNIL);
+    dasm_put(Dst, 13724, LJ_TNIL);
     if (op == BC_JITERL) {
-      dasm_put(Dst, 13438, BC_JLOOP);
+      dasm_put(Dst, 13739, BC_JLOOP);
     } else {
-      dasm_put(Dst, 13452, -BCBIAS_J*4);
+      dasm_put(Dst, 13753, -BCBIAS_J*4);
     }
     dasm_put(Dst, 9805);
     break;
 
   case BC_LOOP:
 #if LJ_HASJIT
-    dasm_put(Dst, 13231, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 13532, HOTCOUNT_PCMASK, GG_DISP2HOT);
 #endif
     break;
 
@@ -2122,12 +2163,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_JLOOP:
 #if LJ_HASJIT
-    dasm_put(Dst, 13468, DISPATCH_J(trace), DtD(->mcode), DISPATCH_GL(jit_base), DISPATCH_GL(jit_L), 9*16+4*8, -1*16, -2*16, -3*16, -4*16, -5*16, -6*16, -7*16, -8*16, -9*16);
+    dasm_put(Dst, 13769, DISPATCH_J(trace), DtD(->mcode), DISPATCH_GL(jit_base), DISPATCH_GL(jit_L), 9*16+4*8, -1*16, -2*16, -3*16, -4*16, -5*16, -6*16, -7*16, -8*16, -9*16);
 #endif
     break;
 
   case BC_JMP:
-    dasm_put(Dst, 13577, -BCBIAS_J*4);
+    dasm_put(Dst, 13878, -BCBIAS_J*4);
     break;
 
   /* -- Function headers -------------------------------------------------- */
@@ -2141,7 +2182,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_FUNCF:
 #if LJ_HASJIT
-    dasm_put(Dst, 13602, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 13903, HOTCOUNT_PCMASK, GG_DISP2HOT);
 #endif
   case BC_FUNCV:  /* NYI: compiled vararg functions. */
     break;
@@ -2151,13 +2192,13 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     break;
 #endif
   case BC_IFUNCF:
-    dasm_put(Dst, 13623, -4+PC2PROTO(k), Dt1(->maxstack), -4+PC2PROTO(numparams));
+    dasm_put(Dst, 13924, -4+PC2PROTO(k), Dt1(->maxstack), -4+PC2PROTO(numparams));
     if (op == BC_JFUNCF) {
-      dasm_put(Dst, 13653, BC_JLOOP);
+      dasm_put(Dst, 13954, BC_JLOOP);
     } else {
       dasm_put(Dst, 9507);
     }
-    dasm_put(Dst, 13662, LJ_TNIL);
+    dasm_put(Dst, 13963, LJ_TNIL);
     break;
 
   case BC_JFUNCV:
@@ -2168,30 +2209,30 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     break;  /* NYI: compiled vararg functions. */
 
   case BC_IFUNCV:
-    dasm_put(Dst, 13684, FRAME_VARG, Dt1(->maxstack), -4+PC2PROTO(numparams), LJ_TNIL);
+    dasm_put(Dst, 13985, FRAME_VARG, Dt1(->maxstack), -4+PC2PROTO(numparams), LJ_TNIL);
     if (op == BC_JFUNCV) {
-      dasm_put(Dst, 13653, BC_JLOOP);
+      dasm_put(Dst, 13954, BC_JLOOP);
     } else {
-      dasm_put(Dst, 13775, -4+PC2PROTO(k));
+      dasm_put(Dst, 14076, -4+PC2PROTO(k));
     }
-    dasm_put(Dst, 13798, LJ_TNIL);
+    dasm_put(Dst, 14099, LJ_TNIL);
     break;
 
   case BC_FUNCC:
   case BC_FUNCCW:
-    dasm_put(Dst, 13820, Dt8(->f), Dt1(->base), 8*LUA_MINSTACK, Dt1(->maxstack), Dt1(->top));
+    dasm_put(Dst, 14121, Dt8(->f), Dt1(->base), 8*LUA_MINSTACK, Dt1(->maxstack), Dt1(->top));
     if (op == BC_FUNCC) {
-      dasm_put(Dst, 13850);
+      dasm_put(Dst, 14151);
     } else {
-      dasm_put(Dst, 13854);
+      dasm_put(Dst, 14155);
     }
-    dasm_put(Dst, 13862, DISPATCH_GL(vmstate), ~LJ_VMST_C);
+    dasm_put(Dst, 14163, DISPATCH_GL(vmstate), ~LJ_VMST_C);
     if (op == BC_FUNCC) {
-      dasm_put(Dst, 13871);
+      dasm_put(Dst, 14172);
     } else {
-      dasm_put(Dst, 13875, DISPATCH_GL(wrapf));
+      dasm_put(Dst, 14176, DISPATCH_GL(wrapf));
     }
-    dasm_put(Dst, 13880, DISPATCH_GL(vmstate), ~LJ_VMST_INTERP, Dt1(->base), Dt1(->top));
+    dasm_put(Dst, 14181, DISPATCH_GL(vmstate), ~LJ_VMST_INTERP, Dt1(->base), Dt1(->top));
     break;
 
   /* ---------------------------------------------------------------------- */
@@ -2219,7 +2260,7 @@ static int build_backend(BuildCtx *ctx)
 
   build_subroutines(ctx, cmov, sse);
 
-  dasm_put(Dst, 13905);
+  dasm_put(Dst, 14206);
   for (op = 0; op < BC__MAX; op++)
     build_ins(ctx, (BCOp)op, op, cmov, sse);
 

src/buildvm_x86.dasc

@@ -4576,6 +4576,109 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     |  ins_call
     break;
 
+  case BC_ITERN:
+    |  ins_A	// RA = base, (RB = nresults+1 (2+1), RC = nargs+1 (2+1))
+#if LJ_HASJIT
+    |  // NYI: add hotloop, record BC_ITERN.
+#endif
+    |  mov TMP1, KBASE			// Need two more free registers.
+    |  mov TMP2, DISPATCH
+    |  mov TAB:RB, [BASE+RA*8-16]
+    |  mov RC, [BASE+RA*8-8]		// Get index from control var.
+    |  mov DISPATCH, TAB:RB->asize
+    |  add PC, 4
+    |  mov KBASE, TAB:RB->array
+    |1:  // Traverse array part.
+    |  cmp RC, DISPATCH; jae >5		// Index points after array part?
+    |  cmp dword [KBASE+RC*8+4], LJ_TNIL; je >4
+    if (sse) {
+      |  cvtsi2sd xmm0, RC
+    } else {
+      |  fild dword [BASE+RA*8-8]
+    }
+    |  // Copy array slot to returned value.
+    |.if X64
+    |  mov RBa, [KBASE+RC*8]
+    |  mov [BASE+RA*8+8], RBa
+    |.else
+    |  mov RB, [KBASE+RC*8+4]
+    |  mov [BASE+RA*8+12], RB
+    |  mov RB, [KBASE+RC*8]
+    |  mov [BASE+RA*8+8], RB
+    |.endif
+    |  add RC, 1
+    |  // Return array index as a numeric key.
+    if (sse) {
+      |  movsd qword [BASE+RA*8], xmm0
+    } else {
+      |  fstp qword [BASE+RA*8]
+    }
+    |  mov [BASE+RA*8-8], RC		// Update control var.
+    |2:
+    |  movzx RD, PC_RD			// Get target from ITERL.
+    |  branchPC RD
+    |3:
+    |  mov DISPATCH, TMP2
+    |  mov KBASE, TMP1
+    |  ins_next
+    |
+    |4:  // Skip holes in array part.
+    |  add RC, 1
+    if (!sse) {
+      |  mov [BASE+RA*8-8], RC
+    }
+    |  jmp <1
+    |
+    |5:  // Traverse hash part.
+    |  sub RC, DISPATCH
+    |6:
+    |  cmp RC, TAB:RB->hmask; ja <3	// End of iteration? Branch to ITERL+1.
+    |  imul KBASE, RC, #NODE
+    |  add NODE:KBASE, TAB:RB->node
+    |  cmp dword NODE:KBASE->val.it, LJ_TNIL; je >7
+    |  lea DISPATCH, [RC+DISPATCH+1]
+    |  // Copy key and value from hash slot.
+    |.if X64
+    |  mov RBa, NODE:KBASE->key
+    |  mov RCa, NODE:KBASE->val
+    |  mov [BASE+RA*8], RBa
+    |  mov [BASE+RA*8+8], RCa
+    |.else
+    |  mov RB, NODE:KBASE->key.gcr
+    |  mov RC, NODE:KBASE->key.it
+    |  mov [BASE+RA*8], RB
+    |  mov [BASE+RA*8+4], RC
+    |  mov RB, NODE:KBASE->val.gcr
+    |  mov RC, NODE:KBASE->val.it
+    |  mov [BASE+RA*8+8], RB
+    |  mov [BASE+RA*8+12], RC
+    |.endif
+    |  mov [BASE+RA*8-8], DISPATCH
+    |  jmp <2
+    |
+    |7:  // Skip holes in hash part.
+    |  add RC, 1
+    |  jmp <6
+    break;
+
+  case BC_ISNEXT:
+    |  ins_AD	// RA = base, RD = target (points to ITERN)
+    |  cmp dword [BASE+RA*8-20], LJ_TFUNC; jne >5
+    |  mov CFUNC:RB, [BASE+RA*8-24]
+    |  cmp dword [BASE+RA*8-12], LJ_TTAB; jne >5
+    |  cmp dword [BASE+RA*8-4], LJ_TNIL; jne >5
+    |  cmp byte CFUNC:RB->ffid, FF_next_N; jne >5
+    |  branchPC RD
+    |  mov dword [BASE+RA*8-8], 0	// Initialize control var.
+    |1:
+    |  ins_next
+    |5:  // Despecialize bytecode if any of the checks fail.
+    |  mov PC_OP, BC_JMP
+    |  branchPC RD
+    |  mov byte [PC], BC_ITERC
+    |  jmp <1
+    break;
+
   case BC_VARG:
     |  ins_ABC	// RA = base, RB = nresults+1, RC = numparams
     |  mov TMP1, KBASE			// Need one more free register.

src/buildvm_x86.h

@@ -12,7 +12,7 @@
 #define DASM_SECTION_CODE_OP	0
 #define DASM_SECTION_CODE_SUB	1
 #define DASM_MAXSECTION		2
-static const unsigned char build_actionlist[15260] = {
+static const unsigned char build_actionlist[15576] = {
   254,1,248,10,252,247,198,237,15,132,244,11,131,230,252,248,41,252,242,141,
   76,49,252,248,139,114,252,252,199,68,10,4,237,248,12,131,192,1,137,68,36,
   20,252,247,198,237,15,132,244,13,248,14,129,252,246,239,252,247,198,237,15,
@@ -699,56 +699,72 @@ static const unsigned char build_actionlist[15260] = {
   139,65,252,236,137,41,137,65,4,139,105,252,240,139,65,252,244,137,105,8,137,
   65,12,139,105,224,139,65,228,137,105,252,248,137,65,252,252,129,252,248,239,
   184,237,15,133,244,28,137,202,137,114,252,252,139,181,233,139,14,15,182,252,
-  233,15,182,205,131,198,4,252,255,36,171,255,15,182,252,236,15,182,192,137,
-  124,36,16,141,188,253,194,233,141,12,202,43,122,252,252,133,252,237,15,132,
-  244,251,141,108,252,233,252,248,57,215,15,131,244,248,248,1,139,71,252,248,
-  137,1,139,71,252,252,131,199,8,137,65,4,131,193,8,57,252,233,15,131,244,249,
-  57,215,15,130,244,1,248,2,199,65,4,237,131,193,8,57,252,233,15,130,244,2,
-  248,3,139,124,36,16,139,6,15,182,204,15,182,232,131,198,4,193,232,16,252,
-  255,36,171,248,5,199,68,36,20,1,0,0,0,137,208,41,252,248,15,134,244,3,137,
-  197,193,252,237,3,131,197,1,137,108,36,20,139,108,36,48,1,200,59,133,233,
-  15,135,244,253,248,6,255,139,71,252,248,137,1,139,71,252,252,131,199,8,137,
-  65,4,131,193,8,57,215,15,130,244,6,252,233,244,3,248,7,137,149,233,137,141,
-  233,137,116,36,24,41,215,139,84,36,20,131,252,234,1,137,252,233,232,251,1,
-  0,139,149,233,139,141,233,1,215,252,233,244,6,255,193,225,3,255,248,1,139,
-  114,252,252,137,68,36,20,252,247,198,237,15,133,244,253,255,248,13,137,215,
-  131,232,1,15,132,244,249,248,2,139,44,15,137,111,252,248,139,108,15,4,137,
-  111,252,252,131,199,8,131,232,1,15,133,244,2,248,3,139,68,36,20,15,182,110,
-  252,255,248,5,57,197,15,135,244,252,255,139,108,10,4,137,106,252,252,139,
-  44,10,137,106,252,248,255,248,5,56,70,252,255,15,135,244,252,255,15,182,78,
-  252,253,252,247,209,141,20,202,139,122,252,248,139,191,233,139,191,233,139,
-  6,15,182,204,15,182,232,131,198,4,193,232,16,252,255,36,171,248,6,255,199,
-  71,252,252,237,131,199,8,255,199,68,194,252,244,237,255,131,192,1,252,233,
-  244,5,248,7,141,174,233,252,247,197,237,15,133,244,14,41,252,234,255,1,252,
-  233,255,137,252,245,209,252,237,129,229,239,102,131,172,253,43,233,1,15,132,
-  244,140,255,141,12,202,255,129,121,253,4,239,15,135,244,53,129,121,253,12,
-  239,15,135,244,53,255,139,105,20,255,129,252,253,239,15,135,244,53,255,252,
-  242,15,16,1,252,242,15,16,73,8,255,252,242,15,88,65,16,252,242,15,17,1,133,
-  252,237,15,136,244,249,255,15,140,244,249,255,102,15,46,200,248,1,252,242,
-  15,17,65,24,255,221,65,8,221,1,255,220,65,16,221,17,221,81,24,133,252,237,
-  15,136,244,247,255,221,81,24,15,140,244,247,255,217,201,248,1,255,15,183,
-  70,252,254,255,15,131,244,248,141,180,253,134,233,255,141,180,253,134,233,
-  15,183,70,252,254,15,131,245,255,15,130,244,248,141,180,253,134,233,255,248,
-  3,102,15,46,193,252,233,244,1,255,141,12,202,139,105,4,129,252,253,239,15,
-  132,244,247,255,137,105,252,252,139,41,137,105,252,248,252,233,245,255,141,
-  180,253,134,233,139,1,137,105,252,252,137,65,252,248,255,139,139,233,139,
-  4,129,139,128,233,139,108,36,48,137,147,233,137,171,233,252,255,224,255,141,
-  180,253,134,233,139,6,15,182,204,15,182,232,131,198,4,193,232,16,252,255,
-  36,171,255,137,252,245,209,252,237,129,229,239,102,131,172,253,43,233,1,15,
-  132,244,142,255,139,190,233,139,108,36,48,141,12,202,59,141,233,15,135,244,
-  23,15,182,142,233,57,200,15,134,244,249,248,2,255,15,183,70,252,254,252,233,
-  245,255,248,3,199,68,194,252,252,237,131,192,1,57,200,15,134,244,3,252,233,
-  244,2,255,141,44,197,237,141,4,194,139,122,252,248,137,104,252,252,137,120,
-  252,248,139,108,36,48,141,12,200,59,141,233,15,135,244,22,137,209,137,194,
-  15,182,174,233,133,252,237,15,132,244,248,248,1,131,193,8,57,209,15,131,244,
-  249,139,121,252,248,137,56,139,121,252,252,137,120,4,131,192,8,199,65,252,
-  252,237,131,252,237,1,15,133,244,1,248,2,255,139,190,233,139,6,15,182,204,
-  15,182,232,131,198,4,193,232,16,252,255,36,171,255,248,3,199,64,4,237,131,
-  192,8,131,252,237,1,15,133,244,3,252,233,244,2,255,139,106,252,248,139,189,
-  233,139,108,36,48,141,68,194,252,248,137,149,233,141,136,233,59,141,233,137,
-  133,233,255,137,44,36,255,137,124,36,4,137,44,36,255,15,135,244,21,199,131,
-  233,237,255,252,255,215,255,252,255,147,233,255,199,131,233,237,139,149,233,
-  141,12,194,252,247,217,3,141,233,139,114,252,252,252,233,244,12,255,254,0
+  233,15,182,205,131,198,4,252,255,36,171,255,137,124,36,16,137,92,36,20,139,
+  108,202,252,240,139,68,202,252,248,139,157,233,131,198,4,139,189,233,248,
+  1,57,216,15,131,244,251,129,124,253,199,4,239,15,132,244,250,255,252,242,
+  15,42,192,255,219,68,202,252,248,255,139,108,199,4,137,108,202,12,139,44,
+  199,137,108,202,8,131,192,1,255,137,68,202,252,248,248,2,15,183,70,252,254,
+  141,180,253,134,233,248,3,139,92,36,20,139,124,36,16,139,6,15,182,204,15,
+  182,232,131,198,4,193,232,16,252,255,36,171,248,4,131,192,1,255,137,68,202,
+  252,248,255,252,233,244,1,248,5,41,216,248,6,59,133,233,15,135,244,3,105,
+  252,248,239,3,189,233,129,191,233,239,15,132,244,253,141,92,24,1,139,175,
+  233,139,135,233,137,44,202,137,68,202,4,139,175,233,139,135,233,137,108,202,
+  8,137,68,202,12,137,92,202,252,248,252,233,244,2,248,7,255,131,192,1,252,
+  233,244,6,255,129,124,253,202,252,236,239,15,133,244,251,139,108,202,232,
+  129,124,253,202,252,244,239,15,133,244,251,129,124,253,202,252,252,239,15,
+  133,244,251,128,189,233,235,15,133,244,251,141,180,253,134,233,199,68,202,
+  252,248,0,0,0,0,248,1,139,6,15,182,204,15,182,232,131,198,4,193,232,16,252,
+  255,36,171,248,5,198,70,252,252,235,141,180,253,134,233,198,6,235,252,233,
+  244,1,255,15,182,252,236,15,182,192,137,124,36,16,141,188,253,194,233,141,
+  12,202,43,122,252,252,133,252,237,15,132,244,251,141,108,252,233,252,248,
+  57,215,15,131,244,248,248,1,139,71,252,248,137,1,139,71,252,252,131,199,8,
+  137,65,4,131,193,8,57,252,233,15,131,244,249,57,215,15,130,244,1,248,2,199,
+  65,4,237,131,193,8,57,252,233,15,130,244,2,248,3,139,124,36,16,139,6,15,182,
+  204,15,182,232,131,198,4,193,232,16,252,255,36,171,248,5,199,68,36,20,1,0,
+  0,0,137,208,41,252,248,15,134,244,3,137,197,193,252,237,3,131,197,1,137,108,
+  36,20,139,108,36,48,1,200,59,133,233,15,135,244,253,248,6,255,139,71,252,
+  248,137,1,139,71,252,252,131,199,8,137,65,4,131,193,8,57,215,15,130,244,6,
+  252,233,244,3,248,7,137,149,233,137,141,233,137,116,36,24,41,215,139,84,36,
+  20,131,252,234,1,137,252,233,232,251,1,0,139,149,233,139,141,233,1,215,252,
+  233,244,6,255,193,225,3,255,248,1,139,114,252,252,137,68,36,20,252,247,198,
+  237,15,133,244,253,255,248,13,137,215,131,232,1,15,132,244,249,248,2,139,
+  44,15,137,111,252,248,139,108,15,4,137,111,252,252,131,199,8,131,232,1,15,
+  133,244,2,248,3,139,68,36,20,15,182,110,252,255,248,5,57,197,15,135,244,252,
+  255,139,108,10,4,137,106,252,252,139,44,10,137,106,252,248,255,248,5,56,70,
+  252,255,15,135,244,252,255,15,182,78,252,253,252,247,209,141,20,202,139,122,
+  252,248,139,191,233,139,191,233,139,6,15,182,204,15,182,232,131,198,4,193,
+  232,16,252,255,36,171,248,6,255,199,71,252,252,237,131,199,8,255,199,68,194,
+  252,244,237,255,131,192,1,252,233,244,5,248,7,141,174,233,252,247,197,237,
+  15,133,244,14,41,252,234,255,1,252,233,255,137,252,245,209,252,237,129,229,
+  239,102,131,172,253,43,233,1,15,132,244,140,255,141,12,202,255,129,121,253,
+  4,239,15,135,244,53,129,121,253,12,239,15,135,244,53,255,139,105,20,255,129,
+  252,253,239,15,135,244,53,255,252,242,15,16,1,252,242,15,16,73,8,255,252,
+  242,15,88,65,16,252,242,15,17,1,133,252,237,15,136,244,249,255,15,140,244,
+  249,255,102,15,46,200,248,1,252,242,15,17,65,24,255,221,65,8,221,1,255,220,
+  65,16,221,17,221,81,24,133,252,237,15,136,244,247,255,221,81,24,15,140,244,
+  247,255,217,201,248,1,255,15,183,70,252,254,255,15,131,244,248,141,180,253,
+  134,233,255,141,180,253,134,233,15,183,70,252,254,15,131,245,255,15,130,244,
+  248,141,180,253,134,233,255,248,3,102,15,46,193,252,233,244,1,255,141,12,
+  202,139,105,4,129,252,253,239,15,132,244,247,255,137,105,252,252,139,41,137,
+  105,252,248,252,233,245,255,141,180,253,134,233,139,1,137,105,252,252,137,
+  65,252,248,255,139,139,233,139,4,129,139,128,233,139,108,36,48,137,147,233,
+  137,171,233,252,255,224,255,141,180,253,134,233,139,6,15,182,204,15,182,232,
+  131,198,4,193,232,16,252,255,36,171,255,137,252,245,209,252,237,129,229,239,
+  102,131,172,253,43,233,1,15,132,244,142,255,139,190,233,139,108,36,48,141,
+  12,202,59,141,233,15,135,244,23,15,182,142,233,57,200,15,134,244,249,248,
+  2,255,15,183,70,252,254,252,233,245,255,248,3,199,68,194,252,252,237,131,
+  192,1,57,200,15,134,244,3,252,233,244,2,255,141,44,197,237,141,4,194,139,
+  122,252,248,137,104,252,252,137,120,252,248,139,108,36,48,141,12,200,59,141,
+  233,15,135,244,22,137,209,137,194,15,182,174,233,133,252,237,15,132,244,248,
+  248,1,131,193,8,57,209,15,131,244,249,139,121,252,248,137,56,139,121,252,
+  252,137,120,4,131,192,8,199,65,252,252,237,131,252,237,1,15,133,244,1,248,
+  2,255,139,190,233,139,6,15,182,204,15,182,232,131,198,4,193,232,16,252,255,
+  36,171,255,248,3,199,64,4,237,131,192,8,131,252,237,1,15,133,244,3,252,233,
+  244,2,255,139,106,252,248,139,189,233,139,108,36,48,141,68,194,252,248,137,
+  149,233,141,136,233,59,141,233,137,133,233,255,137,44,36,255,137,124,36,4,
+  137,44,36,255,15,135,244,21,199,131,233,237,255,252,255,215,255,252,255,147,
+  233,255,199,131,233,237,139,149,233,141,12,194,252,247,217,3,141,233,139,
+  114,252,252,252,233,244,12,255,254,0
 };
 
 enum {
@@ -2115,9 +2131,36 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     dasm_put(Dst, 14164, LJ_TFUNC, 2+1, Dt7(->pc));
     break;
 
+  case BC_ITERN:
+#if LJ_HASJIT
+#endif
+    dasm_put(Dst, 14244, Dt6(->asize), Dt6(->array), LJ_TNIL);
+    if (sse) {
+      dasm_put(Dst, 14290);
+    } else {
+      dasm_put(Dst, 14296);
+    }
+    dasm_put(Dst, 14302);
+    if (sse) {
+      dasm_put(Dst, 11273);
+    } else {
+      dasm_put(Dst, 11285);
+    }
+    dasm_put(Dst, 14321, -BCBIAS_J*4);
+    if (!sse) {
+      dasm_put(Dst, 14372);
+    }
+    dasm_put(Dst, 14378, Dt6(->hmask), sizeof(Node), Dt6(->node), DtB(->val.it), LJ_TNIL, DtB(->key.gcr), DtB(->key.it), DtB(->val.gcr), DtB(->val.it));
+    dasm_put(Dst, 14453);
+    break;
+
+  case BC_ISNEXT:
+    dasm_put(Dst, 14461, LJ_TFUNC, LJ_TTAB, LJ_TNIL, Dt8(->ffid), FF_next_N, -BCBIAS_J*4, BC_JMP, -BCBIAS_J*4, BC_ITERC);
+    break;
+
   case BC_VARG:
-    dasm_put(Dst, 14244, (8+FRAME_VARG), LJ_TNIL, Dt1(->maxstack));
-    dasm_put(Dst, 14408, Dt1(->base), Dt1(->top), Dt1(->base), Dt1(->top));
+    dasm_put(Dst, 14560, (8+FRAME_VARG), LJ_TNIL, Dt1(->maxstack));
+    dasm_put(Dst, 14724, Dt1(->base), Dt1(->top), Dt1(->base), Dt1(->top));
     break;
 
   /* -- Returns ----------------------------------------------------------- */
@@ -2128,30 +2171,30 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_RET: case BC_RET0: case BC_RET1:
     if (op != BC_RET0) {
-      dasm_put(Dst, 14479);
+      dasm_put(Dst, 14795);
     }
-    dasm_put(Dst, 14483, FRAME_TYPE);
+    dasm_put(Dst, 14799, FRAME_TYPE);
     switch (op) {
     case BC_RET:
-      dasm_put(Dst, 14502);
+      dasm_put(Dst, 14818);
       break;
     case BC_RET1:
-      dasm_put(Dst, 14560);
+      dasm_put(Dst, 14876);
       /* fallthrough */
     case BC_RET0:
-      dasm_put(Dst, 14576);
+      dasm_put(Dst, 14892);
     default:
       break;
     }
-    dasm_put(Dst, 14587, Dt7(->pc), PC2PROTO(k));
+    dasm_put(Dst, 14903, Dt7(->pc), PC2PROTO(k));
     if (op == BC_RET) {
-      dasm_put(Dst, 14629, LJ_TNIL);
+      dasm_put(Dst, 14945, LJ_TNIL);
     } else {
-      dasm_put(Dst, 14638, LJ_TNIL);
+      dasm_put(Dst, 14954, LJ_TNIL);
     }
-    dasm_put(Dst, 14645, -FRAME_VARG, FRAME_TYPEP);
+    dasm_put(Dst, 14961, -FRAME_VARG, FRAME_TYPEP);
     if (op != BC_RET0) {
-      dasm_put(Dst, 14669);
+      dasm_put(Dst, 14985);
     }
     dasm_put(Dst, 4689);
     break;
@@ -2161,7 +2204,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_FORL:
 #if LJ_HASJIT
-    dasm_put(Dst, 14673, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 14989, HOTCOUNT_PCMASK, GG_DISP2HOT);
 #endif
     break;
 
@@ -2173,57 +2216,57 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
   case BC_FORI:
   case BC_IFORL:
     vk = (op == BC_IFORL || op == BC_JFORL);
-    dasm_put(Dst, 14694);
+    dasm_put(Dst, 15010);
     if (!vk) {
-      dasm_put(Dst, 14698, LJ_TISNUM, LJ_TISNUM);
+      dasm_put(Dst, 15014, LJ_TISNUM, LJ_TISNUM);
     }
-    dasm_put(Dst, 14717);
+    dasm_put(Dst, 15033);
     if (!vk) {
-      dasm_put(Dst, 14721, LJ_TISNUM);
+      dasm_put(Dst, 15037, LJ_TISNUM);
     }
     if (sse) {
-      dasm_put(Dst, 14730);
+      dasm_put(Dst, 15046);
       if (vk) {
-	dasm_put(Dst, 14742);
+	dasm_put(Dst, 15058);
       } else {
-	dasm_put(Dst, 14761);
+	dasm_put(Dst, 15077);
       }
-      dasm_put(Dst, 14766);
+      dasm_put(Dst, 15082);
     } else {
-      dasm_put(Dst, 14779);
+      dasm_put(Dst, 15095);
       if (vk) {
-	dasm_put(Dst, 14785);
+	dasm_put(Dst, 15101);
       } else {
-	dasm_put(Dst, 14801);
+	dasm_put(Dst, 15117);
       }
-      dasm_put(Dst, 14809);
+      dasm_put(Dst, 15125);
       if (cmov) {
       dasm_put(Dst, 10818);
       } else {
       dasm_put(Dst, 10824);
       }
       if (!cmov) {
-	dasm_put(Dst, 14814);
+	dasm_put(Dst, 15130);
       }
     }
     if (op == BC_FORI) {
-      dasm_put(Dst, 14820, -BCBIAS_J*4);
+      dasm_put(Dst, 15136, -BCBIAS_J*4);
     } else if (op == BC_JFORI) {
-      dasm_put(Dst, 14830, -BCBIAS_J*4, BC_JLOOP);
+      dasm_put(Dst, 15146, -BCBIAS_J*4, BC_JLOOP);
     } else if (op == BC_IFORL) {
-      dasm_put(Dst, 14844, -BCBIAS_J*4);
+      dasm_put(Dst, 15160, -BCBIAS_J*4);
     } else {
-      dasm_put(Dst, 14840, BC_JLOOP);
+      dasm_put(Dst, 15156, BC_JLOOP);
     }
     dasm_put(Dst, 10853);
     if (sse) {
-      dasm_put(Dst, 14854);
+      dasm_put(Dst, 15170);
     }
     break;
 
   case BC_ITERL:
 #if LJ_HASJIT
-    dasm_put(Dst, 14673, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 14989, HOTCOUNT_PCMASK, GG_DISP2HOT);
 #endif
     break;
 
@@ -2232,18 +2275,18 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     break;
 #endif
   case BC_IITERL:
-    dasm_put(Dst, 14865, LJ_TNIL);
+    dasm_put(Dst, 15181, LJ_TNIL);
     if (op == BC_JITERL) {
-      dasm_put(Dst, 14880, BC_JLOOP);
+      dasm_put(Dst, 15196, BC_JLOOP);
     } else {
-      dasm_put(Dst, 14894, -BCBIAS_J*4);
+      dasm_put(Dst, 15210, -BCBIAS_J*4);
     }
     dasm_put(Dst, 11150);
     break;
 
   case BC_LOOP:
 #if LJ_HASJIT
-    dasm_put(Dst, 14673, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 14989, HOTCOUNT_PCMASK, GG_DISP2HOT);
 #endif
     break;
 
@@ -2253,12 +2296,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_JLOOP:
 #if LJ_HASJIT
-    dasm_put(Dst, 14910, DISPATCH_J(trace), DtD(->mcode), DISPATCH_GL(jit_base), DISPATCH_GL(jit_L));
+    dasm_put(Dst, 15226, DISPATCH_J(trace), DtD(->mcode), DISPATCH_GL(jit_base), DISPATCH_GL(jit_L));
 #endif
     break;
 
   case BC_JMP:
-    dasm_put(Dst, 14933, -BCBIAS_J*4);
+    dasm_put(Dst, 15249, -BCBIAS_J*4);
     break;
 
   /* -- Function headers -------------------------------------------------- */
@@ -2272,7 +2315,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
 
   case BC_FUNCF:
 #if LJ_HASJIT
-    dasm_put(Dst, 14957, HOTCOUNT_PCMASK, GG_DISP2HOT);
+    dasm_put(Dst, 15273, HOTCOUNT_PCMASK, GG_DISP2HOT);
 #endif
   case BC_FUNCV:  /* NYI: compiled vararg functions. */
     break;
@@ -2282,13 +2325,13 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     break;
 #endif
   case BC_IFUNCF:
-    dasm_put(Dst, 14978, -4+PC2PROTO(k), Dt1(->maxstack), -4+PC2PROTO(numparams));
+    dasm_put(Dst, 15294, -4+PC2PROTO(k), Dt1(->maxstack), -4+PC2PROTO(numparams));
     if (op == BC_JFUNCF) {
-      dasm_put(Dst, 15008, BC_JLOOP);
+      dasm_put(Dst, 15324, BC_JLOOP);
     } else {
       dasm_put(Dst, 10855);
     }
-    dasm_put(Dst, 15017, LJ_TNIL);
+    dasm_put(Dst, 15333, LJ_TNIL);
     break;
 
   case BC_JFUNCV:
@@ -2299,30 +2342,30 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     break;  /* NYI: compiled vararg functions. */
 
   case BC_IFUNCV:
-    dasm_put(Dst, 15039, FRAME_VARG, Dt1(->maxstack), -4+PC2PROTO(numparams), LJ_TNIL);
+    dasm_put(Dst, 15355, FRAME_VARG, Dt1(->maxstack), -4+PC2PROTO(numparams), LJ_TNIL);
     if (op == BC_JFUNCV) {
-      dasm_put(Dst, 15008, BC_JLOOP);
+      dasm_put(Dst, 15324, BC_JLOOP);
     } else {
-      dasm_put(Dst, 15130, -4+PC2PROTO(k));
+      dasm_put(Dst, 15446, -4+PC2PROTO(k));
     }
-    dasm_put(Dst, 15152, LJ_TNIL);
+    dasm_put(Dst, 15468, LJ_TNIL);
     break;
 
   case BC_FUNCC:
   case BC_FUNCCW:
-    dasm_put(Dst, 15174, Dt8(->f), Dt1(->base), 8*LUA_MINSTACK, Dt1(->maxstack), Dt1(->top));
+    dasm_put(Dst, 15490, Dt8(->f), Dt1(->base), 8*LUA_MINSTACK, Dt1(->maxstack), Dt1(->top));
     if (op == BC_FUNCC) {
-      dasm_put(Dst, 15203);
+      dasm_put(Dst, 15519);
     } else {
-      dasm_put(Dst, 15207);
+      dasm_put(Dst, 15523);
     }
-    dasm_put(Dst, 15215, DISPATCH_GL(vmstate), ~LJ_VMST_C);
+    dasm_put(Dst, 15531, DISPATCH_GL(vmstate), ~LJ_VMST_C);
     if (op == BC_FUNCC) {
-      dasm_put(Dst, 15224);
+      dasm_put(Dst, 15540);
     } else {
-      dasm_put(Dst, 15228, DISPATCH_GL(wrapf));
+      dasm_put(Dst, 15544, DISPATCH_GL(wrapf));
     }
-    dasm_put(Dst, 15233, DISPATCH_GL(vmstate), ~LJ_VMST_INTERP, Dt1(->base), Dt1(->top));
+    dasm_put(Dst, 15549, DISPATCH_GL(vmstate), ~LJ_VMST_INTERP, Dt1(->base), Dt1(->top));
     break;
 
   /* ---------------------------------------------------------------------- */
@@ -2350,7 +2393,7 @@ static int build_backend(BuildCtx *ctx)
 
   build_subroutines(ctx, cmov, sse);
 
-  dasm_put(Dst, 15258);
+  dasm_put(Dst, 15574);
   for (op = 0; op < BC__MAX; op++)
     build_ins(ctx, (BCOp)op, op, cmov, sse);
 

src/lib_base.c

@@ -240,6 +240,9 @@ LJLIB_ASM(tostring)		LJLIB_REC(.)
 
 /* -- Base library: iterators --------------------------------------------- */
 
+/* This solves a circular dependency problem -- change FF_next_N as needed. */
+LJ_STATIC_ASSERT((int)FF_next == FF_next_N);
+
 LJLIB_ASM(next)
 {
   lj_lib_checktab(L, 1);

src/lj_bc.h

@@ -154,6 +154,8 @@
   _(CALLMT,	base,	___,	lit,	call) \
   _(CALLT,	base,	___,	lit,	call) \
   _(ITERC,	base,	lit,	lit,	call) \
+  _(ITERN,	base,	lit,	lit,	call) \
+  _(ISNEXT,	base,	___,	jump,	___) \
   _(VARG,	base,	lit,	lit,	___) \
   \
   /* Returns. */ \
@@ -221,6 +223,9 @@ LJ_STATIC_ASSERT((int)BC_FUNCF + 2 == (int)BC_JFUNCF);
 LJ_STATIC_ASSERT((int)BC_FUNCV + 1 == (int)BC_IFUNCV);
 LJ_STATIC_ASSERT((int)BC_FUNCV + 2 == (int)BC_JFUNCV);
 
+/* This solves a circular dependency problem, change as needed. */
+#define FF_next_N	15
+
 /* Stack slots used by FORI/FORL, relative to operand A. */
 enum {
   FORL_IDX, FORL_STOP, FORL_STEP, FORL_EXT

src/lj_parse.c

@@ -2030,41 +2030,13 @@ static void parse_repeat(LexState *ls, BCLine line)
   scope_end(fs);  /* End loop scope. */
 }
 
-/* Parse body of a 'for' statement. */
-static void parse_for_body(LexState *ls, BCReg base, BCLine line,
-			   BCReg nvars, int isnum)
-{
-  FuncScope bl;
-  FuncState *fs = ls->fs;
-  BCPos loop, loopend;
-  var_add(ls, 3);  /* Hidden control variables. */
-  lex_check(ls, TK_do);
-  loop = isnum ? bcemit_AJ(fs, BC_FORI, base, NO_JMP) :
-		 bcemit_AJ(fs, BC_JMP, fs->freereg, NO_JMP);
-  scope_begin(fs, &bl, 0);  /* Scope for visible variables. */
-  var_add(ls, nvars);
-  bcreg_reserve(fs, nvars);
-  parse_block(ls);
-  scope_end(fs);
-  /* Perform loop inversion. Loop control instructions are at the end. */
-  if (isnum) {
-    loopend = bcemit_AJ(fs, BC_FORL, base, NO_JMP);
-    jmp_patchins(fs, loop, fs->pc);
-  } else {
-    jmp_patchins(fs, loop, fs->pc);
-    bcemit_ABC(fs, BC_ITERC, base+3, nvars+1, 2+1);
-    loopend = bcemit_AJ(fs, BC_ITERL, base+3, NO_JMP);
-    fs->bcbase[loopend-1].line = line;
-  }
-  fs->bcbase[loopend].line = line;  /* Fix line for control ins. */
-  jmp_patchins(fs, loopend, loop+1);
-}
-
 /* Parse numeric 'for'. */
 static void parse_for_num(LexState *ls, GCstr *varname, BCLine line)
 {
   FuncState *fs = ls->fs;
   BCReg base = fs->freereg;
+  FuncScope bl;
+  BCPos loop, loopend;
   /* Hidden control variables. */
   var_new_lit(ls, FORL_IDX, "(for index)");
   var_new_lit(ls, FORL_STOP, "(for limit)");
@@ -2081,7 +2053,51 @@ static void parse_for_num(LexState *ls, GCstr *varname, BCLine line)
     bcemit_AD(fs, BC_KSHORT, fs->freereg, 1);  /* Default step is 1. */
     bcreg_reserve(fs, 1);
   }
-  parse_for_body(ls, base, line, 1, 1);
+  var_add(ls, 3);  /* Hidden control variables. */
+  lex_check(ls, TK_do);
+  loop = bcemit_AJ(fs, BC_FORI, base, NO_JMP);
+  scope_begin(fs, &bl, 0);  /* Scope for visible variables. */
+  var_add(ls, 1);
+  bcreg_reserve(fs, 1);
+  parse_block(ls);
+  scope_end(fs);
+  /* Perform loop inversion. Loop control instructions are at the end. */
+  loopend = bcemit_AJ(fs, BC_FORL, base, NO_JMP);
+  fs->bcbase[loopend].line = line;  /* Fix line for control ins. */
+  jmp_patchins(fs, loopend, loop+1);
+  jmp_patchins(fs, loop, fs->pc);
+}
+
+/* Try to predict whether the iterator is next() and specialize the bytecode.
+** Detecting next() and pairs() by name is simplistic, but quite effective.
+** The interpreter backs off if the check for the closure fails at runtime.
+*/
+static int predict_next(LexState *ls, FuncState *fs, BCPos pc)
+{
+  BCIns ins = fs->bcbase[pc].ins;
+  GCstr *name;
+  cTValue *o;
+  switch (bc_op(ins)) {
+  case BC_MOV:
+    name = gco2str(gcref(var_get(ls, fs, bc_d(ins)).name));
+    break;
+  case BC_UGET:
+    name = gco2str(gcref(ls->vstack[fs->uvloc[bc_d(ins)].vidx].name));
+    break;
+  case BC_GGET:
+    /* There's no inverse index (yet), so lookup the strings. */
+    o = lj_tab_getstr(fs->kt, lj_str_newlit(ls->L, "pairs"));
+    if (o && tvisnum(o) && o->u32.lo == bc_d(ins))
+      return 1;
+    o = lj_tab_getstr(fs->kt, lj_str_newlit(ls->L, "next"));
+    if (o && tvisnum(o) && o->u32.lo == bc_d(ins))
+      return 1;
+    return 0;
+  default:
+    return 0;
+  }
+  return (name->len == 5 && !strcmp(strdata(name), "pairs")) ||
+	 (name->len == 4 && !strcmp(strdata(name), "next"));
 }
 
 /* Parse 'for' iterator. */
@@ -2091,7 +2107,10 @@ static void parse_for_iter(LexState *ls, GCstr *indexname)
   ExpDesc e;
   BCReg nvars = 0;
   BCLine line;
-  BCReg base = fs->freereg;
+  BCReg base = fs->freereg + 3;
+  BCPos loop, loopend, exprpc = fs->pc;
+  FuncScope bl;
+  int isnext;
   /* Hidden control variables. */
   var_new_lit(ls, nvars++, "(for generator)");
   var_new_lit(ls, nvars++, "(for state)");
@@ -2104,7 +2123,22 @@ static void parse_for_iter(LexState *ls, GCstr *indexname)
   line = ls->linenumber;
   assign_adjust(ls, 3, expr_list(ls, &e), &e);
   bcreg_bump(fs, 3);  /* The iterator needs another 3 slots (func + 2 args). */
-  parse_for_body(ls, base, line, nvars - 3, 0);
+  isnext = (nvars <= 5 && predict_next(ls, fs, exprpc));
+  var_add(ls, 3);  /* Hidden control variables. */
+  lex_check(ls, TK_do);
+  loop = bcemit_AJ(fs, isnext ? BC_ISNEXT : BC_JMP, base, NO_JMP);
+  scope_begin(fs, &bl, 0);  /* Scope for visible variables. */
+  var_add(ls, nvars-3);
+  bcreg_reserve(fs, nvars-3);
+  parse_block(ls);
+  scope_end(fs);
+  /* Perform loop inversion. Loop control instructions are at the end. */
+  jmp_patchins(fs, loop, fs->pc);
+  bcemit_ABC(fs, isnext ? BC_ITERN : BC_ITERC, base, nvars-3+1, 2+1);
+  loopend = bcemit_AJ(fs, BC_ITERL, base, NO_JMP);
+  fs->bcbase[loopend-1].line = line;  /* Fix line for control ins. */
+  fs->bcbase[loopend].line = line;
+  jmp_patchins(fs, loopend, loop+1);
 }
 
 /* Parse 'for' statement. */

src/lj_record.c

@@ -2477,6 +2477,8 @@ void lj_record_ins(jit_State *J)
       break;
     }
     /* fallthrough */
+  case BC_ITERN:
+  case BC_ISNEXT:
   case BC_CAT:
   case BC_UCLO:
   case BC_FNEW: