Restructure benchmarks (#1283)

Jint.Benchmark/ArrayStressBenchmark.cs

@@ -5,7 +5,5 @@ namespace Jint.Benchmark;
 [MemoryDiagnoser]
 public class ArrayStressBenchmark : SingleScriptBenchmark
 {
-    protected override string Script => "var ret=[],tmp,num=100,i=256;for(var j1=0;j1<i*15;j1++){ret=[];ret.length=i}for(var j2=0;j2<i*10;j2++){ret=new Array(i)}ret=[];for(var j3=0;j3<i;j3++){ret.unshift(j3)}ret=[];for(var j4=0;j4<i;j4++){ret.splice(0,0,j4)}var a=ret.slice();for(var j5=0;j5<i;j5++){tmp=a.shift()}var b=ret.slice();for(var j6=0;j6<i;j6++){tmp=b.splice(0,1)}ret=[];for(var j7=0;j7<i*25;j7++){ret.push(j7)}var c=ret.slice();for(var j8=0;j8<i*25;j8++){tmp=c.pop()}var done = true;";
-
-    public override int N => 10;
-}
+    protected override string FileName => "array-stress.js";
+}

Jint.Benchmark/DromaeoBenchmark.cs

@@ -22,7 +22,7 @@ public class DromaeoBenchmark
     {
         foreach (var fileName in files.Keys.ToList())
         {
-            files[fileName] = File.ReadAllText($"Scripts/dromaeo/{fileName}.js");
+            files[fileName] = File.ReadAllText($"Scripts/{fileName}.js");
         }
 
         engine = new Engine()
@@ -53,4 +53,4 @@ var prep = function (fn) { fn(); };
     {
         engine.Execute(files[FileName]);
     }
-}
+}

Jint.Benchmark/EngineComparisonBenchmark.cs

@@ -0,0 +1,96 @@
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using Esprima;
+using Esprima.Ast;
+
+namespace Jint.Benchmark;
+
+[RankColumn]
+[MemoryDiagnoser]
+[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByParams)]
+[HideColumns("Error", "Gen0", "Gen1", "Gen2")]
+[BenchmarkCategory("EngineComparison")]
+public class EngineComparisonBenchmark
+{
+    private static readonly Dictionary<string, Script> _parsedScripts = new();
+
+    private static readonly Dictionary<string, string> _files = new()
+    {
+        { "array-stress", null },
+        { "evaluation", null },
+        { "linq-js", null },
+        { "minimal", null },
+        { "stopwatch", null },
+        { "dromaeo-3d-cube", null },
+        { "dromaeo-core-eval", null },
+        { "dromaeo-object-array", null },
+        { "dromaeo-object-regexp", null },
+        { "dromaeo-object-string", null },
+        { "dromaeo-string-base64", null },
+    };
+
+    private static readonly string _dromaeoHelpers = @"
+        var startTest = function () { };
+        var test = function (name, fn) { fn(); };
+        var endTest = function () { };
+        var prep = function (fn) { fn(); };";
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        var javaScriptParser = new JavaScriptParser();
+        foreach (var fileName in _files.Keys.ToList())
+        {
+            var script = File.ReadAllText($"Scripts/{fileName}.js");
+            if (fileName.Contains("dromaeo"))
+            {
+                script = _dromaeoHelpers + Environment.NewLine + Environment.NewLine + script;
+            }
+            _files[fileName] = script;
+            _parsedScripts[fileName] = javaScriptParser.ParseScript(script);
+        }
+    }
+
+    [ParamsSource(nameof(FileNames))]
+    public string FileName { get; set; }
+
+    public IEnumerable<string> FileNames()
+    {
+        return _files.Select(entry => entry.Key);
+    }
+
+    [Benchmark]
+    public void Jint()
+    {
+        var engine = new Engine(static options => options.Strict());
+        engine.Execute(_files[FileName]);
+    }
+
+    [Benchmark]
+    public void Jint_ParsedScript()
+    {
+        var engine = new Engine(static options => options.Strict());
+        engine.Execute(_parsedScripts[FileName]);
+    }
+
+    [Benchmark]
+    public void Jurassic()
+    {
+        var engine = new Jurassic.ScriptEngine { ForceStrictMode = true };
+        engine.Execute(_files[FileName]);
+    }
+
+    [Benchmark]
+    public void NilJS()
+    {
+        var engine = new NiL.JS.Core.Context(strict: true);
+        engine.Eval(_files[FileName]);
+    }
+
+    [Benchmark]
+    public void YantraJS()
+    {
+        var engine = new YantraJS.Core.JSContext();
+        engine.Eval(_files[FileName]);
+    }
+}

Jint.Benchmark/EvaluationBenchmark.cs

@@ -5,26 +5,5 @@ namespace Jint.Benchmark;
 [MemoryDiagnoser]
 public class EvaluationBenchmark : SingleScriptBenchmark
 {
-    protected override string Script => @"
-            var o = {};
-            o.Foo = 'bar';
-            o.Baz = 42.0001;
-            o.Blah = o.Foo + o.Baz;
-
-            if(o.Blah != 'bar42.0001') throw TypeError;
-
-            function fib(n){
-                if(n<2) {
-                    return n;
-                }
-
-                return fib(n-1) + fib(n-2);
-            }
-
-            if(fib(3) != 2) throw TypeError;
-
-            var done = true;
-        ";
-
-    public override int N => 20;
-}
+    protected override string FileName => "evaluation.js";
+}

Jint.Benchmark/Jint.Benchmark.csproj

@@ -18,8 +18,8 @@
     <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
   <ItemGroup>
-    <None Include=".\Scripts\**" CopyToOutputDirectory="PreserveNewest" />
-    <None Include="..\Jint.Tests.CommonScripts\Scripts\**" CopyToOutputDirectory="PreserveNewest" LinkBase="SunSpider" />
+    <None Include=".\Scripts\*.*" CopyToOutputDirectory="PreserveNewest" />
+    <None Include="..\Jint.Tests.CommonScripts\Scripts\**" CopyToOutputDirectory="PreserveNewest" LinkBase="Scripts" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Jint\Jint.csproj" />

Jint.Benchmark/MinimalScriptBenchmark.cs

@@ -5,5 +5,5 @@ namespace Jint.Benchmark;
 [MemoryDiagnoser]
 public class MinimalScriptBenchmark : SingleScriptBenchmark
 {
-    protected override string Script => "var done = 1 + 1 === 2;";
+    protected override string FileName => "minimal.js";
 }

Jint.Benchmark/README.md

@@ -2,4 +2,98 @@ To run tests comparing engines, use:
 
 ```
 dotnet run -c Release --allCategories EngineComparison
-```
+```
+
+## Engine comparison results 
+
+* tests are run in global engine strict mode, as YantraJS always uses strict mode which improves performance
+* `Jint` and `Jint_ParsedScript` shows the difference between always parsing the script source file and reusing parsed `Script` instance.
+
+Last updated 2022-09-03
+
+* Jint main
+* Jurassic 3.2.6
+* NiL.JS.NetCore 2.5.1419
+* YantraJS.Core 1.2.47
+
+
+``` ini
+
+BenchmarkDotNet=v0.13.2, OS=Windows 11 (10.0.22621.457)
+AMD Ryzen 9 5950X, 1 CPU, 32 logical and 16 physical cores
+.NET SDK=6.0.400
+  [Host]     : .NET 6.0.8 (6.0.822.36306), X64 RyuJIT AVX2
+  DefaultJob : .NET 6.0.8 (6.0.822.36306), X64 RyuJIT AVX2
+
+
+```
+|            Method |             FileName |             Mean |         StdDev | Rank |     Allocated |
+|------------------ |--------------------- |-----------------:|---------------:|-----:|--------------:|
+|              **Jint** |         **array-stress** |    **13,507.095 μs** |     **23.0399 μs** |    **5** |     **8745627 B** |
+| Jint_ParsedScript |         array-stress |    13,102.050 μs |     15.5290 μs |    4 |     8723795 B |
+|          Jurassic |         array-stress |    11,213.554 μs |     14.3872 μs |    3 |    11926461 B |
+|             NilJS |         array-stress |     5,334.318 μs |     23.5378 μs |    1 |     4553815 B |
+|          YantraJS |         array-stress |     7,286.025 μs |     32.1957 μs |    2 |     6318022 B |
+|                   |                      |                  |                |      |               |
+|              **Jint** |      **dromaeo-3d-cube** |    **23,587.542 μs** |     **23.6725 μs** |    **4** |     **7247466 B** |
+| Jint_ParsedScript |      dromaeo-3d-cube |    21,772.708 μs |     40.7036 μs |    3 |     7005434 B |
+|          Jurassic |      dromaeo-3d-cube |    39,741.721 μs |     64.2544 μs |    5 |    10925914 B |
+|             NilJS |      dromaeo-3d-cube |     8,708.896 μs |     12.4054 μs |    2 |     4125451 B |
+|          YantraJS |      dromaeo-3d-cube |     5,017.989 μs |     19.0233 μs |    1 |     8786605 B |
+|                   |                      |                  |                |      |               |
+|              **Jint** |    **dromaeo-core-eval** |     **5,705.326 μs** |     **21.1657 μs** |    **2** |      **350102 B** |
+| Jint_ParsedScript |    dromaeo-core-eval |     5,668.950 μs |     12.4161 μs |    2 |      336990 B |
+|          Jurassic |    dromaeo-core-eval |    13,447.357 μs |     30.6629 μs |    4 |     2971062 B |
+|             NilJS |    dromaeo-core-eval |     2,173.203 μs |     46.5428 μs |    1 |     1595219 B |
+|          YantraJS |    dromaeo-core-eval |     8,265.873 μs |     56.7684 μs |    3 |    37131603 B |
+|                   |                      |                  |                |      |               |
+|              **Jint** | **dromaeo-object-array** |   **108,627.168 μs** |    **511.8916 μs** |    **4** |   **103655627 B** |
+| Jint_ParsedScript | dromaeo-object-array |   108,528.990 μs |    288.8372 μs |    4 |   103625558 B |
+|          Jurassic | dromaeo-object-array |    41,209.118 μs |     44.6309 μs |    1 |    26433962 B |
+|             NilJS | dromaeo-object-array |    53,391.875 μs |    732.5365 μs |    2 |    18027054 B |
+|          YantraJS | dromaeo-object-array |    74,332.930 μs |    528.6855 μs |    3 |    24735409 B |
+|                   |                      |                  |                |      |               |
+|              **Jint** | **droma(...)egexp [21]** |   **299,361.023 μs** |  **7,904.1811 μs** |    **1** |   **241632484 B** |
+| Jint_ParsedScript | droma(...)egexp [21] |   295,042.982 μs |  8,424.5753 μs |    1 |   247151352 B |
+|          Jurassic | droma(...)egexp [21] |   683,250.476 μs | 13,213.8098 μs |    2 |   842918784 B |
+|             NilJS | droma(...)egexp [21] |   801,835.247 μs | 13,416.6210 μs |    3 |   901970880 B |
+|          YantraJS | droma(...)egexp [21] | 1,017,060.643 μs |  7,046.5834 μs |    4 |   964707752 B |
+|                   |                      |                  |                |      |               |
+|              **Jint** | **droma(...)tring [21]** |   **628,645.310 μs** | **21,985.7843 μs** |    **4** |  **1363791136 B** |
+| Jint_ParsedScript | droma(...)tring [21] |   524,500.457 μs | 22,516.2241 μs |    2 |  1363541192 B |
+|          Jurassic | droma(...)tring [21] |   557,218.173 μs | 23,822.7984 μs |    3 |  1493044472 B |
+|             NilJS | droma(...)tring [21] |   400,573.819 μs | 12,223.2081 μs |    1 |  1446930864 B |
+|          YantraJS | droma(...)tring [21] | 2,258,766.944 μs | 60,256.3876 μs |    5 | 16092618624 B |
+|                   |                      |                  |                |      |               |
+|              **Jint** | **droma(...)ase64 [21]** |    **66,060.024 μs** |     **79.4200 μs** |    **4** |     **8049804 B** |
+| Jint_ParsedScript | droma(...)ase64 [21] |    64,215.565 μs |     62.6330 μs |    3 |     7968098 B |
+|          Jurassic | droma(...)ase64 [21] |    69,690.458 μs |    199.4819 μs |    5 |    76103531 B |
+|             NilJS | droma(...)ase64 [21] |    45,237.687 μs |    534.6955 μs |    2 |    51047211 B |
+|          YantraJS | droma(...)ase64 [21] |    42,177.560 μs |    246.7996 μs |    1 |   778591540 B |
+|                   |                      |                  |                |      |               |
+|              **Jint** |           **evaluation** |        **28.520 μs** |      **0.0348 μs** |    **2** |       **34784 B** |
+| Jint_ParsedScript |           evaluation |        12.906 μs |      0.0298 μs |    1 |       26752 B |
+|          Jurassic |           evaluation |     1,286.204 μs |      3.0882 μs |    5 |      430506 B |
+|             NilJS |           evaluation |        43.866 μs |      1.2033 μs |    3 |       22456 B |
+|          YantraJS |           evaluation |       178.979 μs |      0.6314 μs |    4 |      178517 B |
+|                   |                      |                  |                |      |               |
+|              **Jint** |              **linq-js** |     **1,876.845 μs** |      **3.2033 μs** |    **3** |     **1301929 B** |
+| Jint_ParsedScript |              linq-js |       123.396 μs |      0.1138 μs |    1 |      230841 B |
+|          Jurassic |              linq-js |    36,984.974 μs |    395.7409 μs |    4 |     9526082 B |
+|             NilJS |              linq-js |               NA |             NA |    ? |             - |
+|          YantraJS |              linq-js |       391.523 μs |      1.8922 μs |    2 |      490377 B |
+|                   |                      |                  |                |      |               |
+|              **Jint** |              **minimal** |         **5.334 μs** |      **0.0065 μs** |    **3** |       **14120 B** |
+| Jint_ParsedScript |              minimal |         3.535 μs |      0.0143 μs |    1 |       12680 B |
+|          Jurassic |              minimal |       234.184 μs |      0.6536 μs |    5 |      395505 B |
+|             NilJS |              minimal |         4.516 μs |      0.0933 μs |    2 |        4272 B |
+|          YantraJS |              minimal |       173.909 μs |      0.1640 μs |    4 |      174668 B |
+|                   |                      |                  |                |      |               |
+|              **Jint** |            **stopwatch** |   **357,288.654 μs** |  **1,123.5726 μs** |    **4** |    **38906408 B** |
+| Jint_ParsedScript |            stopwatch |   362,800.054 μs |    962.7038 μs |    5 |    38890808 B |
+|          Jurassic |            stopwatch |   209,929.047 μs |    346.3075 μs |    2 |   160703632 B |
+|             NilJS |            stopwatch |   247,228.571 μs |    527.9922 μs |    3 |    85866368 B |
+|          YantraJS |            stopwatch |    76,725.368 μs |    194.3363 μs |    1 |   259048377 B |
+
+Benchmarks with issues:
+  EngineComparisonBenchmark.NilJS: DefaultJob [FileName=linq-js]

Jint.Benchmark/Scripts/array-stress.js

@@ -0,0 +1 @@
+var ret=[],tmp,num=100,i=256;for(var j1=0;j1<i*15;j1++){ret=[];ret.length=i}for(var j2=0;j2<i*10;j2++){ret=new Array(i)}ret=[];for(var j3=0;j3<i;j3++){ret.unshift(j3)}ret=[];for(var j4=0;j4<i;j4++){ret.splice(0,0,j4)}var a=ret.slice();for(var j5=0;j5<i;j5++){tmp=a.shift()}var b=ret.slice();for(var j6=0;j6<i;j6++){tmp=b.splice(0,1)}ret=[];for(var j7=0;j7<i*25;j7++){ret.push(j7)}var c=ret.slice();for(var j8=0;j8<i*25;j8++){tmp=c.pop()}

Jint.Benchmark/Scripts/evaluation.js

@@ -0,0 +1,16 @@
+var o = {};
+o.Foo = 'bar';
+o.Baz = 42.0001;
+o.Blah = o.Foo + o.Baz;
+
+if(o.Blah != 'bar42.0001') throw TypeError;
+
+function fib(n){
+    if(n<2) {
+        return n;
+    }
+
+    return fib(n-1) + fib(n-2);
+}
+
+if(fib(3) != 2) throw TypeError;

Jint.Benchmark/Scripts/minimal.js

@@ -0,0 +1 @@
+var x = 1 + 1 === 2;

Jint.Benchmark/Scripts/stopwatch.js

@@ -0,0 +1,56 @@
+function Stopwatch() {
+    var sw = this;
+    var start = null;
+    var stop = null;
+    var isRunning = false;
+
+    sw.Start = function () {
+        if (isRunning)
+            return;
+
+        start = new Date();
+        stop = null;
+        isRunning = true;
+    }
+
+    sw.Stop = function () {
+        if (!isRunning)
+            return;
+
+        stop = new Date();
+        isRunning = false;
+    }
+
+    sw.Reset = function () {
+        start = isRunning ? new Date() : null;
+        stop = null;
+    }
+
+    sw.Restart = function () {
+        isRunning = true;
+        sw.Reset();
+    }
+
+    sw.ElapsedMilliseconds = function () { return (isRunning ? new Date() : stop) - start; }
+    sw.IsRunning = function() { return isRunning; }
+
+}
+
+var sw = new Stopwatch();
+sw.Start();
+for (var x = 0; x < 1021; x++) {
+    for (var y = 0; y < 383; y++) {
+        var z = x ^ y;
+        if (z % 2 == 0)
+            sw.Start();
+        else if(z % 3 == 0)
+            sw.Stop();
+        else if (z % 5 == 0)
+            sw.Reset();
+        else if (z % 7 == 0)
+            sw.Restart();
+        var ms = sw.ElapsedMilliseconds;
+        var rn = sw.IsRunning;
+    }
+}
+sw.Stop();

Jint.Benchmark/SingleScriptBenchmark.cs

@@ -4,61 +4,32 @@ using Esprima.Ast;
 
 namespace Jint.Benchmark;
 
-[RankColumn]
 [MemoryDiagnoser]
-[HideColumns("Error", "StdDev", "Gen0", "Gen1", "Gen2")]
-[BenchmarkCategory("EngineComparison")]
 public abstract class SingleScriptBenchmark
 {
+    private string _script;
     private Script _parsedScript;
 
-    protected abstract string Script { get; }
-
-    public virtual int N => 10;
+    protected abstract string FileName { get; }
 
     [GlobalSetup]
     public void Setup()
     {
-        _parsedScript = new JavaScriptParser().ParseScript(Script);
+        _script = File.ReadAllText($"Scripts/{FileName}");
+        _parsedScript = new JavaScriptParser().ParseScript(_script);
     }
 
     [Benchmark]
-    public bool Jint()
+    public void Execute()
     {
-        var engine = new Engine();
-        engine.Execute(Script);
-        return engine.GetValue("done").AsBoolean();
+        var engine = new Engine(static options => options.Strict());
+        engine.Execute(FileName);
     }
 
     [Benchmark]
-    public bool Jint_ParsedScript()
+    public void Execute_ParsedScript()
     {
-        var engine = new Engine();
+        var engine = new Engine(static options => options.Strict());
         engine.Execute(_parsedScript);
-        return engine.GetValue("done").AsBoolean();
-    }
-
-    [Benchmark]
-    public bool Jurassic()
-    {
-        var engine = new Jurassic.ScriptEngine();
-        engine.Execute(Script);
-        return engine.GetGlobalValue<bool>("done");
-    }
-
-    [Benchmark]
-    public bool NilJS()
-    {
-        var engine = new NiL.JS.Core.Context();
-        engine.Eval(Script);
-        return (bool) engine.GetVariable("done");
-    }
-
-    [Benchmark]
-    public bool YantraJS()
-    {
-        var engine = new YantraJS.Core.JSContext();
-        engine.Eval(Script);
-        return engine["done"].BooleanValue;
     }
 }

Jint.Benchmark/StopwatchBenchmark.cs

@@ -5,66 +5,5 @@ namespace Jint.Benchmark;
 [MemoryDiagnoser]
 public class StopwatchBenchmark : SingleScriptBenchmark
 {
-    public override int N => 10;
-
-    protected override string Script => @"
-function Stopwatch() {
-    var sw = this;
-    var start = null;
-    var stop = null;
-    var isRunning = false;
-
-    sw.Start = function () {
-        if (isRunning)
-            return;
-
-        start = new Date();
-        stop = null;
-        isRunning = true;
-    }
-
-    sw.Stop = function () {
-        if (!isRunning)
-            return;
-
-        stop = new Date();
-        isRunning = false;
-    }
-
-    sw.Reset = function () {
-        start = isRunning ? new Date() : null;
-        stop = null;
-    }
-
-    sw.Restart = function () {
-        isRunning = true;
-        sw.Reset();
-    }
-
-    sw.ElapsedMilliseconds = function () { return (isRunning ? new Date() : stop) - start; }
-    sw.IsRunning = function() { return isRunning; }
-
+    protected override string FileName => "stopwatch.js";
 }
-
-var sw = new Stopwatch();
-sw.Start();
-for (var x = 0; x < 1021; x++) {
-    for (var y = 0; y < 383; y++) {
-        var z = x ^ y;
-        if (z % 2 == 0)
-            sw.Start();
-        else if(z % 3 == 0)
-            sw.Stop();
-        else if (z % 5 == 0)
-            sw.Reset();
-        else if (z % 7 == 0)
-            sw.Restart();
-        var ms = sw.ElapsedMilliseconds;
-        var rn = sw.IsRunning;
-    }
-}
-sw.Stop();
-
-var done = true;
-";
-}

Jint.Benchmark/SunSpiderBenchmark.cs

@@ -1,12 +1,11 @@
 using BenchmarkDotNet.Attributes;
-using Jint;
 
-namespace Esprima.Benchmark;
+namespace Jint.Benchmark;
 
 [MemoryDiagnoser]
 public class SunSpiderBenchmark
 {
-    private static readonly Dictionary<string, string> files = new Dictionary<string, string>
+    private static readonly Dictionary<string, string> files = new()
     {
         {"3d-cube", null},
         {"3d-morph", null},
@@ -43,7 +42,7 @@ public class SunSpiderBenchmark
     {
         foreach (var fileName in files.Keys.ToList())
         {
-            files[fileName] = File.ReadAllText($"SunSpider/{fileName}.js");
+            files[fileName] = File.ReadAllText($"Scripts/{fileName}.js");
         }
 
         engine = new Engine()
@@ -67,4 +66,4 @@ public class SunSpiderBenchmark
     {
         engine.Execute(files[FileName]);
     }
-}
+}

README.md

@@ -6,7 +6,7 @@
 
 # Jint
 
-Jint is a __Javascript interpreter__ for .NET which can run on __any modern .NET platform__ as it supports .NET Standard 2.0 and .NET 4.6.1 targets (and up). Because Jint neither generates any .NET bytecode nor uses the DLR it runs relatively small scripts really fast.
+Jint is a __Javascript interpreter__ for .NET which can run on __any modern .NET platform__ as it supports .NET Standard 2.0 and .NET 4.6.2 targets (and up).
 
 💡 You should prefer 3.x beta over the 2.x legacy version as all new features and improvements are targeted against version 3.x.
 
@@ -117,6 +117,14 @@ The entire execution engine was rebuild with performance in mind, in many cases
 
 > Follow new features as they are being implemented, see https://github.com/sebastienros/jint/issues/343
 
+## Performance
+
+- Because Jint neither generates any .NET bytecode nor uses the DLR it runs relatively small scripts really fast
+- If you repeatedly run the same script, you should cache the `Script` or `Module` instance produced by Esprima and feed it to Jint instead of the content string
+- You should prefer running engine in strict mode, it improves performance
+
+You can check out [the engine comparison results](Jint.Benchmarks), bear in mind that every use case is different and benchmarks might not reflect your real-world usage.
+
 ## Discussion
 
 Join the chat on [Gitter](https://gitter.im/sebastienros/jint) or post your questions with the `jint` tag on [stackoverflow](http://stackoverflow.com/questions/tagged/jint).