Refactoring for custom module loader support (#1731)

Jint.Tests.PublicInterface/ModuleLoaderTests.cs

@@ -0,0 +1,264 @@
+using System.Collections.Concurrent;
+using Jint.Native;
+using Jint.Native.Json;
+using Jint.Runtime.Modules;
+
+#nullable enable
+
+namespace Jint.Tests.PublicInterface;
+
+public class ModuleLoaderTests
+{
+    [Fact]
+    public void CustomModuleLoaderWithUriModuleLocations()
+    {
+        // Dummy module store which shows that different protocols can be
+        // used for modules.
+        var store = new ModuleStore(new Dictionary<string, string>()
+        {
+            ["https://example.com/someModule.js"] = "export const DEFAULT_VALUE = 'remote';",
+            ["https://example.com/test.js"] = "import { DEFAULT_VALUE } from 'someModule.js'; export const value = DEFAULT_VALUE;",
+            ["file:///someModule.js"] = "export const value = 'local';",
+            ["proprietary-protocol:///someModule.js"] = "export const value = 'proprietary';",
+        });
+        var sharedModules = new CachedModuleLoader(store);
+
+        var runA = RunModule("import { value } from 'https://example.com/test.js'; log(value);");
+        var runB = RunModule("import { value } from 'someModule.js'; log(value);");
+        var runC = RunModule("import { value } from 'proprietary-protocol:///someModule.js'; log(value);");
+
+        ExpectLoggedValue(runA, "remote");
+        ExpectLoggedValue(runB, "local");
+        ExpectLoggedValue(runC, "proprietary");
+
+        static void ExpectLoggedValue(ModuleScript executedScript, string expectedValue)
+        {
+            Assert.Single(executedScript.Logs);
+            Assert.Equal(expectedValue, executedScript.Logs[0]);
+        }
+
+        ModuleScript RunModule(string code)
+        {
+            var result = new ModuleScript(code, sharedModules);
+            result.Execute();
+            return result;
+        }
+    }
+
+    [Fact]
+    public void CustomModuleLoaderWithCachingSupport()
+    {
+        // Different engines use the same module loader.
+        // The module loader caches the parsed Esprima.Ast.Module
+        // which allows to re-use these for different engine runs.
+        var store = new ModuleStore(new Dictionary<string, string>()
+        {
+            ["file:///localModule.js"] = "export const value = 'local';",
+        });
+        var sharedModules = new CachedModuleLoader(store);
+
+        // Simulate the re-use by simply running the same main entry point 10 times.
+        foreach (var _ in Enumerable.Range(0, 10))
+        {
+            var runner = new ModuleScript("import { value } from 'localModule.js'; log(value);", sharedModules);
+            runner.Execute();
+        }
+
+        Assert.Equal(1, sharedModules.ModulesParsed);
+    }
+
+    [Fact]
+    public void CustomModuleLoaderCanWorkWithJsonModules()
+    {
+        var store = new ModuleStore(new Dictionary<string, string>()
+        {
+            ["file:///config.json"] = "{ \"value\": \"json\" }",
+        });
+        var sharedModules = new CachedModuleLoader(store);
+
+        var runner = new ModuleScript("import data from 'config.json' with { type: 'json' }; log(data.value);", sharedModules);
+        runner.Execute();
+
+        Assert.Single(runner.Logs);
+        Assert.Equal("json", runner.Logs[0]);
+    }
+
+    /// <summary>
+    /// A simple in-memory store for module sources. The keys
+    /// must be absolute <see cref="Uri.ToString()"/> values.
+    /// </summary>
+    /// <remarks>
+    /// This is just an example and not production ready code. The implementation
+    /// is missing important path traversal checks and other edge cases.
+    /// </remarks>
+    private sealed class ModuleStore
+    {
+        private const string DefaultProtocol = "file:///./";
+        private readonly IReadOnlyDictionary<string, string> _sourceCode;
+
+        public ModuleStore(IReadOnlyDictionary<string, string> sourceCode)
+        {
+            _sourceCode = sourceCode;
+        }
+
+        public ResolvedSpecifier Resolve(string? referencingModuleLocation, ModuleRequest moduleRequest)
+        {
+            Uri uri = Resolve(referencingModuleLocation, moduleRequest.Specifier);
+            return new ResolvedSpecifier(moduleRequest, moduleRequest.Specifier, uri, SpecifierType.Bare);
+        }
+
+        private Uri Resolve(string? referencingModuleLocation, string specifier)
+        {
+            if (Uri.TryCreate(specifier, UriKind.Absolute, out Uri? absoluteLocation))
+                return absoluteLocation;
+
+            if (!string.IsNullOrEmpty(referencingModuleLocation) && Uri.TryCreate(referencingModuleLocation, UriKind.Absolute, out Uri? baseUri))
+            {
+                if (Uri.TryCreate(baseUri, specifier, out Uri? relative))
+                    return relative;
+            }
+
+            return new Uri(DefaultProtocol + specifier);
+        }
+
+        public string GetModuleSource(Uri uri)
+        {
+            if (!_sourceCode.TryGetValue(uri.ToString(), out var result))
+                throw new InvalidOperationException($"Module not found: {uri}");
+            return result;
+        }
+    }
+
+    /// <summary>
+    /// The main entry point for a module script. Allows
+    /// to use a script as a main module.
+    /// </summary>
+    private sealed class ModuleScript : IModuleLoader
+    {
+        private const string MainSpecifier = "____main____";
+        private readonly List<string> _logs = new();
+        private readonly Engine _engine;
+        private readonly string _main;
+        private readonly IModuleLoader _modules;
+
+        public ModuleScript(string main, IModuleLoader modules)
+        {
+            _main = main;
+            _modules = modules;
+
+            _engine = new Engine(options => options.EnableModules(this));
+            _engine.SetValue("log", _logs.Add);
+        }
+
+        public IReadOnlyList<string> Logs => _logs;
+
+        public void Execute()
+        {
+            _engine.Modules.Import(MainSpecifier);
+        }
+
+        ResolvedSpecifier IModuleLoader.Resolve(string? referencingModuleLocation, ModuleRequest moduleRequest)
+        {
+            if (moduleRequest.Specifier == MainSpecifier)
+                return new ResolvedSpecifier(moduleRequest, MainSpecifier, null, SpecifierType.Bare);
+            return _modules.Resolve(referencingModuleLocation, moduleRequest);
+        }
+
+        Module IModuleLoader.LoadModule(Engine engine, ResolvedSpecifier resolved)
+        {
+            if (resolved.ModuleRequest.Specifier == MainSpecifier)
+                return ModuleFactory.BuildSourceTextModule(engine, Engine.PrepareModule(_main, MainSpecifier));
+            return _modules.LoadModule(engine, resolved);
+        }
+    }
+
+    /// <summary>
+    /// <para>
+    /// A simple <see cref="IModuleLoader"/> implementation which will
+    /// re-use prepared <see cref="Esprima.Ast.Module"/> or <see cref="JsValue"/> modules to
+    /// produce <see cref="Jint.Runtime.Modules.Module"/>.
+    /// </para>
+    /// <para>
+    /// The module source gets loaded from <see cref="ModuleStore"/>.
+    /// </para>
+    /// </summary>
+    private sealed class CachedModuleLoader : IModuleLoader
+    {
+        private readonly ConcurrentDictionary<Uri, ParsedModule> _parsedModules = new();
+        private readonly ModuleStore _store;
+        #if NETCOREAPP1_0_OR_GREATER
+        private readonly Func<Uri, ResolvedSpecifier, ParsedModule> _moduleParser;
+        #endif
+        private int _modulesParsed;
+
+        public CachedModuleLoader(ModuleStore store)
+        {
+            _store = store;
+            #if NETCOREAPP1_0_OR_GREATER
+            _moduleParser = GetParsedModule;
+            #endif
+        }
+
+        public int ModulesParsed => _modulesParsed;
+
+        public ResolvedSpecifier Resolve(string? referencingModuleLocation, ModuleRequest moduleRequest)
+        {
+            return _store.Resolve(referencingModuleLocation, moduleRequest);
+        }
+
+        public Module LoadModule(Engine engine, ResolvedSpecifier resolved)
+        {
+            Assert.NotNull(resolved.Uri);
+            #if NETCOREAPP1_0_OR_GREATER
+            var parsedModule = _parsedModules.GetOrAdd(resolved.Uri, _moduleParser, resolved);
+            #else
+            var parsedModule = _parsedModules.GetOrAdd(resolved.Uri, _ => GetParsedModule(resolved.Uri, resolved));
+            #endif
+            return parsedModule.ToModule(engine);
+        }
+
+        private ParsedModule GetParsedModule(Uri uri, ResolvedSpecifier resolved)
+        {
+            var script = _store.GetModuleSource(resolved.Uri!);
+            var result = resolved.ModuleRequest.IsJsonModule()
+                ? ParsedModule.JsonModule(script, resolved.Uri!.ToString())
+                : ParsedModule.TextModule(script, resolved.Uri!.ToString());
+            Interlocked.Increment(ref _modulesParsed);
+            return result;
+        }
+
+        private sealed class ParsedModule
+        {
+            private readonly Esprima.Ast.Module? _textModule;
+            private readonly (JsValue Json, string Location)? _jsonModule;
+
+            private ParsedModule(Esprima.Ast.Module? textModule, (JsValue Json, string Location)? jsonModule)
+            {
+                _textModule = textModule;
+                _jsonModule = jsonModule;
+            }
+
+            public static ParsedModule TextModule(string script, string location)
+                => new(Engine.PrepareModule(script, location), null);
+
+            public static ParsedModule JsonModule(string json, string location)
+                => new(null, (ParseJson(json), location));
+
+            private static JsValue ParseJson(string json)
+            {
+                var engine = new Engine();
+                var parser = new JsonParser(engine);
+                return parser.Parse(json);
+            }
+
+            public Module ToModule(Engine engine)
+            {
+                if (_jsonModule is not null)
+                    return ModuleFactory.BuildJsonModule(engine, _jsonModule.Value.Json, _jsonModule.Value.Location);
+                if (_textModule is not null)
+                    return ModuleFactory.BuildSourceTextModule(engine, _textModule);
+                throw new InvalidOperationException("Unexpected state - no module type available");
+            }
+        }
+    }
+}

Jint/Runtime/Modules/ModuleFactory.cs

@@ -0,0 +1,97 @@
+using Esprima;
+using Jint.Native;
+using Jint.Native.Json;
+
+namespace Jint.Runtime.Modules;
+
+/// <summary>
+/// Factory which creates a single runtime <see cref="Module"/> from a given source.
+/// </summary>
+public static class ModuleFactory
+{
+    /// <summary>
+    /// Creates a <see cref="Module"/> for the usage within the given <paramref name="engine"/>
+    /// from the provided javascript <paramref name="code"/>.
+    /// </summary>
+    /// <remarks>
+    /// The returned modules location (see <see cref="Module.Location"/>) points to
+    /// <see cref="Uri.LocalPath"/> if <see cref="ResolvedSpecifier.Uri"/> is not null. If
+    /// <see cref="ResolvedSpecifier.Uri"/> is null, the modules location source will be null as well.
+    /// </remarks>
+    /// <exception cref="ParserException">Is thrown if the provided <paramref name="code"/> can not be parsed.</exception>
+    /// <exception cref="JavaScriptException">Is thrown if an error occured when parsing <paramref name="code"/>.</exception>
+    public static Module BuildSourceTextModule(Engine engine, ResolvedSpecifier resolved, string code)
+    {
+        var source = resolved.Uri?.LocalPath;
+        Esprima.Ast.Module module;
+        try
+        {
+            module = new JavaScriptParser().ParseModule(code, source);
+        }
+        catch (ParserException ex)
+        {
+            ExceptionHelper.ThrowSyntaxError(engine.Realm, $"Error while loading module: error in module '{source}': {ex.Error}");
+            module = null;
+        }
+        catch (Exception)
+        {
+            ExceptionHelper.ThrowJavaScriptException(engine, $"Could not load module {source}", (Location) default);
+            module = null;
+        }
+
+        return BuildSourceTextModule(engine, module);
+    }
+
+    /// <summary>
+    /// Creates a <see cref="Module"/> for the usage within the given <paramref name="engine"/>
+    /// from the parsed <paramref name="parsedModule"/>.
+    /// </summary>
+    /// <remarks>
+    /// The returned modules location (see <see cref="Module.Location"/>) will be set
+    /// to <see cref="Location.Source"/> of the <paramref name="parsedModule"/>.
+    /// </remarks>
+    public static Module BuildSourceTextModule(Engine engine, Esprima.Ast.Module parsedModule)
+    {
+        return new SourceTextModule(engine, engine.Realm, parsedModule, parsedModule.Location.Source, async: false);
+    }
+
+    /// <summary>
+    /// Creates a <see cref="Module"/> for the usage within the given <paramref name="engine"/> for the
+    /// provided JSON module <paramref name="jsonString"/>.
+    /// </summary>
+    /// <remarks>
+    /// The returned modules location (see <see cref="Module.Location"/>) points to
+    /// <see cref="Uri.LocalPath"/> if <see cref="ResolvedSpecifier.Uri"/> is not null. If
+    /// <see cref="ResolvedSpecifier.Uri"/> is null, the modules location source will be null as well.
+    /// </remarks>
+    /// <exception cref="JavaScriptException">Is thrown if an error occured when parsing <paramref name="jsonString"/>.</exception>
+    public static Module BuildJsonModule(Engine engine, ResolvedSpecifier resolved, string jsonString)
+    {
+        var source = resolved.Uri?.LocalPath;
+        JsValue module;
+        try
+        {
+            module = new JsonParser(engine).Parse(jsonString);
+        }
+        catch (Exception)
+        {
+            ExceptionHelper.ThrowJavaScriptException(engine, $"Could not load module {source}", (Location) default);
+            module = null;
+        }
+
+        return BuildJsonModule(engine, module, resolved.Uri?.LocalPath);
+    }
+
+    /// <summary>
+    /// Creates a <see cref="Module"/> for the usage within the given <paramref name="engine"/>
+    /// from the parsed JSON provided in <paramref name="parsedJson"/>.
+    /// </summary>
+    /// <remarks>
+    /// The returned modules location (see <see cref="Module.Location"/>) will be set
+    /// to <paramref name="location"/>.
+    /// </remarks>
+    public static Module BuildJsonModule(Engine engine, JsValue parsedJson, string? location)
+    {
+        return new SyntheticModule(engine, engine.Realm, parsedJson, location);
+    }
+}

Jint/Runtime/Modules/ModuleLoader.cs

@@ -1,6 +1,4 @@
 using Esprima;
-using Jint.Native;
-using Jint.Native.Json;
 
 namespace Jint.Runtime.Modules;
 
@@ -24,53 +22,13 @@ public abstract class ModuleLoader : IModuleLoader
             return default!;
         }
 
-        var isJson = resolved.ModuleRequest.Attributes != null
-                     && Array.Exists(resolved.ModuleRequest.Attributes, x => string.Equals(x.Key, "type", StringComparison.Ordinal) && string.Equals(x.Value, "json", StringComparison.Ordinal));
-
+        var isJson = resolved.ModuleRequest.IsJsonModule();
         Module moduleRecord = isJson
-            ? BuildJsonModule(engine, resolved, code)
-            : BuildSourceTextModule(engine, resolved, code);
+            ? ModuleFactory.BuildJsonModule(engine, resolved, code)
+            : ModuleFactory.BuildSourceTextModule(engine, resolved, code);
 
         return moduleRecord;
     }
 
     protected abstract string LoadModuleContents(Engine engine, ResolvedSpecifier resolved);
-
-    private static SyntheticModule BuildJsonModule(Engine engine, ResolvedSpecifier resolved, string code)
-    {
-        var source = resolved.Uri?.LocalPath;
-        JsValue module;
-        try
-        {
-            module = new JsonParser(engine).Parse(code);
-        }
-        catch (Exception)
-        {
-            ExceptionHelper.ThrowJavaScriptException(engine, $"Could not load module {source}", (Location) default);
-            module = null;
-        }
-
-        return new SyntheticModule(engine, engine.Realm, module, resolved.Uri?.LocalPath);
-    }
-    private static SourceTextModule BuildSourceTextModule(Engine engine, ResolvedSpecifier resolved, string code)
-    {
-        var source = resolved.Uri?.LocalPath;
-        Esprima.Ast.Module module;
-        try
-        {
-            module = new JavaScriptParser().ParseModule(code, source);
-        }
-        catch (ParserException ex)
-        {
-            ExceptionHelper.ThrowSyntaxError(engine.Realm, $"Error while loading module: error in module '{source}': {ex.Error}");
-            module = null;
-        }
-        catch (Exception)
-        {
-            ExceptionHelper.ThrowJavaScriptException(engine, $"Could not load module {source}", (Location) default);
-            module = null;
-        }
-
-        return new SourceTextModule(engine, engine.Realm, module, resolved.Uri?.LocalPath, async: false);
-    }
 }

Jint/Runtime/Modules/ModuleRequestExtensions.cs

@@ -0,0 +1,21 @@
+namespace Jint.Runtime.Modules;
+
+public static class ModuleRequestExtensions
+{
+    /// <summary>
+    /// Returns true if the provided <paramref name="request"/>
+    /// is a json module, otherwise false.
+    /// </summary>
+    /// <example>
+    /// The following JavaScript import statement imports a JSON module
+    /// for which this method would return true.
+    /// <code>
+    /// import value from 'config.json' with { type: 'json' }
+    /// </code>
+    /// </example>
+    public static bool IsJsonModule(this ModuleRequest request)
+    {
+        return request.Attributes != null
+            && Array.Exists(request.Attributes, x => string.Equals(x.Key, "type", StringComparison.Ordinal) && string.Equals(x.Value, "json", StringComparison.Ordinal));
+    }
+}