Reset static inits in macros when the context is reused (#7554)

* [eval] apply static inits on context reuse (see #5746)

* test modification of macro class too

* fix assertion position

* cause everytime we touch, I get this fstatSync

* allow @:persistent to avoid resetting statics

* remove reuse-callbacks

extra/CHANGES.txt

@@ -1,3 +1,14 @@
+xxxx-xx-xx: 4.0.0-rc.1
+
+	General improvements and optimizations:
+
+	macro : static variables are now always re-initialized when using the compilation server (#5746)
+	macro : support `@:persistent` to keep macro static values across compilations
+
+	Removals :
+
+	macro : deprecated Context.registerModuleReuseCall and onMacroContextReused (#5746)
+
 2018-10-13: 4.0.0-preview.5
 	New features:
 

src/compiler/server.ml

@@ -360,8 +360,7 @@ let rec wait_loop process_params verbose accept =
 					) m.m_types;
 					TypeloadModule.add_module ctx m p;
 					PMap.iter (Hashtbl.replace com2.resources) m.m_extra.m_binded_res;
-					PMap.iter (fun _ m2 -> add_modules (tabs ^ "  ") m0 m2) m.m_extra.m_deps;
-					List.iter (MacroContext.call_init_macro ctx) m.m_extra.m_reuse_macro_calls
+					PMap.iter (fun _ m2 -> add_modules (tabs ^ "  ") m0 m2) m.m_extra.m_deps
 				)
 			end
 		in

src/core/type.ml

@@ -350,7 +350,6 @@ and module_def_extra = {
 	mutable m_processed : int;
 	mutable m_kind : module_kind;
 	mutable m_binded_res : (string, string) PMap.t;
-	mutable m_reuse_macro_calls : string list;
 	mutable m_if_feature : (string *(tclass * tclass_field * bool)) list;
 	mutable m_features : (string,bool) Hashtbl.t;
 }
@@ -479,7 +478,6 @@ let module_extra file sign time kind policy =
 		m_deps = PMap.empty;
 		m_kind = kind;
 		m_binded_res = PMap.empty;
-		m_reuse_macro_calls = [];
 		m_if_feature = [];
 		m_features = Hashtbl.create 0;
 		m_check_policy = policy;
@@ -1564,7 +1562,6 @@ module Printer = struct
 			"m_processed",string_of_int me.m_processed;
 			"m_kind",s_module_kind me.m_kind;
 			"m_binded_res",""; (* TODO *)
-			"m_reuse_macro_calls",String.concat ", " me.m_reuse_macro_calls;
 			"m_if_feature",""; (* TODO *)
 			"m_features",""; (* TODO *)
 		]

src/macro/eval/evalContext.ml

@@ -190,6 +190,7 @@ and context = {
 	mutable static_prototypes : vprototype IntMap.t;
 	mutable constructors : value Lazy.t IntMap.t;
 	get_object_prototype : 'a . context -> (int * 'a) list -> vprototype * (int * 'a) list;
+	mutable static_inits : (vprototype * (vprototype -> unit) list) IntMap.t;
 	(* eval *)
 	toplevel : value;
 	eval : eval;

src/macro/eval/evalMain.ml

@@ -125,6 +125,7 @@ let create com api is_macro =
 		instance_prototypes = IntMap.empty;
 		constructors = IntMap.empty;
 		get_object_prototype = get_object_prototype;
+		static_inits = IntMap.empty;
 		(* eval *)
 		toplevel = 	vobject {
 			ofields = [||];
@@ -355,10 +356,9 @@ let setup get_api =
 	) api;
 	Globals.macro_platform := Globals.Eval
 
-let can_reuse ctx types = true
-
 let do_reuse ctx api =
-	ctx.curapi <- api
+	ctx.curapi <- api;
+	IntMap.iter (fun _ (proto,delays) -> List.iter (fun f -> f proto) delays) ctx.static_inits
 
 let set_error ctx b =
 	(* TODO: Have to reset this somewhere if running compilation server. But where... *)

src/macro/eval/evalPrototype.ml

@@ -172,6 +172,9 @@ end
 let is_removable_field cf =
 	cf.cf_extern || Meta.has Meta.Generic cf.cf_meta
 
+let is_persistent cf =
+	Meta.has (Meta.Custom ":persistent") cf.cf_meta
+
 let create_static_prototype ctx mt =
 	let path = (t_infos mt).mt_path in
 	let key = path_hash path in
@@ -195,7 +198,10 @@ let create_static_prototype ctx mt =
 				let name = hash cf.cf_name in
 				PrototypeBuilder.add_proto_field pctx name (lazy vnull);
 				let i = DynArray.length pctx.PrototypeBuilder.fields - 1 in
-				DynArray.add delays (fun proto -> proto.pfields.(i) <- (match eval_expr ctx key name e with Some e -> e | None -> vnull))
+				let persistent = is_persistent cf in
+				DynArray.add delays (persistent,(fun proto ->
+					proto.pfields.(i) <- (match eval_expr ctx key name e with Some e -> e | None -> vnull)
+				))
 			| _,None when is_physical_field cf ->
 				PrototypeBuilder.add_proto_field pctx (hash cf.cf_name) (lazy vnull);
 			|  _ ->
@@ -203,7 +209,7 @@ let create_static_prototype ctx mt =
 		) fields;
 		begin match c.cl_init with
 			| None -> ()
-			| Some e -> DynArray.add delays (fun _ -> ignore(eval_expr ctx key key___init__ e))
+			| Some e -> DynArray.add delays (false,(fun _ -> ignore(eval_expr ctx key key___init__ e)))
 		end;
 		PrototypeBuilder.finalize pctx,(DynArray.to_list delays)
 	| TEnumDecl en ->
@@ -342,8 +348,11 @@ let add_types ctx types ready =
 		f proto;
 		match delays with
 		| [] -> ()
-		| _ -> DynArray.add fl_static_init (proto,delays)
+		| _ ->
+			DynArray.add fl_static_init (proto,delays);
+			let non_persistent_delays = ExtList.List.filter_map (fun (persistent,f) -> if not persistent then Some f else None) delays in
+			ctx.static_inits <- IntMap.add proto.ppath (proto,non_persistent_delays) ctx.static_inits;
 	) fl_static;
 	(* 4. Initialize static fields. *)
-	DynArray.iter (fun (proto,delays) -> List.iter (fun f -> f proto) delays) fl_static_init;
+	DynArray.iter (fun (proto,delays) -> List.iter (fun (_,f) -> f proto) delays) fl_static_init;
 	t()

src/macro/macroApi.ml

@@ -39,9 +39,7 @@ type 'value compiler_api = {
 	define_type : 'value -> string option -> unit;
 	define_module : string -> 'value list -> ((string * Globals.pos) list * Ast.import_mode) list -> Ast.type_path list -> unit;
 	module_dependency : string -> string -> unit;
-	module_reuse_call : string -> string -> unit;
 	current_module : unit -> module_def;
-	on_reuse : (unit -> bool) -> unit;
 	mutable current_macro_module : unit -> module_def;
 	use_cache : unit -> bool;
 	format_string : string -> Globals.pos -> Ast.expr;
@@ -1789,10 +1787,6 @@ let macro_api ccom get_api =
 			(get_api()).module_dependency (decode_string m) (decode_string file);
 			vnull
 		);
-		"register_module_reuse_call", vfun2 (fun m mcall ->
-			(get_api()).module_reuse_call (decode_string m) (decode_string mcall);
-			vnull
-		);
 		"get_typed_expr", vfun1 (fun e ->
 			let e = decode_texpr e in
 			encode_expr (TExprToExpr.convert_expr e)
@@ -1824,11 +1818,6 @@ let macro_api ccom get_api =
 			let loc = (get_api()).get_pattern_locals (decode_expr e) (decode_type t) in
 			encode_string_map (fun (v,_) -> encode_type v.v_type) loc
 		);
-		"on_macro_context_reused", vfun1 (fun c ->
-			let c = prepare_callback c 0 in
-			(get_api()).on_reuse (fun() -> decode_bool (c []));
-			vnull
-		);
 		"apply_params", vfun3 (fun tpl tl t ->
 			let tl = List.map decode_type (decode_array tl) in
 			let tpl = List.map (fun v -> decode_string (field v "name"), decode_type (field v "t")) (decode_array tpl) in

src/typing/macroContext.ml

@@ -43,8 +43,6 @@ end
 
 let macro_enable_cache = ref false
 let macro_interp_cache = ref None
-let macro_interp_on_reuse = ref []
-let macro_interp_reused = ref false
 
 let safe_decode v t p f =
 	try
@@ -338,10 +336,6 @@ let make_macro_api ctx p =
 			let m = typing_timer ctx false (fun() -> TypeloadModule.load_module ctx (parse_path mpath) p) in
 			add_dependency m (create_fake_module ctx file);
 		);
-		MacroApi.module_reuse_call = (fun mpath call ->
-			let m = typing_timer ctx false (fun() -> TypeloadModule.load_module ctx (parse_path mpath) p) in
-			m.m_extra.m_reuse_macro_calls <- call :: List.filter ((<>) call) m.m_extra.m_reuse_macro_calls
-		);
 		MacroApi.current_module = (fun() ->
 			ctx.m.curmod
 		);
@@ -381,9 +375,6 @@ let make_macro_api ctx p =
 			| CompilationServer.MacroContext -> add_macro ctx
 			| CompilationServer.NormalAndMacroContext -> add ctx; add_macro ctx;
 		);
-		MacroApi.on_reuse = (fun f ->
-			macro_interp_on_reuse := f :: !macro_interp_on_reuse
-		);
 		MacroApi.decode_expr = Interp.decode_expr;
 		MacroApi.encode_expr = Interp.encode_expr;
 		MacroApi.encode_ctype = Interp.encode_ctype;
@@ -399,8 +390,6 @@ let rec init_macro_interp ctx mctx mint =
 	Interp.init mint;
 	if !macro_enable_cache && not (Common.defined mctx.com Define.NoMacroCache) then begin
 		macro_interp_cache := Some mint;
-		macro_interp_on_reuse := [];
-		macro_interp_reused := true;
 	end
 
 and flush_macro_context mint ctx =
@@ -410,26 +399,6 @@ and flush_macro_context mint ctx =
 	let _, types, modules = ctx.g.do_generate mctx in
 	mctx.com.types <- types;
 	mctx.com.Common.modules <- modules;
-	let check_reuse() =
-		if !macro_interp_reused then
-			true
-		else if not (List.for_all (fun f -> f())  !macro_interp_on_reuse) then
-			false
-		else begin
-			macro_interp_reused := true;
-			true;
-		end
-	in
-	(* if one of the type we are using has been modified, we need to create a new macro context from scratch *)
-	let mint = if not (Interp.can_reuse mint types && check_reuse()) then begin
-		let com2 = mctx.com in
-		let mint = Interp.create com2 (make_macro_api ctx Globals.null_pos) true in
-		let macro = ((fun() -> Interp.select mint), mctx) in
-		ctx.g.macros <- Some macro;
-		mctx.g.macros <- Some macro;
-		init_macro_interp ctx mctx mint;
-		mint
-	end else mint in
 	(* we should maybe ensure that all filters in Main are applied. Not urgent atm *)
 	let expr_filters = [VarLazifier.apply mctx.com;AbstractCast.handle_abstract_casts mctx; CapturedVars.captured_vars mctx.com;] in
 
@@ -468,7 +437,6 @@ let create_macro_interp ctx mctx =
 			Interp.select mint;
 			mint, (fun() -> init_macro_interp ctx mctx mint)
 		| Some mint ->
-			macro_interp_reused := false;
 			Interp.do_reuse mint (make_macro_api ctx null_pos);
 			mint, (fun() -> ())
 	) in

std/haxe/macro/Context.hx

@@ -548,30 +548,14 @@ class Context {
 		load("register_module_dependency", 2)(modulePath,externFile);
 	}
 
-	/**
-		Register a macro call to be performed every time the module `modulePath` is reused by the compilation cache,
-		meaning that neither the module itself nor its dependencies was changed since last compilation.
-
-		The `macroCall` should be a String containing valid Haxe expression, similar to `--init` macros (see https://haxe.org/manual/macro-initialization.html).
-		Multiple calls with the exact same `macroCall` value will only register the callback once.
-
-		This also triggers loading of given module and its dependencies, if it's not yet loaded,
-		but given macro call will not be called on the first module load.
-
-		If the compilation cache is not used, `macroCall` expressions will not be called,
-		but calling this function will still trigger loading of given `modulePath`.
-	**/
+	@:deprecated
 	public static function registerModuleReuseCall( modulePath : String, macroCall : String ) {
-		load("register_module_reuse_call", 2)(modulePath,macroCall);
+		throw "This method is no longer supported. See https://github.com/HaxeFoundation/haxe/issues/5746";
 	}
 
-	/**
-		Register a callback function that will be called every time the macro context cached is reused with a new
-		compilation. This enable to reset some static vars since the code might have been changed. If the callback
-		returns false, the macro context is discarded and another one is created.
-	**/
+	@:deprecated
 	public static function onMacroContextReused( callb : Void -> Bool ) {
-		load("on_macro_context_reused", 1)(callb);
+		throw "This method is no longer supported. See https://github.com/HaxeFoundation/haxe/issues/5746";
 	}
 
 	@:allow(haxe.macro.TypeTools)

tests/server/src/HaxeServerTestCase.hx

@@ -1,6 +1,7 @@
 import HaxeServer;
 import haxe.Json;
 import utest.Assert;
+using StringTools;
 
 typedef Message<T> = {
 	kind: String,
@@ -68,6 +69,7 @@ class HaxeServerTestCase {
 			return switch (msg.kind) {
 				case "reusing" | "notCached": data1 == data2;
 				case "skipping": data1.skipped == data2.skipped && data1.dependency == data2.dependency;
+				case "print": Std.string(data1).trim() == Std.string(data2).trim();
 				case _: false;
 			}
 		}
@@ -79,6 +81,10 @@ class HaxeServerTestCase {
 		return false;
 	}
 
+	function assertHasPrint(line:String, ?p:haxe.PosInfos) {
+		Assert.isTrue(hasMessage({kind: "print", data: line}), null, p);
+	}
+
 	function assertReuse(module:String, ?p:haxe.PosInfos) {
 		Assert.isTrue(hasMessage({kind: "reusing", data: module}), null, p);
 	}

tests/server/src/Main.hx

@@ -1,4 +1,5 @@
 import AsyncMacro.async;
+using StringTools;
 
 class NoModification extends HaxeServerTestCase {
 	public function test() {
@@ -46,6 +47,31 @@ class Dependency extends HaxeServerTestCase {
 	}
 }
 
+class Macro extends HaxeServerTestCase {
+	public function test() {
+		async({
+			vfs.putContent("MacroMain.hx", getTemplate("MacroMain.hx"));
+			vfs.putContent("Macro.hx", getTemplate("Macro.hx"));
+			var args = ["-main", "MacroMain.hx", "--no-output", "-js", "no.js"];
+			haxe(args);
+			assertHasPrint("1");
+			vfs.touchFile("MacroMain.hx");
+			haxe(args);
+			assertHasPrint("1");
+			vfs.touchFile("Macro.hx");
+			haxe(args);
+			assertHasPrint("1");
+			vfs.putContent("Macro.hx", getTemplate("Macro.hx").replace("1", "2"));
+			haxe(args);
+			assertHasPrint("2");
+			vfs.touchFile("MacroMain.hx");
+			haxe(args);
+			assertHasPrint("2");
+		});
+	}
+}
+
+
 class Main {
 	static public function main() {
 		Vfs.removeDir("test/cases");
@@ -53,6 +79,7 @@ class Main {
 		runner.addCase(new NoModification());
 		runner.addCase(new Modification());
 		runner.addCase(new Dependency());
+		runner.addCase(new Macro());
 		utest.ui.Report.create(runner);
 		runner.run();
 	}

tests/server/src/Vfs.hx

@@ -16,8 +16,9 @@ class Vfs {
 	public function touchFile(path:String) {
 		var path = getPhysicalPath(path);
 		FileSystem.createDirectory(path.dir);
-		var notNow = js.Date.fromHaxeDate(DateTools.delta(Date.now(), 1000));
 		var file = Fs.openSync(path.dir + "/" + path.file + "." + path.ext, 'a');
+        var last = Date.fromString(Fs.fstatSync(file).mtime.toTimeString().substr(0, 8));
+        var notNow = js.Date.fromHaxeDate(DateTools.delta(last, 1000));
 		Fs.futimesSync(file, notNow, notNow);
 		Fs.closeSync(file);
 	}

tests/server/test/templates/Macro.hx

@@ -0,0 +1,8 @@
+class Macro {
+	static var x = 1;
+
+	macro static public function test() {
+		Sys.println(x++);
+		return macro null;
+	}
+}

tests/server/test/templates/MacroMain.hx

@@ -0,0 +1,5 @@
+class MacroMain {
+	static public function main() {
+		Macro.test();
+	}
+}