[typer] Prepare for typer creation delaying (#11324)

* [debug] add trace utils

* [debug] add compiler stage debug

* [std] ensuring correct macro API usage is now mandatory

* [xmldoc] update macro API usage

* [dune] do not watch non source folders

* [tests] add server tests utils

* [tests] update macro API usage

* [macro] update Context.onAfterInitMacros behavior

* [std] update macro API usage for haxe.macro.CompilationServer

* [std] mark more Context API as after init macros only

* [macro] make Context.parse / Context.parseInlineString available in init macros too

* [tests] fix macro-in-macro when test fails

* [macro] make MacroStringTools.formatString work during init macros too

* Move global_metadata to common context

* [macro] make warning and current_module work (partially) with com_api

* [macro] Compiler: delay type/meta patch instead of aborting

* [tests] adjust for macro API changes

* Revert "Move global_metadata to common context"

This reverts commit 319e0746745c7a3fd30739f30e06f281d34ac62e.

dune

@@ -1 +1,2 @@
-(data_only_dirs extra lib std tests)
+(dirs :standard \ tests std extra)
+(data_only_dirs lib)

extra/ImportAll.hx

@@ -69,42 +69,44 @@ class ImportAll {
 			if (!Context.defined("neko") && !Context.defined("cpp")) return;
 		case "tools", "build-tool", "jar-tool": return;
 		}
-		for( p in Context.getClassPath() ) {
-			if( p == "/" || p == "" )
-				continue;
-			// skip if we have a classpath to haxe
-			if( pack.length == 0 && sys.FileSystem.exists(p+"std") )
-				continue;
-			var p = p + pack.split(".").join("/");
-			if( StringTools.endsWith(p,"/") )
-				p = p.substr(0,-1);
-			if( !sys.FileSystem.exists(p) || !sys.FileSystem.isDirectory(p) )
-				continue;
-			for( file in sys.FileSystem.readDirectory(p) ) {
-				if( file == ".svn" || file == "_std" )
+		Context.onAfterInitMacros(() -> {
+			for( p in Context.getClassPath() ) {
+				if( p == "/" || p == "" )
 					continue;
-				var full = (pack == "") ? file : pack + "." + file;
-				if( StringTools.endsWith(file, ".hx") && file.substr(0, file.length - 3).indexOf(".") < 0 ) {
-					var cl = full.substr(0, full.length - 3);
-					switch( cl ) {
-					case "ImportAll", "neko.db.MacroManager": continue;
-					case "haxe.TimerQueue": if( Context.defined("neko") || Context.defined("php") || Context.defined("cpp") ) continue;
-					case "Sys": if(!isSysTarget()) continue;
-					case "haxe.web.Request": if( !(Context.defined("neko") || Context.defined("php") || Context.defined("js")) ) continue;
-					case "haxe.macro.ExampleJSGenerator","haxe.macro.Context", "haxe.macro.Compiler": if( !Context.defined("eval") ) continue;
-					case "haxe.remoting.SocketWrapper": if( !Context.defined("flash") ) continue;
-					case "haxe.remoting.SyncSocketConnection": if( !(Context.defined("neko") || Context.defined("php") || Context.defined("cpp")) ) continue;
-					case "neko.vm.Ui" | "sys.db.Sqlite" | "sys.db.Mysql" if ( Context.defined("interp") ): continue;
-					case "sys.db.Sqlite" | "sys.db.Mysql" | "cs.db.AdoNet" if ( Context.defined("cs") ): continue;
-					case "haxe.atomic.AtomicBool" if(!Context.defined("target.atomics")): continue;
-					case "haxe.atomic.AtomicInt" if(!Context.defined("target.atomics")): continue;
-					case "haxe.atomic.AtomicObject" if(!Context.defined("target.atomics") || Context.defined("js") || Context.defined("cpp")): continue;
-					}
-					Context.getModule(cl);
-				} else if( sys.FileSystem.isDirectory(p + "/" + file) )
-					run(full);
+				// skip if we have a classpath to haxe
+				if( pack.length == 0 && sys.FileSystem.exists(p+"std") )
+					continue;
+				var p = p + pack.split(".").join("/");
+				if( StringTools.endsWith(p,"/") )
+					p = p.substr(0,-1);
+				if( !sys.FileSystem.exists(p) || !sys.FileSystem.isDirectory(p) )
+					continue;
+				for( file in sys.FileSystem.readDirectory(p) ) {
+					if( file == ".svn" || file == "_std" )
+						continue;
+					var full = (pack == "") ? file : pack + "." + file;
+					if( StringTools.endsWith(file, ".hx") && file.substr(0, file.length - 3).indexOf(".") < 0 ) {
+						var cl = full.substr(0, full.length - 3);
+						switch( cl ) {
+						case "ImportAll", "neko.db.MacroManager": continue;
+						case "haxe.TimerQueue": if( Context.defined("neko") || Context.defined("php") || Context.defined("cpp") ) continue;
+						case "Sys": if(!isSysTarget()) continue;
+						case "haxe.web.Request": if( !(Context.defined("neko") || Context.defined("php") || Context.defined("js")) ) continue;
+						case "haxe.macro.ExampleJSGenerator","haxe.macro.Context", "haxe.macro.Compiler": if( !Context.defined("eval") ) continue;
+						case "haxe.remoting.SocketWrapper": if( !Context.defined("flash") ) continue;
+						case "haxe.remoting.SyncSocketConnection": if( !(Context.defined("neko") || Context.defined("php") || Context.defined("cpp")) ) continue;
+						case "neko.vm.Ui" | "sys.db.Sqlite" | "sys.db.Mysql" if ( Context.defined("interp") ): continue;
+						case "sys.db.Sqlite" | "sys.db.Mysql" | "cs.db.AdoNet" if ( Context.defined("cs") ): continue;
+						case "haxe.atomic.AtomicBool" if(!Context.defined("target.atomics")): continue;
+						case "haxe.atomic.AtomicInt" if(!Context.defined("target.atomics")): continue;
+						case "haxe.atomic.AtomicObject" if(!Context.defined("target.atomics") || Context.defined("js") || Context.defined("cpp")): continue;
+						}
+						Context.getModule(cl);
+					} else if( sys.FileSystem.isDirectory(p + "/" + file) )
+						run(full);
+				}
 			}
-		}
+		});
 	}
 
 }

src/compiler/compiler.ml

@@ -278,6 +278,7 @@ let do_type ctx tctx actx =
 	com.stage <- CInitMacrosStart;
 	List.iter (MacroContext.call_init_macro tctx) (List.rev actx.config_macros);
 	com.stage <- CInitMacrosDone;
+	ServerMessage.compiler_stage com;
 	check_defines ctx.com;
 	CommonCache.lock_signature com "after_init_macros";
 	com.callbacks#run com.callbacks#get_after_init_macros;
@@ -287,6 +288,7 @@ let do_type ctx tctx actx =
 		Finalization.finalize tctx;
 	) ();
 	com.stage <- CTypingDone;
+	ServerMessage.compiler_stage com;
 	(* If we are trying to find references, let's syntax-explore everything we know to check for the
 		identifier we are interested in. We then type only those modules that contain the identifier. *)
 	begin match com.display.dms_kind with
@@ -299,6 +301,7 @@ let finalize_typing ctx tctx =
 	let t = Timer.timer ["finalize"] in
 	let com = ctx.com in
 	com.stage <- CFilteringStart;
+	ServerMessage.compiler_stage com;
 	let main, types, modules = run_or_diagnose ctx Finalization.generate tctx in
 	com.main <- main;
 	com.types <- types;
@@ -343,6 +346,7 @@ let compile ctx actx callbacks =
 	List.iter (fun f -> f()) (List.rev (actx.pre_compilation));
 	t();
 	com.stage <- CInitialized;
+	ServerMessage.compiler_stage com;
 	if actx.classes = [([],"Std")] && not actx.force_typing then begin
 		if actx.cmds = [] && not actx.did_something then actx.raise_usage();
 	end else begin
@@ -350,6 +354,7 @@ let compile ctx actx callbacks =
 		let tctx = Setup.create_typer_context ctx actx.native_libs in
 		tctx.g.macros <- mctx;
 		com.stage <- CTyperCreated;
+		ServerMessage.compiler_stage com;
 		let display_file_dot_path = DisplayProcessing.maybe_load_display_file_before_typing tctx display_file_dot_path in
 		begin try
 			do_type ctx tctx actx
@@ -363,8 +368,10 @@ let compile ctx actx callbacks =
 		if ctx.has_error then raise Abort;
 		Generate.check_auxiliary_output com actx;
 		com.stage <- CGenerationStart;
+		ServerMessage.compiler_stage com;
 		if not actx.no_output then Generate.generate ctx tctx ext actx;
 		com.stage <- CGenerationDone;
+		ServerMessage.compiler_stage com;
 	end;
 	Sys.catch_break false;
 	com.callbacks#run com.callbacks#get_after_generation;

src/compiler/serverMessage.ml

@@ -4,6 +4,7 @@ open CompilationCache
 open Type
 
 type server_message_options = {
+	mutable print_compiler_stage : bool;
 	mutable print_added_directory : bool;
 	mutable print_found_directories : bool;
 	mutable print_changed_directories : bool;
@@ -30,6 +31,7 @@ type server_message_options = {
 }
 
 let config = {
+	print_compiler_stage = false;
 	print_added_directory = false;
 	print_found_directories = false;
 	print_changed_directories = false;
@@ -61,6 +63,9 @@ let sign_string com =
 	let	sign_id = (cs#get_context sign)#get_index in
 	Printf.sprintf "%2i,%3s: " sign_id (short_platform_name com.platform)
 
+let compiler_stage com =
+	if config.print_compiler_stage then print_endline (Printf.sprintf "compiler stage: %s" (s_compiler_stage com.stage))
+
 let added_directory com tabs dir =
 	if config.print_added_directory then print_endline (Printf.sprintf "%sadded directory %s" (sign_string com) dir)
 
@@ -156,6 +161,7 @@ let uncaught_error s =
 	if config.print_uncaught_error then print_endline ("Uncaught Error : " ^ s)
 
 let enable_all () =
+	config.print_compiler_stage <- true;
 	config.print_added_directory <- true;
 	config.print_found_directories <- true;
 	config.print_changed_directories <- true;
@@ -180,6 +186,7 @@ let enable_all () =
 	config.print_new_context <- true
 
 let set_by_name name value = match name with
+	| "compilerStage" -> config.print_compiler_stage <- value
 	| "addedDirectory" -> config.print_added_directory <- value
 	| "foundDirectories" -> config.print_found_directories <- value;
 	| "changedDirectories" -> config.print_changed_directories <- value;

src/context/common.ml

@@ -271,6 +271,24 @@ type compiler_stage =
 	| CGenerationStart  (* Generation is about to begin. *)
 	| CGenerationDone   (* Generation just finished. *)
 
+let s_compiler_stage = function
+	| CCreated          -> "CCreated"
+	| CInitialized      -> "CInitialized"
+	| CTyperCreated     -> "CTyperCreated"
+	| CInitMacrosStart  -> "CInitMacrosStart"
+	| CInitMacrosDone   -> "CInitMacrosDone"
+	| CTypingDone       -> "CTypingDone"
+	| CFilteringStart   -> "CFilteringStart"
+	| CAnalyzerStart    -> "CAnalyzerStart"
+	| CAnalyzerDone     -> "CAnalyzerDone"
+	| CSaveStart        -> "CSaveStart"
+	| CSaveDone         -> "CSaveDone"
+	| CDceStart         -> "CDceStart"
+	| CDceDone          -> "CDceDone"
+	| CFilteringDone    -> "CFilteringDone"
+	| CGenerationStart  -> "CGenerationStart"
+	| CGenerationDone   -> "CGenerationDone"
+
 type report_mode =
 	| RMNone
 	| RMDiagnostics of Path.UniqueKey.t list
@@ -1299,3 +1317,100 @@ let get_entry_point com =
 		let e = Option.get com.main in (* must be present at this point *)
 		(snd path, c, e)
 	) com.main_class
+
+let format_string com s p process_expr =
+	let e = ref None in
+	let pmin = ref p.pmin in
+	let min = ref (p.pmin + 1) in
+	let add_expr (enext,p) len =
+		min := !min + len;
+		let enext = process_expr enext p in
+		match !e with
+		| None -> e := Some enext
+		| Some prev ->
+			e := Some (EBinop (OpAdd,prev,enext),punion (pos prev) p)
+	in
+	let add enext len =
+		let p = { p with pmin = !min; pmax = !min + len } in
+		add_expr (enext,p) len
+	in
+	let add_sub start pos =
+		let len = pos - start in
+		if len > 0 || !e = None then add (EConst (String (String.sub s start len,SDoubleQuotes))) len
+	in
+	let len = String.length s in
+	let rec parse start pos =
+		if pos = len then add_sub start pos else
+		let c = String.unsafe_get s pos in
+		let pos = pos + 1 in
+		if c = '\'' then begin
+			incr pmin;
+			incr min;
+		end;
+		if c <> '$' || pos = len then parse start pos else
+		match String.unsafe_get s pos with
+		| '$' ->
+			(* double $ *)
+			add_sub start pos;
+			parse (pos + 1) (pos + 1)
+		| '{' ->
+			parse_group start pos '{' '}' "brace"
+		| 'a'..'z' | 'A'..'Z' | '_' ->
+			add_sub start (pos - 1);
+			incr min;
+			let rec loop i =
+				if i = len then i else
+				let c = String.unsafe_get s i in
+				match c with
+				| 'a'..'z' | 'A'..'Z' | '0'..'9' | '_' -> loop (i+1)
+				| _ -> i
+			in
+			let iend = loop (pos + 1) in
+			let len = iend - pos in
+			add (EConst (Ident (String.sub s pos len))) len;
+			parse (pos + len) (pos + len)
+		| _ ->
+			(* keep as-it *)
+			parse start pos
+	and parse_group start pos gopen gclose gname =
+		add_sub start (pos - 1);
+		let rec loop groups i =
+			if i = len then
+				match groups with
+				| [] -> die "" __LOC__
+				| g :: _ -> Error.raise_typing_error ("Unclosed " ^ gname) { p with pmin = !pmin + g + 1; pmax = !pmin + g + 2 }
+			else
+				let c = String.unsafe_get s i in
+				if c = gopen then
+					loop (i :: groups) (i + 1)
+				else if c = gclose then begin
+					let groups = List.tl groups in
+					if groups = [] then i else loop groups (i + 1)
+				end else
+					loop groups (i + 1)
+		in
+		let send = loop [pos] (pos + 1) in
+		let slen = send - pos - 1 in
+		let scode = String.sub s (pos + 1) slen in
+		min := !min + 2;
+		begin
+			let e =
+				let ep = { p with pmin = !pmin + pos + 2; pmax = !pmin + send + 1 } in
+				let error msg pos =
+					if Lexer.string_is_whitespace scode then Error.raise_typing_error "Expression cannot be empty" ep
+					else Error.raise_typing_error msg pos
+				in
+				match ParserEntry.parse_expr_string com.defines scode ep error true with
+					| ParseSuccess(data,_,_) -> data
+					| ParseError(_,(msg,p),_) -> error (Parser.error_msg msg) p
+			in
+			add_expr e slen
+		end;
+		min := !min + 1;
+		parse (send + 1) (send + 1)
+	in
+	parse 0 0;
+	match !e with
+	| None -> die "" __LOC__
+	| Some e -> e
+

src/core/globals.ml

@@ -33,6 +33,41 @@ let version_pre = Some "alpha.1"
 
 let null_pos = { pfile = "?"; pmin = -1; pmax = -1 }
 
+let no_color = false
+let c_reset = if no_color then "" else "\x1b[0m"
+let c_dim = if no_color then "" else "\x1b[2m"
+
+let loc_short (loc:Printexc.location) =
+	Printf.sprintf "%s:%d" loc.filename loc.line_number
+
+let loc_to_string (loc:Printexc.location) =
+	Printf.sprintf "%s, line %d, characters %d-%d" loc.filename loc.line_number loc.start_char loc.end_char
+
+let trace s =
+	let stack = Printexc.get_callstack 2 in
+	match Printexc.backtrace_slots stack with
+	| Some [|_; item |] ->
+		(match Printexc.Slot.location item with
+		| Some loc -> print_endline (Printf.sprintf "%s%s:%s %s" c_dim (loc_short loc) c_reset s)
+		| _ -> ())
+	| _ ->
+		()
+
+let trace_call_stack ?(n:int = 5) () =
+	assert (n >= 0);
+	let stack = Printexc.get_callstack (n+2) in
+	let len = Printexc.raw_backtrace_length stack - 1 in
+
+	let slot = Printexc.convert_raw_backtrace_slot (Printexc.get_raw_backtrace_slot stack 1) in
+	let loc = Printexc.Slot.location slot in
+	Option.may (fun loc -> print_endline (Printf.sprintf "%s%s:%s" c_dim (loc_short loc) c_reset)) loc;
+
+	for i = 2 to len do
+		let slot = Printexc.convert_raw_backtrace_slot (Printexc.get_raw_backtrace_slot stack i) in
+		let loc = Printexc.Slot.location slot in
+		Option.may (fun loc -> print_endline (Printf.sprintf "  called from %s" (loc_to_string loc))) loc;
+	done
+
 let macro_platform = ref Neko
 
 let return_partial_type = ref false

src/typing/macroContext.ml

@@ -180,8 +180,26 @@ let make_macro_com_api com p =
 			];
 		);
 		parse_string = (fun s p inl ->
-			(* TODO: typing_timer *)
-			Interp.exc_string "unsupported"
+			let old = com.error_ext in
+			com.error_ext <- (fun err -> raise_error { err with err_from_macro = true });
+			let exit() = com.error_ext <- old in
+
+			try
+				let r = match ParserEntry.parse_expr_string com.defines s p raise_typing_error inl with
+					| ParseSuccess(data,true,_) when inl -> data (* ignore errors when inline-parsing in display file *)
+					| ParseSuccess(data,_,_) -> data
+					| ParseError _ -> raise MacroApi.Invalid_expr in
+				exit();
+				r
+			with Error err ->
+				exit();
+				Interp.compiler_error err
+			| WithTypeError err ->
+				exit();
+				Interp.compiler_error err
+			| e ->
+				exit();
+				raise e
 		);
 		parse = (fun entry s ->
 			match ParserEntry.parse_string entry com.defines s null_pos raise_typing_error false with
@@ -248,13 +266,13 @@ let make_macro_com_api com p =
 			Interp.exc_string "unsupported"
 		);
 		current_module = (fun() ->
-			Interp.exc_string "unsupported"
+			null_module
 		);
 		use_cache = (fun() ->
 			!macro_enable_cache
 		);
 		format_string = (fun s p ->
-			Interp.exc_string "unsupported"
+			Common.format_string com s p (fun e p -> (e,p))
 		);
 		cast_or_unify = (fun t e p ->
 			Interp.exc_string "unsupported"
@@ -282,18 +300,11 @@ let make_macro_com_api com p =
 			com.info ~depth msg p
 		);
 		warning = (fun ?(depth=0) w msg p ->
-			Interp.exc_string "unsupported"
+			com.warning ~depth w [] msg p
 		);
 	}
 
 let make_macro_api ctx p =
-	let parse_expr_string s p inl =
-		typing_timer ctx false (fun() ->
-			match ParserEntry.parse_expr_string ctx.com.defines s p raise_typing_error inl with
-				| ParseSuccess(data,true,_) when inl -> data (* ignore errors when inline-parsing in display file *)
-				| ParseSuccess(data,_,_) -> data
-				| ParseError _ -> raise MacroApi.Invalid_expr)
-	in
 	let parse_metadata s p =
 		try
 			match ParserEntry.parse_string Grammar.parse_meta ctx.com.defines s null_pos raise_typing_error false with
@@ -384,7 +395,6 @@ let make_macro_api ctx p =
 				m
 			)
 		);
-		MacroApi.parse_string = parse_expr_string;
 		MacroApi.type_expr = (fun e ->
 			typing_timer ctx true (fun() -> type_expr ctx e WithType.value)
 		);
@@ -514,9 +524,6 @@ let make_macro_api ctx p =
 		MacroApi.current_module = (fun() ->
 			ctx.m.curmod
 		);
-		MacroApi.format_string = (fun s p ->
-			ctx.g.do_format_string ctx s p
-		);
 		MacroApi.cast_or_unify = (fun t e p ->
 			typing_timer ctx true (fun () ->
 				try

src/typing/typer.ml

@@ -729,104 +729,12 @@ and type_vars ctx vl p =
 		mk (TMeta((Meta.MergeBlock,[],p), e)) e.etype e.epos
 
 and format_string ctx s p =
-	let e = ref None in
-	let pmin = ref p.pmin in
-	let min = ref (p.pmin + 1) in
-	let add_expr (enext,p) len =
-		min := !min + len;
-		let enext = if ctx.in_display && DisplayPosition.display_position#enclosed_in p then
+	Common.format_string ctx.com s p (fun enext p ->
+		if ctx.in_display && DisplayPosition.display_position#enclosed_in p then
 			Display.ExprPreprocessing.process_expr ctx.com (enext,p)
 		else
 			enext,p
-		in
-		match !e with
-		| None -> e := Some enext
-		| Some prev ->
-			e := Some (EBinop (OpAdd,prev,enext),punion (pos prev) p)
-	in
-	let add enext len =
-		let p = { p with pmin = !min; pmax = !min + len } in
-		add_expr (enext,p) len
-	in
-	let add_sub start pos =
-		let len = pos - start in
-		if len > 0 || !e = None then add (EConst (String (String.sub s start len,SDoubleQuotes))) len
-	in
-	let len = String.length s in
-	let rec parse start pos =
-		if pos = len then add_sub start pos else
-		let c = String.unsafe_get s pos in
-		let pos = pos + 1 in
-		if c = '\'' then begin
-			incr pmin;
-			incr min;
-		end;
-		if c <> '$' || pos = len then parse start pos else
-		match String.unsafe_get s pos with
-		| '$' ->
-			(* double $ *)
-			add_sub start pos;
-			parse (pos + 1) (pos + 1)
-		| '{' ->
-			parse_group start pos '{' '}' "brace"
-		| 'a'..'z' | 'A'..'Z' | '_' ->
-			add_sub start (pos - 1);
-			incr min;
-			let rec loop i =
-				if i = len then i else
-				let c = String.unsafe_get s i in
-				match c with
-				| 'a'..'z' | 'A'..'Z' | '0'..'9' | '_' -> loop (i+1)
-				| _ -> i
-			in
-			let iend = loop (pos + 1) in
-			let len = iend - pos in
-			add (EConst (Ident (String.sub s pos len))) len;
-			parse (pos + len) (pos + len)
-		| _ ->
-			(* keep as-it *)
-			parse start pos
-	and parse_group start pos gopen gclose gname =
-		add_sub start (pos - 1);
-		let rec loop groups i =
-			if i = len then
-				match groups with
-				| [] -> die "" __LOC__
-				| g :: _ -> raise_typing_error ("Unclosed " ^ gname) { p with pmin = !pmin + g + 1; pmax = !pmin + g + 2 }
-			else
-				let c = String.unsafe_get s i in
-				if c = gopen then
-					loop (i :: groups) (i + 1)
-				else if c = gclose then begin
-					let groups = List.tl groups in
-					if groups = [] then i else loop groups (i + 1)
-				end else
-					loop groups (i + 1)
-		in
-		let send = loop [pos] (pos + 1) in
-		let slen = send - pos - 1 in
-		let scode = String.sub s (pos + 1) slen in
-		min := !min + 2;
-		begin
-			let e =
-				let ep = { p with pmin = !pmin + pos + 2; pmax = !pmin + send + 1 } in
-				let error msg pos =
-					if Lexer.string_is_whitespace scode then raise_typing_error "Expression cannot be empty" ep
-					else raise_typing_error msg pos
-				in
-				match ParserEntry.parse_expr_string ctx.com.defines scode ep error true with
-					| ParseSuccess(data,_,_) -> data
-					| ParseError(_,(msg,p),_) -> error (Parser.error_msg msg) p
-			in
-			add_expr e slen
-		end;
-		min := !min + 1;
-		parse (send + 1) (send + 1)
-	in
-	parse 0 0;
-	match !e with
-	| None -> die "" __LOC__
-	| Some e -> e
+	)
 
 and type_block ctx el with_type p =
 	let merge acc e = match e.eexpr with

std/haxe/macro/CompilationServer.hx

@@ -101,7 +101,9 @@ class CompilationServer {
 	**/
 	static public function setModuleCheckPolicy(pathFilters:Array<String>, policy:Array<ModuleCheckPolicy>, ?recursive = true,
 			?contextOptions:ContextOptions = NormalContext) {
-		@:privateAccess Compiler.load("server_add_module_check_policy", 4)(pathFilters, policy, recursive, contextOptions);
+		Context.onAfterInitMacros(() -> {
+			@:privateAccess Compiler.load("server_add_module_check_policy", 4)(pathFilters, policy, recursive, contextOptions);
+		});
 	}
 
 	/**

std/haxe/macro/Compiler.hx

@@ -94,7 +94,7 @@ class Compiler {
 		if (!ident.match(field))
 			throw "Invalid " + field;
 		#if (neko || eval)
-		load("type_patch", 4)(className, field, isStatic == true, null);
+		Context.onAfterInitMacros(() -> load("type_patch", 4)(className, field, isStatic == true, null));
 		#else
 		typePatch(className, field, isStatic == true, null);
 		#end
@@ -111,7 +111,7 @@ class Compiler {
 		if (!ident.match((field.charAt(0) == "$") ? field.substr(1) : field))
 			throw "Invalid " + field;
 		#if (neko || eval)
-		load("type_patch", 4)(className, field, isStatic == true, type);
+		Context.onAfterInitMacros(() -> load("type_patch", 4)(className, field, isStatic == true, type));
 		#else
 		typePatch(className, field, isStatic == true, type);
 		#end
@@ -128,7 +128,7 @@ class Compiler {
 		if (field != null && !ident.match(field))
 			throw "Invalid " + field;
 		#if (neko || eval)
-		load("meta_patch", 4)(meta, className, field, isStatic == true);
+		Context.onAfterInitMacros(() -> load("meta_patch", 4)(meta, className, field, isStatic == true));
 		#else
 		metaPatch(meta, className, field, isStatic == true);
 		#end
@@ -299,11 +299,7 @@ class Compiler {
 				Context.error('Package "$pack" was not found in any of class paths', Context.currentPos());
 		}
 
-		if (!Context.initMacrosDone()) {
-			Context.onAfterInitMacros(() -> include(pack, rec, ignore, classPaths, strict));
-		} else {
-			include(pack, rec, ignore, classPaths, strict);
-		}
+		Context.onAfterInitMacros(() -> include(pack, rec, ignore, classPaths, strict));
 	}
 
 	/**

std/haxe/macro/Context.hx

@@ -175,6 +175,7 @@ class Context {
 		Returns `null` if the current macro is not a `@:genericBuild` macro.
 	**/
 	public static function getCallArguments():Null<Array<Expr>> {
+		assertInitMacrosDone(false);
 		return load("get_call_arguments", 0)();
 	}
 
@@ -184,6 +185,7 @@ class Context {
 		If no such class exists, `null` is returned.
 	**/
 	public static function getLocalClass():Null<Type.Ref<Type.ClassType>> {
+		assertInitMacrosDone(false);
 		var l:Type = load("get_local_type", 0)();
 		if (l == null)
 			return null;
@@ -197,6 +199,7 @@ class Context {
 		Returns the current module path in/on which the macro was called.
 	**/
 	public static function getLocalModule():String {
+		assertInitMacrosDone(false);
 		return load("get_local_module", 0)();
 	}
 
@@ -206,6 +209,7 @@ class Context {
 		If no such type exists, `null` is returned.
 	**/
 	public static function getLocalType():Null<Type> {
+		assertInitMacrosDone(false);
 		return load("get_local_type", 0)();
 	}
 
@@ -215,6 +219,7 @@ class Context {
 		If no such method exists, `null` is returned.
 	**/
 	public static function getLocalMethod():Null<String> {
+		assertInitMacrosDone(false);
 		return load("get_local_method", 0)();
 	}
 
@@ -225,6 +230,7 @@ class Context {
 		Modifying the returned array has no effect on the compiler.
 	**/
 	public static function getLocalUsing():Array<Type.Ref<Type.ClassType>> {
+		assertInitMacrosDone(false);
 		return load("get_local_using", 0)();
 	}
 
@@ -234,6 +240,7 @@ class Context {
 		Modifying the returned array has no effect on the compiler.
 	**/
 	public static function getLocalImports():Array<ImportExpr> {
+		assertInitMacrosDone(false);
 		return load("get_local_imports", 0)();
 	}
 
@@ -248,6 +255,7 @@ class Context {
 	**/
 	@:deprecated("Use Context.getLocalTVars() instead")
 	public static function getLocalVars():Map<String, Type> {
+		assertInitMacrosDone(false);
 		return load("local_vars", 1)(false);
 	}
 
@@ -256,6 +264,7 @@ class Context {
 		of `Type`.
 	**/
 	public static function getLocalTVars():Map<String, Type.TVar> {
+		assertInitMacrosDone(false);
 		return load("local_vars", 1)(true);
 	}
 
@@ -459,8 +468,11 @@ class Context {
 		actual typing.
 	**/
 	public static function onAfterInitMacros(callback:Void->Void):Void {
-		assertInitMacro();
-		load("on_after_init_macros", 1)(callback);
+		if (Context.initMacrosDone()) {
+			callback();
+		} else {
+			load("on_after_init_macros", 1)(callback);
+		}
 	}
 
 	/**
@@ -534,6 +546,7 @@ class Context {
 		build any type or trigger macros.
 	**/
 	public static function resolveComplexType(t:ComplexType, p:Position):ComplexType {
+		assertInitMacrosDone(false);
 		return load("resolve_complex_type", 2)(t, p);
 	}
 
@@ -632,6 +645,7 @@ class Context {
 		This is only defined for `@:build/@:autoBuild` macros.
 	**/
 	public static function getBuildFields():Array<Field> {
+		assertInitMacrosDone(false);
 		return load("get_build_fields", 0)();
 	}
 
@@ -773,8 +787,7 @@ class Context {
 		run your code once typer is ready to be used.
 	**/
 	public static function registerModuleDependency(modulePath:String, externFile:String) {
-		assertInitMacrosDone();
-		load("register_module_dependency", 2)(modulePath, externFile);
+		onAfterInitMacros(() -> load("register_module_dependency", 2)(modulePath, externFile));
 	}
 
 	/**
@@ -861,7 +874,7 @@ class Context {
 	}
 
 	private static function sExpr(e:TypedExpr, pretty:Bool):String {
-		return haxe.macro.Context.load("s_expr", 2)(e, pretty);
+		return load("s_expr", 2)(e, pretty);
 	}
 
 	@:allow(haxe.macro.Compiler)
@@ -876,20 +889,19 @@ class Context {
 		}
 	}
 
+	@:allow(haxe.macro.Compiler)
 	private static function assertInitMacrosDone(includeSuggestion = true):Void {
-		#if haxe_next
 		if (!initMacrosDone()) {
 			var stack = getMacroStack();
 			var suggestion = includeSuggestion
 				? "\nUse `Context.onAfterInitMacros` to register a callback to run when context is ready."
 				: "";
 
-			warning(
+			fatalError(
 				"Cannot use this API from initialization macros." + suggestion,
 				if (stack.length > 2) stack[2] else currentPos()
 			);
 		}
-		#end
 	}
 	#end
 }

tests/misc/projects/Issue10844/user-defined-define-json-fail.hxml.stderr

@@ -1,3 +1,3 @@
 (unknown) : Uncaught exception Could not read file define.jsno
-$$normPath(::std::)/haxe/macro/Compiler.hx:509: characters 11-39 : Called from here
+$$normPath(::std::)/haxe/macro/Compiler.hx:505: characters 11-39 : Called from here
 (unknown) : Called from here

tests/misc/projects/Issue10844/user-defined-meta-json-fail.hxml.stderr

@@ -1,3 +1,3 @@
 (unknown) : Uncaught exception Could not read file meta.jsno
-$$normPath(::std::)/haxe/macro/Compiler.hx:498: characters 11-39 : Called from here
+$$normPath(::std::)/haxe/macro/Compiler.hx:494: characters 11-39 : Called from here
 (unknown) : Called from here

tests/misc/projects/Issue10844/user-defined-meta-json-indent-fail.hxml.stderr

@@ -1,3 +1,3 @@
 (unknown) : Uncaught exception Could not read file meta.jsno
-  $$normPath(::std::)/haxe/macro/Compiler.hx:498: characters 11-39 : Called from here
+  $$normPath(::std::)/haxe/macro/Compiler.hx:494: characters 11-39 : Called from here
   (unknown) : Called from here

tests/misc/projects/Issue10844/user-defined-meta-json-pretty-fail.hxml.stderr

@@ -2,9 +2,9 @@
 
    | Uncaught exception Could not read file meta.jsno
 
-    ->  $$normPath(::std::)/haxe/macro/Compiler.hx:498: characters 11-39
+    ->  $$normPath(::std::)/haxe/macro/Compiler.hx:494: characters 11-39
 
-    498 |   var f = sys.io.File.getContent(path);
+    494 |   var f = sys.io.File.getContent(path);
         |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
         | Called from here
 

tests/misc/projects/Issue3500/Main.hx

@@ -1,18 +1,21 @@
 class Main {
 	static function test() {
-		var t = haxe.macro.Context.getType("A");
 		function fail(msg) {
 			Sys.println(msg);
 			Sys.exit(1);
 		}
-		switch (t) {
-			case TAbstract(a, _):
-				var hasTestMeta = Lambda.exists(a.get().impl.get().meta.get(), function(m) return m.name == ":test");
-				if (!hasTestMeta) {
-					fail("Abstract implementation class has no @:test metadata");
-				}
-			case _:
-				fail("Should be abstract");
-		}
+
+		haxe.macro.Context.onAfterInitMacros(() -> {
+			var t = haxe.macro.Context.getType("A");
+			switch (t) {
+				case TAbstract(a, _):
+					var hasTestMeta = Lambda.exists(a.get().impl.get().meta.get(), function(m) return m.name == ":test");
+					if (!hasTestMeta) {
+						fail("Abstract implementation class has no @:test metadata");
+					}
+				case _:
+					fail("Should be abstract");
+			}
+		});
 	}
-}
+}

tests/misc/projects/Issue5002/Main3.hx

@@ -0,0 +1,24 @@
+import haxe.macro.Context;
+
+class Main3 {
+	static function init() {
+		Context.onAfterInitMacros(() -> {
+			function defineType(name, kind) {
+				Context.defineType({
+					pack: [],
+					name: name,
+					pos: (macro 0).pos,
+					kind: kind,
+					fields: []
+				});
+			}
+
+			defineType("lowercase", TDClass());
+
+			defineType("0_class", TDClass());
+			defineType("0_enum", TDEnum);
+			defineType("0_struct", TDStructure);
+			defineType("0_abstract", TDAbstract(TPath({pack: [], name: "Int"})));
+		});
+	}
+}

tests/misc/projects/Issue5002/compile3-fail.hxml.stderr

@@ -0,0 +1,6 @@
+Main3.hx:10: characters 18-19 : Module "lowercase" does not have a valid name. Module name should start with an uppercase letter: "lowercase"
+Main3.hx:10: characters 18-19 : Module "0_class" does not have a valid name. "0_class" is not a valid module name.
+Main3.hx:10: characters 18-19 : Module "0_enum" does not have a valid name. "0_enum" is not a valid module name.
+Main3.hx:10: characters 18-19 : Module "0_struct" does not have a valid name. "0_struct" is not a valid module name.
+Main3.hx:10: characters 18-19 : "0_abstract_Impl_" is not a valid type name.
+Main3.hx:10: characters 18-19 : Module "0_abstract" does not have a valid name. "0_abstract" is not a valid module name.

tests/misc/projects/Issue7871/Macro.hx

@@ -2,14 +2,11 @@ import haxe.macro.Compiler;
 import haxe.macro.Context;
 
 function init() {
-	var e = macro 42;
-	Context.typeof(e);
 	if (Context.defined("haxe")) Context.warning("ok", (macro 0).pos);
-	Compiler.define("foo", "foo");
 
 	Context.onAfterInitMacros(() -> {
 		Context.warning("after init 1", (macro 0).pos);
-		Compiler.define("bar", "bar");
+		var e = macro 42;
 		Context.typeof(e);
 	});
 	Compiler.include("hax.ds", true, true);
@@ -17,3 +14,13 @@ function init() {
 		Context.warning("after init 2", (macro 0).pos);
 	});
 }
+
+function init_fail() {
+	var e = macro 42;
+	Context.typeof(e);
+	Compiler.define("foo", "foo");
+
+	Context.onAfterInitMacros(() -> {
+		Compiler.define("bar", "bar");
+	});
+}

tests/misc/projects/Issue7871/compile-1-fail.hxml

@@ -0,0 +1 @@
+--macro Macro.init()

tests/misc/projects/Issue7871/compile-1-fail.hxml.stderr

@@ -0,0 +1,4 @@
+Macro.hx:5: characters 60-61 : Warning : ok
+Macro.hx:8: characters 42-43 : Warning : after init 1
+Package "hax.ds" was not found in any of class paths
+Macro.hx:14: characters 42-43 : Warning : after init 2

tests/misc/projects/Issue7871/compile-fail.hxml

@@ -1 +1 @@
---macro Macro.init()
+--macro Macro.init_fail()

tests/misc/projects/Issue7871/compile-fail.hxml.stderr

@@ -1,5 +1,2 @@
-Macro.hx:7: characters 60-61 : Warning : ok
-Macro.hx:11: characters 42-43 : Warning : after init 1
-Macro.hx:12: characters 3-32 : Warning : This API should only be used from initialization macros.
-Package "hax.ds" was not found in any of class paths
-Macro.hx:17: characters 42-43 : Warning : after init 2
+Macro.hx:20: characters 2-19 : Cannot use this API from initialization macros.
+Macro.hx:20: characters 2-19 : ... Use `Context.onAfterInitMacros` to register a callback to run when context is ready.

tests/misc/projects/Issue7871/compile-next-fail.hxml

@@ -1,2 +0,0 @@
-compile-fail.hxml
--D haxe-next

tests/misc/projects/Issue7871/compile-next-fail.hxml.stderr

@@ -1,7 +0,0 @@
-Macro.hx:6: characters 2-19 : Warning : Cannot use this API from initialization macros.
-Macro.hx:6: characters 2-19 : ... Use `Context.onAfterInitMacros` to register a callback to run when context is ready.
-Macro.hx:7: characters 60-61 : Warning : ok
-Macro.hx:11: characters 42-43 : Warning : after init 1
-Macro.hx:12: characters 3-32 : Warning : This API should only be used from initialization macros.
-Package "hax.ds" was not found in any of class paths
-Macro.hx:17: characters 42-43 : Warning : after init 2

tests/misc/projects/Issue8019/Macro.hx

@@ -3,32 +3,34 @@ import haxe.macro.Context;
 
 class Macro {
 	static function init() {
-		function defineType(pack, kind) {
-			Context.defineType({
-				pos: (macro 0).pos,
-				pack: pack,
-				name: 'Test',
-				kind: kind,
-				fields: []
-			});
-		}
+		Context.onAfterInitMacros(() -> {
+			function defineType(pack, kind) {
+				Context.defineType({
+					pos: (macro 0).pos,
+					pack: pack,
+					name: 'Test',
+					kind: kind,
+					fields: []
+				});
+			}
 
-		function defineClass(pack) {
-			defineType(pack, TDClass());
-		}
+			function defineClass(pack) {
+				defineType(pack, TDClass());
+			}
 
-		defineClass(["Module"]);
-		defineClass(["pack", "Module"]);
+			defineClass(["Module"]);
+			defineClass(["pack", "Module"]);
 
-		defineClass([""]);
-		defineClass(["\n"]);
-		defineClass(["pack\n"]);
-		defineClass(["pack~"]);
-		defineClass(["Foo", "Bar"]);
-		
-		defineClass(["0_class"]);
-		defineType(["0_enum"], TDEnum);
-		defineType(["0_structure"], TDStructure);
-		defineType(["0_abstract"], TDAbstract(TPath({pack: [], name: "Int"})));
+			defineClass([""]);
+			defineClass(["\n"]);
+			defineClass(["pack\n"]);
+			defineClass(["pack~"]);
+			defineClass(["Foo", "Bar"]);
+
+			defineClass(["0_class"]);
+			defineType(["0_enum"], TDEnum);
+			defineType(["0_structure"], TDStructure);
+			defineType(["0_abstract"], TDAbstract(TPath({pack: [], name: "Int"})));
+		});
 	}
 }

tests/misc/projects/Issue8019/Macro2.hx

@@ -2,20 +2,22 @@ import haxe.macro.Context;
 
 class Macro2 {
 	static function init() {
-		function defineModule(name:String) {
-			Context.defineModule(name, [{
-				pos: (macro 0).pos,
-				pack: [],
-				name: name,
-				kind: TDClass(),
-				fields: []
-			}]);
-		}
+		Context.onAfterInitMacros(() -> {
+			function defineModule(name:String) {
+				Context.defineModule(name, [{
+					pos: (macro 0).pos,
+					pack: [],
+					name: name,
+					kind: TDClass(),
+					fields: []
+				}]);
+			}
 
-		defineModule("Valid");
+			defineModule("Valid");
 
-		defineModule("");
-		defineModule("0");
-		defineModule("Type+");
+			defineModule("");
+			defineModule("0");
+			defineModule("Type+");
+		});
 	}
 }

tests/misc/projects/Issue8019/compile-fail.hxml.stderr

@@ -1,18 +1,18 @@
-Macro.hx:8: characters 17-18 : "" is not a valid package name:
-Macro.hx:8: characters 17-18 : Package name must not be empty
-Macro.hx:8: characters 17-18 : "\n" is not a valid package name:
-Macro.hx:8: characters 17-18 : Package name must start with a lowercase letter
-Macro.hx:8: characters 17-18 : "pack\n" is not a valid package name:
-Macro.hx:8: characters 17-18 : Invalid character: \n
-Macro.hx:8: characters 17-18 : "pack~" is not a valid package name:
-Macro.hx:8: characters 17-18 : Invalid character: ~
-Macro.hx:8: characters 17-18 : "Foo" is not a valid package name:
-Macro.hx:8: characters 17-18 : Package name must start with a lowercase letter
-Macro.hx:8: characters 17-18 : "0_class" is not a valid package name:
-Macro.hx:8: characters 17-18 : Package name must start with a lowercase letter
-Macro.hx:8: characters 17-18 : "0_enum" is not a valid package name:
-Macro.hx:8: characters 17-18 : Package name must start with a lowercase letter
-Macro.hx:8: characters 17-18 : "0_structure" is not a valid package name:
-Macro.hx:8: characters 17-18 : Package name must start with a lowercase letter
-Macro.hx:8: characters 17-18 : "0_abstract" is not a valid package name:
-Macro.hx:8: characters 17-18 : Package name must start with a lowercase letter
+Macro.hx:9: characters 18-19 : "" is not a valid package name:
+Macro.hx:9: characters 18-19 : Package name must not be empty
+Macro.hx:9: characters 18-19 : "\n" is not a valid package name:
+Macro.hx:9: characters 18-19 : Package name must start with a lowercase letter
+Macro.hx:9: characters 18-19 : "pack\n" is not a valid package name:
+Macro.hx:9: characters 18-19 : Invalid character: \n
+Macro.hx:9: characters 18-19 : "pack~" is not a valid package name:
+Macro.hx:9: characters 18-19 : Invalid character: ~
+Macro.hx:9: characters 18-19 : "Foo" is not a valid package name:
+Macro.hx:9: characters 18-19 : Package name must start with a lowercase letter
+Macro.hx:9: characters 18-19 : "0_class" is not a valid package name:
+Macro.hx:9: characters 18-19 : Package name must start with a lowercase letter
+Macro.hx:9: characters 18-19 : "0_enum" is not a valid package name:
+Macro.hx:9: characters 18-19 : Package name must start with a lowercase letter
+Macro.hx:9: characters 18-19 : "0_structure" is not a valid package name:
+Macro.hx:9: characters 18-19 : Package name must start with a lowercase letter
+Macro.hx:9: characters 18-19 : "0_abstract" is not a valid package name:
+Macro.hx:9: characters 18-19 : Package name must start with a lowercase letter

tests/misc/projects/Issue8019/compile3-fail.hxml.stderr

@@ -1,3 +1,3 @@
-Macro2.hx:7: characters 17-18 : Module "" does not have a valid name. Module name must not be empty.
-Macro2.hx:7: characters 17-18 : Module "0" does not have a valid name. "0" is not a valid module name.
-Macro2.hx:7: characters 17-18 : Module "Type+" does not have a valid name. "Type+" is not a valid module name.
+Macro2.hx:8: characters 18-19 : Module "" does not have a valid name. Module name must not be empty.
+Macro2.hx:8: characters 18-19 : Module "0" does not have a valid name. "0" is not a valid module name.
+Macro2.hx:8: characters 18-19 : Module "Type+" does not have a valid name. "Type+" is not a valid module name.

tests/misc/projects/issue5002/Main3.hx

@@ -1,22 +0,0 @@
-import haxe.macro.Context;
-
-class Main3 {
-	static function init() {
-		function defineType(name, kind) {
-			Context.defineType({
-				pack: [],
-				name: name,
-				pos: (macro 0).pos,
-				kind: kind,
-				fields: []
-			});
-		}
-
-		defineType("lowercase", TDClass());
-
-		defineType("0_class", TDClass());
-		defineType("0_enum", TDEnum);
-		defineType("0_struct", TDStructure);
-		defineType("0_abstract", TDAbstract(TPath({pack: [], name: "Int"})));
-	}
-}

tests/misc/projects/issue5002/compile3-fail.hxml.stderr

@@ -1,6 +0,0 @@
-Main3.hx:9: characters 17-18 : Module "lowercase" does not have a valid name. Module name should start with an uppercase letter: "lowercase"
-Main3.hx:9: characters 17-18 : Module "0_class" does not have a valid name. "0_class" is not a valid module name.
-Main3.hx:9: characters 17-18 : Module "0_enum" does not have a valid name. "0_enum" is not a valid module name.
-Main3.hx:9: characters 17-18 : Module "0_struct" does not have a valid name. "0_struct" is not a valid module name.
-Main3.hx:9: characters 17-18 : "0_abstract_Impl_" is not a valid type name.
-Main3.hx:9: characters 17-18 : Module "0_abstract" does not have a valid name. "0_abstract" is not a valid module name.

tests/misc/projects/recursive-callbacks/compile.hxml.stderr

@@ -1,5 +1,4 @@
 Main.hx:5: characters 20-24 : Warning : 1
-Main.hx:9: lines 9-11 : Warning : This API should only be used from initialization macros.
 Main.hx:5: characters 20-24 : Warning : 2
 Main.hx:5: characters 20-24 : Warning : 3
 Main.hx:5: characters 20-24 : Warning : 4

tests/optimization/src/Macro.hx

@@ -15,20 +15,22 @@ class Macro {
 		if (classes.length == 0) {
 			Context.onAfterGenerate(run);
 		}
-		if (className.charAt(0).toLowerCase() == className.charAt(0)) {
-			var dir = sys.FileSystem.readDirectory("src/" +className.replace(".", "/"));
-			for (file in dir) {
-				if (file.endsWith(".hx")) {
-					var name = className + "." + file.substr(0, -3);
-					Context.getType(name);
-					classes.push(name);
+		Context.onAfterInitMacros(() -> {
+			if (className.charAt(0).toLowerCase() == className.charAt(0)) {
+				var dir = sys.FileSystem.readDirectory("src/" +className.replace(".", "/"));
+				for (file in dir) {
+					if (file.endsWith(".hx")) {
+						var name = className + "." + file.substr(0, -3);
+						Context.getType(name);
+						classes.push(name);
 
+					}
 				}
+			} else {
+				Context.getType(className);
+				classes.push(className);
 			}
-		} else {
-			Context.getType(className);
-			classes.push(className);
-		}
+		});
 	}
 
 	static function run() {

tests/server/src/TestCase.hx

@@ -36,6 +36,22 @@ class TestCase implements ITest {
 
 	public function new() {}
 
+	function debugMessages(?pos:PosInfos) {
+		for (m in messages) haxe.Log.trace(m, pos);
+	}
+
+	function debugErrorMessages(?pos:PosInfos) {
+		for (m in errorMessages) haxe.Log.trace(m, pos);
+	}
+
+	function messagesWith(s:String, ?pos:PosInfos) {
+		for (m in messages) if (m.contains(s)) haxe.Log.trace(m, pos);
+	}
+
+	function errorMessagesWith(s:String, ?pos:PosInfos) {
+		for (m in errorMessages) if (m.contains(s)) haxe.Log.trace(m, pos);
+	}
+
 	static public function printSkipReason(ddr:SkipReason) {
 		return switch (ddr) {
 			case DependencyDirty(path): 'DependencyDirty $path';

tests/sourcemaps/src/Validator.hx

@@ -98,7 +98,7 @@ class Validator {
 		} else if(Context.defined('php')) {
 			Php;
 		} else {
-			throw new InvalidTargetException('Current target is not supported', haxe.macro.PositionTools.here());
+			throw new InvalidTargetException('Current target is not supported', (macro 0).pos);
 		}
 	}