[macro] Avoid polluting lexer cache with Context.parseInlineString (#11920)

* [macro] Do not pollute lexer cache with Context.parse() calls

* [macro] Expose Context.loadLexerLines(file,content) instead

* [tests] Add test for 3188

* Rename to registerFileContents

src/macro/macroApi.ml

@@ -33,6 +33,7 @@ type 'value compiler_api = {
 	after_generate : (unit -> unit) -> unit;
 	on_type_not_found : (string -> 'value) -> unit;
 	parse_string : string -> Globals.pos -> bool -> Ast.expr;
+	register_file_contents : string -> string -> unit;
 	parse : 'a . ((Ast.token * Globals.pos) Stream.t -> 'a) -> string -> 'a;
 	type_expr : Ast.expr -> Type.texpr;
 	resolve_type  : Ast.complex_type -> Globals.pos -> t;
@@ -1908,6 +1909,12 @@ let macro_api ccom get_api =
 			if s = "" then (get_api()).exc_string "Invalid expression";
 			encode_expr ((get_api()).parse_string s (decode_pos p) (decode_bool b))
 		);
+		"register_file_contents", vfun2 (fun f c ->
+			let f = decode_string f in
+			let content = decode_string c in
+			(get_api()).register_file_contents f content;
+			vnull
+		);
 		"make_expr", vfun2 (fun v p ->
 			encode_expr (value_to_expr v (decode_pos p))
 		);

src/syntax/lexer.ml

@@ -210,27 +210,19 @@ let find_line p f =
 		loop 0 (Array.length f.lalines)
 
 (* resolve a position within a non-haxe file by counting newlines *)
-let resolve_pos file =
-	let ch = open_in_bin file in
-	let f = make_file file in
+let resolve_pos f next skip =
 	let rec loop p =
 		let inc i () =
 			f.lline <- f.lline + 1;
 			f.llines <- (p + i,f.lline) :: f.llines;
 			i
 		in
-		let i = match input_char ch with
+		let i = match next() with
 			| '\n' -> inc 1
 			| '\r' ->
-				ignore(input_char ch);
+				skip 1;
 				inc 2
 			| c -> (fun () ->
-				let rec skip n =
-					if n > 0 then begin
-						ignore(input_char ch);
-						skip (n - 1)
-					end
-				in
 				let code = int_of_char c in
 				if code < 0xC0 then ()
 				else if code < 0xE0 then skip 1
@@ -241,8 +233,35 @@ let resolve_pos file =
 		in
 		loop (p + i())
 	in
+	loop 0
+
+let resolve_file_content_pos file content =
+	let f = make_file file in
+	let i = ref 0 in
+	let next () =
+		try
+			let ret = String.get content !i in
+			incr i;
+			ret
+		with Invalid_argument _ -> raise End_of_file
+	in
+	let skip n =
+		i := !i + n
+	in
+	try resolve_pos f next skip with End_of_file -> f
+
+let resolve_file_pos file =
+	let ch = open_in_bin file in
+	let f = make_file file in
+	let next () = input_char ch in
+	let rec skip n =
+		if n > 0 then begin
+			ignore(next ());
+			skip (n - 1)
+		end
+	in
 	try
-		loop 0
+		resolve_pos f next skip
 	with End_of_file ->
 		close_in ch;
 		f
@@ -252,7 +271,7 @@ let find_file file =
 		Hashtbl.find all_files file
 	with Not_found ->
 		try
-			let f = resolve_pos file in
+			let f = resolve_file_pos file in
 			Hashtbl.add all_files file f;
 			f
 		with Sys_error _ ->

src/syntax/parserEntry.ml

@@ -401,7 +401,7 @@ let parse_string entry com s p error inlined =
 	syntax_errors := [];
 	let restore() =
 		(match old_file with
-		| None -> ()
+		| None -> Hashtbl.remove Lexer.all_files p.pfile
 		| Some f -> Hashtbl.replace Lexer.all_files p.pfile f);
 		if not inlined then begin
 			display_position#set old_display;

src/typing/macroContext.ml

@@ -196,6 +196,10 @@ let make_macro_com_api com mcom p =
 			| ParseSuccess(r,_,_) -> r
 			| ParseError(_,(msg,p),_) -> Parser.error msg p
 		);
+		register_file_contents = (fun file content ->
+			let f = Lexer.resolve_file_content_pos file content in
+			Hashtbl.add Lexer.all_files file f;
+		);
 		type_expr = (fun e ->
 			Interp.exc_string "unsupported"
 		);

std/haxe/macro/Context.hx

@@ -395,6 +395,15 @@ class Context {
 		return load("do_parse", 3)(expr, pos, true);
 	}
 
+	/**
+		Parse file content for newlines, allowing positions to be resolved
+		properly inside that file later on (using `Context.parseInlineString`
+		for example). Works with both real and virtual files.
+	**/
+	public static function registerFileContents(file:String, content:String):Void {
+		load("register_file_contents", 2)(file, content);
+	}
+
 	/**
 		Builds an expression from `v`.
 

tests/misc/projects/Issue3188/Main.hx

@@ -0,0 +1,7 @@
+import haxe.macro.Context;
+
+macro function init():Void {
+	Context.parseInlineString("null", Context.makePosition( { file: "external.txt", min: 14, max: 18, }));
+	Context.warning("Correct line number", Context.makePosition( { file: "external.txt", min: 24, max: 27, }));
+	Context.warning("Correct line number", Context.makePosition( { file: "./external.txt", min: 24, max: 27, }));
+}

tests/misc/projects/Issue3188/compile.hxml

@@ -0,0 +1,3 @@
+--macro Main.init()
+-D message.reporting=pretty
+-D message.no-color

tests/misc/projects/Issue3188/compile.hxml.stderr

@@ -0,0 +1,12 @@
+[WARNING] external.txt:3: characters 5-8
+
+ 3 | 012345678
+   |     ^^^
+   | Correct line number
+
+[WARNING] ./external.txt:3: characters 5-8
+
+ 3 | 012345678
+   |     ^^^
+   | Correct line number
+

tests/misc/projects/Issue3188/external.txt

@@ -0,0 +1,10 @@
+012345678
+012345678
+012345678
+012345678
+012345678
+012345678
+012345678
+012345678
+012345678
+012345678

tests/unit/src/unit/issues/Issue3387.hx

@@ -6,7 +6,8 @@ class Issue3387 extends unit.Test {
 	}
 
 	static macro function getPos(source:String, file:String) {
+		haxe.macro.Context.registerFileContents(file, source);
 		var e = haxe.macro.Context.parseInlineString(source, haxe.macro.Context.makePosition({min: 0, max: 0, file: file}));
 		return macro $v{Std.string(e.pos)};
 	}
-}
+}