handle utf8 for position creation in non-hx files

fixes #8533

src/syntax/lexer.ml

@@ -198,7 +198,20 @@ let resolve_pos file =
 			| '\r' ->
 				ignore(input_char ch);
 				inc 2
-			| _ -> fun () -> 1
+			| c -> (fun () ->
+				let rec skip n =
+					if n > 0 then begin
+						ignore(input_char ch);
+						skip (n - 1)
+					end
+				in
+				let code = int_of_char c in
+				if code < 0xC0 then ()
+				else if code < 0xE0 then skip 1
+				else if code < 0xF0 then skip 2
+				else skip 3;
+				1
+			)
 		in
 		loop (p + i())
 	in

tests/misc/projects/Issue8533/Main.hx

@@ -0,0 +1,36 @@
+import haxe.display.Position;
+import haxe.macro.Context;
+import haxe.macro.PositionTools;
+
+class Main {
+	static function main() {
+		var s = "待到来年九月八";
+		posChecking("我花开后百花杀");
+	}
+
+	macro static function posChecking(expr) {
+		var info = PositionTools.getInfos(expr.pos);
+		var txtPos = PositionTools.make({
+			min: info.min,
+			max: info.max,
+			file: info.file + '.txt'
+		});
+
+		var hxLoc = PositionTools.toLocation(expr.pos);
+		var txtLoc = PositionTools.toLocation(txtPos);
+		if(!equalRanges(hxLoc.range, txtLoc.range)) {
+			var msg = 'position numbers should be the same';
+			Context.warning(msg, expr.pos);
+			Context.error(msg, txtPos);
+		}
+		return expr;
+	}
+
+	static function equalRanges(r1:Range, r2:Range):Bool {
+		return equalPositions(r1.start, r2.start) && equalPositions(r1.end, r2.end);
+	}
+
+	static function equalPositions(p1:Position, p2:Position) {
+		return p1.line == p2.line && p1.character == p2.character;
+	}
+}

tests/misc/projects/Issue8533/Main.hx.txt

@@ -0,0 +1,36 @@
+import haxe.display.Position;
+import haxe.macro.Context;
+import haxe.macro.PositionTools;
+
+class Main {
+	static function main() {
+		var s = "待到来年九月八";
+		posChecking("我花开后百花杀");
+	}
+
+	macro static function posChecking(expr) {
+		var info = PositionTools.getInfos(expr.pos);
+		var txtPos = PositionTools.make({
+			min: info.min,
+			max: info.max,
+			file: info.file + '.txt'
+		});
+
+		var hxLoc = PositionTools.toLocation(expr.pos);
+		var txtLoc = PositionTools.toLocation(txtPos);
+		if(!equalRanges(hxLoc.range, txtLoc.range)) {
+			var msg = 'position numbers should be equal';
+			Context.warning(msg, expr.pos);
+			Context.error(msg, txtPos);
+		}
+		return expr;
+	}
+
+	static function equalRanges(r1:Range, r2:Range):Bool {
+		return equalPositions(r1.start, r2.start) && equalPositions(r1.end, r2.end);
+	}
+
+	static function equalPositions(p1:Position, p2:Position) {
+		return p1.line == p2.line && p1.character == p2.character;
+	}
+}

tests/misc/projects/Issue8533/compile.hxml

@@ -0,0 +1 @@
+-main Main