Fix losing X11 window normal size hint properties

This was caused by `XSetWMNormalHints` being called multiple times, each time with different values. Calling the method replaces the old data completely, resulting in some of the settings being lost.

Since the method was called 3 times before the window was mapped, this resulted in the position hint being lost and the window always getting opened at a position determined by the WM.

platform/linuxbsd/display_server_x11.cpp

@@ -881,6 +881,46 @@ void DisplayServerX11::window_set_transient(WindowID p_window, WindowID p_parent
 	}
 }
 
+// Helper method. Assumes that the window id has already been checked and exists.
+void DisplayServerX11::_update_size_hints(WindowID p_window) {
+	WindowData &wd = windows[p_window];
+	WindowMode window_mode = window_get_mode(p_window);
+	XSizeHints *xsh = XAllocSizeHints();
+
+	// Always set the position and size hints - they should be synchronized with the actual values after the window is mapped anyway
+	xsh->flags |= PPosition | PSize;
+	xsh->x = wd.position.x;
+	xsh->y = wd.position.y;
+	xsh->width = wd.size.width;
+	xsh->height = wd.size.height;
+
+	if (window_mode == WINDOW_MODE_FULLSCREEN) {
+		// Do not set any other hints to prevent the window manager from ignoring the fullscreen flags
+	} else if (window_get_flag(WINDOW_FLAG_RESIZE_DISABLED, p_window)) {
+		// If resizing is disabled, use the forced size
+		xsh->flags |= PMinSize | PMaxSize;
+		xsh->min_width = wd.size.x;
+		xsh->max_width = wd.size.x;
+		xsh->min_height = wd.size.y;
+		xsh->max_height = wd.size.y;
+	} else {
+		// Otherwise, just respect min_size and max_size
+		if (wd.min_size != Size2i()) {
+			xsh->flags |= PMinSize;
+			xsh->min_width = wd.min_size.x;
+			xsh->min_height = wd.min_size.y;
+		}
+		if (wd.max_size != Size2i()) {
+			xsh->flags |= PMaxSize;
+			xsh->max_width = wd.max_size.x;
+			xsh->max_height = wd.max_size.y;
+		}
+	}
+
+	XSetWMNormalHints(x11_display, wd.x11_window, xsh);
+	XFree(xsh);
+}
+
 Point2i DisplayServerX11::window_get_position(WindowID p_window) const {
 	_THREAD_SAFE_METHOD_
 
@@ -934,25 +974,8 @@ void DisplayServerX11::window_set_max_size(const Size2i p_size, WindowID p_windo
 	}
 	wd.max_size = p_size;
 
-	if (!window_get_flag(WINDOW_FLAG_RESIZE_DISABLED, p_window)) {
-		XSizeHints *xsh;
-		xsh = XAllocSizeHints();
-		xsh->flags = 0L;
-		if (wd.min_size != Size2i()) {
-			xsh->flags |= PMinSize;
-			xsh->min_width = wd.min_size.x;
-			xsh->min_height = wd.min_size.y;
-		}
-		if (wd.max_size != Size2i()) {
-			xsh->flags |= PMaxSize;
-			xsh->max_width = wd.max_size.x;
-			xsh->max_height = wd.max_size.y;
-		}
-		XSetWMNormalHints(x11_display, wd.x11_window, xsh);
-		XFree(xsh);
-
-		XFlush(x11_display);
-	}
+	_update_size_hints(p_window);
+	XFlush(x11_display);
 }
 
 Size2i DisplayServerX11::window_get_max_size(WindowID p_window) const {
@@ -976,25 +999,8 @@ void DisplayServerX11::window_set_min_size(const Size2i p_size, WindowID p_windo
 	}
 	wd.min_size = p_size;
 
-	if (!window_get_flag(WINDOW_FLAG_RESIZE_DISABLED, p_window)) {
-		XSizeHints *xsh;
-		xsh = XAllocSizeHints();
-		xsh->flags = 0L;
-		if (wd.min_size != Size2i()) {
-			xsh->flags |= PMinSize;
-			xsh->min_width = wd.min_size.x;
-			xsh->min_height = wd.min_size.y;
-		}
-		if (wd.max_size != Size2i()) {
-			xsh->flags |= PMaxSize;
-			xsh->max_width = wd.max_size.x;
-			xsh->max_height = wd.max_size.y;
-		}
-		XSetWMNormalHints(x11_display, wd.x11_window, xsh);
-		XFree(xsh);
-
-		XFlush(x11_display);
-	}
+	_update_size_hints(p_window);
+	XFlush(x11_display);
 }
 
 Size2i DisplayServerX11::window_get_min_size(WindowID p_window) const {
@@ -1027,37 +1033,15 @@ void DisplayServerX11::window_set_size(const Size2i p_size, WindowID p_window) {
 	int old_w = xwa.width;
 	int old_h = xwa.height;
 
-	// If window resizable is disabled we need to update the attributes first
-	XSizeHints *xsh;
-	xsh = XAllocSizeHints();
-	if (!window_get_flag(WINDOW_FLAG_RESIZE_DISABLED, p_window)) {
-		xsh->flags = PMinSize | PMaxSize;
-		xsh->min_width = size.x;
-		xsh->max_width = size.x;
-		xsh->min_height = size.y;
-		xsh->max_height = size.y;
-	} else {
-		xsh->flags = 0L;
-		if (wd.min_size != Size2i()) {
-			xsh->flags |= PMinSize;
-			xsh->min_width = wd.min_size.x;
-			xsh->min_height = wd.min_size.y;
-		}
-		if (wd.max_size != Size2i()) {
-			xsh->flags |= PMaxSize;
-			xsh->max_width = wd.max_size.x;
-			xsh->max_height = wd.max_size.y;
-		}
-	}
-	XSetWMNormalHints(x11_display, wd.x11_window, xsh);
-	XFree(xsh);
+	// Update our videomode width and height
+	wd.size = size;
+
+	// Update the size hints first to make sure the window size can be set
+	_update_size_hints(p_window);
 
 	// Resize the window
 	XResizeWindow(x11_display, wd.x11_window, size.x, size.y);
 
-	// Update our videomode width and height
-	wd.size = size;
-
 	for (int timeout = 0; timeout < 50; ++timeout) {
 		XSync(x11_display, False);
 		XGetWindowAttributes(x11_display, wd.x11_window, &xwa);
@@ -1213,14 +1197,9 @@ void DisplayServerX11::_set_wm_fullscreen(WindowID p_window, bool p_enabled) {
 		XChangeProperty(x11_display, wd.x11_window, property, property, 32, PropModeReplace, (unsigned char *)&hints, 5);
 	}
 
-	if (p_enabled && window_get_flag(WINDOW_FLAG_RESIZE_DISABLED, p_window)) {
+	if (p_enabled) {
 		// Set the window as resizable to prevent window managers to ignore the fullscreen state flag.
-		XSizeHints *xsh;
-
-		xsh = XAllocSizeHints();
-		xsh->flags = 0L;
-		XSetWMNormalHints(x11_display, wd.x11_window, xsh);
-		XFree(xsh);
+		_update_size_hints(p_window);
 	}
 
 	// Using EWMH -- Extended Window Manager Hints
@@ -1248,30 +1227,7 @@ void DisplayServerX11::_set_wm_fullscreen(WindowID p_window, bool p_enabled) {
 
 	if (!p_enabled) {
 		// Reset the non-resizable flags if we un-set these before.
-		Size2i size = window_get_size(p_window);
-		XSizeHints *xsh;
-		xsh = XAllocSizeHints();
-		if (window_get_flag(WINDOW_FLAG_RESIZE_DISABLED, p_window)) {
-			xsh->flags = PMinSize | PMaxSize;
-			xsh->min_width = size.x;
-			xsh->max_width = size.x;
-			xsh->min_height = size.y;
-			xsh->max_height = size.y;
-		} else {
-			xsh->flags = 0L;
-			if (wd.min_size != Size2i()) {
-				xsh->flags |= PMinSize;
-				xsh->min_width = wd.min_size.x;
-				xsh->min_height = wd.min_size.y;
-			}
-			if (wd.max_size != Size2i()) {
-				xsh->flags |= PMaxSize;
-				xsh->max_width = wd.max_size.x;
-				xsh->max_height = wd.max_size.y;
-			}
-		}
-		XSetWMNormalHints(x11_display, wd.x11_window, xsh);
-		XFree(xsh);
+		_update_size_hints(p_window);
 
 		// put back or remove decorations according to the last set borderless state
 		Hints hints;
@@ -1329,13 +1285,13 @@ void DisplayServerX11::window_set_mode(WindowMode p_mode, WindowID p_window) {
 		} break;
 		case WINDOW_MODE_FULLSCREEN: {
 			//Remove full-screen
+			wd.fullscreen = false;
+
 			_set_wm_fullscreen(p_window, false);
 
 			//un-maximize required for always on top
 			bool on_top = window_get_flag(WINDOW_FLAG_ALWAYS_ON_TOP, p_window);
 
-			wd.fullscreen = false;
-
 			window_set_position(wd.last_position_before_fs, p_window);
 
 			if (on_top) {
@@ -1381,15 +1337,16 @@ void DisplayServerX11::window_set_mode(WindowMode p_mode, WindowID p_window) {
 		} break;
 		case WINDOW_MODE_FULLSCREEN: {
 			wd.last_position_before_fs = wd.position;
+
 			if (window_get_flag(WINDOW_FLAG_ALWAYS_ON_TOP, p_window)) {
 				_set_wm_maximized(p_window, true);
 			}
-			_set_wm_fullscreen(p_window, true);
+
 			wd.fullscreen = true;
+			_set_wm_fullscreen(p_window, true);
 		} break;
 		case WINDOW_MODE_MAXIMIZED: {
 			_set_wm_maximized(p_window, true);
-
 		} break;
 	}
 }
@@ -1456,37 +1413,11 @@ void DisplayServerX11::window_set_flag(WindowFlags p_flag, bool p_enabled, Windo
 
 	switch (p_flag) {
 		case WINDOW_FLAG_RESIZE_DISABLED: {
-			XSizeHints *xsh;
-			xsh = XAllocSizeHints();
-			if (p_enabled) {
-				Size2i size = window_get_size(p_window);
-
-				xsh->flags = PMinSize | PMaxSize;
-				xsh->min_width = size.x;
-				xsh->max_width = size.x;
-				xsh->min_height = size.y;
-				xsh->max_height = size.y;
-			} else {
-				xsh->flags = 0L;
-				if (wd.min_size != Size2i()) {
-					xsh->flags |= PMinSize;
-					xsh->min_width = wd.min_size.x;
-					xsh->min_height = wd.min_size.y;
-				}
-				if (wd.max_size != Size2i()) {
-					xsh->flags |= PMaxSize;
-					xsh->max_width = wd.max_size.x;
-					xsh->max_height = wd.max_size.y;
-				}
-			}
-
-			XSetWMNormalHints(x11_display, wd.x11_window, xsh);
-			XFree(xsh);
-
 			wd.resize_disabled = p_enabled;
 
-			XFlush(x11_display);
+			_update_size_hints(p_window);
 
+			XFlush(x11_display);
 		} break;
 		case WINDOW_FLAG_BORDERLESS: {
 			Hints hints;
@@ -3305,20 +3236,6 @@ DisplayServerX11::WindowID DisplayServerX11::_create_window(WindowMode p_mode, u
 		windows[id] = wd;
 
 		{
-			if (p_flags & WINDOW_FLAG_RESIZE_DISABLED_BIT) {
-				XSizeHints *xsh;
-				xsh = XAllocSizeHints();
-
-				xsh->flags = PMinSize | PMaxSize;
-				xsh->min_width = p_rect.size.width;
-				xsh->max_width = p_rect.size.width;
-				xsh->min_height = p_rect.size.height;
-				xsh->max_height = p_rect.size.height;
-
-				XSetWMNormalHints(x11_display, wd.x11_window, xsh);
-				XFree(xsh);
-			}
-
 			bool make_utility = false;
 
 			if (p_flags & WINDOW_FLAG_BORDERLESS_BIT) {
@@ -3368,18 +3285,7 @@ DisplayServerX11::WindowID DisplayServerX11::_create_window(WindowMode p_mode, u
 			}
 		}
 
-		if (id != MAIN_WINDOW_ID) {
-			XSizeHints my_hints = XSizeHints();
-
-			my_hints.flags = PPosition | PSize; /* I want to specify position and size */
-			my_hints.x = p_rect.position.x; /* The origin and size coords I want */
-			my_hints.y = p_rect.position.y;
-			my_hints.width = p_rect.size.width;
-			my_hints.height = p_rect.size.height;
-
-			XSetNormalHints(x11_display, wd.x11_window, &my_hints);
-			XMoveWindow(x11_display, wd.x11_window, p_rect.position.x, p_rect.position.y);
-		}
+		_update_size_hints(id);
 
 #if defined(VULKAN_ENABLED)
 		if (context_vulkan) {

platform/linuxbsd/display_server_x11.h

@@ -235,6 +235,7 @@ class DisplayServerX11 : public DisplayServer {
 
 	void _update_real_mouse_position(const WindowData &wd);
 	bool _window_maximize_check(WindowID p_window, const char *p_atom_name) const;
+	void _update_size_hints(WindowID p_window);
 	void _set_wm_fullscreen(WindowID p_window, bool p_enabled);
 	void _set_wm_maximized(WindowID p_window, bool p_enabled);