The wayland messagebox should only be used under wayland

src/video/wayland/SDL_waylandmessagebox.c

@@ -0,0 +1,42 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2025 Sam Lantinga <[email protected]>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+
+#include "SDL_internal.h"
+
+#ifdef SDL_VIDEO_DRIVER_WAYLAND
+
+#include "SDL_waylandmessagebox.h"
+#include "../../dialog/unix/SDL_zenitymessagebox.h"
+
+bool Wayland_ShowMessageBox(const SDL_MessageBoxData *messageboxdata, int *buttonID)
+{
+    // Are we trying to connect to or are currently in a Wayland session?
+    if (!SDL_getenv("WAYLAND_DISPLAY")) {
+        const char *session = SDL_getenv("XDG_SESSION_TYPE");
+        if (session && SDL_strcasecmp(session, "wayland") != 0) {
+            return SDL_SetError("Not on a wayland display");
+        }
+    }
+
+    return SDL_Zenity_ShowMessageBox(messageboxdata, buttonID);
+}
+
+#endif // SDL_VIDEO_DRIVER_WAYLAND

src/video/wayland/SDL_waylandmessagebox.h

@@ -0,0 +1,31 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2025 Sam Lantinga <[email protected]>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+
+#ifndef SDL_waylandmessagebox_h_
+#define SDL_waylandmessagebox_h_
+
+#ifdef SDL_VIDEO_DRIVER_WAYLAND
+
+extern bool Wayland_ShowMessageBox(const SDL_MessageBoxData *messageboxdata, int *buttonID);
+
+#endif // SDL_VIDEO_DRIVER_WAYLAND
+
+#endif // SDL_waylandmessagebox_h_

src/video/wayland/SDL_waylandvideo.c

@@ -25,13 +25,13 @@
 
 #include "../../core/linux/SDL_system_theme.h"
 #include "../../core/linux/SDL_progressbar.h"
-#include "../../dialog/unix/SDL_zenitymessagebox.h"
 #include "../../events/SDL_events_c.h"
 
 #include "SDL_waylandclipboard.h"
 #include "SDL_waylandcolor.h"
 #include "SDL_waylandevents_c.h"
 #include "SDL_waylandkeyboard.h"
+#include "SDL_waylandmessagebox.h"
 #include "SDL_waylandmouse.h"
 #include "SDL_waylandopengles.h"
 #include "SDL_waylandvideo.h"
@@ -708,14 +708,14 @@ static SDL_VideoDevice *Wayland_Fallback_CreateDevice(void)
 VideoBootStrap Wayland_preferred_bootstrap = {
     WAYLANDVID_DRIVER_NAME, "SDL Wayland video driver",
     Wayland_Preferred_CreateDevice,
-    SDL_Zenity_ShowMessageBox,
+    Wayland_ShowMessageBox,
     true
 };
 
 VideoBootStrap Wayland_bootstrap = {
     WAYLANDVID_DRIVER_NAME, "SDL Wayland video driver",
     Wayland_Fallback_CreateDevice,
-    SDL_Zenity_ShowMessageBox,
+    Wayland_ShowMessageBox,
     false
 };