cocoa: Don't minimize fullscreen windows for a modal file dialog.

macOS sends a focus loss event when the dialog is created, which causes SDL
to try to minimize the window, which confuses the entire system. So in this
special case, don't do the minimization.

Fixes #13168.

(cherry picked from commit 9af93abd4f94ed52c6dd2859805aa832fafc4a3b)

src/dialog/cocoa/SDL_cocoadialog.m

@@ -27,6 +27,8 @@
 #import <Cocoa/Cocoa.h>
 #import <UniformTypeIdentifiers/UTType.h>
 
+extern void Cocoa_SetWindowHasModalDialog(SDL_Window *window, bool has_modal);
+
 static void AddFileExtensionType(NSMutableArray *types, const char *pattern_ptr)
 {
     if (!*pattern_ptr) {
@@ -163,6 +165,9 @@ void SDL_SYS_ShowFileDialogWithProperties(SDL_FileDialogType type, SDL_DialogFil
 
     if (window) {
         w = (__bridge NSWindow *)SDL_GetPointerProperty(SDL_GetWindowProperties(window), SDL_PROP_WINDOW_COCOA_WINDOW_POINTER, NULL);
+        if (w) {
+            Cocoa_SetWindowHasModalDialog(window, true);
+        }
     }
 
     if (w) {
@@ -186,6 +191,7 @@ void SDL_SYS_ShowFileDialogWithProperties(SDL_FileDialogType type, SDL_DialogFil
                 callback(userdata, files, -1);
             }
 
+            Cocoa_SetWindowHasModalDialog(window, false);
             ReactivateAfterDialog();
         }];
     } else {
@@ -206,6 +212,7 @@ void SDL_SYS_ShowFileDialogWithProperties(SDL_FileDialogType type, SDL_DialogFil
             const char *files[1] = { NULL };
             callback(userdata, files, -1);
         }
+
         ReactivateAfterDialog();
     }
 }

src/video/SDL_video.c

@@ -171,9 +171,10 @@ static VideoBootStrap *bootstrap[] = {
     }
 
 #if defined(SDL_PLATFORM_MACOS) && defined(SDL_VIDEO_DRIVER_COCOA)
-// Support for macOS fullscreen spaces
+// Support for macOS fullscreen spaces, etc.
 extern bool Cocoa_IsWindowInFullscreenSpace(SDL_Window *window);
 extern bool Cocoa_SetWindowFullscreenSpace(SDL_Window *window, bool state, bool blocking);
+extern bool Cocoa_IsShowingModalDialog(SDL_Window *window);
 #endif
 
 #ifdef SDL_VIDEO_DRIVER_UIKIT
@@ -4171,7 +4172,9 @@ static bool SDL_ShouldMinimizeOnFocusLoss(SDL_Window *window)
 
 #if defined(SDL_PLATFORM_MACOS) && defined(SDL_VIDEO_DRIVER_COCOA)
     if (SDL_strcmp(_this->name, "cocoa") == 0) { // don't do this for X11, etc
-        if (Cocoa_IsWindowInFullscreenSpace(window)) {
+        if (Cocoa_IsShowingModalDialog(window)) {
+            return false;  // modal system dialogs can live over fullscreen windows, don't minimize.
+        } else if (Cocoa_IsWindowInFullscreenSpace(window)) {
             return false;
         }
     }

src/video/cocoa/SDL_cocoawindow.h

@@ -152,6 +152,7 @@ typedef enum
 @property(nonatomic) bool pending_size;
 @property(nonatomic) bool pending_position;
 @property(nonatomic) bool border_toggled;
+@property(nonatomic) bool has_modal_dialog;
 
 #ifdef SDL_VIDEO_OPENGL_EGL
 @property(nonatomic) EGLSurface egl_surface;

src/video/cocoa/SDL_cocoawindow.m

@@ -432,6 +432,18 @@ bool Cocoa_IsWindowZoomed(SDL_Window *window)
     return zoomed;
 }
 
+bool Cocoa_IsShowingModalDialog(SDL_Window *window)
+{
+    SDL_CocoaWindowData *data = (__bridge SDL_CocoaWindowData *)window->internal;
+    return data.has_modal_dialog;
+}
+
+void Cocoa_SetWindowHasModalDialog(SDL_Window *window, bool has_modal)
+{
+    SDL_CocoaWindowData *data = (__bridge SDL_CocoaWindowData *)window->internal;
+    data.has_modal_dialog = has_modal;
+}
+
 typedef enum CocoaMenuVisibility
 {
     COCOA_MENU_VISIBILITY_AUTO = 0,