x11: Better handle XInput2 mouse tracking outside the window

There is a quirk with XInput2 mouse capture that causes a leave event to be sent if the pointer moves out->in->out, which breaks mouse tracking outside the window. If the mouse leaves the window with buttons pressed, continue tracking it until the buttons are released.

(cherry picked from commit 8c733d1f7bdb03b2082028349e97137c0bdea13b)

src/video/x11/SDL_x11events.c

@@ -492,7 +492,17 @@ static void X11_DispatchFocusOut(SDL_VideoDevice *_this, SDL_WindowData *data)
     /* If another window has already processed a focus in, then don't try to
      * remove focus here.  Doing so will incorrectly remove focus from that
      * window, and the focus lost event for this window will have already
-     * been dispatched anyway. */
+     * been dispatched anyway.
+     */
+    if (data->tracking_mouse_outside_window && data->window == SDL_GetMouseFocus()) {
+        // If tracking the pointer and keyboard focus is lost, raise all buttons and relinquish mouse focus.
+        SDL_SendMouseButton(0, data->window, SDL_GLOBAL_MOUSE_ID, SDL_BUTTON_LEFT, false);
+        SDL_SendMouseButton(0, data->window, SDL_GLOBAL_MOUSE_ID, SDL_BUTTON_MIDDLE, false);
+        SDL_SendMouseButton(0, data->window, SDL_GLOBAL_MOUSE_ID, SDL_BUTTON_RIGHT, false);
+        SDL_SendMouseButton(0, data->window, SDL_GLOBAL_MOUSE_ID, SDL_BUTTON_X1, false);
+        SDL_SendMouseButton(0, data->window, SDL_GLOBAL_MOUSE_ID, SDL_BUTTON_X2, false);
+        SDL_SetMouseFocus(NULL);
+    }
     if (data->window == SDL_GetKeyboardFocus()) {
         SDL_SetKeyboardFocus(NULL);
     }
@@ -1074,6 +1084,16 @@ void X11_HandleButtonRelease(SDL_VideoDevice *_this, SDL_WindowData *windowdata,
             // see explanation at case ButtonPress
             button -= (8 - SDL_BUTTON_X1);
         }
+
+        /* If the mouse is captured and all buttons are now released, clear the capture
+         * flag so the focus will be cleared if the mouse is outside the window.
+         */
+        if ((window->flags & SDL_WINDOW_MOUSE_CAPTURE)  &&
+            !(SDL_GetMouseState(NULL, NULL) & ~SDL_BUTTON_MASK(button))) {
+            window->flags &= ~SDL_WINDOW_MOUSE_CAPTURE;
+            windowdata->tracking_mouse_outside_window = false;
+        }
+
         SDL_SendMouseButton(timestamp, window, mouseID, button, false);
     }
 }
@@ -1319,6 +1339,8 @@ static void X11_DispatchEvent(SDL_VideoDevice *_this, XEvent *xevent)
             SDL_Log("Mode: NotifyUngrab");
         }
 #endif
+        data->tracking_mouse_outside_window = false;
+
         SDL_SetMouseFocus(data->window);
 
         mouse->last_x = xevent->xcrossing.x;
@@ -1365,14 +1387,17 @@ static void X11_DispatchEvent(SDL_VideoDevice *_this, XEvent *xevent)
         if (xevent->xcrossing.mode != NotifyGrab &&
             xevent->xcrossing.mode != NotifyUngrab &&
             xevent->xcrossing.detail != NotifyInferior) {
+            if (!(data->window->flags & SDL_WINDOW_MOUSE_CAPTURE)) {
+                /* In order for interaction with the window decorations and menu to work properly
+                   on Mutter, we need to ungrab the keyboard when the mouse leaves. */
+                if (!(data->window->flags & SDL_WINDOW_FULLSCREEN)) {
+                    X11_SetWindowKeyboardGrab(_this, data->window, false);
+                }
 
-            /* In order for interaction with the window decorations and menu to work properly
-               on Mutter, we need to ungrab the keyboard when the mouse leaves. */
-            if (!(data->window->flags & SDL_WINDOW_FULLSCREEN)) {
-                X11_SetWindowKeyboardGrab(_this, data->window, false);
+                SDL_SetMouseFocus(NULL);
+            } else {
+                data->tracking_mouse_outside_window = true;
             }
-
-            SDL_SetMouseFocus(NULL);
         }
     } break;
 

src/video/x11/SDL_x11window.h

@@ -118,6 +118,7 @@ struct SDL_WindowData
     bool fullscreen_borders_forced_on;
     bool was_shown;
     bool emit_size_move_after_property_notify;
+    bool tracking_mouse_outside_window;
     SDL_HitTestResult hit_test_result;
 
     XPoint xim_spot;

src/video/x11/SDL_x11xinput2.c

@@ -467,15 +467,17 @@ void X11_HandleXinput2Event(SDL_VideoDevice *_this, XGenericEventCookie *cookie)
                     SDL_SendPenAxis(0, pen->pen, window, (SDL_PenAxis) i, axes[i]);
                 }
             }
-        } else if (!pointer_emulated && xev->deviceid == videodata->xinput_master_pointer_device) {
-            // Use the master device for non-relative motion, as the slave devices can seemingly lag behind.
+        } else {
             SDL_Mouse *mouse = SDL_GetMouse();
-            if (!mouse->relative_mode) {
-                SDL_Window *window = xinput2_get_sdlwindow(videodata, xev->event);
-                if (window) {
-                    X11_ProcessHitTest(_this, window->internal, (float)xev->event_x, (float)xev->event_y, false);
-                    SDL_SendMouseMotion(0, window, SDL_GLOBAL_MOUSE_ID, false, (float)xev->event_x, (float)xev->event_y);
-                }
+            SDL_Window *window = xinput2_get_sdlwindow(videodata, xev->event);
+            if (!mouse->relative_mode && !pointer_emulated && window &&
+                (xev->deviceid == videodata->xinput_master_pointer_device || window->internal->tracking_mouse_outside_window)) {
+                /* Use the master device for non-relative motion, as the slave devices can seemingly lag behind, unless
+                 * tracking the mouse outside the window, in which case the slave devices deliver coordinates, while the
+                 * master does not.
+                 */
+                X11_ProcessHitTest(_this, window->internal, (float)xev->event_x, (float)xev->event_y, false);
+                SDL_SendMouseMotion(0, window, SDL_GLOBAL_MOUSE_ID, false, (float)xev->event_x, (float)xev->event_y);
             }
         }
     } break;