Merge pull request #1220 from floooh/issue1211/win32_mouse_lock_behaviour

sokol_app.h win32: fix mouse lock edge cases (fixes #1211)

CHANGELOG.md

@@ -16,6 +16,25 @@ unfortunately slipped through testing:
 
 Both fixes are in this PR: https://github.com/floooh/sokol/pull/1219
 
+- sokol_app.h win32: Fix mouse locking behaviour in edge cases: an assert could
+  be triggered on Win32 when the mouse is currently locked and the window focus
+  is stolen via Ctrl-Shift-Esc or Ctrl-Alt-Del (basically: opening the task manager).
+  Also, even without the assert, the mouse might remain stuck in 'mouse lock mode'
+  while the task manager is open. The behaviour has been worked around by the following
+  changes:
+
+  - a return value `false` from GetCursorPos() will be handled instead of asserted
+  - a return value `false` from SetCursorPos() will be ignored, and SetCursorPos()
+    will only be called to restore the mouse position when the previous GetCursorPos()
+    had succeeded.
+  - trying to lock the mouse while the application window isn't in the foreground
+    is now ignored
+  - the check whether a locked mouse must be unlocked now happens via polling
+    the current foreground window instead of WM_KILLFOCUS
+
+  See PR https://github.com/floooh/sokol/pull/1220 for details. Many thanks to
+  @Hisashimaru for bringing up the issue!
+
 ### 08-Mar-2025
 
 Initial compute shader support has been merged into sokol_gfx.h.

sokol_app.h

@@ -2632,6 +2632,7 @@ typedef struct {
     HCURSOR cursors[_SAPP_MOUSECURSOR_NUM];
     UINT orig_codepage;
     LONG mouse_locked_x, mouse_locked_y;
+    bool mouse_locked_pos_valid;
     RECT stored_window_rect;    // used to restore window pos/size when toggling fullscreen => windowed
     bool is_win10_or_greater;
     bool in_create_window;
@@ -7193,33 +7194,46 @@ _SOKOL_PRIVATE void _sapp_win32_release_mouse(uint8_t btn_mask) {
     }
 }
 
+_SOKOL_PRIVATE bool _sapp_win32_is_foreground_window(void) {
+    return _sapp.win32.hwnd == GetForegroundWindow();
+}
+
 _SOKOL_PRIVATE void _sapp_win32_lock_mouse(bool lock) {
     if (lock == _sapp.mouse.locked) {
         return;
     }
     _sapp.mouse.dx = 0.0f;
     _sapp.mouse.dy = 0.0f;
-    _sapp.mouse.locked = lock;
     _sapp_win32_release_mouse(0xFF);
-    if (_sapp.mouse.locked) {
+    if (lock) {
+        // don't allow locking the mouse unless we're the active window
+        if (!_sapp_win32_is_foreground_window()) {
+            return;
+        }
+
+        _sapp.mouse.locked = true;
         /* store the current mouse position, so it can be restored when unlocked */
         POINT pos;
         BOOL res = GetCursorPos(&pos);
-        SOKOL_ASSERT(res); _SOKOL_UNUSED(res);
-        _sapp.win32.mouse_locked_x = pos.x;
-        _sapp.win32.mouse_locked_y = pos.y;
-
-        /* while the mouse is locked, make the mouse cursor invisible and
-           confine the mouse movement to a small rectangle inside our window
-           (so that we don't miss any mouse up events)
-        */
-        RECT client_rect = {
-            _sapp.win32.mouse_locked_x,
-            _sapp.win32.mouse_locked_y,
-            _sapp.win32.mouse_locked_x,
-            _sapp.win32.mouse_locked_y
-        };
-        ClipCursor(&client_rect);
+        if (res) {
+            _sapp.win32.mouse_locked_x = pos.x;
+            _sapp.win32.mouse_locked_y = pos.y;
+            _sapp.win32.mouse_locked_pos_valid = true;
+
+            /* while the mouse is locked, make the mouse cursor invisible and
+               confine the mouse movement to a small rectangle inside our window
+               (so that we don't miss any mouse up events)
+            */
+            RECT client_rect = {
+                _sapp.win32.mouse_locked_x,
+                _sapp.win32.mouse_locked_y,
+                _sapp.win32.mouse_locked_x,
+                _sapp.win32.mouse_locked_y
+            };
+            ClipCursor(&client_rect);
+        } else {
+            _sapp.win32.mouse_locked_pos_valid = false;
+        }
 
         /* make the mouse cursor invisible, this will stack with sapp_show_mouse() */
         ShowCursor(FALSE);
@@ -7238,8 +7252,8 @@ _SOKOL_PRIVATE void _sapp_win32_lock_mouse(bool lock) {
            we need to skip the dx/dy compution for the first WM_INPUT event
         */
         _sapp.win32.raw_input_mousepos_valid = false;
-    }
-    else {
+    } else {
+        _sapp.mouse.locked = false;
         /* disable raw input for mouse */
         const RAWINPUTDEVICE rid = { 0x01, 0x02, RIDEV_REMOVE, NULL };
         if (!RegisterRawInputDevices(&rid, 1, sizeof(rid))) {
@@ -7251,8 +7265,10 @@ _SOKOL_PRIVATE void _sapp_win32_lock_mouse(bool lock) {
         ShowCursor(TRUE);
 
         /* restore the 'pre-locked' mouse position */
-        BOOL res = SetCursorPos(_sapp.win32.mouse_locked_x, _sapp.win32.mouse_locked_y);
-        SOKOL_ASSERT(res); _SOKOL_UNUSED(res);
+        if (_sapp.win32.mouse_locked_pos_valid) {
+            SetCursorPos(_sapp.win32.mouse_locked_x, _sapp.win32.mouse_locked_y);
+            _sapp.win32.mouse_locked_pos_valid = false;
+        }
     }
 }
 
@@ -7502,10 +7518,6 @@ _SOKOL_PRIVATE LRESULT CALLBACK _sapp_win32_wndproc(HWND hWnd, UINT uMsg, WPARAM
                 _sapp_win32_app_event(SAPP_EVENTTYPE_FOCUSED);
                 break;
             case WM_KILLFOCUS:
-                /* if focus is lost for any reason, and we're in mouse locked mode, disable mouse lock */
-                if (_sapp.mouse.locked) {
-                    _sapp_win32_lock_mouse(false);
-                }
                 _sapp_win32_app_event(SAPP_EVENTTYPE_UNFOCUSED);
                 break;
             case WM_SETCURSOR:
@@ -8106,6 +8118,12 @@ _SOKOL_PRIVATE void _sapp_win32_run(const sapp_desc* desc) {
         if (_sapp.quit_requested) {
             PostMessage(_sapp.win32.hwnd, WM_CLOSE, 0, 0);
         }
+        // unlock mouse if window doesn't have focus
+        if (_sapp.mouse.locked) {
+            if (!_sapp_win32_is_foreground_window()) {
+                _sapp_win32_lock_mouse(false);
+            }
+        }
     }
     _sapp_call_cleanup();