Win32: Fix GLFW_MOUSE_PASSTHROUGH dropping events

Returning HTTRANSPARENT from WM_NCHITTEST does cause the window to be
transparent for some hit-testing APIs but does not make it pass mouse
input through to whatever window is below it.

For that to work on modern Windows, the window needs to be both layered
and extended-window-style-transparent.

Additional logic changes to ensure mouse input passthrough, framebuffer
transparency and window opacity are mindful of one another when
modifying WS_EX_LAYERED.

Related to #1568.

src/win32_window.c

@@ -417,10 +417,15 @@ static void updateFramebufferTransparency(const _GLFWwindow* window)
     else
     {
         LONG exStyle = GetWindowLongW(window->win32.handle, GWL_EXSTYLE);
-        exStyle &= ~WS_EX_LAYERED;
-        SetWindowLongW(window->win32.handle, GWL_EXSTYLE, exStyle);
-        RedrawWindow(window->win32.handle, NULL, NULL,
-                     RDW_ERASE | RDW_INVALIDATE | RDW_FRAME);
+        if (exStyle & WS_EX_TRANSPARENT)
+            SetLayeredWindowAttributes(window->win32.handle, 0, 0, 0);
+        else
+        {
+            exStyle &= ~WS_EX_LAYERED;
+            SetWindowLongW(window->win32.handle, GWL_EXSTYLE, exStyle);
+            RedrawWindow(window->win32.handle, NULL, NULL,
+                         RDW_ERASE | RDW_INVALIDATE | RDW_FRAME);
+        }
     }
 }
 
@@ -1201,13 +1206,6 @@ static LRESULT CALLBACK windowProc(HWND hWnd, UINT uMsg,
             DragFinish(drop);
             return 0;
         }
-
-        case WM_NCHITTEST:
-        {
-            if (window->mousePassthrough)
-                return HTTRANSPARENT;
-            break;
-        }
     }
 
     return DefWindowProcW(hWnd, uMsg, wParam, lParam);
@@ -1863,6 +1861,33 @@ void _glfwPlatformSetWindowFloating(_GLFWwindow* window, GLFWbool enabled)
 
 void _glfwPlatformSetWindowMousePassthrough(_GLFWwindow* window, GLFWbool enabled)
 {
+    COLORREF key = 0;
+    BYTE alpha = 0;
+    DWORD flags = 0;
+    DWORD exStyle = GetWindowLongW(window->win32.handle, GWL_EXSTYLE);
+
+    if (exStyle & WS_EX_LAYERED)
+        GetLayeredWindowAttributes(window->win32.handle, &key, &alpha, &flags);
+
+    if (enabled)
+        exStyle |= (WS_EX_TRANSPARENT | WS_EX_LAYERED);
+    else
+    {
+        exStyle &= ~WS_EX_TRANSPARENT;
+        // NOTE: Window opacity and framebuffer transparency also need to
+        //       control the layered style so avoid stepping on their feet
+        if (exStyle & WS_EX_LAYERED)
+        {
+            if (!(flags & (LWA_ALPHA | LWA_COLORKEY)))
+                exStyle &= ~WS_EX_LAYERED;
+        }
+    }
+
+    SetWindowLongW(window->win32.handle, GWL_EXSTYLE, exStyle);
+
+    if (enabled)
+        SetLayeredWindowAttributes(window->win32.handle, key, alpha, flags);
+
     window->mousePassthrough = enabled;
 }
 
@@ -1883,19 +1908,22 @@ float _glfwPlatformGetWindowOpacity(_GLFWwindow* window)
 
 void _glfwPlatformSetWindowOpacity(_GLFWwindow* window, float opacity)
 {
-    if (opacity < 1.f)
+    LONG exStyle = GetWindowLongW(window->win32.handle, GWL_EXSTYLE);
+    if (opacity < 1.f || (exStyle & WS_EX_TRANSPARENT))
     {
         const BYTE alpha = (BYTE) (255 * opacity);
-        DWORD style = GetWindowLongW(window->win32.handle, GWL_EXSTYLE);
-        style |= WS_EX_LAYERED;
-        SetWindowLongW(window->win32.handle, GWL_EXSTYLE, style);
+        exStyle |= WS_EX_LAYERED;
+        SetWindowLongW(window->win32.handle, GWL_EXSTYLE, exStyle);
         SetLayeredWindowAttributes(window->win32.handle, 0, alpha, LWA_ALPHA);
     }
+    else if (exStyle & WS_EX_TRANSPARENT)
+    {
+        SetLayeredWindowAttributes(window->win32.handle, 0, 0, 0);
+    }
     else
     {
-        DWORD style = GetWindowLongW(window->win32.handle, GWL_EXSTYLE);
-        style &= ~WS_EX_LAYERED;
-        SetWindowLongW(window->win32.handle, GWL_EXSTYLE, style);
+        exStyle &= ~WS_EX_LAYERED;
+        SetWindowLongW(window->win32.handle, GWL_EXSTYLE, exStyle);
     }
 }