Switch uiArea from using a custom message pump to using a message filter for input handling. This allows us to rewrite the dialog system to avoid needing to block ALL windows and set a proper parent window.

windows/area.c

@@ -511,6 +511,9 @@ keyFound:
 	return (*(a->ah->KeyEvent))(a->ah, a, &ke);
 }
 
+// We don't handle the standard Windows keyboard messages directly, to avoid both the dialog manager and TranslateMessage().
+// Instead, we set up a message filter and do things there.
+// That stuff is later in this file.
 enum {
 	// start at 0x40 to avoid clobbering dialog messages
 	msgAreaKeyDown = WM_USER + 0x40,
@@ -662,33 +665,6 @@ static void minimumSize(uiWindowsControl *c, uiWindowsSizing *d, intmax_t *width
 	*height = 0;
 }
 
-// TODO affect visibility properly
-// TODO what did this mean
-void processAreaMessage(HWND active, MSG *msg)
-{
-	LRESULT handled;
-
-	handled = 0;
-	switch (msg->message) {
-	case WM_KEYDOWN:
-	case WM_SYSKEYDOWN:
-		handled = SendMessageW(msg->hwnd, msgAreaKeyDown, msg->wParam, msg->lParam);
-		break;
-	case WM_KEYUP:
-	case WM_SYSKEYUP:
-		handled = SendMessageW(msg->hwnd, msgAreaKeyUp, msg->wParam, msg->lParam);
-		break;
-	}
-	if (handled)
-		return;
-
-	// don't call TranslateMessage(); we do our own keyboard handling
-	// TODO should we just return to the standard message loop?
-	if (IsDialogMessage(active, msg) != 0)
-		return;
-	DispatchMessageW(msg);
-}
-
 ATOM registerAreaClass(HICON hDefaultIcon, HCURSOR hDefaultCursor)
 {
 	WNDCLASSW wc;
@@ -704,10 +680,59 @@ ATOM registerAreaClass(HICON hDefaultIcon, HCURSOR hDefaultCursor)
 	return RegisterClassW(&wc);
 }
 
-void unregisterAreaClass(void)
+static HHOOK areaFilter;
+
+// TODO affect visibility properly
+// TODO what did this mean
+static LRESULT CALLBACK areaFilterProc(int code, WPARAM wParam, LPARAM lParam)
 {
+	MSG *msg = (MSG *) lParam;
+	LRESULT handled;
+
+	if (code < 0)
+		goto callNext;
+
+	// is the recipient an area?
+	if (windowClassOf(msg->hwnd, areaClass, NULL) != 0)
+		goto callNext;		// nope
+
+	handled = 0;
+	switch (msg->message) {
+	case WM_KEYDOWN:
+	case WM_SYSKEYDOWN:
+		handled = SendMessageW(msg->hwnd, msgAreaKeyDown, msg->wParam, msg->lParam);
+		break;
+	case WM_KEYUP:
+	case WM_SYSKEYUP:
+		handled = SendMessageW(msg->hwnd, msgAreaKeyUp, msg->wParam, msg->lParam);
+		break;
+	// otherwise handled remains 0, as we didn't handle this
+	}
+	if (handled)
+		goto callNext;
+
+	// we handled it; discard the message so the dialog manager doesn't see it
+	return 1;
+
+callNext:
+	return CallNextHookEx(areaFilter, code, wParam, lParam);
+}
+
+int registerAreaFilter(void)
+{
+	areaFilter = SetWindowsHookExW(WH_MSGFILTER,
+		areaFilterProc,
+		hInstance,
+		GetCurrentThreadId());
+	return areaFilter != NULL;
+}
+
+void unregisterArea(void)
+{
+	if (UnhookWindowsHookEx(areaFilter) == 0)
+		logLastError("error unregistering uiArea message filter in unregisterArea()");
 	if (UnregisterClassW(areaClass, hInstance) == 0)
-		logLastError("error unregistering uiArea window class in unregisterAreaClass()");
+		logLastError("error unregistering uiArea window class in unregisterArea()");
 }
 
 void uiAreaUpdateScroll(uiArea *a)

windows/init.c

@@ -161,6 +161,8 @@ const char *uiInit(uiInitOptions *o)
 
 	if (registerAreaClass(hDefaultIcon, hDefaultCursor) == 0)
 		return loadLastError("registering uiArea window class");
+	if (registerAreaFilter() == 0)
+		return loadLastError("registering uiArea message filter");
 
 	return NULL;
 }
@@ -168,7 +170,7 @@ const char *uiInit(uiInitOptions *o)
 void uiUninit(void)
 {
 	uninitMenus();
-	unregisterAreaClass();
+	unregisterArea();
 	uninitDraw();
 	CoUninitialize();
 	uninitDialogHelper();

windows/main.c

@@ -13,7 +13,7 @@ void uiMain(void)
 {
 	MSG msg;
 	int res;
-	HWND active, focus;
+	HWND active;
 
 	for (;;) {
 		SetLastError(0);
@@ -28,18 +28,8 @@ void uiMain(void)
 			continue;
 		}
 
-		// bit of logic involved here:
-		// we don't want dialog messages passed into Areas, so we don't call IsDialogMessageW() there
-		// as for Tabs, we can't have both WS_TABSTOP and WS_EX_CONTROLPARENT set at the same time, so we hotswap the two styles to get the behavior we want
-		focus = GetFocus();
-		if (focus != NULL) {
-			switch (windowClassOf(focus, areaClass, NULL)) {
-			case 0:		// uiArea
-				processAreaMessage(active, &msg);
-				continue;
-			}
-			// else fall through
-		}
+		// TODO find documentation that says IsDialogMessage() calls CallMsgFilter() for us, because that's what's happening
+		// TODO rewrite this whole function to compensate
 
 		if (IsDialogMessage(active, &msg) != 0)
 			continue;

windows/uipriv_windows.h

@@ -124,9 +124,9 @@ extern HWND newTabPage(void);
 
 // area.c
 #define areaClass L"libui_uiAreaClass"
-extern void processAreaMessage(HWND, MSG *);
 extern ATOM registerAreaClass(HICON, HCURSOR);
-extern void unregisterAreaClass(void);
+extern int registerAreaFilter(void);
+extern void unregisterArea(void);
 
 // draw.c
 extern HRESULT initDraw(void);