Tried to resolve checkboxes appearing selected. It doesn't work fully yet, so the actual drawing that makes it work is disabled for now. But the handler for NM_CUSTOMDRAW now has a hook to become cleaner in the future.

windows/table.cpp

@@ -143,10 +143,12 @@ static LRESULT onNM_CUSTOMDRAW(uiTable *t, NMLVCUSTOMDRAW *nm)
 	uiTableData *data;
 	double r, g, b, a;
 	LRESULT ret;
+	HRESULT hr;
 
 	switch (nm->nmcd.dwDrawStage) {
 	case CDDS_PREPAINT:
-		return CDRF_NOTIFYITEMDRAW;
+		ret = CDRF_NOTIFYITEMDRAW;
+		break;
 	case CDDS_ITEMPREPAINT:
 		if (t->backgroundColumn != -1) {
 			data = (*(t->model->mh->CellValue))(t->model->mh, t->model, nm->nmcd.dwItemSpec, t->backgroundColumn);
@@ -157,7 +159,8 @@ static LRESULT onNM_CUSTOMDRAW(uiTable *t, NMLVCUSTOMDRAW *nm)
 			}
 		}
 		t->clrItemText = nm->clrText;
-		return CDRF_NEWFONT | CDRF_NOTIFYSUBITEMDRAW;
+		ret = CDRF_NEWFONT | CDRF_NOTIFYSUBITEMDRAW;
+		break;
 	case CDDS_SUBITEM | CDDS_ITEMPREPAINT:
 		p = (*(t->columns))[nm->iSubItem];
 		// we need this as previous subitems will persist their colors
@@ -171,9 +174,17 @@ static LRESULT onNM_CUSTOMDRAW(uiTable *t, NMLVCUSTOMDRAW *nm)
 			}
 		}
 		// TODO draw background on image columns if needed
-		return CDRF_NEWFONT;
+		ret = CDRF_NEWFONT;
+		break;
+	default:
+		ret = CDRF_DODEFAULT;
 	}
-	return CDRF_DODEFAULT;
+
+	hr = uiprivNM_CUSTOMDRAWImagesCheckboxes(t, nm, &ret);
+	if (hr != S_OK) {
+		// TODO
+	}
+	return ret;
 }
 
 static BOOL onWM_NOTIFY(uiControl *c, HWND hwnd, NMHDR *nmhdr, LRESULT *lResult)
@@ -364,6 +375,7 @@ uiTable *uiNewTable(uiTableModel *model)
 	uiWindowsRegisterWM_NOTIFYHandler(t->hwnd, onWM_NOTIFY, uiControl(t));
 
 	// TODO: try LVS_EX_AUTOSIZECOLUMNS
+	// TODO check error
 	SendMessageW(t->hwnd, LVM_SETEXTENDEDLISTVIEWSTYLE,
 		(WPARAM) (LVS_EX_FULLROWSELECT | LVS_EX_LABELTIP | LVS_EX_SUBITEMIMAGES),
 		(LPARAM) (LVS_EX_FULLROWSELECT | LVS_EX_LABELTIP | LVS_EX_SUBITEMIMAGES));

windows/table.hpp

@@ -46,4 +46,5 @@ struct uiTable {
 
 // tableimages.cpp
 extern HRESULT uiprivLVN_GETDISPINFOImagesCheckboxes(uiTable *t, NMLVDISPINFOW *nm, uiprivTableColumnParams *p);
+extern HRESULT uiprivNM_CUSTOMDRAWImagesCheckboxes(uiTable *t, NMLVCUSTOMDRAW *nm, LRESULT *lResult);
 extern HRESULT uiprivTableSetupImagesCheckboxes(uiTable *t);

windows/tableimages.cpp

@@ -15,7 +15,6 @@ We'll use the small image list. For this, the first few items will be reserved f
 
 // checkboxes TODOs:
 // - see if we need to get rid of the extra margin in subitems
-// - see if we need to get rid of the glow effect
 
 #define nCheckboxImages 4
 
@@ -59,6 +58,36 @@ static HRESULT setCellImage(uiTable *t, NMLVDISPINFOW *nm, uiprivTableColumnPara
 	return S_OK;
 }
 
+#define stateUnchecked 0
+#define stateChecked 1
+#define stateDisabled 2
+
+static int checkboxIndex(uiTableModel *m, int row, int checkboxModelColumn, int checkboxEditableColumn)
+{
+	uiTableData *data;
+	int ret;
+
+	ret = stateUnchecked;
+	data = (*(m->mh->CellValue))(m->mh, m, row, checkboxModelColumn);
+	if (uiTableDataInt(data) != 0)
+		ret = stateChecked;
+	uiFreeTableData(data);
+
+	switch (checkboxEditableColumn) {
+	case uiTableModelColumnNeverEditable:
+		ret += stateDisabled;
+		break;
+	case uiTableModelColumnAlwaysEditable:
+		break;
+	default:
+		data = (*(m->mh->CellValue))(m->mh, m, row, checkboxEditableColumn);
+		if (uiTableDataInt(data) != 0)
+			ret += stateDisabled;
+	}
+
+	return ret;
+}
+
 HRESULT uiprivLVN_GETDISPINFOImagesCheckboxes(uiTable *t, NMLVDISPINFOW *nm, uiprivTableColumnParams *p)
 {
 	uiTableData *data;
@@ -104,6 +133,48 @@ HRESULT uiprivLVN_GETDISPINFOImagesCheckboxes(uiTable *t, NMLVDISPINFOW *nm, uip
 	return S_OK;
 }
 
+// in order to properly look like checkboxes, we need to exclude them from being colored in by the selection rect
+// however, there seems to be no way to do this natively, so we have to draw over ourselves (TODO?)
+// hopefully the performance won't be too bad
+// see also https://www.codeproject.com/Articles/79/Neat-Stuff-to-Do-in-List-Controls-Using-Custom-Dra
+HRESULT uiprivNM_CUSTOMDRAWImagesCheckboxes(uiTable *t, NMLVCUSTOMDRAW *nm, LRESULT *lResult)
+{
+	uiprivTableColumnParams *p;
+	int index;
+	RECT r;
+
+	if (nm->nmcd.dwDrawStage == (CDDS_SUBITEM | CDDS_ITEMPREPAINT)) {
+		*lResult |= CDRF_NOTIFYPOSTPAINT;
+		return S_OK;
+	}
+	if (nm->nmcd.dwDrawStage != (CDDS_SUBITEM | CDDS_ITEMPOSTPAINT))
+		return S_OK;
+
+	// only draw over checkboxes
+	p = (*(t->columns))[nm->iSubItem];
+	if (p->checkboxModelColumn == -1)
+		return S_OK;
+
+	index = checkboxIndex(t->model, nm->nmcd.dwItemSpec,
+		p->checkboxModelColumn, p->checkboxEditableColumn);
+	ZeroMemory(&r, sizeof (RECT));
+	r.left = LVIR_ICON;
+	r.top = nm->iSubItem;
+	if (SendMessageW(t->hwnd, LVM_GETSUBITEMRECT, nm->nmcd.dwItemSpec, (LPARAM) (&r)) == 0) {
+		logLastError(L"LVM_GETSUBITEMRECT");
+		return E_FAIL;
+	}
+#if 0
+	// TODO this is offset by one pixel on my system and everything I've found indicates this should not be happening???
+	if (ImageList_Draw(t->smallImages, index, nm->nmcd.hdc,
+		r.left, r.top, ILD_NORMAL) == 0) {
+		logLastError(L"ImageList_Draw()");
+		return E_FAIL;
+	}
+#endif
+	return S_OK;
+}
+
 // references for checkbox drawing:
 // - https://blogs.msdn.microsoft.com/oldnewthing/20171129-00/?p=97485
 // - https://blogs.msdn.microsoft.com/oldnewthing/20171201-00/?p=97505