Wrote the new hit-testing API on OS X, the easiest target. Also updated the example. Had to slightly modify one function for this all to work.

darwin/drawtext.m

@@ -280,6 +280,8 @@ void uiDrawTextLayoutLineGetMetrics(uiDrawTextLayout *tl, int line, uiDrawTextLa
 	*m = tl->lineMetrics[line];
 }
 
+#if 0 /* TODO */
+
 // TODO note that in some cases lines can overlap slightly
 // in our case, we read lines first to last and use their bottommost point (Y + Height) to determine where the next line should start for hit-testing
 void uiDrawTextLayoutHitTest(uiDrawTextLayout *tl, double x, double y, uiDrawTextLayoutHitTestResult *result)
@@ -381,3 +383,48 @@ void uiDrawTextLayoutByteRangeToRectangle(uiDrawTextLayout *tl, size_t start, si
 	r->RealStart = tl->u16tou8[r->RealStart];
 	r->RealEnd = tl->u16tou8[r->RealEnd];
 }
+
+#endif
+
+// TODO note that in some cases lines can overlap slightly
+// in our case, we read lines first to last and use their bottommost point (Y + Height) to determine where the next line should start for hit-testing
+void uiDrawTextLayoutHitTest(uiDrawTextLayout *tl, double x, double y, size_t *pos, int *line)
+{
+	int i;
+	CTLineRef ln;
+	CFIndex p;
+
+	for (i = 0; i < tl->nLines; i++) {
+		double ltop, lbottom;
+
+		ltop = tl->lineMetrics[i].Y;
+		lbottom = ltop + tl->lineMetrics[i].Height;
+		// y will already >= ltop at this point since the past lbottom should == (or at least >=, see above) ltop
+		if (y < lbottom)
+			break;
+	}
+	if (i == tl->nLines)
+		i--;
+	if (line != NULL)
+		*line = i;
+
+	// TODO do the hit-test unconditionally?
+	if (pos != NULL) {
+		ln = (CTLineRef) CFArrayGetValueAtIndex(tl->lines, i);
+		p = CTLineGetStringIndexForPosition(ln, CGPointMake(x, 0));
+		if (p == kCFNotFound) {
+			// TODO
+		}
+		*pos = tl->u16tou8[p];
+	}
+}
+
+double uiDrawTextLayoutByteLocationInLine(uiDrawTextLayout *tl, size_t pos, int line)
+{
+	CTLineRef ln;
+
+	ln = (CTLineRef) CFArrayGetValueAtIndex(tl->lines, line);
+	// TODO what happens if the byte location is not in this line? a return of 0?
+	// TODO check return? see if this can return 0, anyway, and if so make a note I guess
+	return CTLineGetOffsetForStringIndex(ln, tl->u8tou16[pos], NULL);
+}

examples/drawtext/hittest.c

@@ -33,9 +33,8 @@ static uiBox *panel;
 static uiLabel *caretLabel;
 static uiCheckbox *showLineBounds;
 
-static int caretInit = 0;
-// TODO rename all this to caret, as well as in the text above
-static double cursorX, cursorY, cursorHeight;
+static int caretLine = -1;
+static double caretX;
 
 // TODO should be const?
 static uiDrawBrush fillBrushes[4] = {
@@ -84,6 +83,7 @@ static void draw(uiAreaDrawParams *p)
 {
 	uiDrawPath *path;
 	uiDrawTextLayout *layout;
+	uiDrawTextLayoutLineMetrics m;
 	uiDrawBrush brush;
 
 	// only clip the text, not the guides
@@ -104,21 +104,16 @@ static void draw(uiAreaDrawParams *p)
 
 	uiDrawRestore(p->Context);
 
-	if (!caretInit) {
-		uiDrawTextLayoutByteRangeRectangle r;
-
-		uiDrawTextLayoutByteRangeToRectangle(layout,
+	if (caretLine == -1) {
+		caretLine = uiDrawTextLayoutNumLines(layout) - 1;
+		caretX = uiDrawTextLayoutByteLocationInLine(layout,
 			uiAttributedStringLen(attrstr),
-			uiAttributedStringLen(attrstr),
-			&r);
-		cursorX = r.X;
-		cursorY = r.Y;
-		cursorHeight = r.Height;
-		caretInit = 1;
+			caretLine);
 	}
+	uiDrawTextLayoutLineGetMetrics(layout, caretLine, &m);
 	path = uiDrawNewPath(uiDrawFillModeWinding);
-	uiDrawPathNewFigure(path, margins + cursorX, margins + cursorY);
-	uiDrawPathLineTo(path, margins + cursorX, margins + cursorY + cursorHeight);
+	uiDrawPathNewFigure(path, margins + caretX, margins + m.Y);
+	uiDrawPathLineTo(path, margins + caretX, margins + m.Y + m.Height);
 	uiDrawPathEnd(path);
 	brush.Type = uiDrawBrushTypeSolid;
 	brush.R = 0.0;
@@ -129,7 +124,6 @@ static void draw(uiAreaDrawParams *p)
 	uiDrawFreePath(path);
 
 	if (uiCheckboxChecked(showLineBounds)) {
-		uiDrawTextLayoutLineMetrics m;
 		int i, n;
 		int fill = 0;
 
@@ -158,7 +152,8 @@ static const char *positions[] = {
 static void mouse(uiAreaMouseEvent *e)
 {
 	uiDrawTextLayout *layout;
-	uiDrawTextLayoutHitTestResult res;
+//	uiDrawTextLayoutHitTestResult res;
+	size_t pos;
 	char labelText[128];
 
 	if (e->Down != 1)
@@ -169,22 +164,25 @@ static void mouse(uiAreaMouseEvent *e)
 		e->AreaWidth - 2 * margins);
 	uiDrawTextLayoutHitTest(layout,
 		e->X - margins, e->Y - margins,
-		&res);
+//		&res);
+		&pos, &caretLine);
+	caretX = uiDrawTextLayoutByteLocationInLine(layout, pos, caretLine);
 	uiDrawFreeTextLayout(layout);
 
 	// urgh %zd is not supported by MSVC with sprintf()
 	// TODO get that part in test/ about having no other option
 	// TODO byte 1 is actually byte 684?!
-	sprintf(labelText, "pos %d line %d x position %s y position %s",
+/*	sprintf(labelText, "pos %d line %d x position %s y position %s",
 		(int) (res.Pos), res.Line,
 		positions[res.XPosition],
 		positions[res.YPosition]);
+*/	sprintf(labelText, "TODO\n");
 	uiLabelSetText(caretLabel, labelText);
 
-	cursorX = res.CaretX;
+/*	cursorX = res.CaretX;
 	cursorY = res.CaretY;
 	cursorHeight = res.CaretHeight;
-	redraw();
+*/	redraw();
 }
 
 static struct example hitTestExample;

ui_attrstr.h

@@ -187,5 +187,4 @@ _UI_EXTERN void uiDrawTextLayoutLineGetMetrics(uiDrawTextLayout *tl, int line, u
 // TODO number of lines visible for clipping rect, range visible for clipping rect?
 
 _UI_EXTERN void uiDrawTextLayoutHitTest(uiDrawTextLayout *tl, double x, double y, size_t *pos, int *line);
-// TODO line first?
-_UI_EXTERN void uiDrawTextLayoutByteLocation(uiDrawTextLayout *tl, size_t pos, double *x, int *line);
+_UI_EXTERN double uiDrawTextLayoutByteLocationInLine(uiDrawTextLayout *tl, size_t pos, int line);