git: 35b262e57114 - stable/13 - rb_tree: avoid extra reads in rebalancing

From: Doug Moore <dougm_at_FreeBSD.org>
Date: Mon, 19 Sep 2022 02:39:41 UTC
The branch stable/13 has been updated by dougm:

URL: https://cgit.FreeBSD.org/src/commit/?id=35b262e57114adfa8064c66b9ab83fc60e11fa81

commit 35b262e57114adfa8064c66b9ab83fc60e11fa81
Author:     Doug Moore <dougm@FreeBSD.org>
AuthorDate: 2022-08-29 16:11:31 +0000
Commit:     Doug Moore <dougm@FreeBSD.org>
CommitDate: 2022-09-19 02:39:21 +0000

    rb_tree: avoid extra reads in rebalancing
    
    In RB_INSERT_COLOR and RB_REMOVE_COLOR, avoid reading a parent pointer
    from memory, and then reading the left-color bit from memory, and then
    reading the right-color bit from memory, since they're all in the same
    field. The compiler can't infer that only the first read is really
    necessary, so write the code in a way so that it doesn't have to.
    
    Drop RB_RED_LEFT and RB_RED_RIGHT macros that reach into memory to get
    those bits.  Drop RB_COLOR, the only thing left using RB_RED_LEFT and
    RB_RED_RIGHT after the other changes, and go straight to DIAGNOSTIC
    code in subr_stats to implement RB_COLOR for its single, dubious use
    there.
    
    Reviewed by:    alc
    MFC after:      3 weeks
    Differential Revision:  https://reviews.freebsd.org/D36353
    
    (cherry picked from commit 5d91386826d0f360d87ce3f39c9ebf266f149325)
---
 sys/kern/subr_stats.c |  9 +++++-
 sys/sys/tree.h        | 82 +++++++++++++++++++++++++++------------------------
 2 files changed, 52 insertions(+), 39 deletions(-)

diff --git a/sys/kern/subr_stats.c b/sys/kern/subr_stats.c
index 946999263898..0984d2a21014 100644
--- a/sys/kern/subr_stats.c
+++ b/sys/kern/subr_stats.c
@@ -3406,6 +3406,13 @@ stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst,
 
 				Q_TOSTR(rbctd64->mu, -1, 10, qstr,
 				    sizeof(qstr));
+				struct voistatdata_tdgstctd64 *parent;
+				parent = RB_PARENT(rbctd64, rblnk);
+				int rb_color =
+					parent == NULL ? 0 :
+					RB_LEFT(parent, rblnk) == rbctd64 ?
+					(RB_BITS(parent, rblnk) & RB_RED_L) != 0 :
+ 					(RB_BITS(parent, rblnk) & RB_RED_R) != 0;
 				printf(" RB ctd=%3d p=%3d l=%3d r=%3d c=%2d "
 				    "mu=%s\n",
 				    (int)ARB_SELFIDX(ctd64tree, rbctd64),
@@ -3415,7 +3422,7 @@ stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst,
 				      RB_LEFT(rbctd64, rblnk)),
 				    (int)ARB_SELFIDX(ctd64tree,
 				      RB_RIGHT(rbctd64, rblnk)),
-				    RB_COLOR(rbctd64, rblnk),
+				    rb_color,
 				    qstr);
 
 				panic("RB@%p and ARB@%p trees differ\n",
diff --git a/sys/sys/tree.h b/sys/sys/tree.h
index 38c82c4e4a52..3f1735d2a837 100644
--- a/sys/sys/tree.h
+++ b/sys/sys/tree.h
@@ -341,8 +341,6 @@ struct {								\
 #define RB_FLIP_LEFT(elm, field)	(RB_BITS(elm, field) ^= RB_RED_L)
 #define RB_FLIP_RIGHT(elm, field)	(RB_BITS(elm, field) ^= RB_RED_R)
 #define RB_FLIP_ALL(elm, field)		(RB_BITS(elm, field) ^= RB_RED_MASK)
-#define RB_RED_LEFT(elm, field)		((RB_BITS(elm, field) & RB_RED_L) != 0)
-#define RB_RED_RIGHT(elm, field)	((RB_BITS(elm, field) & RB_RED_R) != 0)
 #define _RB_PARENT_ONLY(elm)		(__typeof(elm))			\
 					((__uintptr_t)elm & ~RB_RED_MASK)
 #define RB_PARENT(elm, field)		_RB_PARENT_ONLY(RB_UP(elm, field))
@@ -359,11 +357,6 @@ struct {								\
 	RB_LEFT(elm, field) = RB_RIGHT(elm, field) = NULL;		\
 } while (/*CONSTCOND*/ 0)
 
-#define RB_COLOR(elm, field)	(RB_PARENT(elm, field) == NULL ? 0 :	\
-				RB_LEFT(RB_PARENT(elm, field), field) == elm ? \
-				RB_RED_LEFT(RB_PARENT(elm, field), field) : \
-				RB_RED_RIGHT(RB_PARENT(elm, field), field))
-
 /*
  * Something to be invoked in a loop at the root of every modified subtree,
  * from the bottom up to the root, to update augmented node data.
@@ -489,67 +482,72 @@ name##_RB_INSERT_COLOR(struct name *head, struct type *elm)		\
 	 * when a value has been assigned to 'child' in the previous    \
 	 * one.								\
 	 */								\
-	struct type *child, *gpar, *parent;				\
-	while ((parent = RB_PARENT(elm, field)) != NULL) {		\
-		gpar = RB_PARENT(parent, field);			\
+	struct type *child, *gpar = RB_UP(elm, field), *parent;		\
+	__uintptr_t red;						\
+									\
+	while ((parent = gpar) != NULL) {				\
+		red = RB_BITS(parent, field);				\
+		gpar = RB_UP(parent, field);				\
 		if (RB_LEFT(parent, field) == elm) {			\
-			if (RB_RED_LEFT(parent, field)) {		\
+			if (red & RB_RED_L) {				\
 				RB_FLIP_LEFT(parent, field);		\
 				return;					\
 			}						\
 			RB_FLIP_RIGHT(parent, field);			\
-			if (RB_RED_RIGHT(parent, field)) {		\
+			if ((red & RB_RED_MASK) == 0) {			\
 				child = elm;				\
 				elm = parent;				\
 				continue;				\
 			}						\
-			if (RB_RED_RIGHT(elm, field))			\
+			red = RB_BITS(elm, field);			\
+			if (red & RB_RED_R)				\
 				child = elm;				\
 			else {						\
 				/* coverity[uninit_use] */		\
 				RB_ROTATE_LEFT(elm, child, field);	\
-				if (RB_RED_RIGHT(child, field))		\
+				red = RB_BITS(child, field);		\
+				if (red & RB_RED_R)			\
 					RB_FLIP_LEFT(parent, field);	\
-				if (RB_RED_LEFT(child, field))		\
+				if (red & RB_RED_L)			\
 					RB_FLIP_ALL(elm, field);	\
 				else					\
 					RB_FLIP_LEFT(elm, field);	\
-				if ((RB_BITS(child, field) &		\
-				    RB_RED_MASK) == 0)			\
+				if ((red & RB_RED_MASK) == 0)		\
 					elm = child;			\
 			}						\
 			RB_ROTATE_RIGHT(parent, child, field);		\
 		} else {						\
-			if (RB_RED_RIGHT(parent, field)) {		\
+			if (red & RB_RED_R) {				\
 				RB_FLIP_RIGHT(parent, field);		\
 				return;					\
 			}						\
 			RB_FLIP_LEFT(parent, field);			\
-			if (RB_RED_LEFT(parent, field)) {		\
+			if ((red & RB_RED_MASK) == 0) {			\
 				child = elm;				\
 				elm = parent;				\
 				continue;				\
 			}						\
-			if (RB_RED_LEFT(elm, field))			\
+			red = RB_BITS(elm, field);			\
+			if (red & RB_RED_L)				\
 				child = elm;				\
 			else {						\
 				/* coverity[uninit_use] */		\
 				RB_ROTATE_RIGHT(elm, child, field);	\
-				if (RB_RED_LEFT(child, field))		\
+				red = RB_BITS(child, field);		\
+				if (red & RB_RED_L)			\
 					RB_FLIP_RIGHT(parent, field);	\
-				if (RB_RED_RIGHT(child, field))		\
+				if (red & RB_RED_R)			\
 					RB_FLIP_ALL(elm, field);	\
 				else					\
 					RB_FLIP_RIGHT(elm, field);	\
-				if ((RB_BITS(child, field) &		\
-				    RB_RED_MASK) == 0)			\
+				if ((red & RB_RED_MASK) == 0)		\
 					elm = child;			\
 			}						\
 			RB_ROTATE_LEFT(parent, child, field);		\
 		}							\
-		RB_SET_PARENT(child, gpar, field);			\
+		gpar = _RB_PARENT_ONLY(gpar);				\
+		RB_UP(child, field) = gpar;				\
 		RB_SWAP_CHILD(head, gpar, parent, child, field);	\
-		RB_BITS(child, field) &= ~RB_RED_MASK;			\
 		if (elm != child)					\
 			RB_AUGMENT(elm);				\
 		RB_AUGMENT(parent);					\
@@ -574,6 +572,8 @@ name##_RB_REMOVE_COLOR(struct name *head,				\
     struct type *parent, struct type *elm)				\
 {									\
 	struct type *gpar, *sib;					\
+	__uintptr_t red;						\
+									\
 	if (RB_LEFT(parent, field) == elm &&				\
 	    RB_RIGHT(parent, field) == elm) {				\
 		RB_BITS(parent, field) &= ~RB_RED_MASK;			\
@@ -583,19 +583,21 @@ name##_RB_REMOVE_COLOR(struct name *head,				\
 			return;						\
 	}								\
 	do {								\
-		gpar = RB_PARENT(parent, field);			\
+		red = RB_BITS(parent, field);				\
+		gpar = RB_UP(parent, field);				\
 		if (RB_LEFT(parent, field) == elm) {			\
-			if (!RB_RED_LEFT(parent, field)) {		\
+			if (~red & RB_RED_L) {				\
 				RB_FLIP_LEFT(parent, field);		\
 				return;					\
 			}						\
-			if (RB_RED_RIGHT(parent, field)) {		\
+			if ((~red & RB_RED_MASK) == 0) {		\
 				RB_FLIP_RIGHT(parent, field);		\
 				elm = parent;				\
 				continue;				\
 			}						\
 			sib = RB_RIGHT(parent, field);			\
-			switch (RB_BITS(sib, field) & RB_RED_MASK) {	\
+			red = RB_BITS(sib, field);			\
+			switch (red & RB_RED_MASK) {			\
 			case RB_RED_MASK:				\
 				RB_FLIP_ALL(sib, field);		\
 				elm = parent;				\
@@ -603,11 +605,12 @@ name##_RB_REMOVE_COLOR(struct name *head,				\
 			case RB_RED_R:					\
 				elm = RB_LEFT(sib, field);		\
 				RB_ROTATE_RIGHT(sib, elm, field);	\
-				if (RB_RED_LEFT(elm, field))		\
+				red = RB_BITS(elm, field);		\
+				if (red & RB_RED_L)			\
 					RB_FLIP_ALL(parent, field);	\
 				else					\
 					RB_FLIP_LEFT(parent, field);	\
-				if (RB_RED_RIGHT(elm, field))		\
+				if (red & RB_RED_R)			\
 					RB_FLIP_ALL(sib, field);	\
 				else					\
 					RB_FLIP_RIGHT(sib, field);	\
@@ -629,17 +632,18 @@ name##_RB_REMOVE_COLOR(struct name *head,				\
 			}						\
 			RB_ROTATE_LEFT(parent, elm, field);		\
 		} else {						\
-			if (!RB_RED_RIGHT(parent, field)) {		\
+			if (~red & RB_RED_R) {				\
 				RB_FLIP_RIGHT(parent, field);		\
 				return;					\
 			}						\
-			if (RB_RED_LEFT(parent, field)) {		\
+			if ((~red & RB_RED_MASK) == 0) {		\
 				RB_FLIP_LEFT(parent, field);		\
 				elm = parent;				\
 				continue;				\
 			}						\
 			sib = RB_LEFT(parent, field);			\
-			switch (RB_BITS(sib, field) & RB_RED_MASK) {	\
+			red = RB_BITS(sib, field);			\
+			switch (red & RB_RED_MASK) {			\
 			case RB_RED_MASK:				\
 				RB_FLIP_ALL(sib, field);		\
 				elm = parent;				\
@@ -647,11 +651,12 @@ name##_RB_REMOVE_COLOR(struct name *head,				\
 			case RB_RED_L:					\
 				elm = RB_RIGHT(sib, field);		\
 				RB_ROTATE_LEFT(sib, elm, field);	\
-				if (RB_RED_RIGHT(elm, field))		\
+				red = RB_BITS(elm, field);		\
+				if (red & RB_RED_R)			\
 					RB_FLIP_ALL(parent, field);	\
 				else					\
 					RB_FLIP_RIGHT(parent, field);	\
-				if (RB_RED_LEFT(elm, field))		\
+				if (red & RB_RED_L)			\
 					RB_FLIP_ALL(sib, field);	\
 				else					\
 					RB_FLIP_LEFT(sib, field);	\
@@ -673,12 +678,13 @@ name##_RB_REMOVE_COLOR(struct name *head,				\
 			}						\
 			RB_ROTATE_RIGHT(parent, elm, field);		\
 		}							\
+		gpar = _RB_PARENT_ONLY(gpar);				\
 		RB_SET_PARENT(elm, gpar, field);			\
 		RB_SWAP_CHILD(head, gpar, parent, elm, field);		\
 		if (sib != elm)						\
 			RB_AUGMENT(sib);				\
 		break;							\
-	} while ((parent = RB_PARENT(elm, field)) != NULL);		\
+	} while ((parent = _RB_PARENT_ONLY(gpar)) != NULL);		\
 }
 
 #define RB_GENERATE_REMOVE(name, type, field, attr)			\