svn commit: r228314 - user/attilio/vmcontention/sys/vm

[Attilio Rao]

Author: attilio
Date: Tue Dec  6 22:57:48 2011
New Revision: 228314
URL: http://svn.freebsd.org/changeset/base/228314

Log:
  Use atomics for rn_count on leaf node because RED operations happen
  without the VM_OBJECT_LOCK held, thus can be concurrent with BLACK ones.
  However, also use a write memory barrier in order to not reorder the
  operation of decrementing rn_count in respect fetching the pointer.
  
  Discussed with:	jeff

Modified:
  user/attilio/vmcontention/sys/vm/vm_radix.c
  user/attilio/vmcontention/sys/vm/vm_radix.h

Modified: user/attilio/vmcontention/sys/vm/vm_radix.c
==============================================================================
--- user/attilio/vmcontention/sys/vm/vm_radix.c	Tue Dec  6 20:55:20 2011	(r228313)
+++ user/attilio/vmcontention/sys/vm/vm_radix.c	Tue Dec  6 22:57:48 2011	(r228314)
@@ -344,7 +344,7 @@ vm_radix_insert(struct vm_radix *rtree, 
 	    rnode->rn_child[slot], (u_long)index));
 	val = (void *)((uintptr_t)val | VM_RADIX_BLACK);
 	rnode->rn_child[slot] = val;
-	rnode->rn_count++;
+	atomic_add_32(&rnode->rn_count, 1);
 	CTR6(KTR_VM,
 	    "insert: tree %p, index %ju, level %d, slot %d, rnode %p, count %u",
 	    rtree, (uintmax_t)index, level, slot, rnode, rnode->rn_count);
@@ -681,7 +681,18 @@ vm_radix_remove(struct vm_radix *rtree, 
 		    (rnode != NULL) ? rnode->rn_child[slot] : NULL,
 		    (rnode != NULL) ? rnode->rn_count : 0);
 		rnode->rn_child[slot] = NULL;
-		rnode->rn_count--;
+		/*
+		 * Use atomics for the last level since red and black
+		 * will both adjust it.
+		 * Use a write memory barrier here in order to avoid
+		 * rn_count reaching 0 before to fetch the actual pointer.
+		 * Concurrent black removal, infact, may want to reclaim
+		 * the radix node itself before to read it.
+		 */
+		if (level == 0)
+			atomic_add_rel_32(&rnode->rn_count, -1);
+		else
+			rnode->rn_count--;
 		/*
 		 * Only allow black removes to prune the tree.
 		 */

Modified: user/attilio/vmcontention/sys/vm/vm_radix.h
==============================================================================
--- user/attilio/vmcontention/sys/vm/vm_radix.h	Tue Dec  6 20:55:20 2011	(r228313)
+++ user/attilio/vmcontention/sys/vm/vm_radix.h	Tue Dec  6 22:57:48 2011	(r228314)
@@ -62,7 +62,7 @@ CTASSERT(VM_RADIX_HEIGHT >= VM_RADIX_LIM
 
 struct vm_radix_node {
 	void		*rn_child[VM_RADIX_COUNT];	/* Child nodes. */
-    	uint32_t	 rn_count;			/* Valid children. */
+    	volatile uint32_t rn_count;			/* Valid children. */
 };
 
 void	vm_radix_init(void);