svn commit: r302131 - head/sys/vm

[Konstantin Belousov]

Author: kib
Date: Thu Jun 23 08:28:13 2016
New Revision: 302131
URL: https://svnweb.freebsd.org/changeset/base/302131

Log:
  In vm_page_xunbusy_maybelocked(), add fast path for unbusy when no
  waiters exist, same as for vm_page_xunbusy().  If previous value of
  busy_lock was VPB_SINGLE_EXCLUSIVER, no waiters existed and wakeup is
  not needed.
  
  Move common code from vm_page_xunbusy_maybelocked() and
  vm_page_xunbusy_hard() to vm_page_xunbusy_locked().
  
  Reviewed by:	alc
  Sponsored by:	The FreeBSD Foundation
  MFC after:	2 weeks
  Approved by:	re (gjb)

Modified:
  head/sys/vm/vm_page.c

Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c	Thu Jun 23 08:27:38 2016	(r302130)
+++ head/sys/vm/vm_page.c	Thu Jun 23 08:28:13 2016	(r302131)
@@ -760,17 +760,36 @@ vm_page_trysbusy(vm_page_t m)
 }
 
 static void
+vm_page_xunbusy_locked(vm_page_t m)
+{
+
+	vm_page_assert_xbusied(m);
+	vm_page_assert_locked(m);
+
+	atomic_store_rel_int(&m->busy_lock, VPB_UNBUSIED);
+	/* There is a waiter, do wakeup() instead of vm_page_flash(). */
+	wakeup(m);
+}
+
+static void
 vm_page_xunbusy_maybelocked(vm_page_t m)
 {
 	bool lockacq;
 
 	vm_page_assert_xbusied(m);
 
+	/*
+	 * Fast path for unbusy.  If it succeeds, we know that there
+	 * are no waiters, so we do not need a wakeup.
+	 */
+	if (atomic_cmpset_rel_int(&m->busy_lock, VPB_SINGLE_EXCLUSIVER,
+	    VPB_UNBUSIED))
+		return;
+
 	lockacq = !mtx_owned(vm_page_lockptr(m));
 	if (lockacq)
 		vm_page_lock(m);
-	vm_page_flash(m);
-	atomic_store_rel_int(&m->busy_lock, VPB_UNBUSIED);
+	vm_page_xunbusy_locked(m);
 	if (lockacq)
 		vm_page_unlock(m);
 }
@@ -788,8 +807,7 @@ vm_page_xunbusy_hard(vm_page_t m)
 	vm_page_assert_xbusied(m);
 
 	vm_page_lock(m);
-	atomic_store_rel_int(&m->busy_lock, VPB_UNBUSIED);
-	wakeup(m);
+	vm_page_xunbusy_locked(m);
 	vm_page_unlock(m);
 }