svn commit: r327786 - stable/11/sys/vm

[Mark Johnston]

Author: markj
Date: Wed Jan 10 20:48:21 2018
New Revision: 327786
URL: https://svnweb.freebsd.org/changeset/base/327786

Log:
  MFC r327168, r327213:
  Fix two problems with the page daemon control loop.

Modified:
  stable/11/sys/vm/vm_page.c
  stable/11/sys/vm/vm_pageout.c
  stable/11/sys/vm/vm_pageout.h
Directory Properties:
  stable/11/   (props changed)

Modified: stable/11/sys/vm/vm_page.c
==============================================================================
--- stable/11/sys/vm/vm_page.c	Wed Jan 10 20:39:26 2018	(r327785)
+++ stable/11/sys/vm/vm_page.c	Wed Jan 10 20:48:21 2018	(r327786)
@@ -2580,15 +2580,9 @@ _vm_wait(void)
 		msleep(&vm_pageout_pages_needed, &vm_page_queue_free_mtx,
 		    PDROP | PSWP, "VMWait", 0);
 	} else {
-		if (__predict_false(pageproc == NULL))
+		if (pageproc == NULL)
 			panic("vm_wait in early boot");
-		if (!vm_pageout_wanted) {
-			vm_pageout_wanted = true;
-			wakeup(&vm_pageout_wanted);
-		}
-		vm_pages_needed = true;
-		msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PVM,
-		    "vmwait", 0);
+		pagedaemon_wait(PVM, "vmwait");
 	}
 }
 
@@ -2618,7 +2612,6 @@ vm_page_alloc_fail(vm_object_t object, int req)
 
 	atomic_add_int(&vm_pageout_deficit,
 	    max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
-	pagedaemon_wakeup();
 	if (req & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) {
 		if (object != NULL) 
 			VM_OBJECT_WUNLOCK(object);
@@ -2627,8 +2620,10 @@ vm_page_alloc_fail(vm_object_t object, int req)
 			VM_OBJECT_WLOCK(object);
 		if (req & VM_ALLOC_WAITOK)
 			return (EAGAIN);
-	} else
+	} else {
 		mtx_unlock(&vm_page_queue_free_mtx);
+		pagedaemon_wakeup();
+	}
 	return (0);
 }
 
@@ -2647,13 +2642,7 @@ vm_waitpfault(void)
 {
 
 	mtx_lock(&vm_page_queue_free_mtx);
-	if (!vm_pageout_wanted) {
-		vm_pageout_wanted = true;
-		wakeup(&vm_pageout_wanted);
-	}
-	vm_pages_needed = true;
-	msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PUSER,
-	    "pfault", 0);
+	pagedaemon_wait(PUSER, "pfault");
 }
 
 struct vm_pagequeue *

Modified: stable/11/sys/vm/vm_pageout.c
==============================================================================
--- stable/11/sys/vm/vm_pageout.c	Wed Jan 10 20:39:26 2018	(r327785)
+++ stable/11/sys/vm/vm_pageout.c	Wed Jan 10 20:48:21 2018	(r327786)
@@ -1768,10 +1768,16 @@ vm_pageout_worker(void *arg)
 			pass++;
 		} else {
 			/*
-			 * Yes.  Sleep until pages need to be reclaimed or
+			 * Yes.  If threads are still sleeping in VM_WAIT
+			 * then we immediately start a new scan.  Otherwise,
+			 * sleep until the next wakeup or until pages need to
 			 * have their reference stats updated.
 			 */
-			if (mtx_sleep(&vm_pageout_wanted,
+			if (vm_pages_needed) {
+				mtx_unlock(&vm_page_queue_free_mtx);
+				if (pass == 0)
+					pass++;
+			} else if (mtx_sleep(&vm_pageout_wanted,
 			    &vm_page_queue_free_mtx, PDROP | PVM, "psleep",
 			    hz) == 0) {
 				PCPU_INC(cnt.v_pdwakeups);
@@ -1879,17 +1885,42 @@ vm_pageout(void)
 }
 
 /*
- * Unless the free page queue lock is held by the caller, this function
- * should be regarded as advisory.  Specifically, the caller should
- * not msleep() on &vm_cnt.v_free_count following this function unless
- * the free page queue lock is held until the msleep() is performed.
+ * Perform an advisory wakeup of the page daemon.
  */
 void
 pagedaemon_wakeup(void)
 {
 
+	mtx_assert(&vm_page_queue_free_mtx, MA_NOTOWNED);
+
 	if (!vm_pageout_wanted && curthread->td_proc != pageproc) {
 		vm_pageout_wanted = true;
 		wakeup(&vm_pageout_wanted);
 	}
+}
+
+/*
+ * Wake up the page daemon and wait for it to reclaim free pages.
+ *
+ * This function returns with the free queues mutex unlocked.
+ */
+void
+pagedaemon_wait(int pri, const char *wmesg)
+{
+
+	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
+
+	/*
+	 * vm_pageout_wanted may have been set by an advisory wakeup, but if the
+	 * page daemon is running on a CPU, the wakeup will have been lost.
+	 * Thus, deliver a potentially spurious wakeup to ensure that the page
+	 * daemon has been notified of the shortage.
+	 */
+	if (!vm_pageout_wanted || !vm_pages_needed) {
+		vm_pageout_wanted = true;
+		wakeup(&vm_pageout_wanted);
+	}
+	vm_pages_needed = true;
+	msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | pri,
+	    wmesg, 0);
 }

Modified: stable/11/sys/vm/vm_pageout.h
==============================================================================
--- stable/11/sys/vm/vm_pageout.h	Wed Jan 10 20:39:26 2018	(r327785)
+++ stable/11/sys/vm/vm_pageout.h	Wed Jan 10 20:48:21 2018	(r327786)
@@ -94,11 +94,12 @@ extern bool vm_pages_needed;
  *	Signal pageout-daemon and wait for it.
  */
 
-extern void pagedaemon_wakeup(void);
+void pagedaemon_wait(int pri, const char *wmesg);
+void pagedaemon_wakeup(void);
 #define VM_WAIT vm_wait()
 #define VM_WAITPFAULT vm_waitpfault()
-extern void vm_wait(void);
-extern void vm_waitpfault(void);
+void vm_wait(void);
+void vm_waitpfault(void);
 
 #ifdef _KERNEL
 int vm_pageout_flush(vm_page_t *, int, int, int, int *, boolean_t *);