Sleeping thread held mutex in vm_pageout_oom()

[Konstantin Belousov]

On Wed, Jan 21, 2015 at 09:04:51PM -0500, Ryan Stone wrote:
> Thanks for the patch.  I tried it out this afternoon and got the
> following panic:
> 
> panic: PHOLD of exiting process^M^M
> cpuid = 9^M^M
> curthread = pagedaemon/pagedaemon (8/100154)^M^M
> cpu_ticks = 979566002403^M^M
> KDB: stack backtrace:^M^M
> db_trace_self_wrapper() at 0xffffffff801e672a = db_trace_self_wrapper+0x2a^M^M
> panic() at 0xffffffff80480818 = panic+0x228^M^M
> vm_pageout_oom() at 0xffffffff80669a9b = vm_pageout_oom+0x58b^M^M
> vm_pageout() at 0xffffffff8066aa35 = vm_pageout+0x975^M^M
> fork_exit() at 0xffffffff80454ada = fork_exit+0x12a^M^M
> fork_trampoline() at 0xffffffff8067b37e = fork_trampoline+0xe^M^M
> 
> 
> I believe that we need to check for P_WEXIT before considering p as a
> candidate to be killed:

Sure, you are right.  I also took opportunity to reformat the condition
and to remove excessive () there.

diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index ca9d7f9..d2e0483 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -1516,15 +1542,15 @@ vm_pageout_oom(int shortage)
 	FOREACH_PROC_IN_SYSTEM(p) {
 		int breakout;
 
-		if (PROC_TRYLOCK(p) == 0)
-			continue;
+		PROC_LOCK(p);
+
 		/*
 		 * If this is a system, protected or killed process, skip it.
 		 */
-		if (p->p_state != PRS_NORMAL ||
-		    (p->p_flag & (P_INEXEC | P_PROTECTED | P_SYSTEM)) ||
-		    (p->p_pid == 1) || P_KILLED(p) ||
-		    ((p->p_pid < 48) && (swap_pager_avail != 0))) {
+		if (p->p_state != PRS_NORMAL || (p->p_flag & (P_INEXEC |
+		    P_PROTECTED | P_SYSTEM | P_WEXIT)) != 0 ||
+		    p->p_pid == 1 || P_KILLED(p) ||
+		    (p->p_pid < 48 && swap_pager_avail != 0)) {
 			PROC_UNLOCK(p);
 			continue;
 		}
@@ -1557,11 +1583,14 @@ vm_pageout_oom(int shortage)
 			PROC_UNLOCK(p);
 			continue;
 		}
+		_PHOLD(p);
 		if (!vm_map_trylock_read(&vm->vm_map)) {
-			vmspace_free(vm);
+			_PRELE(p);
 			PROC_UNLOCK(p);
+			vmspace_free(vm);
 			continue;
 		}
+		PROC_UNLOCK(p);
 		size = vmspace_swap_count(vm);
 		vm_map_unlock_read(&vm->vm_map);
 		if (shortage == VM_OOM_MEM)
@@ -1573,16 +1602,19 @@ vm_pageout_oom(int shortage)
 		 */
 		if (size > bigsize) {
 			if (bigproc != NULL)
-				PROC_UNLOCK(bigproc);
+				PRELE(bigproc);
 			bigproc = p;
 			bigsize = size;
-		} else
-			PROC_UNLOCK(p);
+		} else {
+			PRELE(p);
+		}
 	}
 	sx_sunlock(&allproc_lock);
 	if (bigproc != NULL) {
+		PROC_LOCK(bigproc);
 		killproc(bigproc, "out of swap space");
 		sched_nice(bigproc, PRIO_MIN);
+		_PRELE(bigproc);
 		PROC_UNLOCK(bigproc);
 		wakeup(&vm_cnt.v_free_count);
 	}