Sleeping thread held mutex in vm_pageout_oom()

[Ryan Stone]

Thanks for the patch.  I tried it out this afternoon and got the
following panic:

panic: PHOLD of exiting process^M^M
cpuid = 9^M^M
curthread = pagedaemon/pagedaemon (8/100154)^M^M
cpu_ticks = 979566002403^M^M
KDB: stack backtrace:^M^M
db_trace_self_wrapper() at 0xffffffff801e672a = db_trace_self_wrapper+0x2a^M^M
panic() at 0xffffffff80480818 = panic+0x228^M^M
vm_pageout_oom() at 0xffffffff80669a9b = vm_pageout_oom+0x58b^M^M
vm_pageout() at 0xffffffff8066aa35 = vm_pageout+0x975^M^M
fork_exit() at 0xffffffff80454ada = fork_exit+0x12a^M^M
fork_trampoline() at 0xffffffff8067b37e = fork_trampoline+0xe^M^M


I believe that we need to check for P_WEXIT before considering p as a
candidate to be killed:

diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index ca9d7f9..64d4277 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -1516,13 +1516,13 @@ vm_pageout_oom(int shortage)
        FOREACH_PROC_IN_SYSTEM(p) {
                int breakout;

-               if (PROC_TRYLOCK(p) == 0)
-                       continue;
+               PROC_LOCK(p);
                /*
                 * If this is a system, protected or killed process, skip it.
                 */
                if (p->p_state != PRS_NORMAL ||
-                   (p->p_flag & (P_INEXEC | P_PROTECTED | P_SYSTEM)) ||
+                   (p->p_flag &
+                     (P_INEXEC | P_PROTECTED | P_SYSTEM | P_WEXIT)) ||
                    (p->p_pid == 1) || P_KILLED(p) ||
                    ((p->p_pid < 48) && (swap_pager_avail != 0))) {
                        PROC_UNLOCK(p);
@@ -1557,11 +1557,14 @@ vm_pageout_oom(int shortage)
                        PROC_UNLOCK(p);
                        continue;
                }
+               _PHOLD(p);
                if (!vm_map_trylock_read(&vm->vm_map)) {
-                       vmspace_free(vm);
+                       _PRELE(p);
                        PROC_UNLOCK(p);
+                       vmspace_free(vm);
                        continue;
                }
+               PROC_UNLOCK(p);
                size = vmspace_swap_count(vm);
                vm_map_unlock_read(&vm->vm_map);
                if (shortage == VM_OOM_MEM)
@@ -1573,16 +1576,18 @@ vm_pageout_oom(int shortage)
                 */
                if (size > bigsize) {
                        if (bigproc != NULL)
-                               PROC_UNLOCK(bigproc);
+                               PRELE(bigproc);
                        bigproc = p;
                        bigsize = size;
                } else
-                       PROC_UNLOCK(p);
+                       PRELE(p);
        }
        sx_sunlock(&allproc_lock);
        if (bigproc != NULL) {
+               PROC_LOCK(bigproc);
                killproc(bigproc, "out of swap space");
                sched_nice(bigproc, PRIO_MIN);
+               _PRELE(bigproc);
                PROC_UNLOCK(bigproc);
                wakeup(&vm_cnt.v_free_count);
        }