PERFORCE change 129905 for review
Peter Wemm
peter at FreeBSD.org
Sat Dec 1 14:00:38 PST 2007
http://perforce.freebsd.org/chv.cgi?CH=129905
Change 129905 by peter at peter_daintree on 2007/12/01 21:59:45
IFC
Affected files ...
.. //depot/projects/bike_sched/sys/kern/kern_thread.c#5 integrate
Differences ...
==== //depot/projects/bike_sched/sys/kern/kern_thread.c#5 (text+ko) ====
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__FBSDID("$FreeBSD: src/sys/kern/kern_thread.c,v 1.234 2006/06/30 08:10:55 maxim Exp $");
+__FBSDID("$FreeBSD: src/sys/kern/kern_thread.c,v 1.262 2007/11/15 14:20:06 rrs Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -49,33 +49,26 @@
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/uma.h>
+#include <sys/eventhandler.h>
/*
* thread related storage.
*/
static uma_zone_t thread_zone;
-/* DEBUG ONLY */
SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation");
-static int thread_debug = 0;
-SYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW,
- &thread_debug, 0, "thread debug");
int max_threads_per_proc = 1500;
SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW,
&max_threads_per_proc, 0, "Limit on threads per proc");
-int max_groups_per_proc = 1500;
-SYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW,
- &max_groups_per_proc, 0, "Limit on thread groups per proc");
-
int max_threads_hits;
SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD,
&max_threads_hits, 0, "");
TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads);
-struct mtx kse_zombie_lock;
-MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN);
+static struct mtx zombie_lock;
+MTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN);
struct mtx tid_lock;
static struct unrhdr *tid_unrhdr;
@@ -93,24 +86,19 @@
td->td_oncpu = NOCPU;
td->td_tid = alloc_unr(tid_unrhdr);
+ td->td_syscalls = 0;
/*
* Note that td_critnest begins life as 1 because the thread is not
* running and is thereby implicitly waiting to be on the receiving
- * end of a context switch. A context switch must occur inside a
- * critical section, and in fact, includes hand-off of the sched_lock.
- * After a context switch to a newly created thread, it will release
- * sched_lock for the first time, and its td_critnest will hit 0 for
- * the first time. This happens on the far end of a context switch,
- * and when it context switches away from itself, it will in fact go
- * back into a critical section, and hand off the sched lock to the
- * next thread.
+ * end of a context switch.
*/
td->td_critnest = 1;
-
+ EVENTHANDLER_INVOKE(thread_ctor, td);
#ifdef AUDIT
audit_thread_alloc(td);
#endif
+ umtx_thread_alloc(td);
return (0);
}
@@ -147,6 +135,7 @@
#ifdef AUDIT
audit_thread_free(td);
#endif
+ EVENTHANDLER_INVOKE(thread_dtor, td);
free_unr(tid_unrhdr, td->td_tid);
sched_newthread(td);
}
@@ -161,13 +150,13 @@
td = (struct thread *)mem;
- vm_thread_new(td, 0);
- cpu_thread_setup(td);
td->td_sleepqueue = sleepq_alloc();
td->td_turnstile = turnstile_alloc();
- td->td_umtxq = umtxq_alloc();
+ EVENTHANDLER_INVOKE(thread_init, td);
td->td_sched = (struct td_sched *)&td[1];
sched_newthread(td);
+ umtx_thread_init(td);
+ td->td_kstack = 0;
return (0);
}
@@ -180,10 +169,10 @@
struct thread *td;
td = (struct thread *)mem;
+ EVENTHANDLER_INVOKE(thread_fini, td);
turnstile_free(td->td_turnstile);
sleepq_free(td->td_sleepqueue);
- umtxq_free(td->td_umtxq);
- vm_thread_dispose(td);
+ umtx_thread_fini(td);
}
/*
@@ -195,11 +184,16 @@
* proc_init()
*/
void
+proc_linkup0(struct proc *p, struct thread *td)
+{
+ TAILQ_INIT(&p->p_threads); /* all threads in proc */
+ proc_linkup(p, td);
+}
+
+void
proc_linkup(struct proc *p, struct thread *td)
{
- TAILQ_INIT(&p->p_threads); /* all threads in proc */
- TAILQ_INIT(&p->p_suspended); /* Threads suspended */
sigqueue_init(&p->p_sigqueue, p);
p->p_ksi = ksiginfo_alloc(1);
if (p->p_ksi != NULL) {
@@ -208,7 +202,6 @@
}
LIST_INIT(&p->p_mqnotifier);
p->p_numthreads = 0;
-
thread_link(td, p);
}
@@ -224,19 +217,20 @@
thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(),
thread_ctor, thread_dtor, thread_init, thread_fini,
- UMA_ALIGN_CACHE, 0);
+ 16 - 1, 0);
}
/*
- * Stash an embarasingly extra thread into the zombie thread queue.
+ * Place an unused thread on the zombie list.
+ * Use the slpq as that must be unused by now.
*/
void
-thread_stash(struct thread *td)
+thread_zombie(struct thread *td)
{
- mtx_lock_spin(&kse_zombie_lock);
- TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq);
- mtx_unlock_spin(&kse_zombie_lock);
+ mtx_lock_spin(&zombie_lock);
+ TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq);
+ mtx_unlock_spin(&zombie_lock);
}
/*
@@ -252,19 +246,22 @@
* we really don't care about the next instant..
*/
if (!TAILQ_EMPTY(&zombie_threads)) {
- mtx_lock_spin(&kse_zombie_lock);
+ mtx_lock_spin(&zombie_lock);
td_first = TAILQ_FIRST(&zombie_threads);
if (td_first)
TAILQ_INIT(&zombie_threads);
- mtx_unlock_spin(&kse_zombie_lock);
+ mtx_unlock_spin(&zombie_lock);
while (td_first) {
- td_next = TAILQ_NEXT(td_first, td_runq);
+ td_next = TAILQ_NEXT(td_first, td_slpq);
if (td_first->td_ucred)
crfree(td_first->td_ucred);
thread_free(td_first);
td_first = td_next;
}
}
+#ifdef KSE
+ upcall_reap();
+#endif
}
/*
@@ -273,19 +270,31 @@
struct thread *
thread_alloc(void)
{
+ struct thread *td;
thread_reap(); /* check if any zombies to get */
- return (uma_zalloc(thread_zone, M_WAITOK));
+
+ td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK);
+ KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack"));
+ if (!vm_thread_new(td, 0)) {
+ uma_zfree(thread_zone, td);
+ return (NULL);
+ }
+ cpu_thread_alloc(td);
+ return (td);
}
-/*
* Deallocate a thread.
*/
void
thread_free(struct thread *td)
{
- cpu_thread_clean(td);
+ cpu_thread_free(td);
+ if (td->td_altkstack != 0)
+ vm_thread_dispose_altkstack(td);
+ if (td->td_kstack != 0)
+ vm_thread_dispose(td);
uma_zfree(thread_zone, td);
}
@@ -321,23 +330,27 @@
{
uint64_t new_switchtime;
struct thread *td;
+ struct thread *td2;
struct proc *p;
td = curthread;
p = td->td_proc;
- mtx_assert(&sched_lock, MA_OWNED);
+ PROC_SLOCK_ASSERT(p, MA_OWNED);
mtx_assert(&Giant, MA_NOTOWNED);
+
PROC_LOCK_ASSERT(p, MA_OWNED);
KASSERT(p != NULL, ("thread exiting without a process"));
CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td,
- (long)p->p_pid, p->p_comm);
+ (long)p->p_pid, td->td_name);
KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending"));
#ifdef AUDIT
AUDIT_SYSCALL_EXIT(0, td);
#endif
+ umtx_thread_exit(td);
+
/*
* drop FPU & debug register state storage, or any other
* architecture specific resources that
@@ -348,17 +361,12 @@
/* Do the same timestamp bookkeeping that mi_switch() would do. */
new_switchtime = cpu_ticks();
p->p_rux.rux_runtime += (new_switchtime - PCPU_GET(switchtime));
- p->p_rux.rux_uticks += td->td_uticks;
- p->p_rux.rux_sticks += td->td_sticks;
- p->p_rux.rux_iticks += td->td_iticks;
PCPU_SET(switchtime, new_switchtime);
PCPU_SET(switchticks, ticks);
- cnt.v_swtch++;
-
- /* Add our usage into the usage of all our children. */
- if (p->p_numthreads == 1)
- ruadd(p->p_ru, &p->p_rux, &p->p_stats->p_cru, &p->p_crux);
-
+ PCPU_INC(cnt.v_swtch);
+ /* Save our resource usage in our process. */
+ td->td_ru.ru_nvcsw++;
+ rucollect(&p->p_ru, &td->td_ru);
/*
* The last thread is left attached to the process
* So that the whole bundle gets recycled. Skip
@@ -369,7 +377,9 @@
*/
if (p->p_flag & P_HADTHREADS) {
if (p->p_numthreads > 1) {
+ thread_lock(td);
thread_unlink(td);
+ thread_unlock(td);
sched_exit(p, td);
/*
@@ -378,10 +388,13 @@
* in exit1() after it is the only survivor.
*/
if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
- if (p->p_numthreads == p->p_suspcount)
+ if (p->p_numthreads == p->p_suspcount) {
+ thread_lock(p->p_singlethread);
thread_unsuspend_one(p->p_singlethread);
+ thread_unlock(p->p_singlethread);
+ }
}
- PROC_UNLOCK(p);
+ atomic_add_int(&td->td_proc->p_exitthreads, 1);
PCPU_SET(deadthread, td);
} else {
/*
@@ -395,17 +408,15 @@
*/
panic ("thread_exit: Last thread exiting on its own");
}
- } else {
- /*
- * non threaded process comes here.
- * This includes an EX threaded process that is coming
- * here via exit1(). (exit1 dethreads the proc first).
- */
- PROC_UNLOCK(p);
- }
+ }
+ PROC_UNLOCK(p);
+ thread_lock(td);
+ /* Save our tick information with both the thread and proc locked */
+ ruxagg(&p->p_rux, td);
+ PROC_SUNLOCK(p);
td->td_state = TDS_INACTIVE;
CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td);
- cpu_throw(td, choosethread());
+ sched_throw(td);
panic("I'm a teapot!");
/* NOTREACHED */
}
@@ -421,10 +432,15 @@
mtx_assert(&Giant, MA_NOTOWNED);
KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()"));
- FOREACH_THREAD_IN_PROC(p, td) {
- cpu_thread_clean(td);
- crfree(td->td_ucred);
- }
+ td = FIRST_THREAD_IN_PROC(p);
+ /* Lock the last thread so we spin until it exits cpu_throw(). */
+ thread_lock(td);
+ thread_unlock(td);
+ /* Wait for any remaining threads to exit cpu_throw(). */
+ while (p->p_exitthreads)
+ sched_relinquish(curthread);
+ cpu_thread_clean(td);
+ crfree(td->td_ucred);
thread_reap(); /* check for zombie threads etc. */
}
@@ -443,9 +459,14 @@
thread_link(struct thread *td, struct proc *p)
{
+ /*
+ * XXX This can't be enabled because it's called for proc0 before
+ * it's spinlock has been created.
+ * PROC_SLOCK_ASSERT(p, MA_OWNED);
+ */
td->td_state = TDS_INACTIVE;
td->td_proc = p;
- td->td_flags = 0;
+ td->td_flags = TDF_INMEM;
LIST_INIT(&td->td_contested);
sigqueue_init(&td->td_sigqueue, p);
@@ -466,7 +487,13 @@
struct proc *p = td->td_proc;
KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads"));
+#ifdef KSE
+ thread_lock(td);
+ thread_unlock(td);
+ thread_zombie(td->td_standin);
+#else
p->p_flag &= ~P_HADTHREADS;
+#endif
}
/*
@@ -478,7 +505,7 @@
{
struct proc *p = td->td_proc;
- mtx_assert(&sched_lock, MA_OWNED);
+ PROC_SLOCK_ASSERT(p, MA_OWNED);
TAILQ_REMOVE(&p->p_threads, td, td_plist);
p->p_numthreads--;
/* could clear a few other things here */
@@ -530,7 +557,7 @@
p->p_flag &= ~P_SINGLE_BOUNDARY;
}
p->p_flag |= P_STOPPED_SINGLE;
- mtx_lock_spin(&sched_lock);
+ PROC_SLOCK(p);
p->p_singlethread = td;
if (mode == SINGLE_EXIT)
remaining = p->p_numthreads;
@@ -544,6 +571,7 @@
FOREACH_THREAD_IN_PROC(p, td2) {
if (td2 == td)
continue;
+ thread_lock(td2);
td2->td_flags |= TDF_ASTPENDING;
if (TD_IS_INHIBITED(td2)) {
switch (mode) {
@@ -557,18 +585,14 @@
sleepq_abort(td2, EINTR);
break;
case SINGLE_BOUNDARY:
- if (TD_IS_SUSPENDED(td2) &&
- !(td2->td_flags & TDF_BOUNDARY))
- thread_unsuspend_one(td2);
- if (TD_ON_SLEEPQ(td2) &&
- (td2->td_flags & TDF_SINTR))
- sleepq_abort(td2, ERESTART);
break;
default:
- if (TD_IS_SUSPENDED(td2))
+ if (TD_IS_SUSPENDED(td2)) {
+ thread_unlock(td2);
continue;
+ }
/*
- * maybe other inhibitted states too?
+ * maybe other inhibited states too?
*/
if ((td2->td_flags & TDF_SINTR) &&
(td2->td_inhibitors &
@@ -582,6 +606,7 @@
forward_signal(td2);
}
#endif
+ thread_unlock(td2);
}
if (mode == SINGLE_EXIT)
remaining = p->p_numthreads;
@@ -601,13 +626,7 @@
* Wake us up when everyone else has suspended.
* In the mean time we suspend as well.
*/
- thread_stopped(p);
- thread_suspend_one(td);
- PROC_UNLOCK(p);
- mi_switch(SW_VOL, NULL);
- mtx_unlock_spin(&sched_lock);
- PROC_LOCK(p);
- mtx_lock_spin(&sched_lock);
+ thread_suspend_switch(td);
if (mode == SINGLE_EXIT)
remaining = p->p_numthreads;
else if (mode == SINGLE_BOUNDARY)
@@ -626,7 +645,7 @@
p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT);
thread_unthread(td);
}
- mtx_unlock_spin(&sched_lock);
+ PROC_SUNLOCK(p);
return (0);
}
@@ -699,7 +718,7 @@
if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td))
sigqueue_flush(&td->td_sigqueue);
- mtx_lock_spin(&sched_lock);
+ PROC_SLOCK(p);
thread_stopped(p);
/*
* If the process is waiting for us to exit,
@@ -708,44 +727,75 @@
*/
if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td))
thread_exit();
-
+ if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
+ if (p->p_numthreads == p->p_suspcount + 1) {
+ thread_lock(p->p_singlethread);
+ thread_unsuspend_one(p->p_singlethread);
+ thread_unlock(p->p_singlethread);
+ }
+ }
+ PROC_UNLOCK(p);
+ thread_lock(td);
/*
* When a thread suspends, it just
- * moves to the processes's suspend queue
- * and stays there.
+ * gets taken off all queues.
*/
thread_suspend_one(td);
if (return_instead == 0) {
p->p_boundary_count++;
td->td_flags |= TDF_BOUNDARY;
}
- if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
- if (p->p_numthreads == p->p_suspcount)
- thread_unsuspend_one(p->p_singlethread);
- }
- PROC_UNLOCK(p);
+ PROC_SUNLOCK(p);
mi_switch(SW_INVOL, NULL);
- if (return_instead == 0) {
- p->p_boundary_count--;
+ if (return_instead == 0)
td->td_flags &= ~TDF_BOUNDARY;
- }
- mtx_unlock_spin(&sched_lock);
+ thread_unlock(td);
PROC_LOCK(p);
+ if (return_instead == 0)
+ p->p_boundary_count--;
}
return (0);
}
void
+thread_suspend_switch(struct thread *td)
+{
+ struct proc *p;
+
+ p = td->td_proc;
+ KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
+ PROC_LOCK_ASSERT(p, MA_OWNED);
+ PROC_SLOCK_ASSERT(p, MA_OWNED);
+ /*
+ * We implement thread_suspend_one in stages here to avoid
+ * dropping the proc lock while the thread lock is owned.
+ */
+ thread_stopped(p);
+ p->p_suspcount++;
+ PROC_UNLOCK(p);
+ thread_lock(td);
+ sched_sleep(td);
+ TD_SET_SUSPENDED(td);
+ PROC_SUNLOCK(p);
+ DROP_GIANT();
+ mi_switch(SW_VOL, NULL);
+ thread_unlock(td);
+ PICKUP_GIANT();
+ PROC_LOCK(p);
+ PROC_SLOCK(p);
+}
+
+void
thread_suspend_one(struct thread *td)
{
struct proc *p = td->td_proc;
- mtx_assert(&sched_lock, MA_OWNED);
- PROC_LOCK_ASSERT(p, MA_OWNED);
+ PROC_SLOCK_ASSERT(p, MA_OWNED);
+ THREAD_LOCK_ASSERT(td, MA_OWNED);
KASSERT(!TD_IS_SUSPENDED(td), ("already suspended"));
p->p_suspcount++;
+ sched_sleep(td);
TD_SET_SUSPENDED(td);
- TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq);
}
void
@@ -753,9 +803,9 @@
{
struct proc *p = td->td_proc;
- mtx_assert(&sched_lock, MA_OWNED);
- PROC_LOCK_ASSERT(p, MA_OWNED);
- TAILQ_REMOVE(&p->p_suspended, td, td_runq);
+ PROC_SLOCK_ASSERT(p, MA_OWNED);
+ THREAD_LOCK_ASSERT(td, MA_OWNED);
+ KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended"));
TD_CLR_SUSPENDED(td);
p->p_suspcount--;
setrunnable(td);
@@ -769,11 +819,15 @@
{
struct thread *td;
- mtx_assert(&sched_lock, MA_OWNED);
PROC_LOCK_ASSERT(p, MA_OWNED);
+ PROC_SLOCK_ASSERT(p, MA_OWNED);
if (!P_SHOULDSTOP(p)) {
- while ((td = TAILQ_FIRST(&p->p_suspended))) {
- thread_unsuspend_one(td);
+ FOREACH_THREAD_IN_PROC(p, td) {
+ thread_lock(td);
+ if (TD_IS_SUSPENDED(td)) {
+ thread_unsuspend_one(td);
+ }
+ thread_unlock(td);
}
} else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) &&
(p->p_numthreads == p->p_suspcount)) {
@@ -782,7 +836,9 @@
* threading request. Now we've downgraded to single-threaded,
* let it continue.
*/
+ thread_lock(p->p_singlethread);
thread_unsuspend_one(p->p_singlethread);
+ thread_unlock(p->p_singlethread);
}
}
@@ -799,7 +855,7 @@
p = td->td_proc;
PROC_LOCK_ASSERT(p, MA_OWNED);
p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY);
- mtx_lock_spin(&sched_lock);
+ PROC_SLOCK(p);
p->p_singlethread = NULL;
/*
* If there are other threads they mey now run,
@@ -808,11 +864,15 @@
* to continue however as this is a bad place to stop.
*/
if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) {
- while ((td = TAILQ_FIRST(&p->p_suspended))) {
- thread_unsuspend_one(td);
+ FOREACH_THREAD_IN_PROC(p, td) {
+ thread_lock(td);
+ if (TD_IS_SUSPENDED(td)) {
+ thread_unsuspend_one(td);
+ }
+ thread_unlock(td);
}
}
- mtx_unlock_spin(&sched_lock);
+ PROC_SUNLOCK(p);
}
struct thread *
@@ -821,11 +881,11 @@
struct thread *td;
PROC_LOCK_ASSERT(p, MA_OWNED);
- mtx_lock_spin(&sched_lock);
+ PROC_SLOCK(p);
FOREACH_THREAD_IN_PROC(p, td) {
if (td->td_tid == tid)
break;
}
- mtx_unlock_spin(&sched_lock);
+ PROC_SUNLOCK(p);
return (td);
}
More information about the p4-projects
mailing list