svn commit: r262917 - head/sys/kern

[Jeff Roberson]

Author: jeff
Date: Sat Mar  8 00:35:06 2014
New Revision: 262917
URL: http://svnweb.freebsd.org/changeset/base/262917

Log:
   - Make runq_steal_from more aggressive.  Previously it would examine only
     a single priority queue.  If that queue had a thread or threads which
     could not be migrated we would fail to steal load.  This could cause
     starvation in situations where cores are idle.
  
  Submitted by:	Doug Kilpatrick <dkilpatrick at isilon.com>
  Tested by:	pho
  Reviewed by:	mav
  Sponsored by:	EMC / Isilon Storage Division

Modified:
  head/sys/kern/sched_ule.c

Modified: head/sys/kern/sched_ule.c
==============================================================================
--- head/sys/kern/sched_ule.c	Sat Mar  8 00:14:40 2014	(r262916)
+++ head/sys/kern/sched_ule.c	Sat Mar  8 00:35:06 2014	(r262917)
@@ -1057,32 +1057,27 @@ runq_steal_from(struct runq *rq, int cpu
 	struct rqhead *rqh;
 	struct thread *td, *first;
 	int bit;
-	int pri;
 	int i;
 
 	rqb = &rq->rq_status;
 	bit = start & (RQB_BPW -1);
-	pri = 0;
 	first = NULL;
 again:
 	for (i = RQB_WORD(start); i < RQB_LEN; bit = 0, i++) {
 		if (rqb->rqb_bits[i] == 0)
 			continue;
-		if (bit != 0) {
-			for (pri = bit; pri < RQB_BPW; pri++)
-				if (rqb->rqb_bits[i] & (1ul << pri))
-					break;
-			if (pri >= RQB_BPW)
+		if (bit == 0)
+			bit = RQB_FFS(rqb->rqb_bits[i]);
+		for (; bit < RQB_BPW; bit++) {
+			if ((rqb->rqb_bits[i] & (1ul << bit)) == 0)
 				continue;
-		} else
-			pri = RQB_FFS(rqb->rqb_bits[i]);
-		pri += (i << RQB_L2BPW);
-		rqh = &rq->rq_queues[pri];
-		TAILQ_FOREACH(td, rqh, td_runq) {
-			if (first && THREAD_CAN_MIGRATE(td) &&
-			    THREAD_CAN_SCHED(td, cpu))
-				return (td);
-			first = td;
+			rqh = &rq->rq_queues[bit + (i << RQB_L2BPW)];
+			TAILQ_FOREACH(td, rqh, td_runq) {
+				if (first && THREAD_CAN_MIGRATE(td) &&
+				    THREAD_CAN_SCHED(td, cpu))
+					return (td);
+				first = td;
+			}
 		}
 	}
 	if (start != 0) {