svn commit: r256208 - stable/9/sys/kern

[Alexander Motin]

Author: mav
Date: Wed Oct  9 18:29:06 2013
New Revision: 256208
URL: http://svnweb.freebsd.org/changeset/base/256208

Log:
  MFC r255363:
  Micro-optimize cpu_search(), allowing compiler to use more efficient inline
  ffsl() implementation, when it is available, instead of homegrown iteration.
  
  On dual-E5645 amd64 system (2x6x2 cores) under heavy I/O load that reduces
  time spent inside cpu_search() from 19% to 13%, while IOPS increased by 5%.

Modified:
  stable/9/sys/kern/sched_ule.c

Modified: stable/9/sys/kern/sched_ule.c
==============================================================================
--- stable/9/sys/kern/sched_ule.c	Wed Oct  9 18:23:30 2013	(r256207)
+++ stable/9/sys/kern/sched_ule.c	Wed Oct  9 18:29:06 2013	(r256208)
@@ -632,10 +632,14 @@ cpu_search(const struct cpu_group *cg, s
 	}
 
 	/* Iterate through the child CPU groups and then remaining CPUs. */
-	for (i = cg->cg_children, cpu = mp_maxid; i >= 0; ) {
+	for (i = cg->cg_children, cpu = mp_maxid; ; ) {
 		if (i == 0) {
+#ifdef HAVE_INLINE_FFSL
+			cpu = CPU_FFS(&cpumask) - 1;
+#else
 			while (cpu >= 0 && !CPU_ISSET(cpu, &cpumask))
 				cpu--;
+#endif
 			if (cpu < 0)
 				break;
 			child = NULL;
@@ -660,6 +664,7 @@ cpu_search(const struct cpu_group *cg, s
 				break;
 			}
 		} else {			/* Handle child CPU. */
+			CPU_CLR(cpu, &cpumask);
 			tdq = TDQ_CPU(cpu);
 			load = tdq->tdq_load * 256;
 			rndptr = DPCPU_PTR(randomval);
@@ -707,8 +712,11 @@ cpu_search(const struct cpu_group *cg, s
 			i--;
 			if (i == 0 && CPU_EMPTY(&cpumask))
 				break;
-		} else
+		}
+#ifndef HAVE_INLINE_FFSL
+		else
 			cpu--;
+#endif
 	}
 	return (total);
 }