svn commit: r255541 - stable/9/sys/kern

[Alexander Motin]

Author: mav
Date: Sat Sep 14 08:45:15 2013
New Revision: 255541
URL: http://svnweb.freebsd.org/changeset/base/255541

Log:
  MFC r255363:
  Micro-optimize cpu_search(), allowing compiler to use more efficient inline
  ffsl() implementation, when it is available, instead of homegrown iteration.
  
  On dual-E5645 amd64 system (2x6x2 cores) under heavy I/O load that reduces
  time spent inside cpu_search() from 19% to 13%, while IOPS increased by 5%.

Modified:
  stable/9/sys/kern/sched_ule.c
Directory Properties:
  stable/9/sys/   (props changed)

Modified: stable/9/sys/kern/sched_ule.c
==============================================================================
--- stable/9/sys/kern/sched_ule.c	Sat Sep 14 08:37:49 2013	(r255540)
+++ stable/9/sys/kern/sched_ule.c	Sat Sep 14 08:45:15 2013	(r255541)
@@ -632,10 +632,14 @@ cpu_search(const struct cpu_group *cg, s
 	}
 
 	/* Iterate through the child CPU groups and then remaining CPUs. */
-	for (i = cg->cg_children, cpu = mp_maxid; i >= 0; ) {
+	for (i = cg->cg_children, cpu = mp_maxid; ; ) {
 		if (i == 0) {
+#ifdef HAVE_INLINE_FFSL
+			cpu = CPU_FFS(&cpumask) - 1;
+#else
 			while (cpu >= 0 && !CPU_ISSET(cpu, &cpumask))
 				cpu--;
+#endif
 			if (cpu < 0)
 				break;
 			child = NULL;
@@ -660,6 +664,7 @@ cpu_search(const struct cpu_group *cg, s
 				break;
 			}
 		} else {			/* Handle child CPU. */
+			CPU_CLR(cpu, &cpumask);
 			tdq = TDQ_CPU(cpu);
 			load = tdq->tdq_load * 256;
 			rndptr = DPCPU_PTR(randomval);
@@ -707,8 +712,11 @@ cpu_search(const struct cpu_group *cg, s
 			i--;
 			if (i == 0 && CPU_EMPTY(&cpumask))
 				break;
-		} else
+		}
+#ifndef HAVE_INLINE_FFSL
+		else
 			cpu--;
+#endif
 	}
 	return (total);
 }