svn commit: r241251 - stable/8/sys/kern

[Alexander Motin]

Author: mav
Date: Sat Oct  6 13:38:49 2012
New Revision: 241251
URL: http://svn.freebsd.org/changeset/base/241251

Log:
  MFC r212455:
  Merge some SCHED_ULE features to SCHED_4BSD:
  - Teach SCHED_4BSD to inform cpu_idle() about high sleep/wakeup rate to
  choose optimized handler. In case of x86 it is MONITOR/MWAIT. Also it
  will be needed to bypass forthcoming idle tick skipping logic to not
  consume resources on events rescheduling when it won't give any benefits.
  - Teach SCHED_4BSD to wake up idle CPUs without using IPI. In case of x86,
  when MONITOR/MWAIT is active, it require just single memory write. This
  doubles performance on some heavily switching test loads.

Modified:
  stable/8/sys/kern/sched_4bsd.c
Directory Properties:
  stable/8/sys/   (props changed)
  stable/8/sys/kern/   (props changed)

Modified: stable/8/sys/kern/sched_4bsd.c
==============================================================================
--- stable/8/sys/kern/sched_4bsd.c	Sat Oct  6 13:01:08 2012	(r241250)
+++ stable/8/sys/kern/sched_4bsd.c	Sat Oct  6 13:38:49 2012	(r241251)
@@ -158,6 +158,12 @@ static struct runq runq_pcpu[MAXCPU];
 long runq_length[MAXCPU];
 #endif
 
+struct pcpuidlestat {
+	u_int idlecalls;
+	u_int oldidlecalls;
+};
+static DPCPU_DEFINE(struct pcpuidlestat, idlestat);
+
 static void
 setup_runqs(void)
 {
@@ -709,6 +715,7 @@ sched_rr_interval(void)
 void
 sched_clock(struct thread *td)
 {
+	struct pcpuidlestat *stat;
 	struct td_sched *ts;
 
 	THREAD_LOCK_ASSERT(td, MA_OWNED);
@@ -728,6 +735,10 @@ sched_clock(struct thread *td)
 	if (!TD_IS_IDLETHREAD(td) &&
 	    ticks - PCPU_GET(switchticks) >= sched_quantum)
 		td->td_flags |= TDF_NEEDRESCHED;
+
+	stat = DPCPU_PTR(idlestat);
+	stat->oldidlecalls = stat->idlecalls;
+	stat->idlecalls = 0;
 }
 
 /*
@@ -1168,7 +1179,15 @@ forward_wakeup(int cpunum)
 	}
 	if (map) {
 		forward_wakeups_delivered++;
-		ipi_selected(map, IPI_AST);
+		SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
+			id = pc->pc_cpumask;
+			if ((map & id) == 0)
+				continue;
+			if (cpu_idle_wakeup(pc->pc_cpuid))
+				map &= ~id;
+		}
+		if (map)
+			ipi_selected(map, IPI_AST);
 		return (1);
 	}
 	if (cpunum == NOCPU)
@@ -1185,7 +1204,8 @@ kick_other_cpu(int pri, int cpuid)
 	pcpu = pcpu_find(cpuid);
 	if (idle_cpus_mask & pcpu->pc_cpumask) {
 		forward_wakeups_delivered++;
-		ipi_cpu(cpuid, IPI_AST);
+		if (!cpu_idle_wakeup(cpuid))
+			ipi_cpu(cpuid, IPI_AST);
 		return;
 	}
 
@@ -1577,12 +1597,16 @@ sched_tick(void)
 void
 sched_idletd(void *dummy)
 {
+	struct pcpuidlestat *stat;
 
+	stat = DPCPU_PTR(idlestat);
 	for (;;) {
 		mtx_assert(&Giant, MA_NOTOWNED);
 
-		while (sched_runnable() == 0)
-			cpu_idle(0);
+		while (sched_runnable() == 0) {
+			cpu_idle(stat->idlecalls + stat->oldidlecalls > 64);
+			stat->idlecalls++;
+		}
 
 		mtx_lock_spin(&sched_lock);
 		mi_switch(SW_VOL | SWT_IDLE, NULL);