svn commit: r340168 - head/sys/x86/x86

[John Baldwin]

Author: jhb
Date: Mon Nov  5 22:51:45 2018
New Revision: 340168
URL: https://svnweb.freebsd.org/changeset/base/340168

Log:
  Add a delay_tsc() static function for when DELAY() uses the TSC.
  
  This uses slightly simpler logic than the existing code by using the
  full 64-bit counter and thus not having to worry about counter
  overflow.
  
  Reviewed by:	kib
  MFC after:	3 days
  Differential Revision:	https://reviews.freebsd.org/D17850

Modified:
  head/sys/x86/x86/delay.c

Modified: head/sys/x86/x86/delay.c
==============================================================================
--- head/sys/x86/x86/delay.c	Mon Nov  5 22:51:44 2018	(r340167)
+++ head/sys/x86/x86/delay.c	Mon Nov  5 22:51:45 2018	(r340168)
@@ -51,11 +51,23 @@ __FBSDID("$FreeBSD$");
 #include <machine/cpu.h>
 #include <x86/init.h>
 
-static u_int
-get_tsc(__unused struct timecounter *tc)
+static void
+delay_tsc(int n)
 {
+	uint64_t end, now;
 
-	return (rdtsc32());
+	/*
+	 * Pin the current thread ensure correct behavior if the TSCs
+	 * on different CPUs are not in sync.
+	 */
+	sched_pin();
+	now = rdtsc();
+	end = now + tsc_freq * n / 1000000;
+	do {
+		cpu_spinwait();
+		now = rdtsc();
+	} while (now < end);
+	sched_unpin();
 }
 
 static int
@@ -66,22 +78,24 @@ delay_tc(int n)
 	uint64_t end, freq, now;
 	u_int last, mask, u;
 
-	tc = timecounter;
-	freq = atomic_load_acq_64(&tsc_freq);
-	if (tsc_is_invariant && freq != 0) {
-		func = get_tsc;
-		mask = ~0u;
-	} else {
-		if (tc->tc_quality <= 0)
-			return (0);
-		func = tc->tc_get_timecount;
-		mask = tc->tc_counter_mask;
-		freq = tc->tc_frequency;
+	/*
+	 * Only use the TSC if it is P-state invariant.  If the TSC is
+	 * not P-state invariant and the CPU is not running at the
+	 * "full" P-state, then the TSC will increment at some rate
+	 * less than tsc_freq and delay_tsc() will wait too long.
+	 */
+	if (tsc_is_invariant && tsc_freq != 0) {
+		delay_tsc(n);
+		return (1);
 	}
+	tc = timecounter;
+	if (tc->tc_quality <= 0)
+		return (0);
+	func = tc->tc_get_timecount;
+	mask = tc->tc_counter_mask;
+	freq = tc->tc_frequency;
 	now = 0;
 	end = freq * n / 1000000;
-	if (func == get_tsc)
-		sched_pin();
 	last = func(tc) & mask;
 	do {
 		cpu_spinwait();
@@ -92,8 +106,6 @@ delay_tc(int n)
 			now += u - last;
 		last = u;
 	} while (now < end);
-	if (func == get_tsc)
-		sched_unpin();
 	return (1);
 }