git: 553af8f1ec71 - main - x86: Perform late TSC calibration before LAPIC timer calibration

The branch main has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=553af8f1ec71d397b5b4fd5876622b9269936e63

commit 553af8f1ec71d397b5b4fd5876622b9269936e63
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2021-12-06 15:42:19 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2021-12-06 15:42:19 +0000

    x86: Perform late TSC calibration before LAPIC timer calibration
    
    This ensures that LAPIC calibration is done using the correct tsc_freq
    value, i.e., the one associated with the TSC timecounter.  It does mean
    though that TSC calibration cannot use sbinuptime() to read the
    reference timecounter, as timehands are not yet set up.
    
    Reviewed by:    kib, jhb
    MFC after:      2 weeks
    Sponsored by:   The FreeBSD Foundation
    Differential Revision:  https://reviews.freebsd.org/D33209
---
 sys/x86/include/clock.h |  1 +
 sys/x86/isa/clock.c     |  2 ++
 sys/x86/x86/tsc.c       | 26 +++++++++++++++++---------
 3 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/sys/x86/include/clock.h b/sys/x86/include/clock.h
index d492196bac85..83c8351ed31c 100644
--- a/sys/x86/include/clock.h
+++ b/sys/x86/include/clock.h
@@ -28,6 +28,7 @@ void	i8254_init(void);
 void	i8254_delay(int);
 void	clock_init(void);
 void	lapic_calibrate(void);
+void	tsc_calibrate(void);
 
 /*
  * Driver to clock driver interface.
diff --git a/sys/x86/isa/clock.c b/sys/x86/isa/clock.c
index 2eb1c343f4db..513174626892 100644
--- a/sys/x86/isa/clock.c
+++ b/sys/x86/isa/clock.c
@@ -413,6 +413,7 @@ cpu_initclocks(void)
 
 	td = curthread;
 
+	tsc_calibrate();
 	lapic_calibrate_timer();
 	cpu_initclocks_bsp();
 	CPU_FOREACH(i) {
@@ -428,6 +429,7 @@ cpu_initclocks(void)
 		sched_unbind(td);
 	thread_unlock(td);
 #else
+	tsc_calibrate();
 	lapic_calibrate_timer();
 	cpu_initclocks_bsp();
 #endif
diff --git a/sys/x86/x86/tsc.c b/sys/x86/x86/tsc.c
index bba5e352b031..693e9a9f2c75 100644
--- a/sys/x86/x86/tsc.c
+++ b/sys/x86/x86/tsc.c
@@ -695,23 +695,27 @@ tsc_update_freq(uint64_t new_freq)
 
 /*
  * Perform late calibration of the TSC frequency once ACPI-based timecounters
- * are available.
+ * are available.  At this point timehands are not set up, so we read the
+ * highest-quality timecounter directly rather than using (s)binuptime().
  */
-static void
-tsc_calib(void *arg __unused)
+void
+tsc_calibrate(void)
 {
-	sbintime_t t_start, t_end;
+	struct timecounter *tc;
 	uint64_t freq_khz, tsc_start, tsc_end;
+	u_int t_start, t_end;
 	register_t flags;
 	int cpu;
 
 	if (tsc_disabled)
 		return;
 
+	tc = atomic_load_ptr(&timecounter);
+
 	flags = intr_disable();
 	cpu = curcpu;
 	tsc_start = rdtsc_ordered();
-	t_start = sbinuptime();
+	t_start = tc->tc_get_timecount(tc) & tc->tc_counter_mask;
 	intr_restore(flags);
 
 	DELAY(1000000);
@@ -721,19 +725,23 @@ tsc_calib(void *arg __unused)
 
 	flags = intr_disable();
 	tsc_end = rdtsc_ordered();
-	t_end = sbinuptime();
+	t_end = tc->tc_get_timecount(tc) & tc->tc_counter_mask;
 	intr_restore(flags);
 
 	sched_unbind(curthread);
 	thread_unlock(curthread);
 
-	freq_khz = (SBT_1S / 1024) * (tsc_end - tsc_start) / (t_end - t_start);
+	if (t_end <= t_start) {
+		/* Assume that the counter has wrapped around at most once. */
+		t_end += (uint64_t)tc->tc_counter_mask + 1;
+	}
+
+	freq_khz = tc->tc_frequency * (tsc_end - tsc_start) / (t_end - t_start);
 
-	tsc_update_freq(freq_khz * 1024);
+	tsc_update_freq(freq_khz);
 	tc_init(&tsc_timecounter);
 	set_cputicker(rdtsc, tsc_freq, !tsc_is_invariant);
 }
-SYSINIT(tsc_calib, SI_SUB_CLOCKS + 1, SI_ORDER_ANY, tsc_calib, NULL);
 
 void
 resume_TSC(void)