git: cd240c9cf100 - main - x86 vdso gettc: Add RDTSCP support

[Konstantin Belousov]

The branch main has been updated by kib:

URL: https://cgit.FreeBSD.org/src/commit/?id=cd240c9cf100bec3def38ceb4a320611b1d02693

commit cd240c9cf100bec3def38ceb4a320611b1d02693
Author:     Konstantin Belousov <kib at FreeBSD.org>
AuthorDate: 2021-01-06 15:05:09 +0000
Commit:     Konstantin Belousov <kib at FreeBSD.org>
CommitDate: 2021-01-10 02:42:34 +0000

    x86 vdso gettc: Add RDTSCP support
    
    Detect and use RDTSCP if available, instead of fence+RDTSC.  For AMD Zens+,
    use LFENCE+RDTSC instead of RDTSCP (or MFENCE;RDTSC previously).
    
    Reviewed by:    gallatin, markj
    Tested by:      pho
    MFC after:      1 week
    Sponsored by:   The FreeBSD Foundation
    Differential Revision:  https://reviews.freebsd.org/D27986
---
 lib/libc/x86/sys/__vdso_gettc.c | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/lib/libc/x86/sys/__vdso_gettc.c b/lib/libc/x86/sys/__vdso_gettc.c
index 4e0fc9f7e841..7f224f8758cb 100644
--- a/lib/libc/x86/sys/__vdso_gettc.c
+++ b/lib/libc/x86/sys/__vdso_gettc.c
@@ -63,6 +63,16 @@ rdtsc_low(const struct vdso_timehands *th)
 	return (rv);
 }
 
+static inline u_int
+rdtscp_low(const struct vdso_timehands *th)
+{
+	u_int rv;
+
+	__asm __volatile("rdtscp; movl %%edi,%%ecx; shrd %%cl, %%edx, %0"
+	    : "=a" (rv) : "D" (th->th_x86_shift) : "ecx", "edx");
+	return (rv);
+}
+
 static u_int
 rdtsc_low_mb_lfence(const struct vdso_timehands *th)
 {
@@ -103,6 +113,12 @@ rdtsc32_mb_none(void)
 	return (rdtsc32());
 }
 
+static u_int
+rdtscp32_(void)
+{
+	return (rdtscp32());
+}
+
 struct tsc_selector_tag {
 	u_int (*ts_rdtsc32)(void);
 	u_int (*ts_rdtsc_low)(const struct vdso_timehands *);
@@ -121,6 +137,10 @@ static const struct tsc_selector_tag tsc_selector[] = {
 		.ts_rdtsc32 = rdtsc32_mb_none,
 		.ts_rdtsc_low = rdtsc_low_mb_none,
 	},
+	[3] = {				/* RDTSCP */
+		.ts_rdtsc32 =	rdtscp32_,
+		.ts_rdtsc_low =	rdtscp_low,
+	},
 };
 
 static int
@@ -144,6 +164,9 @@ tsc_selector_idx(u_int cpu_feature)
 	do_cpuid(1, p);
 	cpu_id = p[0];
 
+	if (amd_cpu && CPUID_TO_FAMILY(cpu_id) >= 0x17)
+		return (0);
+
 	if (cpu_feature != 0) {
 		do_cpuid(0x80000000, p);
 		cpu_exthigh = p[0];
@@ -157,6 +180,8 @@ tsc_selector_idx(u_int cpu_feature)
 		amd_feature = 0;
 	}
 
+	if ((amd_feature & AMDID_RDTSCP) != 0)
+		return (3);
 	if ((cpu_feature & CPUID_SSE2) == 0)
 		return (2);
 	return (amd_cpu ? 1 : 0);