git: 652ae7b11485 - main - x86: cpufunc: Add rdtsc_ordered()

[Konstantin Belousov]

The branch main has been updated by kib:

URL: https://cgit.FreeBSD.org/src/commit/?id=652ae7b11485d9f991ea66a56aa79d4dd9cd1103

commit 652ae7b11485d9f991ea66a56aa79d4dd9cd1103
Author:     Adam Fenn <adam at fenn.io>
AuthorDate: 2021-07-28 17:12:00 +0000
Commit:     Konstantin Belousov <kib at FreeBSD.org>
CommitDate: 2021-08-14 12:57:53 +0000

    x86: cpufunc: Add rdtsc_ordered()
    
    Add a variant of 'rdtsc()' that performs the ordered version of 'rdtsc'
    appropriate for the invoking x86 variant.
    
    Also, expose the 'lfence'-ed and 'mfence'-ed 'rdtsc()' variants needed
    by 'rdtsc_ordered()' for general use.
    
    Sponsored by:   Juniper Networks, Inc.
    Sponsored by:   Klara, Inc.
    Reviewed by:    kib
    Differential Revision:  https://reviews.freebsd.org/D31416
---
 sys/amd64/include/cpufunc.h | 14 ++++++++++++++
 sys/i386/include/cpufunc.h  | 14 ++++++++++++++
 sys/x86/include/x86_var.h   |  1 +
 sys/x86/x86/cpu_machdep.c   | 16 ++++++++++++++++
 4 files changed, 45 insertions(+)

diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h
index 49975e826fb9..bca74d8ead67 100644
--- a/sys/amd64/include/cpufunc.h
+++ b/sys/amd64/include/cpufunc.h
@@ -355,6 +355,20 @@ rdtsc(void)
 	return (low | ((uint64_t)high << 32));
 }
 
+static __inline uint64_t
+rdtsc_ordered_lfence(void)
+{
+	lfence();
+	return (rdtsc());
+}
+
+static __inline uint64_t
+rdtsc_ordered_mfence(void)
+{
+	mfence();
+	return (rdtsc());
+}
+
 static __inline uint64_t
 rdtscp(void)
 {
diff --git a/sys/i386/include/cpufunc.h b/sys/i386/include/cpufunc.h
index 7b99978facc3..79cdd3004b77 100644
--- a/sys/i386/include/cpufunc.h
+++ b/sys/i386/include/cpufunc.h
@@ -394,6 +394,20 @@ rdtsc(void)
 	return (rv);
 }
 
+static __inline uint64_t
+rdtsc_ordered_lfence(void)
+{
+	lfence();
+	return (rdtsc());
+}
+
+static __inline uint64_t
+rdtsc_ordered_mfence(void)
+{
+	mfence();
+	return (rdtsc());
+}
+
 static __inline uint64_t
 rdtscp(void)
 {
diff --git a/sys/x86/include/x86_var.h b/sys/x86/include/x86_var.h
index ec268503634b..d695822ddb7f 100644
--- a/sys/x86/include/x86_var.h
+++ b/sys/x86/include/x86_var.h
@@ -154,6 +154,7 @@ int	user_dbreg_trap(register_t dr6);
 int	minidumpsys(struct dumperinfo *);
 struct pcb *get_pcb_td(struct thread *td);
 void	x86_set_fork_retval(struct thread *td);
+uint64_t rdtsc_ordered(void);
 
 /*
  * MSR ops for x86_msr_op()
diff --git a/sys/x86/x86/cpu_machdep.c b/sys/x86/x86/cpu_machdep.c
index bfc1a697cde0..1924342c8e83 100644
--- a/sys/x86/x86/cpu_machdep.c
+++ b/sys/x86/x86/cpu_machdep.c
@@ -76,6 +76,7 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/clock.h>
 #include <machine/cpu.h>
+#include <machine/cpufunc.h>
 #include <machine/cputypes.h>
 #include <machine/specialreg.h>
 #include <machine/md_var.h>
@@ -88,6 +89,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/elan_mmcr.h>
 #endif
 #include <x86/acpica_machdep.h>
+#include <x86/ifunc.h>
 
 #include <vm/vm.h>
 #include <vm/vm_extern.h>
@@ -1508,3 +1510,17 @@ acpi_get_fadt_bootflags(uint16_t *flagsp)
 	return (false);
 #endif
 }
+
+DEFINE_IFUNC(, uint64_t, rdtsc_ordered, (void))
+{
+	bool cpu_is_amd = cpu_vendor_id == CPU_VENDOR_AMD ||
+	    cpu_vendor_id == CPU_VENDOR_HYGON;
+
+	if ((amd_feature & AMDID_RDTSCP) != 0)
+		return (rdtscp);
+	else if ((cpu_feature & CPUID_SSE2) != 0)
+		return (cpu_is_amd ? rdtsc_ordered_mfence :
+		    rdtsc_ordered_lfence);
+	else
+		return (rdtsc);
+}