git: a93941b439fc - main - Switch to an ifunc in the kernel for crc32c

[Andrew Turner]

The branch main has been updated by andrew:

URL: https://cgit.FreeBSD.org/src/commit/?id=a93941b439fce7047dffad6bc380cc9454b967cd

commit a93941b439fce7047dffad6bc380cc9454b967cd
Author:     Andrew Turner <andrew at FreeBSD.org>
AuthorDate: 2021-07-22 10:24:33 +0000
Commit:     Andrew Turner <andrew at FreeBSD.org>
CommitDate: 2021-07-22 20:54:21 +0000

    Switch to an ifunc in the kernel for crc32c
    
    There is no need to read the same variable to check if the CPU supports
    crc32c instructions.
    
    Reviewed by:    arichardson, kib, markj
    Sponsored by:   The FreeBSD Foundation
    Differential Revision: https://reviews.freebsd.org/D31274
---
 sys/libkern/gsb_crc32.c | 56 +++++++++++++++++++++++++++++++++----------------
 1 file changed, 38 insertions(+), 18 deletions(-)

diff --git a/sys/libkern/gsb_crc32.c b/sys/libkern/gsb_crc32.c
index 170ceb3aa710..27b9a926888b 100644
--- a/sys/libkern/gsb_crc32.c
+++ b/sys/libkern/gsb_crc32.c
@@ -55,11 +55,12 @@ __FBSDID("$FreeBSD$");
 #if defined(__amd64__) || defined(__i386__)
 #include <machine/md_var.h>
 #include <machine/specialreg.h>
+#include <x86/ifunc.h>
 #endif
 
 #if defined(__aarch64__)
-#include <machine/elf.h>
-#include <machine/md_var.h>
+#include <machine/armreg.h>
+#include <machine/ifunc.h>
 #endif
 #endif /* _KERNEL */
 
@@ -750,29 +751,48 @@ multitable_crc32c(uint32_t crc32c,
 	return (crc32c_sb8_64_bit(crc32c, buffer, length, to_even_word));
 }
 
-uint32_t
-calculate_crc32c(uint32_t crc32c,
-    const unsigned char *buffer,
-    unsigned int length)
+static uint32_t
+table_crc32c(uint32_t crc32c, const unsigned char *buffer, unsigned int length)
 {
-#ifdef _KERNEL
-#if defined(__amd64__) || defined(__i386__)
-	if ((cpu_feature2 & CPUID2_SSE42) != 0) {
-		return (sse42_crc32c(crc32c, buffer, length));
-	} else
-#endif
-#if defined(__aarch64__)
-	if ((elf_hwcap & HWCAP_CRC32) != 0) {
-		return (armv8_crc32c(crc32c, buffer, length));
-	} else
-#endif
-#endif /* _KERNEL */
 	if (length < 4) {
 		return (singletable_crc32c(crc32c, buffer, length));
 	} else {
 		return (multitable_crc32c(crc32c, buffer, length));
 	}
 }
+
+#if defined(_KERNEL) && defined(__aarch64__)
+DEFINE_IFUNC(, uint32_t, calculate_crc32c,
+    (uint32_t crc32c, const unsigned char *buffer, unsigned int length))
+{
+	uint64_t reg;
+
+	if (get_kernel_reg(ID_AA64ISAR0_EL1, &reg)) {
+		if (ID_AA64ISAR0_CRC32_VAL(reg) >= ID_AA64ISAR0_CRC32_BASE)
+			return (armv8_crc32c);
+	}
+
+	return (table_crc32c);
+}
+#elif defined(_KERNEL) && (defined(__amd64__) || defined(__i386__))
+DEFINE_IFUNC(, uint32_t, calculate_crc32c,
+    (uint32_t crc32c, const unsigned char *buffer, unsigned int length))
+{
+	if ((cpu_feature2 & CPUID2_SSE42) != 0)
+		return (sse42_crc32c);
+
+	return (table_crc32c);
+}
+#else
+uint32_t
+calculate_crc32c(uint32_t crc32c,
+    const unsigned char *buffer,
+    unsigned int length)
+{
+	return (table_crc32c(crc32c, buffer, length));
+}
+#endif /* _KERNEL && __aarch64__ */
+
 #else
 uint32_t
 calculate_crc32c(uint32_t crc32c, const unsigned char *buffer, unsigned int length)