Author: andrew
Date: Tue May 19 16:04:27 2020
New Revision: 361259
URL: https://svnweb.freebsd.org/changeset/base/361259
Log:
Stop performing a full icache sync when the DIC and IDC flags are set
The DIC and IDC bits in the CTR_EL0 register signal to the kernel when it
can relax the instruction cache synchronisation operations. The IDC bit
means we can relax cleaning the data cache to the point of unification
while the DIC bit means we don't need to invalidate the instruction cache
for data coherence. In both cases an appropriate barrier is still needed.
For now only implement the case where both bits are set, as is the case
on the Neoverse-N1 as used in the Amazon AWS Graviton 2 CPU. Note that
this behaviour is a optional on the N1 so we may later need to implement
only one or the other bit being set.
There is a tunable to disable each flag on boot.
Testing on a 4 core Graviton 2 instance found a significant improvement
in sys and real time when running "make buildkernel -j4", with no
significant difference in user time.
Reviewed by: markj
Sponsored by: Innovate UK
Differential Revision: https://reviews.freebsd.org/D24853
Modified:
head/sys/arm64/arm64/cpufunc_asm.S
head/sys/arm64/arm64/identcpu.c
head/sys/arm64/include/cpufunc.h
Modified: head/sys/arm64/arm64/cpufunc_asm.S
==============================================================================
--- head/sys/arm64/arm64/cpufunc_asm.S Tue May 19 15:27:20 2020 (r361258)
+++ head/sys/arm64/arm64/cpufunc_asm.S Tue May 19 16:04:27 2020 (r361259)
@@ -133,9 +133,20 @@ ENTRY(arm64_dcache_inv_range)
END(arm64_dcache_inv_range)
/*
- * void arm64_icache_sync_range(vm_offset_t, vm_size_t)
+ * void arm64_dic_idc_icache_sync_range(vm_offset_t, vm_size_t)
+ * When the CTR_EL0.IDC bit is set cleaning to PoU becomes a dsb.
+ * When the CTR_EL0.DIC bit is set icache invalidation becomes an isb.
*/
-ENTRY(arm64_icache_sync_range)
+ENTRY(arm64_dic_idc_icache_sync_range)
+ dsb ishst
+ isb
+ ret
+END(arm64_dic_idc_icache_sync_range)
+
+/*
+ * void arm64_aliasing_icache_sync_range(vm_offset_t, vm_size_t)
+ */
+ENTRY(arm64_aliasing_icache_sync_range)
/*
* XXX Temporary solution - I-cache flush should be range based for
* PIPT cache or IALLUIS for VIVT or VIPT caches
@@ -146,7 +157,7 @@ ENTRY(arm64_icache_sync_range)
dsb ish
isb
ret
-END(arm64_icache_sync_range)
+END(arm64_aliasing_icache_sync_range)
/*
* int arm64_icache_sync_range_checked(vm_offset_t, vm_size_t)
Modified: head/sys/arm64/arm64/identcpu.c
==============================================================================
--- head/sys/arm64/arm64/identcpu.c Tue May 19 15:27:20 2020 (r361258)
+++ head/sys/arm64/arm64/identcpu.c Tue May 19 16:04:27 2020 (r361259)
@@ -56,6 +56,24 @@ char machine[] = "arm64";
extern int adaptive_machine_arch;
#endif
+static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+ "Cache management tuning");
+
+static int allow_dic = 1;
+SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
+ "Allow optimizations based on the DIC cache bit");
+
+static int allow_idc = 1;
+SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
+ "Allow optimizations based on the IDC cache bit");
+
+/*
+ * The default implementation of I-cache sync assumes we have an
+ * aliasing cache until we know otherwise.
+ */
+void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t) =
+ &arm64_aliasing_icache_sync_range;
+
static int
sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
{
@@ -977,6 +995,7 @@ identify_cpu_sysinit(void *dummy __unused)
{
int cpu;
u_long hwcap;
+ bool dic, idc;
/* Create a user visible cpu description with safe values */
memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
@@ -985,6 +1004,8 @@ identify_cpu_sysinit(void *dummy __unused)
ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 | ID_AA64PFR0_EL0_64;
user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
+ dic = (allow_dic != 0);
+ idc = (allow_idc != 0);
CPU_FOREACH(cpu) {
print_cpu_features(cpu);
hwcap = parse_cpu_features_hwcap(cpu);
@@ -993,6 +1014,17 @@ identify_cpu_sysinit(void *dummy __unused)
else
elf_hwcap &= hwcap;
update_user_regs(cpu);
+
+ if (CTR_DIC_VAL(cpu_desc[cpu].ctr) == 0)
+ dic = false;
+ if (CTR_IDC_VAL(cpu_desc[cpu].ctr) == 0)
+ idc = false;
+ }
+
+ if (dic && idc) {
+ arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range;
+ if (bootverbose)
+ printf("Enabling DIC & IDC ICache sync\n");
}
if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
Modified: head/sys/arm64/include/cpufunc.h
==============================================================================
--- head/sys/arm64/include/cpufunc.h Tue May 19 15:27:20 2020 (r361258)
+++ head/sys/arm64/include/cpufunc.h Tue May 19 16:04:27 2020 (r361259)
@@ -216,12 +216,15 @@ extern int64_t dczva_line_size;
#define cpu_dcache_inv_range(a, s) arm64_dcache_inv_range((a), (s))
#define cpu_dcache_wb_range(a, s) arm64_dcache_wb_range((a), (s))
+extern void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t);
+
#define cpu_icache_sync_range(a, s) arm64_icache_sync_range((a), (s))
#define cpu_icache_sync_range_checked(a, s) arm64_icache_sync_range_checked((a), (s))
void arm64_nullop(void);
void arm64_tlb_flushID(void);
-void arm64_icache_sync_range(vm_offset_t, vm_size_t);
+void arm64_dic_idc_icache_sync_range(vm_offset_t, vm_size_t);
+void arm64_aliasing_icache_sync_range(vm_offset_t, vm_size_t);
int arm64_icache_sync_range_checked(vm_offset_t, vm_size_t);
void arm64_dcache_wbinv_range(vm_offset_t, vm_size_t);
void arm64_dcache_inv_range(vm_offset_t, vm_size_t);