git: 9e0716f46101 - main - Add arm64 CnP support

The branch main has been updated by andrew:

URL: https://cgit.FreeBSD.org/src/commit/?id=9e0716f4610158936c0b9cec1a9930c4ebe924fb

commit 9e0716f4610158936c0b9cec1a9930c4ebe924fb
Author:     Andrew Turner <andrew@FreeBSD.org>
AuthorDate: 2022-05-25 09:55:40 +0000
Commit:     Andrew Turner <andrew@FreeBSD.org>
CommitDate: 2022-05-25 09:56:13 +0000

    Add arm64 CnP support
    
    Set the Common not Private bit in the ttbr registers when supported on
    arm64. This tells the hardware it can share the translation table
    entries on multiple CPUs.
    
    Reviewed by:    alc, kib
    Sponsored by:   The FreeBSD Foundation
---
 sys/arm64/arm64/pmap.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 64 insertions(+), 2 deletions(-)

diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 3a356e8b720b..3b37ce214664 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -6724,6 +6724,8 @@ out:
 	mtx_unlock_spin(&set->asid_set_mutex);
 }
 
+static uint64_t __read_mostly ttbr_flags;
+
 /*
  * Compute the value that should be stored in ttbr0 to activate the specified
  * pmap.  This value may change from time to time.
@@ -6731,10 +6733,70 @@ out:
 uint64_t
 pmap_to_ttbr0(pmap_t pmap)
 {
+	uint64_t ttbr;
+
+	ttbr = pmap->pm_ttbr;
+	ttbr |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));
+	ttbr |= ttbr_flags;
+
+	return (ttbr);
+}
+
+static void
+pmap_set_cnp(void *arg)
+{
+	uint64_t ttbr0, ttbr1;
+	u_int cpuid;
+
+	cpuid = *(u_int *)arg;
+	if (cpuid == curcpu) {
+		/*
+		 * Set the flags while all CPUs are handling the
+		 * smp_rendezvous so will not call pmap_to_ttbr0. Any calls
+		 * to pmap_to_ttbr0 after this will have the CnP flag set.
+		 * The dsb after invalidating the TLB will act as a barrier
+		 * to ensure all CPUs can observe this change.
+		 */
+		ttbr_flags |= TTBR_CnP;
+	}
+
+	ttbr0 = READ_SPECIALREG(ttbr0_el1);
+	ttbr0 |= TTBR_CnP;
+
+	ttbr1 = READ_SPECIALREG(ttbr1_el1);
+	ttbr1 |= TTBR_CnP;
+
+	/* Update ttbr{0,1}_el1 with the CnP flag */
+	WRITE_SPECIALREG(ttbr0_el1, ttbr0);
+	WRITE_SPECIALREG(ttbr1_el1, ttbr1);
+	isb();
+	__asm __volatile("tlbi vmalle1is");
+	dsb(ish);
+	isb();
+}
+
+/*
+ * Defer enabling CnP until we have read the ID registers to know if it's
+ * supported on all CPUs.
+ */
+static void
+pmap_init_cnp(void *dummy __unused)
+{
+	uint64_t reg;
+	u_int cpuid;
+
+	if (!get_kernel_reg(ID_AA64MMFR2_EL1, &reg))
+		return;
+
+	if (ID_AA64MMFR2_CnP_VAL(reg) != ID_AA64MMFR2_CnP_NONE) {
+		if (bootverbose)
+			printf("Enabling CnP\n");
+		cpuid = curcpu;
+		smp_rendezvous(NULL, pmap_set_cnp, NULL, &cpuid);
+	}
 
-	return (ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)) |
-	    pmap->pm_ttbr);
 }
+SYSINIT(pmap_init_cnp, SI_SUB_SMP, SI_ORDER_ANY, pmap_init_cnp, NULL);
 
 static bool
 pmap_activate_int(pmap_t pmap)