git: bcd763b642ab - main - Move the arm64 DMAP creation to C
Date: Wed, 06 Apr 2022 13:24:00 UTC
The branch main has been updated by andrew:
URL: https://cgit.FreeBSD.org/src/commit/?id=bcd763b642abb4b236902e84184c579225265fb7
commit bcd763b642abb4b236902e84184c579225265fb7
Author: Andrew Turner <andrew@FreeBSD.org>
AuthorDate: 2022-03-15 17:05:48 +0000
Commit: Andrew Turner <andrew@FreeBSD.org>
CommitDate: 2022-04-06 13:09:49 +0000
Move the arm64 DMAP creation to C
To simplify the creation of the direct map (DMAP) region on arm64 move
it from the pre-C code into pmap. This simplifies the DMAP creation
as we can use the notmal index macros, and should reduce the number
of pages needed to hold the level 1 tables to just those needed.
Reviewed by: alc, dch
Tested by: dch, kevans
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D34568
---
sys/arm64/arm64/locore.S | 13 +--
sys/arm64/arm64/pmap.c | 265 ++++++++++++++++++++++++++++++++++++-----------
2 files changed, 205 insertions(+), 73 deletions(-)
diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S
index 534c89ac32dc..e1b0530006cb 100644
--- a/sys/arm64/arm64/locore.S
+++ b/sys/arm64/arm64/locore.S
@@ -38,7 +38,6 @@
#include <machine/vmparam.h>
#define VIRT_BITS 48
-#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT)
.globl kernbase
.set kernbase, KERNBASE
@@ -378,7 +377,6 @@ LEND(get_virt_delta)
* - The Kernel L0 table (TTBR1)
* - The identity (PA = VA) L1 table
* - The identity (PA = VA) L0 table (TTBR0)
- * - The DMAP L1 tables
*/
LENTRY(create_pagetables)
/* Save the Link register */
@@ -476,13 +474,6 @@ common:
mov x10, #1
bl link_l0_pagetable
- /* Link the DMAP tables */
- ldr x8, =DMAP_MIN_ADDRESS
- adrp x9, pagetable_dmap
- add x9, x9, :lo12:pagetable_dmap
- mov x10, #DMAP_TABLES
- bl link_l0_pagetable
-
/*
* Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG.
* They are only needed early on, so the VA = PA map is uncached.
@@ -771,6 +762,7 @@ END(abort)
* L0 bootstrap for user
* L0 for user
*/
+ .globl pagetable_l0_ttbr1
pagetable:
.space PAGE_SIZE
pagetable_l1_ttbr1:
@@ -786,9 +778,6 @@ pagetable_l0_ttbr0_boostrap:
pagetable_l0_ttbr0:
.space PAGE_SIZE
- .globl pagetable_dmap
-pagetable_dmap:
- .space PAGE_SIZE * DMAP_TABLES
pagetable_end:
el2_pagetable:
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 9776c36b3ef4..3a356e8b720b 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -290,8 +290,7 @@ vm_offset_t dmap_max_addr; /* The virtual address limit of the dmap */
CTASSERT((DMAP_MIN_ADDRESS & ~L0_OFFSET) == DMAP_MIN_ADDRESS);
CTASSERT((DMAP_MAX_ADDRESS & ~L0_OFFSET) == DMAP_MAX_ADDRESS);
-#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT)
-extern pt_entry_t pagetable_dmap[];
+extern pt_entry_t pagetable_l0_ttbr1[];
#define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1))
static vm_paddr_t physmap[PHYSMAP_SIZE];
@@ -808,104 +807,248 @@ pmap_early_vtophys(vm_offset_t va)
return (pa_page | (va & PAR_LOW_MASK));
}
-static vm_offset_t
-pmap_bootstrap_dmap_l2(vm_offset_t *va, vm_paddr_t *pa, u_int *prev_l1_slot,
- pt_entry_t **l2p, int i, vm_offset_t freemempos)
+/* State of the bootstrapped DMAP page tables */
+struct dmap_bootstrap_state {
+ vm_offset_t va;
+ vm_paddr_t pa;
+ pt_entry_t *l1;
+ pt_entry_t *l2;
+ pt_entry_t *l3;
+ u_int l0_slot;
+ u_int l1_slot;
+ u_int l2_slot;
+ vm_offset_t freemempos;
+};
+
+static void
+pmap_bootstrap_dmap_l0_table(struct dmap_bootstrap_state *state)
+{
+ vm_paddr_t l1_pa;
+ u_int l0_slot;
+
+ /* Link the level 0 table to a level 1 table */
+ l0_slot = pmap_l0_index(state->va);
+ if (l0_slot != state->l0_slot) {
+ MPASS(state->l0_slot < l0_slot ||
+ state->l0_slot == L0_ENTRIES);
+
+ /* Create a new L0 table entry */
+ state->l0_slot = l0_slot;
+ state->l1 = (pt_entry_t *)state->freemempos;
+ memset(state->l1, 0, PAGE_SIZE);
+ state->freemempos += PAGE_SIZE;
+
+ /* Reset lower levels */
+ state->l2 = NULL;
+ state->l3 = NULL;
+ state->l1_slot = Ln_ENTRIES;
+ state->l2_slot = Ln_ENTRIES;
+
+ l1_pa = pmap_early_vtophys((vm_offset_t)state->l1);
+ MPASS((l1_pa & Ln_TABLE_MASK) == 0);
+ MPASS(pagetable_l0_ttbr1[l0_slot] == 0);
+ pmap_store(&pagetable_l0_ttbr1[l0_slot], l1_pa |
+ TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0 | L0_TABLE);
+ }
+ KASSERT(state->l1 != NULL, ("%s: NULL l1", __func__));
+}
+
+static void
+pmap_bootstrap_dmap_l1_table(struct dmap_bootstrap_state *state)
{
- pt_entry_t *l2;
vm_paddr_t l2_pa;
- u_int l1_slot, l2_slot;
- bool first;
+ u_int l1_slot;
+
+ /* Make sure there is a valid L0 -> L1 table */
+ pmap_bootstrap_dmap_l0_table(state);
+
+ /* Link the level 1 table to a level 2 table */
+ l1_slot = pmap_l1_index(state->va);
+ if (l1_slot != state->l1_slot) {
+ MPASS(state->l1_slot < l1_slot ||
+ state->l1_slot == Ln_ENTRIES);
+
+ /* Create a new L1 table entry */
+ state->l1_slot = l1_slot;
+ state->l2 = (pt_entry_t *)state->freemempos;
+ memset(state->l2, 0, PAGE_SIZE);
+ state->freemempos += PAGE_SIZE;
+
+ /* Reset lower levels */
+ state->l3 = NULL;
+ state->l2_slot = Ln_ENTRIES;
+
+ l2_pa = pmap_early_vtophys((vm_offset_t)state->l2);
+ MPASS((l2_pa & Ln_TABLE_MASK) == 0);
+ MPASS(state->l1[l1_slot] == 0);
+ pmap_store(&state->l1[l1_slot], l2_pa | TATTR_PXN_TABLE |
+ L1_TABLE);
+ }
+ KASSERT(state->l2 != NULL, ("%s: NULL l2", __func__));
+}
- l2 = *l2p;
- l1_slot = ((*va - DMAP_MIN_ADDRESS) >> L1_SHIFT);
- if (l1_slot != *prev_l1_slot) {
- *prev_l1_slot = l1_slot;
- l2 = (pt_entry_t *)freemempos;
- l2_pa = pmap_early_vtophys((vm_offset_t)l2);
- freemempos += PAGE_SIZE;
+static void
+pmap_bootstrap_dmap_l2_table(struct dmap_bootstrap_state *state)
+{
+ vm_paddr_t l3_pa;
+ u_int l2_slot;
+
+ /* Make sure there is a valid L1 -> L2 table */
+ pmap_bootstrap_dmap_l1_table(state);
- pmap_store(&pagetable_dmap[l1_slot],
- (l2_pa & ~Ln_TABLE_MASK) |
- TATTR_PXN_TABLE | L1_TABLE);
+ /* Link the level 2 table to a level 3 table */
+ l2_slot = pmap_l2_index(state->va);
+ if (l2_slot != state->l2_slot) {
+ MPASS(state->l2_slot < l2_slot ||
+ state->l2_slot == Ln_ENTRIES);
- memset(l2, 0, PAGE_SIZE);
+ /* Create a new L2 table entry */
+ state->l2_slot = l2_slot;
+ state->l3 = (pt_entry_t *)state->freemempos;
+ memset(state->l3, 0, PAGE_SIZE);
+ state->freemempos += PAGE_SIZE;
+
+ l3_pa = pmap_early_vtophys((vm_offset_t)state->l3);
+ MPASS((l3_pa & Ln_TABLE_MASK) == 0);
+ MPASS(state->l2[l2_slot] == 0);
+ pmap_store(&state->l2[l2_slot], l3_pa | TATTR_PXN_TABLE |
+ L2_TABLE);
}
- KASSERT(l2 != NULL,
- ("pmap_bootstrap_dmap_l2: NULL l2 map"));
- for (first = true; *va < DMAP_MAX_ADDRESS && *pa < physmap[i + 1];
- *pa += L2_SIZE, *va += L2_SIZE) {
+ KASSERT(state->l3 != NULL, ("%s: NULL l3", __func__));
+}
+
+static void
+pmap_bootstrap_dmap_l2_block(struct dmap_bootstrap_state *state, int i)
+{
+ u_int l2_slot;
+ bool first;
+
+ if ((physmap[i + 1] - state->pa) < L2_SIZE)
+ return;
+
+ /* Make sure there is a valid L1 table */
+ pmap_bootstrap_dmap_l1_table(state);
+
+ MPASS((state->va & L2_OFFSET) == 0);
+ for (first = true;
+ state->va < DMAP_MAX_ADDRESS &&
+ (physmap[i + 1] - state->pa) >= L2_SIZE;
+ state->va += L2_SIZE, state->pa += L2_SIZE) {
/*
* Stop if we are about to walk off the end of what the
* current L1 slot can address.
*/
- if (!first && (*pa & L1_OFFSET) == 0)
+ if (!first && (state->pa & L1_OFFSET) == 0)
break;
first = false;
- l2_slot = pmap_l2_index(*va);
- pmap_store(&l2[l2_slot],
- (*pa & ~L2_OFFSET) | ATTR_DEFAULT |
- ATTR_S1_XN |
- ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
+ l2_slot = pmap_l2_index(state->va);
+ MPASS((state->pa & L2_OFFSET) == 0);
+ MPASS(state->l2[l2_slot] == 0);
+ pmap_store(&state->l2[l2_slot], state->pa | ATTR_DEFAULT |
+ ATTR_S1_XN | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
L2_BLOCK);
}
- MPASS(*va == (*pa - dmap_phys_base + DMAP_MIN_ADDRESS));
- *l2p = l2;
+ MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS));
+}
- return (freemempos);
+static void
+pmap_bootstrap_dmap_l3_page(struct dmap_bootstrap_state *state, int i)
+{
+ u_int l3_slot;
+ bool first;
+
+ if ((physmap[i + 1] - state->pa) < L3_SIZE)
+ return;
+
+ /* Make sure there is a valid L2 table */
+ pmap_bootstrap_dmap_l2_table(state);
+
+ MPASS((state->va & L3_OFFSET) == 0);
+ for (first = true;
+ state->va < DMAP_MAX_ADDRESS &&
+ (physmap[i + 1] - state->pa) >= L3_SIZE;
+ state->va += L3_SIZE, state->pa += L3_SIZE) {
+ /*
+ * Stop if we are about to walk off the end of what the
+ * current L2 slot can address.
+ */
+ if (!first && (state->pa & L2_OFFSET) == 0)
+ break;
+
+ first = false;
+ l3_slot = pmap_l3_index(state->va);
+ MPASS((state->pa & L3_OFFSET) == 0);
+ MPASS(state->l3[l3_slot] == 0);
+ pmap_store(&state->l3[l3_slot], state->pa | ATTR_DEFAULT |
+ ATTR_S1_XN | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
+ L3_PAGE);
+ }
+ MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS));
}
static vm_offset_t
pmap_bootstrap_dmap(vm_offset_t kern_l1, vm_paddr_t min_pa,
vm_offset_t freemempos)
{
- pt_entry_t *l2;
- vm_offset_t va;
- vm_paddr_t pa;
- u_int l1_slot, prev_l1_slot;
+ struct dmap_bootstrap_state state;
int i;
dmap_phys_base = min_pa & ~L1_OFFSET;
dmap_phys_max = 0;
dmap_max_addr = 0;
- l2 = NULL;
- prev_l1_slot = -1;
+
+ state.l1 = state.l2 = state.l3 = NULL;
+ state.l0_slot = L0_ENTRIES;
+ state.l1_slot = Ln_ENTRIES;
+ state.l2_slot = Ln_ENTRIES;
+ state.freemempos = freemempos;
for (i = 0; i < (physmap_idx * 2); i += 2) {
- pa = physmap[i] & ~L2_OFFSET;
- va = pa - dmap_phys_base + DMAP_MIN_ADDRESS;
+ state.pa = physmap[i] & ~L3_OFFSET;
+ state.va = state.pa - dmap_phys_base + DMAP_MIN_ADDRESS;
- /* Create L2 mappings at the start of the region */
- if ((pa & L1_OFFSET) != 0) {
- freemempos = pmap_bootstrap_dmap_l2(&va, &pa,
- &prev_l1_slot, &l2, i, freemempos);
- }
+ /* Create L3 mappings at the start of the region */
+ if ((state.pa & L2_OFFSET) != 0)
+ pmap_bootstrap_dmap_l3_page(&state, i);
+ MPASS(state.pa <= physmap[i + 1]);
- for (; va < DMAP_MAX_ADDRESS && pa < physmap[i + 1] &&
- (physmap[i + 1] - pa) >= L1_SIZE;
- pa += L1_SIZE, va += L1_SIZE) {
- l1_slot = ((va - DMAP_MIN_ADDRESS) >> L1_SHIFT);
- pmap_store(&pagetable_dmap[l1_slot],
- (pa & ~L1_OFFSET) | ATTR_DEFAULT | ATTR_S1_XN |
- ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L1_BLOCK);
- }
+ /* Create L2 mappings at the start of the region */
+ if ((state.pa & L1_OFFSET) != 0)
+ pmap_bootstrap_dmap_l2_block(&state, i);
+ MPASS(state.pa <= physmap[i + 1]);
+
+ /* Create the main L1 block mappings */
+ for (; state.va < DMAP_MAX_ADDRESS &&
+ (physmap[i + 1] - state.pa) >= L1_SIZE;
+ state.va += L1_SIZE, state.pa += L1_SIZE) {
+ /* Make sure there is a valid L1 table */
+ pmap_bootstrap_dmap_l0_table(&state);
+ MPASS((state.pa & L1_OFFSET) == 0);
+ pmap_store(&state.l1[pmap_l1_index(state.va)],
+ state.pa | ATTR_DEFAULT | ATTR_S1_XN |
+ ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
+ L1_BLOCK);
+ }
+ MPASS(state.pa <= physmap[i + 1]);
/* Create L2 mappings at the end of the region */
- if (pa < physmap[i + 1]) {
- freemempos = pmap_bootstrap_dmap_l2(&va, &pa,
- &prev_l1_slot, &l2, i, freemempos);
- }
+ pmap_bootstrap_dmap_l2_block(&state, i);
+ MPASS(state.pa <= physmap[i + 1]);
+
+ /* Create L3 mappings at the end of the region */
+ pmap_bootstrap_dmap_l3_page(&state, i);
+ MPASS(state.pa == physmap[i + 1]);
- if (pa > dmap_phys_max) {
- dmap_phys_max = pa;
- dmap_max_addr = va;
+ if (state.pa > dmap_phys_max) {
+ dmap_phys_max = state.pa;
+ dmap_max_addr = state.va;
}
}
cpu_tlb_flushID();
- return (freemempos);
+ return (state.freemempos);
}
static vm_offset_t