PERFORCE change 94980 for review
Alan Cox
alc at FreeBSD.org
Tue Apr 11 08:45:25 UTC 2006
http://perforce.freebsd.org/chv.cgi?CH=94980
Change 94980 by alc at alc_home on 2006/04/11 08:44:23
IF user/alc/superpages
Various fixes to promotion and demotion of kernel pmap pages.
Some code clean up and optimization.
Catch up with the change to pmap_remove_pages()'s parameters.
Affected files ...
.. //depot/projects/superpages/src/sys/i386/i386/pmap.c#12 integrate
Differences ...
==== //depot/projects/superpages/src/sys/i386/i386/pmap.c#12 (text+ko) ====
@@ -259,6 +259,8 @@
static pv_entry_t get_pv_entry(pmap_t locked_pmap);
static void pmap_clear_ptes(vm_page_t m, int bit);
+static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
+static void pmap_promote_pde(pmap_t pmap, vm_offset_t va, reservation_t reserv);
static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva);
static void pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva);
static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva);
@@ -281,9 +283,6 @@
static void *pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait);
#endif
-static void mach_promote(pmap_t pmap, vm_offset_t va, reservation_t reserv);
-static boolean_t pmap_demote(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
-
CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t));
@@ -1187,7 +1186,7 @@
* normal 4K page.
*/
if (ptepa & PG_PS) {
- pmap_demote(pmap, &pmap->pm_pdir[ptepindex], va);
+ pmap_demote_pde(pmap, &pmap->pm_pdir[ptepindex], va);
ptepa = pmap->pm_pdir[ptepindex];
}
@@ -1777,15 +1776,21 @@
printf("pmap_remove: superpage at %x to destroy.\n",
sva);
#endif
+
+ /*
+ * The TLB entry for a PG_G mapping is
+ * invalidated by pmap_remove_pde().
+ */
+ if ((ptpaddr & PG_G) == 0)
+ anyvalid = 1;
pmap_remove_pde(pmap, pmap_pde(pmap, sva), sva);
- anyvalid = 1;
continue;
} else {
#ifdef INVARIANTS
printf("pmap_remove: superpage at %x to demote !!!\n",
sva);
#endif
- if (!pmap_demote(pmap, pmap_pde(pmap, sva), sva)) {
+ if (!pmap_demote_pde(pmap, pmap_pde(pmap, sva), sva)) {
anyvalid = 1; /* XXX */
continue;
}
@@ -1862,7 +1867,7 @@
pde = pmap_pde(pmap, pv->pv_va);
if (*pde & PG_PS) {
printf("pmap_remove_all: superpage to demote !!!\n");
- if (!pmap_demote(pmap, pde, pv->pv_va)) {
+ if (!pmap_demote_pde(pmap, pde, pv->pv_va)) {
/*
* All mappings within the same 4mpage were
* destroyed and pv was freed.
@@ -1991,7 +1996,7 @@
anychanged = 1;
continue;
} else {
- if (!pmap_demote(pmap, pmap_pde(pmap, sva), sva)) {
+ if (!pmap_demote_pde(pmap, pmap_pde(pmap, sva), sva)) {
anychanged = 1; /* XXX */
continue;
}
@@ -2250,7 +2255,7 @@
printf("%s: pmap %p va %x XXX\n", __func__, pmap, va);
#endif
KASSERT(m->object->flags & OBJ_SUPERPAGES, ("pmap_enter: xxx"));
- mach_promote(pmap, va, m->reserv);
+ pmap_promote_pde(pmap, va, m->reserv);
}
sched_unpin();
vm_page_unlock_queues();
@@ -2380,7 +2385,7 @@
#endif
KASSERT(m->object->flags & OBJ_SUPERPAGES,
("pmap_enter_quick: xxx"));
- mach_promote(pmap, va, m->reserv);
+ pmap_promote_pde(pmap, va, m->reserv);
}
out:
PMAP_UNLOCK(pmap);
@@ -2801,28 +2806,13 @@
npv = TAILQ_NEXT(pv, pv_plist);
continue;
}
- if (sva <= trunc_4mpage(pv->pv_va) &&
- eva >= round_4mpage(pv->pv_va + 1)) {
-#ifdef INVARIANTS
- printf("pmap_remove_pages: superpage at %x to destroy.\n",
- trunc_4mpage(pv->pv_va));
-#endif
- pmap_remove_pde(pmap, pde, trunc_4mpage(pv->pv_va));
- npv = TAILQ_FIRST(&pmap->pm_pvlist);
- continue;
- }
#ifdef INVARIANTS
- printf("pmap_remove_pages: superpage at %x to demote !!!\n",
- pv->pv_va);
+ printf("pmap_remove_pages: superpage at %x to destroy.\n",
+ trunc_4mpage(pv->pv_va));
#endif
- if (!pmap_demote(pmap, pde, pv->pv_va)) {
- /*
- * All mappings within the same 2mpage were
- * destroyed and pv was freed.
- */
- npv = TAILQ_FIRST(&pmap->pm_pvlist);
- continue;
- }
+ pmap_remove_pde(pmap, pde, trunc_4mpage(pv->pv_va));
+ npv = TAILQ_FIRST(&pmap->pm_pvlist);
+ continue;
}
#ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY
@@ -2982,7 +2972,7 @@
if (*pde & PG_PS) {
printf("pmap_clear_ptes: superpage to demote !!!\n");
if ((*pde & bit) == 0 ||
- !pmap_demote(pmap, pde, pv->pv_va)) {
+ !pmap_demote_pde(pmap, pde, pv->pv_va)) {
/*
* All mappings within the same 2mpage were
* destroyed and pv was freed.
@@ -3292,7 +3282,7 @@
#define COMPATIBLE_PTE(a,b) ((a & COMPATIBLE_PTE_MASK) == (b & COMPATIBLE_PTE_MASK))
static void
-mach_promote(pmap_t pmap, vm_offset_t va, reservation_t reserv)
+pmap_promote_pde(pmap_t pmap, vm_offset_t va, reservation_t reserv)
{
vm_paddr_t pa;
pmap_t allpmaps_entry;
@@ -3314,8 +3304,9 @@
pa += PAGE_SIZE;
page_pa = PHYS_TO_VM_PAGE(*pte & PG_FRAME);
- KASSERT(page_pa->reserv,("mach_promote: page has no reservation"));
- KASSERT(page_pa->reserv == reserv,("mach_promote: reservation mismatch"));
+ KASSERT(page_pa->reserv,("pmap_promote_pde: page has no reservation"));
+ KASSERT(page_pa->reserv == reserv,
+ ("pmap_promote_pde: reservation mismatch"));
if ((*pte & PG_V) == 0 || !COMPATIBLE_PTE(*pte, flags))
return;
@@ -3341,15 +3332,10 @@
/* Invalidate old TLB entries */
pmap_invalidate_all(pmap);
- /*
- * XXX
- *
- * File system corruption occurs if pte pages belonging to the
- * kernel pmap are freed.
- */
- if (pmap != kernel_pmap) {
+ /* This leaks up to nkpt kernel pmap page table pages. XXX */
+ if (pmap != kernel_pmap || tofree->wire_count == NPTEPG) {
KASSERT(tofree->wire_count == NPTEPG,
- ("pmap_promote: pte page wire count error"));
+ ("pmap_promote_pde: pte page wire count error"));
tofree->wire_count = 0;
vm_page_free(tofree);
atomic_subtract_int(&cnt.v_wire_count, 1);
@@ -3361,8 +3347,9 @@
}
static boolean_t
-pmap_demote(pmap_t pmap, pd_entry_t *pde0, vm_offset_t va)
+pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
{
+ pmap_t allpmaps_entry;
pd_entry_t save_pde_value, new_pte_value ;
pt_entry_t *pte_page_va, *new_pte_va;
vm_paddr_t pte_page_pa;
@@ -3370,21 +3357,21 @@
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
- KASSERT((*pde0 & PG_PS) != 0,
- ("pmap_demote: not a superpage, impossible to demote"));
+ KASSERT((*pde & PG_PS) != 0,
+ ("pmap_demote_pde: not a superpage, impossible to demote"));
/* STEP 1
* Allocate the PTE page
*/
if ((pte_page = vm_page_alloc(NULL, va >> PDRSHIFT,
VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL | VM_ALLOC_WIRED)) == NULL) {
- pmap_remove_pde(pmap, pde0, trunc_4mpage(va));
+ pmap_remove_pde(pmap, pde, trunc_4mpage(va));
pmap_invalidate_all(pmap);
return (FALSE);
}
pte_page->wire_count += NPTEPG - 1;
KASSERT(pte_page->wire_count == NPTEPG,
- ("pmap_demote: page table page %p has wire count %d",
+ ("pmap_demote_pde: page table page %p has wire count %d",
pte_page, pte_page->wire_count));
if (pmap != kernel_pmap)
pmap->pm_stats.resident_count++;
@@ -3407,7 +3394,7 @@
#endif
PMAP1unchanged++;
pte_page_va = PADDR1;
- pte_page_pa |= PG_U | PG_RW | PG_V | PG_A | PG_M;
+ pte_page_pa |= PG_M | PG_A | (*pde & PG_U) | PG_RW | PG_V;
repeat:
@@ -3415,7 +3402,7 @@
* Save the value of the pde entry
* Define the value of the first pte entry
*/
- save_pde_value = *pde0;
+ save_pde_value = *pde;
/* STEP 3
* Fill the PTE page with the physical address of the base pages
@@ -3433,10 +3420,23 @@
* If not, assign the new pde value.
* If yes, repeat the pte assignment loop.
*/
- if (!atomic_cmpset_int(pde0, save_pde_value, pte_page_pa))
- goto repeat;
+ if (pmap == kernel_pmap) {
+ mtx_lock_spin(&allpmaps_lock);
+ LIST_FOREACH(allpmaps_entry, &allpmaps, pm_list) {
+ pde = pmap_pde(allpmaps_entry, va);
+ KASSERT(COMPATIBLE_PTE(*pde, save_pde_value),
+ ("pmap_demote_pde: pde was %#jx, expected %#jx",
+ (uintmax_t)*pde, (uintmax_t)save_pde_value));
+ pde_store(pde, pte_page_pa);
+ }
+ mtx_unlock_spin(&allpmaps_lock);
+ } else {
+ if (!atomic_cmpset_int(pde, save_pde_value, pte_page_pa))
+ goto repeat;
+ }
- pmap_invalidate_all(pmap);
+ /* Invalidate the recursive mapping of the page table page. */
+ pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));
return (TRUE);
}
More information about the p4-projects
mailing list