svn commit: r233911 - stable/9/sys/powerpc/aim
Nathan Whitehorn
nwhitehorn at FreeBSD.org
Thu Apr 5 00:53:22 UTC 2012
Author: nwhitehorn
Date: Thu Apr 5 00:53:21 2012
New Revision: 233911
URL: http://svn.freebsd.org/changeset/base/233911
Log:
MFC 232980,233011,233017,233117,233434,233436,233454,233529,233530,233618:
Major pmap performance, concurrency, and correctness improvements, mostly
for the 64-bit PMAP module (64-bit-capable CPUs with either a 32-bit or
64-bit kernel). Thanks to alc for his help and prodding.
Modified:
stable/9/sys/powerpc/aim/mmu_oea.c
stable/9/sys/powerpc/aim/mmu_oea64.c
stable/9/sys/powerpc/aim/moea64_native.c
Directory Properties:
stable/9/sys/ (props changed)
Modified: stable/9/sys/powerpc/aim/mmu_oea.c
==============================================================================
--- stable/9/sys/powerpc/aim/mmu_oea.c Thu Apr 5 00:08:16 2012 (r233910)
+++ stable/9/sys/powerpc/aim/mmu_oea.c Thu Apr 5 00:53:21 2012 (r233911)
@@ -1760,7 +1760,7 @@ moea_release(mmu_t mmu, pmap_t pmap)
void
moea_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva)
{
- struct pvo_entry *pvo;
+ struct pvo_entry *pvo, *tpvo;
int pteidx;
vm_page_lock_queues();
@@ -1772,7 +1772,7 @@ moea_remove(mmu_t mmu, pmap_t pm, vm_off
moea_pvo_remove(pvo, pteidx);
}
} else {
- LIST_FOREACH(pvo, &pm->pmap_pvo, pvo_plink) {
+ LIST_FOREACH_SAFE(pvo, &pm->pmap_pvo, pvo_plink, tpvo) {
if (PVO_VADDR(pvo) < sva || PVO_VADDR(pvo) >= eva)
continue;
moea_pvo_remove(pvo, -1);
Modified: stable/9/sys/powerpc/aim/mmu_oea64.c
==============================================================================
--- stable/9/sys/powerpc/aim/mmu_oea64.c Thu Apr 5 00:08:16 2012 (r233910)
+++ stable/9/sys/powerpc/aim/mmu_oea64.c Thu Apr 5 00:53:21 2012 (r233911)
@@ -125,6 +125,7 @@ __FBSDID("$FreeBSD$");
#include <sys/msgbuf.h>
#include <sys/mutex.h>
#include <sys/proc.h>
+#include <sys/rwlock.h>
#include <sys/sched.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
@@ -172,9 +173,20 @@ uintptr_t moea64_get_unique_vsid(void);
#define VSID_TO_HASH(vsid) (((vsid) >> 4) & 0xfffff)
#define VSID_HASH_MASK 0x0000007fffffffffULL
-#define LOCK_TABLE() mtx_lock(&moea64_table_mutex)
-#define UNLOCK_TABLE() mtx_unlock(&moea64_table_mutex);
-#define ASSERT_TABLE_LOCK() mtx_assert(&moea64_table_mutex, MA_OWNED)
+/*
+ * Locking semantics:
+ * -- Read lock: if no modifications are being made to either the PVO lists
+ * or page table or if any modifications being made result in internal
+ * changes (e.g. wiring, protection) such that the existence of the PVOs
+ * is unchanged and they remain associated with the same pmap (in which
+ * case the changes should be protected by the pmap lock)
+ * -- Write lock: required if PTEs/PVOs are being inserted or removed.
+ */
+
+#define LOCK_TABLE_RD() rw_rlock(&moea64_table_lock)
+#define UNLOCK_TABLE_RD() rw_runlock(&moea64_table_lock)
+#define LOCK_TABLE_WR() rw_wlock(&moea64_table_lock)
+#define UNLOCK_TABLE_WR() rw_wunlock(&moea64_table_lock)
struct ofw_map {
cell_t om_va;
@@ -197,7 +209,7 @@ extern void bs_remap_earlyboot(void);
/*
* Lock for the pteg and pvo tables.
*/
-struct mtx moea64_table_mutex;
+struct rwlock moea64_table_lock;
struct mtx moea64_slb_mutex;
/*
@@ -307,6 +319,7 @@ void moea64_qenter(mmu_t, vm_offset_t, v
void moea64_qremove(mmu_t, vm_offset_t, int);
void moea64_release(mmu_t, pmap_t);
void moea64_remove(mmu_t, pmap_t, vm_offset_t, vm_offset_t);
+void moea64_remove_pages(mmu_t, pmap_t);
void moea64_remove_all(mmu_t, vm_page_t);
void moea64_remove_write(mmu_t, vm_page_t);
void moea64_zero_page(mmu_t, vm_page_t);
@@ -349,6 +362,7 @@ static mmu_method_t moea64_methods[] = {
MMUMETHOD(mmu_qremove, moea64_qremove),
MMUMETHOD(mmu_release, moea64_release),
MMUMETHOD(mmu_remove, moea64_remove),
+ MMUMETHOD(mmu_remove_pages, moea64_remove_pages),
MMUMETHOD(mmu_remove_all, moea64_remove_all),
MMUMETHOD(mmu_remove_write, moea64_remove_write),
MMUMETHOD(mmu_sync_icache, moea64_sync_icache),
@@ -393,35 +407,10 @@ vm_page_to_pvoh(vm_page_t m)
}
static __inline void
-moea64_attr_clear(vm_page_t m, u_int64_t ptebit)
-{
-
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
- m->md.mdpg_attrs &= ~ptebit;
-}
-
-static __inline u_int64_t
-moea64_attr_fetch(vm_page_t m)
-{
-
- return (m->md.mdpg_attrs);
-}
-
-static __inline void
-moea64_attr_save(vm_page_t m, u_int64_t ptebit)
-{
-
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
- m->md.mdpg_attrs |= ptebit;
-}
-
-static __inline void
moea64_pte_create(struct lpte *pt, uint64_t vsid, vm_offset_t va,
uint64_t pte_lo, int flags)
{
- ASSERT_TABLE_LOCK();
-
/*
* Construct a PTE. Default to IMB initially. Valid bit only gets
* set when the real pte is set in memory.
@@ -614,6 +603,7 @@ moea64_setup_direct_map(mmu_t mmup, vm_o
DISABLE_TRANS(msr);
if (hw_direct_map) {
+ LOCK_TABLE_WR();
PMAP_LOCK(kernel_pmap);
for (i = 0; i < pregions_sz; i++) {
for (pa = pregions[i].mr_start; pa < pregions[i].mr_start +
@@ -638,6 +628,7 @@ moea64_setup_direct_map(mmu_t mmup, vm_o
}
}
PMAP_UNLOCK(kernel_pmap);
+ UNLOCK_TABLE_WR();
} else {
size = sizeof(struct pvo_head) * moea64_pteg_count;
off = (vm_offset_t)(moea64_pvo_table);
@@ -796,8 +787,7 @@ moea64_mid_bootstrap(mmu_t mmup, vm_offs
* Initialize the lock that synchronizes access to the pteg and pvo
* tables.
*/
- mtx_init(&moea64_table_mutex, "pmap table", NULL, MTX_DEF |
- MTX_RECURSE);
+ rw_init_flags(&moea64_table_lock, "pmap tables", RW_RECURSE);
mtx_init(&moea64_slb_mutex, "SLB table", NULL, MTX_DEF);
/*
@@ -976,7 +966,7 @@ moea64_late_bootstrap(mmu_t mmup, vm_off
moea64_scratchpage_pvo[i] = moea64_pvo_find_va(
kernel_pmap, (vm_offset_t)moea64_scratchpage_va[i]);
- LOCK_TABLE();
+ LOCK_TABLE_RD();
moea64_scratchpage_pte[i] = MOEA64_PVO_TO_PTE(
mmup, moea64_scratchpage_pvo[i]);
moea64_scratchpage_pvo[i]->pvo_pte.lpte.pte_hi
@@ -984,7 +974,7 @@ moea64_late_bootstrap(mmu_t mmup, vm_off
MOEA64_PTE_CHANGE(mmup, moea64_scratchpage_pte[i],
&moea64_scratchpage_pvo[i]->pvo_pte.lpte,
moea64_scratchpage_pvo[i]->pvo_vpn);
- UNLOCK_TABLE();
+ UNLOCK_TABLE_RD();
}
}
}
@@ -1030,11 +1020,11 @@ moea64_change_wiring(mmu_t mmu, pmap_t p
uint64_t vsid;
int i, ptegidx;
+ LOCK_TABLE_WR();
PMAP_LOCK(pm);
pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF);
if (pvo != NULL) {
- LOCK_TABLE();
pt = MOEA64_PVO_TO_PTE(mmu, pvo);
if (wired) {
@@ -1070,8 +1060,8 @@ moea64_change_wiring(mmu_t mmu, pmap_t p
}
}
- UNLOCK_TABLE();
}
+ UNLOCK_TABLE_WR();
PMAP_UNLOCK(pm);
}
@@ -1182,10 +1172,10 @@ moea64_enter(mmu_t mmu, pmap_t pmap, vm_
vm_prot_t prot, boolean_t wired)
{
- vm_page_lock_queues();
+ LOCK_TABLE_WR();
PMAP_LOCK(pmap);
moea64_enter_locked(mmu, pmap, va, m, prot, wired);
- vm_page_unlock_queues();
+ UNLOCK_TABLE_WR();
PMAP_UNLOCK(pmap);
}
@@ -1194,7 +1184,7 @@ moea64_enter(mmu_t mmu, pmap_t pmap, vm_
* target pmap with the protection requested. If specified the page
* will be wired down.
*
- * The page queues and pmap must be locked.
+ * The table (write) and pmap must be locked.
*/
static void
@@ -1220,8 +1210,6 @@ moea64_enter_locked(mmu_t mmu, pmap_t pm
pvo_flags = PVO_MANAGED;
}
- if (pmap_bootstrapped)
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
KASSERT((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) != 0 ||
VM_OBJECT_LOCKED(m->object),
@@ -1319,14 +1307,14 @@ moea64_enter_object(mmu_t mmu, pmap_t pm
psize = atop(end - start);
m = m_start;
- vm_page_lock_queues();
+ LOCK_TABLE_WR();
PMAP_LOCK(pm);
while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
moea64_enter_locked(mmu, pm, start + ptoa(diff), m, prot &
(VM_PROT_READ | VM_PROT_EXECUTE), FALSE);
m = TAILQ_NEXT(m, listq);
}
- vm_page_unlock_queues();
+ UNLOCK_TABLE_WR();
PMAP_UNLOCK(pm);
}
@@ -1335,11 +1323,11 @@ moea64_enter_quick(mmu_t mmu, pmap_t pm,
vm_prot_t prot)
{
- vm_page_lock_queues();
+ LOCK_TABLE_WR();
PMAP_LOCK(pm);
moea64_enter_locked(mmu, pm, va, m,
prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE);
- vm_page_unlock_queues();
+ UNLOCK_TABLE_WR();
PMAP_UNLOCK(pm);
}
@@ -1349,6 +1337,7 @@ moea64_extract(mmu_t mmu, pmap_t pm, vm_
struct pvo_entry *pvo;
vm_paddr_t pa;
+ LOCK_TABLE_RD();
PMAP_LOCK(pm);
pvo = moea64_pvo_find_va(pm, va);
if (pvo == NULL)
@@ -1356,6 +1345,7 @@ moea64_extract(mmu_t mmu, pmap_t pm, vm_
else
pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) |
(va - PVO_VADDR(pvo));
+ UNLOCK_TABLE_RD();
PMAP_UNLOCK(pm);
return (pa);
}
@@ -1374,6 +1364,7 @@ moea64_extract_and_hold(mmu_t mmu, pmap_
m = NULL;
pa = 0;
+ LOCK_TABLE_RD();
PMAP_LOCK(pmap);
retry:
pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF);
@@ -1387,6 +1378,7 @@ retry:
vm_page_hold(m);
}
PA_UNLOCK_COND(pa);
+ UNLOCK_TABLE_RD();
PMAP_UNLOCK(pmap);
return (m);
}
@@ -1411,9 +1403,6 @@ moea64_uma_page_alloc(uma_zone_t zone, i
*flags = UMA_SLAB_PRIV;
needed_lock = !PMAP_LOCKED(kernel_pmap);
- if (needed_lock)
- PMAP_LOCK(kernel_pmap);
-
if ((wait & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT)
pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED;
else
@@ -1433,12 +1422,17 @@ moea64_uma_page_alloc(uma_zone_t zone, i
va = VM_PAGE_TO_PHYS(m);
+ LOCK_TABLE_WR();
+ if (needed_lock)
+ PMAP_LOCK(kernel_pmap);
+
moea64_pvo_enter(installed_mmu, kernel_pmap, moea64_upvo_zone,
&moea64_pvo_kunmanaged, va, VM_PAGE_TO_PHYS(m), LPTE_M,
PVO_WIRED | PVO_BOOTSTRAP);
if (needed_lock)
PMAP_UNLOCK(kernel_pmap);
+ UNLOCK_TABLE_WR();
if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
bzero((void *)va, PAGE_SIZE);
@@ -1502,10 +1496,12 @@ moea64_is_prefaultable(mmu_t mmu, pmap_t
struct pvo_entry *pvo;
boolean_t rv;
+ LOCK_TABLE_RD();
PMAP_LOCK(pmap);
pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF);
rv = pvo == NULL || (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0;
PMAP_UNLOCK(pmap);
+ UNLOCK_TABLE_RD();
return (rv);
}
@@ -1547,7 +1543,7 @@ moea64_remove_write(mmu_t mmu, vm_page_t
struct pvo_entry *pvo;
uintptr_t pt;
pmap_t pmap;
- uint64_t lo;
+ uint64_t lo = 0;
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
("moea64_remove_write: page %p is not managed", m));
@@ -1561,13 +1557,11 @@ moea64_remove_write(mmu_t mmu, vm_page_t
if ((m->oflags & VPO_BUSY) == 0 &&
(m->aflags & PGA_WRITEABLE) == 0)
return;
- vm_page_lock_queues();
- lo = moea64_attr_fetch(m);
powerpc_sync();
+ LOCK_TABLE_RD();
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap);
- LOCK_TABLE();
if ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) {
pt = MOEA64_PVO_TO_PTE(mmu, pvo);
pvo->pvo_pte.lpte.pte_lo &= ~LPTE_PP;
@@ -1582,15 +1576,12 @@ moea64_remove_write(mmu_t mmu, vm_page_t
isync();
}
}
- UNLOCK_TABLE();
+ if ((lo & LPTE_CHG) != 0)
+ vm_page_dirty(m);
PMAP_UNLOCK(pmap);
}
- if ((lo & LPTE_CHG) != 0) {
- moea64_attr_clear(m, LPTE_CHG);
- vm_page_dirty(m);
- }
+ UNLOCK_TABLE_RD();
vm_page_aflag_clear(m, PGA_WRITEABLE);
- vm_page_unlock_queues();
}
/*
@@ -1631,13 +1622,12 @@ moea64_page_set_memattr(mmu_t mmu, vm_pa
return;
}
- vm_page_lock_queues();
pvo_head = vm_page_to_pvoh(m);
lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m), ma);
+ LOCK_TABLE_RD();
LIST_FOREACH(pvo, pvo_head, pvo_vlink) {
pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap);
- LOCK_TABLE();
pt = MOEA64_PVO_TO_PTE(mmu, pvo);
pvo->pvo_pte.lpte.pte_lo &= ~LPTE_WIMG;
pvo->pvo_pte.lpte.pte_lo |= lo;
@@ -1647,11 +1637,10 @@ moea64_page_set_memattr(mmu_t mmu, vm_pa
if (pvo->pvo_pmap == kernel_pmap)
isync();
}
- UNLOCK_TABLE();
PMAP_UNLOCK(pmap);
}
+ UNLOCK_TABLE_RD();
m->md.mdpg_cache_attrs = ma;
- vm_page_unlock_queues();
}
/*
@@ -1665,9 +1654,12 @@ moea64_kenter_attr(mmu_t mmu, vm_offset_
pte_lo = moea64_calc_wimg(pa, ma);
+ LOCK_TABLE_WR();
PMAP_LOCK(kernel_pmap);
error = moea64_pvo_enter(mmu, kernel_pmap, moea64_upvo_zone,
&moea64_pvo_kunmanaged, va, pa, pte_lo, PVO_WIRED);
+ PMAP_UNLOCK(kernel_pmap);
+ UNLOCK_TABLE_WR();
if (error != 0 && error != ENOENT)
panic("moea64_kenter: failed to enter va %#zx pa %#zx: %d", va,
@@ -1678,7 +1670,6 @@ moea64_kenter_attr(mmu_t mmu, vm_offset_
*/
if ((pte_lo & (LPTE_I | LPTE_G)) == 0)
__syncicache((void *)va, PAGE_SIZE);
- PMAP_UNLOCK(kernel_pmap);
}
void
@@ -1705,11 +1696,13 @@ moea64_kextract(mmu_t mmu, vm_offset_t v
if (va < VM_MIN_KERNEL_ADDRESS)
return (va);
+ LOCK_TABLE_RD();
PMAP_LOCK(kernel_pmap);
pvo = moea64_pvo_find_va(kernel_pmap, va);
KASSERT(pvo != NULL, ("moea64_kextract: no addr found for %#" PRIxPTR,
va));
pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | (va - PVO_VADDR(pvo));
+ UNLOCK_TABLE_RD();
PMAP_UNLOCK(kernel_pmap);
return (pa);
}
@@ -1765,7 +1758,7 @@ moea64_page_exists_quick(mmu_t mmu, pmap
("moea64_page_exists_quick: page %p is not managed", m));
loops = 0;
rv = FALSE;
- vm_page_lock_queues();
+ LOCK_TABLE_RD();
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
if (pvo->pvo_pmap == pmap) {
rv = TRUE;
@@ -1774,7 +1767,7 @@ moea64_page_exists_quick(mmu_t mmu, pmap
if (++loops >= 16)
break;
}
- vm_page_unlock_queues();
+ UNLOCK_TABLE_RD();
return (rv);
}
@@ -1791,11 +1784,11 @@ moea64_page_wired_mappings(mmu_t mmu, vm
count = 0;
if ((m->oflags & VPO_UNMANAGED) != 0)
return (count);
- vm_page_lock_queues();
+ LOCK_TABLE_RD();
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink)
if ((pvo->pvo_vaddr & PVO_WIRED) != 0)
count++;
- vm_page_unlock_queues();
+ UNLOCK_TABLE_RD();
return (count);
}
@@ -1903,16 +1896,73 @@ moea64_pinit0(mmu_t mmu, pmap_t pm)
/*
* Set the physical protection on the specified range of this map as requested.
*/
+static void
+moea64_pvo_protect(mmu_t mmu, pmap_t pm, struct pvo_entry *pvo, vm_prot_t prot)
+{
+ uintptr_t pt;
+ uint64_t oldlo;
+
+ PMAP_LOCK_ASSERT(pm, MA_OWNED);
+
+ /*
+ * Grab the PTE pointer before we diddle with the cached PTE
+ * copy.
+ */
+ pt = MOEA64_PVO_TO_PTE(mmu, pvo);
+
+ /*
+ * Change the protection of the page.
+ */
+ oldlo = pvo->pvo_pte.lpte.pte_lo;
+ pvo->pvo_pte.lpte.pte_lo &= ~LPTE_PP;
+ pvo->pvo_pte.lpte.pte_lo &= ~LPTE_NOEXEC;
+ if ((prot & VM_PROT_EXECUTE) == 0)
+ pvo->pvo_pte.lpte.pte_lo |= LPTE_NOEXEC;
+ if (prot & VM_PROT_WRITE)
+ pvo->pvo_pte.lpte.pte_lo |= LPTE_BW;
+ else
+ pvo->pvo_pte.lpte.pte_lo |= LPTE_BR;
+
+ /*
+ * If the PVO is in the page table, update that pte as well.
+ */
+ if (pt != -1) {
+ MOEA64_PTE_CHANGE(mmu, pt, &pvo->pvo_pte.lpte,
+ pvo->pvo_vpn);
+ if ((pvo->pvo_pte.lpte.pte_lo &
+ (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
+ moea64_syncicache(mmu, pm, PVO_VADDR(pvo),
+ pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN,
+ PAGE_SIZE);
+ }
+ }
+
+ /*
+ * Update vm about the REF/CHG bits if the page is managed and we have
+ * removed write access.
+ */
+ if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED &&
+ (oldlo & LPTE_PP) != LPTE_BR && !(prot && VM_PROT_WRITE)) {
+ struct vm_page *pg;
+
+ pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
+ if (pg != NULL) {
+ if (pvo->pvo_pte.lpte.pte_lo & LPTE_CHG)
+ vm_page_dirty(pg);
+ if (pvo->pvo_pte.lpte.pte_lo & LPTE_REF)
+ vm_page_aflag_set(pg, PGA_REFERENCED);
+ }
+ }
+}
+
void
moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
vm_prot_t prot)
{
- struct pvo_entry *pvo;
- uintptr_t pt;
-
- CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm, sva,
- eva, prot);
+ struct pvo_entry *pvo, *tpvo;
+ CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm,
+ sva, eva, prot);
KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap,
("moea64_protect: non current pmap"));
@@ -1922,45 +1972,30 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_
return;
}
- vm_page_lock_queues();
+ LOCK_TABLE_RD();
PMAP_LOCK(pm);
- for (; sva < eva; sva += PAGE_SIZE) {
- pvo = moea64_pvo_find_va(pm, sva);
- if (pvo == NULL)
- continue;
-
- /*
- * Grab the PTE pointer before we diddle with the cached PTE
- * copy.
- */
- LOCK_TABLE();
- pt = MOEA64_PVO_TO_PTE(mmu, pvo);
-
- /*
- * Change the protection of the page.
- */
- pvo->pvo_pte.lpte.pte_lo &= ~LPTE_PP;
- pvo->pvo_pte.lpte.pte_lo |= LPTE_BR;
- pvo->pvo_pte.lpte.pte_lo &= ~LPTE_NOEXEC;
- if ((prot & VM_PROT_EXECUTE) == 0)
- pvo->pvo_pte.lpte.pte_lo |= LPTE_NOEXEC;
-
- /*
- * If the PVO is in the page table, update that pte as well.
- */
- if (pt != -1) {
- MOEA64_PTE_CHANGE(mmu, pt, &pvo->pvo_pte.lpte,
- pvo->pvo_vpn);
- if ((pvo->pvo_pte.lpte.pte_lo &
- (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
- moea64_syncicache(mmu, pm, sva,
- pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN,
- PAGE_SIZE);
+ if ((eva - sva)/PAGE_SIZE < pm->pm_stats.resident_count) {
+ while (sva < eva) {
+ #ifdef __powerpc64__
+ if (pm != kernel_pmap &&
+ user_va_to_slb_entry(pm, sva) == NULL) {
+ sva = roundup2(sva + 1, SEGMENT_LENGTH);
+ continue;
}
+ #endif
+ pvo = moea64_pvo_find_va(pm, sva);
+ if (pvo != NULL)
+ moea64_pvo_protect(mmu, pm, pvo, prot);
+ sva += PAGE_SIZE;
+ }
+ } else {
+ LIST_FOREACH_SAFE(pvo, &pm->pmap_pvo, pvo_plink, tpvo) {
+ if (PVO_VADDR(pvo) < sva || PVO_VADDR(pvo) >= eva)
+ continue;
+ moea64_pvo_protect(mmu, pm, pvo, prot);
}
- UNLOCK_TABLE();
}
- vm_page_unlock_queues();
+ UNLOCK_TABLE_RD();
PMAP_UNLOCK(pm);
}
@@ -2028,29 +2063,61 @@ moea64_release(mmu_t mmu, pmap_t pmap)
}
/*
+ * Remove all pages mapped by the specified pmap
+ */
+void
+moea64_remove_pages(mmu_t mmu, pmap_t pm)
+{
+ struct pvo_entry *pvo, *tpvo;
+
+ LOCK_TABLE_WR();
+ PMAP_LOCK(pm);
+ LIST_FOREACH_SAFE(pvo, &pm->pmap_pvo, pvo_plink, tpvo) {
+ if (!(pvo->pvo_vaddr & PVO_WIRED))
+ moea64_pvo_remove(mmu, pvo);
+ }
+ UNLOCK_TABLE_WR();
+ PMAP_UNLOCK(pm);
+}
+
+/*
* Remove the given range of addresses from the specified map.
*/
void
moea64_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva)
{
- struct pvo_entry *pvo;
+ struct pvo_entry *pvo, *tpvo;
+
+ /*
+ * Perform an unsynchronized read. This is, however, safe.
+ */
+ if (pm->pm_stats.resident_count == 0)
+ return;
- vm_page_lock_queues();
+ LOCK_TABLE_WR();
PMAP_LOCK(pm);
- if ((eva - sva)/PAGE_SIZE < 10) {
- for (; sva < eva; sva += PAGE_SIZE) {
+ if ((eva - sva)/PAGE_SIZE < pm->pm_stats.resident_count) {
+ while (sva < eva) {
+ #ifdef __powerpc64__
+ if (pm != kernel_pmap &&
+ user_va_to_slb_entry(pm, sva) == NULL) {
+ sva = roundup2(sva + 1, SEGMENT_LENGTH);
+ continue;
+ }
+ #endif
pvo = moea64_pvo_find_va(pm, sva);
if (pvo != NULL)
moea64_pvo_remove(mmu, pvo);
+ sva += PAGE_SIZE;
}
} else {
- LIST_FOREACH(pvo, &pm->pmap_pvo, pvo_plink) {
+ LIST_FOREACH_SAFE(pvo, &pm->pmap_pvo, pvo_plink, tpvo) {
if (PVO_VADDR(pvo) < sva || PVO_VADDR(pvo) >= eva)
continue;
moea64_pvo_remove(mmu, pvo);
}
}
- vm_page_unlock_queues();
+ UNLOCK_TABLE_WR();
PMAP_UNLOCK(pm);
}
@@ -2065,8 +2132,8 @@ moea64_remove_all(mmu_t mmu, vm_page_t m
struct pvo_entry *pvo, *next_pvo;
pmap_t pmap;
- vm_page_lock_queues();
pvo_head = vm_page_to_pvoh(m);
+ LOCK_TABLE_WR();
for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) {
next_pvo = LIST_NEXT(pvo, pvo_vlink);
@@ -2075,12 +2142,10 @@ moea64_remove_all(mmu_t mmu, vm_page_t m
moea64_pvo_remove(mmu, pvo);
PMAP_UNLOCK(pmap);
}
- if ((m->aflags & PGA_WRITEABLE) && moea64_is_modified(mmu, m)) {
- moea64_attr_clear(m, LPTE_CHG);
+ UNLOCK_TABLE_WR();
+ if ((m->aflags & PGA_WRITEABLE) && moea64_is_modified(mmu, m))
vm_page_dirty(m);
- }
vm_page_aflag_clear(m, PGA_WRITEABLE);
- vm_page_unlock_queues();
}
/*
@@ -2156,6 +2221,9 @@ moea64_pvo_enter(mmu_t mmu, pmap_t pm, u
if (!moea64_initialized)
bootstrap = 1;
+ PMAP_LOCK_ASSERT(pm, MA_OWNED);
+ rw_assert(&moea64_table_lock, RA_WLOCKED);
+
/*
* Compute the PTE Group index.
*/
@@ -2167,8 +2235,6 @@ moea64_pvo_enter(mmu_t mmu, pmap_t pm, u
* Remove any existing mapping for this page. Reuse the pvo entry if
* there is a mapping.
*/
- LOCK_TABLE();
-
moea64_pvo_enter_calls++;
LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) {
@@ -2184,7 +2250,6 @@ moea64_pvo_enter(mmu_t mmu, pmap_t pm, u
PVO_PTEGIDX_SET(pvo, i);
moea64_pte_overflow--;
}
- UNLOCK_TABLE();
return (0);
}
moea64_pvo_remove(mmu, pvo);
@@ -2211,15 +2276,11 @@ moea64_pvo_enter(mmu_t mmu, pmap_t pm, u
* table. The mapping we are working with is already
* protected by the PMAP lock.
*/
- UNLOCK_TABLE();
pvo = uma_zalloc(zone, M_NOWAIT);
- LOCK_TABLE();
}
- if (pvo == NULL) {
- UNLOCK_TABLE();
+ if (pvo == NULL)
return (ENOMEM);
- }
moea64_pvo_entries++;
pvo->pvo_vaddr = va;
@@ -2274,8 +2335,6 @@ moea64_pvo_enter(mmu_t mmu, pmap_t pm, u
if (pm == kernel_pmap)
isync();
- UNLOCK_TABLE();
-
#ifdef __powerpc64__
/*
* Make sure all our bootstrap mappings are in the SLB as soon
@@ -2293,11 +2352,13 @@ moea64_pvo_remove(mmu_t mmu, struct pvo_
{
uintptr_t pt;
+ PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
+ rw_assert(&moea64_table_lock, RA_WLOCKED);
+
/*
* If there is an active pte entry, we need to deactivate it (and
* save the ref & cfg bits).
*/
- LOCK_TABLE();
pt = MOEA64_PVO_TO_PTE(mmu, pvo);
if (pt != -1) {
MOEA64_PTE_UNSET(mmu, pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn);
@@ -2314,19 +2375,6 @@ moea64_pvo_remove(mmu_t mmu, struct pvo_
pvo->pvo_pmap->pm_stats.wired_count--;
/*
- * Save the REF/CHG bits into their cache if the page is managed.
- */
- if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED) {
- struct vm_page *pg;
-
- pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
- if (pg != NULL) {
- moea64_attr_save(pg, pvo->pvo_pte.lpte.pte_lo &
- (LPTE_REF | LPTE_CHG));
- }
- }
-
- /*
* Remove this PVO from the PV and pmap lists.
*/
LIST_REMOVE(pvo, pvo_vlink);
@@ -2338,11 +2386,27 @@ moea64_pvo_remove(mmu_t mmu, struct pvo_
*/
LIST_REMOVE(pvo, pvo_olink);
+ /*
+ * Update vm about the REF/CHG bits if the page is managed.
+ */
+ if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED &&
+ (pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) {
+ struct vm_page *pg;
+
+ pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
+ if (pg != NULL) {
+ if (pvo->pvo_pte.lpte.pte_lo & LPTE_CHG)
+ vm_page_dirty(pg);
+ if (pvo->pvo_pte.lpte.pte_lo & LPTE_REF)
+ vm_page_aflag_set(pg, PGA_REFERENCED);
+ if (LIST_EMPTY(vm_page_to_pvoh(pg)))
+ vm_page_aflag_clear(pg, PGA_WRITEABLE);
+ }
+ }
+
moea64_pvo_entries--;
moea64_pvo_remove_calls++;
- UNLOCK_TABLE();
-
if (!(pvo->pvo_vaddr & PVO_BOOTSTRAP))
uma_zfree((pvo->pvo_vaddr & PVO_MANAGED) ? moea64_mpvo_zone :
moea64_upvo_zone, pvo);
@@ -2380,12 +2444,10 @@ moea64_pvo_find_va(pmap_t pm, vm_offset_
ptegidx = va_to_pteg(vsid, va, 0);
#endif
- LOCK_TABLE();
LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) {
if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va)
break;
}
- UNLOCK_TABLE();
return (pvo);
}
@@ -2396,20 +2458,13 @@ moea64_query_bit(mmu_t mmu, vm_page_t m,
struct pvo_entry *pvo;
uintptr_t pt;
- if (moea64_attr_fetch(m) & ptebit)
- return (TRUE);
-
- vm_page_lock_queues();
-
+ LOCK_TABLE_RD();
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
-
/*
- * See if we saved the bit off. If so, cache it and return
- * success.
+ * See if we saved the bit off. If so, return success.
*/
if (pvo->pvo_pte.lpte.pte_lo & ptebit) {
- moea64_attr_save(m, ptebit);
- vm_page_unlock_queues();
+ UNLOCK_TABLE_RD();
return (TRUE);
}
}
@@ -2425,24 +2480,22 @@ moea64_query_bit(mmu_t mmu, vm_page_t m,
/*
* See if this pvo has a valid PTE. if so, fetch the
* REF/CHG bits from the valid PTE. If the appropriate
- * ptebit is set, cache it and return success.
+ * ptebit is set, return success.
*/
- LOCK_TABLE();
+ PMAP_LOCK(pvo->pvo_pmap);
pt = MOEA64_PVO_TO_PTE(mmu, pvo);
if (pt != -1) {
MOEA64_PTE_SYNCH(mmu, pt, &pvo->pvo_pte.lpte);
if (pvo->pvo_pte.lpte.pte_lo & ptebit) {
- UNLOCK_TABLE();
-
- moea64_attr_save(m, ptebit);
- vm_page_unlock_queues();
+ PMAP_UNLOCK(pvo->pvo_pmap);
+ UNLOCK_TABLE_RD();
return (TRUE);
}
}
- UNLOCK_TABLE();
+ PMAP_UNLOCK(pvo->pvo_pmap);
}
- vm_page_unlock_queues();
+ UNLOCK_TABLE_RD();
return (FALSE);
}
@@ -2453,13 +2506,6 @@ moea64_clear_bit(mmu_t mmu, vm_page_t m,
struct pvo_entry *pvo;
uintptr_t pt;
- vm_page_lock_queues();
-
- /*
- * Clear the cached value.
- */
- moea64_attr_clear(m, ptebit);
-
/*
* Sync so that any pending REF/CHG bits are flushed to the PTEs (so
* we can reset the right ones). note that since the pvo entries and
@@ -2474,9 +2520,9 @@ moea64_clear_bit(mmu_t mmu, vm_page_t m,
* valid pte clear the ptebit from the valid pte.
*/
count = 0;
+ LOCK_TABLE_RD();
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
-
- LOCK_TABLE();
+ PMAP_LOCK(pvo->pvo_pmap);
pt = MOEA64_PVO_TO_PTE(mmu, pvo);
if (pt != -1) {
MOEA64_PTE_SYNCH(mmu, pt, &pvo->pvo_pte.lpte);
@@ -2487,10 +2533,10 @@ moea64_clear_bit(mmu_t mmu, vm_page_t m,
}
}
pvo->pvo_pte.lpte.pte_lo &= ~ptebit;
- UNLOCK_TABLE();
+ PMAP_UNLOCK(pvo->pvo_pmap);
}
- vm_page_unlock_queues();
+ UNLOCK_TABLE_RD();
return (count);
}
@@ -2501,6 +2547,7 @@ moea64_dev_direct_mapped(mmu_t mmu, vm_o
vm_offset_t ppa;
int error = 0;
+ LOCK_TABLE_RD();
PMAP_LOCK(kernel_pmap);
for (ppa = pa & ~ADDR_POFF; ppa < pa + size; ppa += PAGE_SIZE) {
pvo = moea64_pvo_find_va(kernel_pmap, ppa);
@@ -2510,6 +2557,7 @@ moea64_dev_direct_mapped(mmu_t mmu, vm_o
break;
}
}
+ UNLOCK_TABLE_RD();
PMAP_UNLOCK(kernel_pmap);
return (error);
@@ -2528,7 +2576,7 @@ moea64_mapdev_attr(mmu_t mmu, vm_offset_
ppa = trunc_page(pa);
offset = pa & PAGE_MASK;
- size = roundup(offset + size, PAGE_SIZE);
+ size = roundup2(offset + size, PAGE_SIZE);
va = kmem_alloc_nofault(kernel_map, size);
@@ -2559,7 +2607,7 @@ moea64_unmapdev(mmu_t mmu, vm_offset_t v
base = trunc_page(va);
offset = va & PAGE_MASK;
- size = roundup(offset + size, PAGE_SIZE);
+ size = roundup2(offset + size, PAGE_SIZE);
kmem_free(kernel_map, base, size);
}
@@ -2572,6 +2620,7 @@ moea64_sync_icache(mmu_t mmu, pmap_t pm,
vm_paddr_t pa;
vm_size_t len;
+ LOCK_TABLE_RD();
PMAP_LOCK(pm);
while (sz > 0) {
lim = round_page(va);
@@ -2585,5 +2634,6 @@ moea64_sync_icache(mmu_t mmu, pmap_t pm,
va += len;
sz -= len;
}
+ UNLOCK_TABLE_RD();
PMAP_UNLOCK(pm);
}
Modified: stable/9/sys/powerpc/aim/moea64_native.c
==============================================================================
--- stable/9/sys/powerpc/aim/moea64_native.c Thu Apr 5 00:08:16 2012 (r233910)
+++ stable/9/sys/powerpc/aim/moea64_native.c Thu Apr 5 00:53:21 2012 (r233911)
@@ -103,6 +103,7 @@ __FBSDID("$FreeBSD$");
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
+#include <sys/sched.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
@@ -138,7 +139,7 @@ __FBSDID("$FreeBSD$");
* Just to add to the fun, exceptions must be off as well
* so that we can't trap in 64-bit mode. What a pain.
*/
-struct mtx tlbie_mutex;
+static struct mtx tlbie_mutex;
static __inline void
TLBIE(uint64_t vpn) {
@@ -151,19 +152,20 @@ TLBIE(uint64_t vpn) {
vpn <<= ADDR_PIDX_SHFT;
vpn &= ~(0xffffULL << 48);
- mtx_lock_spin(&tlbie_mutex);
#ifdef __powerpc64__
- __asm __volatile("\
- ptesync; \
- tlbie %0; \
- eieio; \
- tlbsync; \
- ptesync;"
- :: "r"(vpn) : "memory");
+ sched_pin();
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-stable-9
mailing list