svn commit: r203881 - projects/ppc64/sys/powerpc/aim
Nathan Whitehorn
nwhitehorn at FreeBSD.org
Sun Feb 14 16:24:10 UTC 2010
Author: nwhitehorn
Date: Sun Feb 14 16:24:10 2010
New Revision: 203881
URL: http://svn.freebsd.org/changeset/base/203881
Log:
Revisit page table synchronization requirements while trying to track
down a memory corruption bug during page out/in. This does not fix that
bug, but does give both a performance and correctness improvement.
Modified:
projects/ppc64/sys/powerpc/aim/mmu_oea64.c
Modified: projects/ppc64/sys/powerpc/aim/mmu_oea64.c
==============================================================================
--- projects/ppc64/sys/powerpc/aim/mmu_oea64.c Sun Feb 14 16:21:00 2010 (r203880)
+++ projects/ppc64/sys/powerpc/aim/mmu_oea64.c Sun Feb 14 16:24:10 2010 (r203881)
@@ -168,6 +168,7 @@ cntlzd(volatile register_t a) {
return b;
}
+#define PTESYNC() __asm __volatile("ptesync");
#define TLBSYNC() __asm __volatile("tlbsync; ptesync");
#define SYNC() __asm __volatile("sync");
#define EIEIO() __asm __volatile("eieio");
@@ -196,6 +197,7 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
vpn = (uint64_t)(va & ADDR_PIDX);
if (pmap != NULL)
vpn |= (va_to_vsid(pmap,va) << 28);
+ vpn &= ~(0xffffULL << 48);
mtx_lock_spin(&tlbie_mutex);
#ifdef __powerpc64__
@@ -205,15 +207,13 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
eieio; \
tlbsync; \
ptesync;"
- :: "r"(vpn));
+ :: "r"(vpn) : "memory");
#else
vpn_hi = (uint32_t)(vpn >> 32);
vpn_lo = (uint32_t)vpn;
__asm __volatile("\
mfmsr %0; \
- clrldi %1,%0,49; \
- mtmsr %1; \
insrdi %1,%5,1,0; \
mtmsrd %1; \
ptesync; \
@@ -226,7 +226,8 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
eieio; \
tlbsync; \
ptesync;"
- : "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1));
+ : "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1)
+ : "memory");
#endif
mtx_unlock_spin(&tlbie_mutex);
}
@@ -237,13 +238,13 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
#define VSID_MAKE(sr, hash) ((sr) | (((hash) & 0xfffff) << 4))
#define VSID_TO_HASH(vsid) (((vsid) >> 4) & 0xfffff)
-#define PVO_PTEGIDX_MASK 0x007 /* which PTEG slot */
-#define PVO_PTEGIDX_VALID 0x008 /* slot is valid */
-#define PVO_WIRED 0x010 /* PVO entry is wired */
-#define PVO_MANAGED 0x020 /* PVO entry is managed */
-#define PVO_BOOTSTRAP 0x080 /* PVO entry allocated during
+#define PVO_PTEGIDX_MASK 0x007UL /* which PTEG slot */
+#define PVO_PTEGIDX_VALID 0x008UL /* slot is valid */
+#define PVO_WIRED 0x010UL /* PVO entry is wired */
+#define PVO_MANAGED 0x020UL /* PVO entry is managed */
+#define PVO_BOOTSTRAP 0x080UL /* PVO entry allocated during
bootstrap */
-#define PVO_FAKE 0x100 /* fictitious phys page */
+#define PVO_FAKE 0x100UL /* fictitious phys page */
#define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF)
#define PVO_ISFAKE(pvo) ((pvo)->pvo_vaddr & PVO_FAKE)
#define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
@@ -533,23 +534,6 @@ moea64_attr_save(vm_page_t m, u_int64_t
m->md.mdpg_attrs |= ptebit;
}
-static __inline int
-moea64_pte_compare(const struct lpte *pt, const struct lpte *pvo_pt)
-{
- if (pt->pte_hi == pvo_pt->pte_hi)
- return (1);
-
- return (0);
-}
-
-static __inline int
-moea64_pte_match(struct lpte *pt, uint64_t vsid, vm_offset_t va, int which)
-{
- return (pt->pte_hi & ~LPTE_VALID) ==
- ((vsid << LPTE_VSID_SHIFT) |
- ((uint64_t)(va >> ADDR_API_SHFT64) & LPTE_API) | which);
-}
-
static __inline void
moea64_pte_create(struct lpte *pt, uint64_t vsid, vm_offset_t va,
uint64_t pte_lo)
@@ -604,7 +588,7 @@ moea64_pte_set(struct lpte *pt, struct l
pt->pte_lo = pvo_pt->pte_lo;
EIEIO();
pt->pte_hi = pvo_pt->pte_hi;
- SYNC();
+ PTESYNC();
moea64_pte_valid++;
}
@@ -623,7 +607,6 @@ moea64_pte_unset(struct lpte *pt, struct
* Invalidate the pte.
*/
pt->pte_hi &= ~LPTE_VALID;
-
TLBIE(pmap,va);
/*
@@ -642,6 +625,8 @@ moea64_pte_change(struct lpte *pt, struc
*/
moea64_pte_unset(pt, pvo_pt, pmap, va);
moea64_pte_set(pt, pvo_pt);
+ if (pmap == kernel_pmap)
+ isync();
}
static __inline uint64_t
@@ -744,7 +729,7 @@ moea64_bridge_cpu_bootstrap(mmu_t mmup,
* Install page table
*/
- __asm __volatile ("sync; mtsdr1 %0; isync"
+ __asm __volatile ("ptesync; mtsdr1 %0; isync"
:: "r"((uintptr_t)moea64_pteg_table
| (64 - cntlzd(moea64_pteg_mask >> 11))));
tlbia();
@@ -1243,7 +1228,7 @@ void moea64_set_scratchpage_pa(int which
EIEIO();
moea64_scratchpage_pte[which]->pte_hi |= LPTE_VALID;
- TLBIE(kernel_pmap, moea64_scratchpage_va[which]);
+ PTESYNC(); isync();
}
void
@@ -1263,8 +1248,6 @@ moea64_copy_page(mmu_t mmu, vm_page_t ms
kcopy((void *)moea64_scratchpage_va[0],
(void *)moea64_scratchpage_va[1], PAGE_SIZE);
- __syncicache((void *)moea64_scratchpage_va[1],PAGE_SIZE);
-
mtx_unlock(&moea64_scratchpage_mtx);
}
@@ -1282,8 +1265,6 @@ moea64_zero_page_area(mmu_t mmu, vm_page
moea64_set_scratchpage_pa(0,pa);
bzero((caddr_t)moea64_scratchpage_va[0] + off, size);
- __syncicache((void *)moea64_scratchpage_va[0],PAGE_SIZE);
-
mtx_unlock(&moea64_scratchpage_mtx);
}
@@ -1374,9 +1355,6 @@ moea64_enter_locked(pmap_t pmap, vm_offs
error = moea64_pvo_enter(pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m),
pte_lo, pvo_flags);
- if (pmap == kernel_pmap)
- TLBIE(pmap, va);
-
/*
* Flush the page from the instruction cache if this page is
* mapped executable and cacheable.
@@ -1549,8 +1527,6 @@ moea64_uma_page_alloc(uma_zone_t zone, i
&moea64_pvo_kunmanaged, va, VM_PAGE_TO_PHYS(m), LPTE_M,
PVO_WIRED | PVO_BOOTSTRAP);
- TLBIE(kernel_pmap, va);
-
if (needed_lock)
PMAP_UNLOCK(kernel_pmap);
@@ -1639,7 +1615,7 @@ moea64_remove_write(mmu_t mmu, vm_page_t
lo |= pvo->pvo_pte.lpte.pte_lo;
pvo->pvo_pte.lpte.pte_lo &= ~LPTE_CHG;
moea64_pte_change(pt, &pvo->pvo_pte.lpte,
- pvo->pvo_pmap, pvo->pvo_vaddr);
+ pvo->pvo_pmap, PVO_VADDR(pvo));
}
UNLOCK_TABLE();
}
@@ -1699,8 +1675,6 @@ moea64_kenter(mmu_t mmu, vm_offset_t va,
&moea64_pvo_kunmanaged, va, pa, pte_lo,
PVO_WIRED | VM_PROT_EXECUTE);
- TLBIE(kernel_pmap, va);
-
if (error != 0 && error != ENOENT)
panic("moea64_kenter: failed to enter va %#zx pa %#zx: %d", va,
pa, error);
@@ -1959,7 +1933,7 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_
*/
if (pt != NULL) {
moea64_pte_change(pt, &pvo->pvo_pte.lpte,
- pvo->pvo_pmap, pvo->pvo_vaddr);
+ pvo->pvo_pmap, PVO_VADDR(pvo));
if ((pvo->pvo_pte.lpte.pte_lo &
(LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
moea64_syncicache(pm, sva,
@@ -2197,7 +2171,6 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t z
*/
va &= ~ADDR_POFF;
vsid = va_to_vsid(pm, va);
-
ptegidx = va_to_pteg(vsid, va);
/*
@@ -2276,7 +2249,7 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t z
first = 1;
LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink);
- if (pvo->pvo_pte.lpte.pte_lo & PVO_WIRED)
+ if (pvo->pvo_vaddr & PVO_WIRED)
pm->pm_stats.wired_count++;
pm->pm_stats.resident_count++;
@@ -2291,6 +2264,9 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t z
moea64_pte_overflow++;
}
+ if (pm == kernel_pmap)
+ isync();
+
UNLOCK_TABLE();
return (first ? ENOENT : 0);
@@ -2309,7 +2285,7 @@ moea64_pvo_remove(struct pvo_entry *pvo,
pt = moea64_pvo_to_pte(pvo, pteidx);
if (pt != NULL) {
moea64_pte_unset(pt, &pvo->pvo_pte.lpte, pvo->pvo_pmap,
- pvo->pvo_vaddr);
+ PVO_VADDR(pvo));
PVO_PTEGIDX_CLR(pvo);
} else {
moea64_pte_overflow--;
@@ -2320,7 +2296,7 @@ moea64_pvo_remove(struct pvo_entry *pvo,
* Update our statistics.
*/
pvo->pvo_pmap->pm_stats.resident_count--;
- if (pvo->pvo_pte.lpte.pte_lo & PVO_WIRED)
+ if (pvo->pvo_vaddr & PVO_WIRED)
pvo->pvo_pmap->pm_stats.wired_count--;
/*
@@ -2347,7 +2323,7 @@ moea64_pvo_remove(struct pvo_entry *pvo,
*/
LIST_REMOVE(pvo, pvo_olink);
if (!(pvo->pvo_vaddr & PVO_BOOTSTRAP))
- uma_zfree(pvo->pvo_vaddr & PVO_MANAGED ? moea64_mpvo_zone :
+ uma_zfree((pvo->pvo_vaddr & PVO_MANAGED) ? moea64_mpvo_zone :
moea64_upvo_zone, pvo);
moea64_pvo_entries--;
moea64_pvo_remove_calls++;
@@ -2406,8 +2382,8 @@ moea64_pvo_to_pte(const struct pvo_entry
int ptegidx;
uint64_t vsid;
- vsid = va_to_vsid(pvo->pvo_pmap, pvo->pvo_vaddr);
- ptegidx = va_to_pteg(vsid, pvo->pvo_vaddr);
+ vsid = va_to_vsid(pvo->pvo_pmap, PVO_VADDR(pvo));
+ ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo));
pteidx = moea64_pvo_pte_index(pvo, ptegidx);
}
More information about the svn-src-projects
mailing list