svn commit: r324400 - in stable/11/sys: amd64/amd64 arm/arm arm64/arm64 i386/i386 powerpc/booke riscv/riscv sparc64/sparc64 vm
Alan Cox
alc at FreeBSD.org
Sat Oct 7 21:13:57 UTC 2017
Author: alc
Date: Sat Oct 7 21:13:54 2017
New Revision: 324400
URL: https://svnweb.freebsd.org/changeset/base/324400
Log:
MFC r305685
Various changes to pmap_ts_referenced()
Move PMAP_TS_REFERENCED_MAX out of the various pmap implementations and
into vm/pmap.h, and describe what its purpose is. Eliminate the archaic
"XXX" comment about its value. I don't believe that its exact value, e.g.,
5 versus 6, matters.
Update the arm64 and riscv pmap implementations of pmap_ts_referenced()
to opportunistically update the page's dirty field.
On amd64, use the PDE value already cached in a local variable rather than
dereferencing a pointer again and again.
Modified:
stable/11/sys/amd64/amd64/pmap.c
stable/11/sys/arm/arm/pmap-v6.c
stable/11/sys/arm64/arm64/pmap.c
stable/11/sys/i386/i386/pmap.c
stable/11/sys/powerpc/booke/pmap.c
stable/11/sys/riscv/riscv/pmap.c
stable/11/sys/sparc64/sparc64/pmap.c
stable/11/sys/vm/pmap.h
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/sys/amd64/amd64/pmap.c
==============================================================================
--- stable/11/sys/amd64/amd64/pmap.c Sat Oct 7 20:22:04 2017 (r324399)
+++ stable/11/sys/amd64/amd64/pmap.c Sat Oct 7 21:13:54 2017 (r324400)
@@ -5986,8 +5986,6 @@ safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte)
return (FALSE);
}
-#define PMAP_TS_REFERENCED_MAX 5
-
/*
* pmap_ts_referenced:
*
@@ -5996,10 +5994,6 @@ safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte)
* is necessary that 0 only be returned when there are truly no
* reference bits set.
*
- * XXX: The exact number of bits to check and clear is a matter that
- * should be tested and standardized at some point in the future for
- * optimal aging of shared pages.
- *
* As an optimization, update the page's dirty field if a modified bit is
* found while counting reference bits. This opportunistic update can be
* performed at low cost and can eliminate the need for some future calls
@@ -6068,7 +6062,7 @@ retry:
*/
vm_page_dirty(m);
}
- if ((*pde & PG_A) != 0) {
+ if ((oldpde & PG_A) != 0) {
/*
* Since this reference bit is shared by 512 4KB
* pages, it should not be cleared every time it is
@@ -6089,7 +6083,7 @@ retry:
*/
if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> PDRSHIFT) ^
(uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&
- (*pde & PG_W) == 0) {
+ (oldpde & PG_W) == 0) {
if (safe_to_clear_referenced(pmap, oldpde)) {
atomic_clear_long(pde, PG_A);
pmap_invalidate_page(pmap, pv->pv_va);
Modified: stable/11/sys/arm/arm/pmap-v6.c
==============================================================================
--- stable/11/sys/arm/arm/pmap-v6.c Sat Oct 7 20:22:04 2017 (r324399)
+++ stable/11/sys/arm/arm/pmap-v6.c Sat Oct 7 21:13:54 2017 (r324400)
@@ -5198,8 +5198,6 @@ pmap_is_referenced(vm_page_t m)
return (rv);
}
-#define PMAP_TS_REFERENCED_MAX 5
-
/*
* pmap_ts_referenced:
*
@@ -5207,10 +5205,6 @@ pmap_is_referenced(vm_page_t m)
* It is not necessary for every reference bit to be cleared, but it
* is necessary that 0 only be returned when there are truly no
* reference bits set.
- *
- * XXX: The exact number of bits to check and clear is a matter that
- * should be tested and standardized at some point in the future for
- * optimal aging of shared pages.
*
* As an optimization, update the page's dirty field if a modified bit is
* found while counting reference bits. This opportunistic update can be
Modified: stable/11/sys/arm64/arm64/pmap.c
==============================================================================
--- stable/11/sys/arm64/arm64/pmap.c Sat Oct 7 20:22:04 2017 (r324399)
+++ stable/11/sys/arm64/arm64/pmap.c Sat Oct 7 21:13:54 2017 (r324400)
@@ -4021,8 +4021,6 @@ safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte)
return (FALSE);
}
-#define PMAP_TS_REFERENCED_MAX 5
-
/*
* pmap_ts_referenced:
*
@@ -4031,9 +4029,13 @@ safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte)
* is necessary that 0 only be returned when there are truly no
* reference bits set.
*
- * XXX: The exact number of bits to check and clear is a matter that
- * should be tested and standardized at some point in the future for
- * optimal aging of shared pages.
+ * As an optimization, update the page's dirty field if a modified bit is
+ * found while counting reference bits. This opportunistic update can be
+ * performed at low cost and can eliminate the need for some future calls
+ * to pmap_is_modified(). However, since this function stops after
+ * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
+ * dirty pages. Those dirty pages will only be detected by a future call
+ * to pmap_is_modified().
*/
int
pmap_ts_referenced(vm_page_t m)
@@ -4088,6 +4090,14 @@ retry:
("pmap_ts_referenced: found an invalid l1 table"));
pte = pmap_l1_to_l2(pde, pv->pv_va);
tpte = pmap_load(pte);
+ if (pmap_page_dirty(tpte)) {
+ /*
+ * Although "tpte" is mapping a 2MB page, because
+ * this function is called at a 4KB page granularity,
+ * we only update the 4KB page under test.
+ */
+ vm_page_dirty(m);
+ }
if ((tpte & ATTR_AF) != 0) {
/*
* Since this reference bit is shared by 512 4KB
@@ -4184,6 +4194,8 @@ small_mappings:
("pmap_ts_referenced: found an invalid l2 table"));
pte = pmap_l2_to_l3(pde, pv->pv_va);
tpte = pmap_load(pte);
+ if (pmap_page_dirty(tpte))
+ vm_page_dirty(m);
if ((tpte & ATTR_AF) != 0) {
if (safe_to_clear_referenced(pmap, tpte)) {
/*
Modified: stable/11/sys/i386/i386/pmap.c
==============================================================================
--- stable/11/sys/i386/i386/pmap.c Sat Oct 7 20:22:04 2017 (r324399)
+++ stable/11/sys/i386/i386/pmap.c Sat Oct 7 21:13:54 2017 (r324400)
@@ -4836,8 +4836,6 @@ retry:
rw_wunlock(&pvh_global_lock);
}
-#define PMAP_TS_REFERENCED_MAX 5
-
/*
* pmap_ts_referenced:
*
@@ -4845,10 +4843,6 @@ retry:
* It is not necessary for every reference bit to be cleared, but it
* is necessary that 0 only be returned when there are truly no
* reference bits set.
- *
- * XXX: The exact number of bits to check and clear is a matter that
- * should be tested and standardized at some point in the future for
- * optimal aging of shared pages.
*
* As an optimization, update the page's dirty field if a modified bit is
* found while counting reference bits. This opportunistic update can be
Modified: stable/11/sys/powerpc/booke/pmap.c
==============================================================================
--- stable/11/sys/powerpc/booke/pmap.c Sat Oct 7 20:22:04 2017 (r324399)
+++ stable/11/sys/powerpc/booke/pmap.c Sat Oct 7 21:13:54 2017 (r324400)
@@ -2527,9 +2527,13 @@ mmu_booke_clear_modify(mmu_t mmu, vm_page_t m)
* is necessary that 0 only be returned when there are truly no
* reference bits set.
*
- * XXX: The exact number of bits to check and clear is a matter that
- * should be tested and standardized at some point in the future for
- * optimal aging of shared pages.
+ * As an optimization, update the page's dirty field if a modified bit is
+ * found while counting reference bits. This opportunistic update can be
+ * performed at low cost and can eliminate the need for some future calls
+ * to pmap_is_modified(). However, since this function stops after
+ * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
+ * dirty pages. Those dirty pages will only be detected by a future call
+ * to pmap_is_modified().
*/
static int
mmu_booke_ts_referenced(mmu_t mmu, vm_page_t m)
@@ -2546,6 +2550,8 @@ mmu_booke_ts_referenced(mmu_t mmu, vm_page_t m)
PMAP_LOCK(pv->pv_pmap);
if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL &&
PTE_ISVALID(pte)) {
+ if (PTE_ISMODIFIED(pte))
+ vm_page_dirty(m);
if (PTE_ISREFERENCED(pte)) {
mtx_lock_spin(&tlbivax_mutex);
tlb_miss_lock();
@@ -2556,7 +2562,7 @@ mmu_booke_ts_referenced(mmu_t mmu, vm_page_t m)
tlb_miss_unlock();
mtx_unlock_spin(&tlbivax_mutex);
- if (++count > 4) {
+ if (++count >= PMAP_TS_REFERENCED_MAX) {
PMAP_UNLOCK(pv->pv_pmap);
break;
}
Modified: stable/11/sys/riscv/riscv/pmap.c
==============================================================================
--- stable/11/sys/riscv/riscv/pmap.c Sat Oct 7 20:22:04 2017 (r324399)
+++ stable/11/sys/riscv/riscv/pmap.c Sat Oct 7 21:13:54 2017 (r324400)
@@ -2990,8 +2990,6 @@ safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte)
return (FALSE);
}
-#define PMAP_TS_REFERENCED_MAX 5
-
/*
* pmap_ts_referenced:
*
@@ -3000,9 +2998,13 @@ safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte)
* is necessary that 0 only be returned when there are truly no
* reference bits set.
*
- * XXX: The exact number of bits to check and clear is a matter that
- * should be tested and standardized at some point in the future for
- * optimal aging of shared pages.
+ * As an optimization, update the page's dirty field if a modified bit is
+ * found while counting reference bits. This opportunistic update can be
+ * performed at low cost and can eliminate the need for some future calls
+ * to pmap_is_modified(). However, since this function stops after
+ * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
+ * dirty pages. Those dirty pages will only be detected by a future call
+ * to pmap_is_modified().
*/
int
pmap_ts_referenced(vm_page_t m)
@@ -3011,7 +3013,7 @@ pmap_ts_referenced(vm_page_t m)
pmap_t pmap;
struct rwlock *lock;
pd_entry_t *l2;
- pt_entry_t *l3;
+ pt_entry_t *l3, old_l3;
vm_paddr_t pa;
int cleared, md_gen, not_cleared;
struct spglist free;
@@ -3049,15 +3051,18 @@ retry:
("pmap_ts_referenced: found an invalid l2 table"));
l3 = pmap_l2_to_l3(l2, pv->pv_va);
- if ((pmap_load(l3) & PTE_REF) != 0) {
- if (safe_to_clear_referenced(pmap, pmap_load(l3))) {
+ old_l3 = pmap_load(l3);
+ if (pmap_page_dirty(old_l3))
+ vm_page_dirty(m);
+ if ((old_l3 & PTE_REF) != 0) {
+ if (safe_to_clear_referenced(pmap, old_l3)) {
/*
* TODO: We don't handle the access flag
* at all. We need to be able to set it in
* the exception handler.
*/
panic("RISCVTODO: safe_to_clear_referenced\n");
- } else if ((pmap_load(l3) & PTE_SW_WIRED) == 0) {
+ } else if ((old_l3 & PTE_SW_WIRED) == 0) {
/*
* Wired pages cannot be paged out so
* doing accessed bit emulation for
Modified: stable/11/sys/sparc64/sparc64/pmap.c
==============================================================================
--- stable/11/sys/sparc64/sparc64/pmap.c Sat Oct 7 20:22:04 2017 (r324399)
+++ stable/11/sys/sparc64/sparc64/pmap.c Sat Oct 7 21:13:54 2017 (r324400)
@@ -2102,17 +2102,11 @@ pmap_page_is_mapped(vm_page_t m)
return (rv);
}
-#define PMAP_TS_REFERENCED_MAX 5
-
/*
* Return a count of reference bits for a page, clearing those bits.
* It is not necessary for every reference bit to be cleared, but it
* is necessary that 0 only be returned when there are truly no
* reference bits set.
- *
- * XXX: The exact number of bits to check and clear is a matter that
- * should be tested and standardized at some point in the future for
- * optimal aging of shared pages.
*
* As an optimization, update the page's dirty field if a modified bit is
* found while counting reference bits. This opportunistic update can be
Modified: stable/11/sys/vm/pmap.h
==============================================================================
--- stable/11/sys/vm/pmap.h Sat Oct 7 20:22:04 2017 (r324399)
+++ stable/11/sys/vm/pmap.h Sat Oct 7 21:13:54 2017 (r324400)
@@ -106,6 +106,16 @@ extern vm_offset_t kernel_vm_end;
#define PMAP_ENTER_WIRED 0x00000200
#define PMAP_ENTER_RESERVED 0xFF000000
+/*
+ * Define the maximum number of machine-dependent reference bits that are
+ * cleared by a call to pmap_ts_referenced(). This limit serves two purposes.
+ * First, it bounds the cost of reference bit maintenance on widely shared
+ * pages. Second, it prevents numeric overflow during maintenance of a
+ * widely shared page's "act_count" field. An overflow could result in the
+ * premature deactivation of the page.
+ */
+#define PMAP_TS_REFERENCED_MAX 5
+
void pmap_activate(struct thread *td);
void pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
int advice);
More information about the svn-src-all
mailing list