svn commit: r318716 - in stable/11: sys/amd64/amd64 sys/arm64/arm64 sys/cddl/compat/opensolaris/sys sys/cddl/contrib/opensolaris/uts/common/fs/zfs sys/fs/tmpfs sys/i386/i386 sys/kern sys/sys sys/vm...
Mark Johnston
markj at FreeBSD.org
Tue May 23 07:27:33 UTC 2017
Author: markj
Date: Tue May 23 07:27:30 2017
New Revision: 318716
URL: https://svnweb.freebsd.org/changeset/base/318716
Log:
MFC r308474, r308691, r309203, r309365, r309703, r309898, r310720,
r308489, r308706:
Add PQ_LAUNDRY and remove PG_CACHED pages.
Modified:
stable/11/sys/amd64/amd64/pmap.c
stable/11/sys/arm64/arm64/pmap.c
stable/11/sys/cddl/compat/opensolaris/sys/vnode.h
stable/11/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
stable/11/sys/fs/tmpfs/tmpfs_subr.c
stable/11/sys/i386/i386/pmap.c
stable/11/sys/kern/kern_exec.c
stable/11/sys/kern/uipc_shm.c
stable/11/sys/sys/vmmeter.h
stable/11/sys/vm/_vm_radix.h
stable/11/sys/vm/swap_pager.c
stable/11/sys/vm/vm_fault.c
stable/11/sys/vm/vm_map.c
stable/11/sys/vm/vm_meter.c
stable/11/sys/vm/vm_mmap.c
stable/11/sys/vm/vm_object.c
stable/11/sys/vm/vm_object.h
stable/11/sys/vm/vm_page.c
stable/11/sys/vm/vm_page.h
stable/11/sys/vm/vm_pageout.c
stable/11/sys/vm/vm_phys.c
stable/11/sys/vm/vm_radix.c
stable/11/sys/vm/vm_radix.h
stable/11/sys/vm/vm_reserv.c
stable/11/sys/vm/vm_reserv.h
stable/11/sys/vm/vnode_pager.c
stable/11/usr.bin/systat/systat.1
stable/11/usr.bin/systat/vmstat.c
stable/11/usr.bin/top/machine.c
stable/11/usr.bin/top/top.local.1
stable/11/usr.bin/vmstat/vmstat.c
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/sys/amd64/amd64/pmap.c
==============================================================================
--- stable/11/sys/amd64/amd64/pmap.c Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/amd64/amd64/pmap.c Tue May 23 07:27:30 2017 (r318716)
@@ -614,7 +614,6 @@ static vm_page_t pmap_enter_quick_locked
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
-static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va);
static void pmap_pde_attr(pd_entry_t *pde, int cache_bits, int mask);
static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
struct rwlock **lockp);
@@ -625,7 +624,7 @@ static int pmap_remove_pde(pmap_t pmap,
struct spglist *free, struct rwlock **lockp);
static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp);
-static void pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte);
+static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
struct spglist *free);
static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
@@ -2218,29 +2217,17 @@ pmap_insert_pt_page(pmap_t pmap, vm_page
}
/*
- * Looks for a page table page mapping the specified virtual address in the
- * specified pmap's collection of idle page table pages. Returns NULL if there
- * is no page table page corresponding to the specified virtual address.
+ * Removes the page table page mapping the specified virtual address from the
+ * specified pmap's collection of idle page table pages, and returns it.
+ * Otherwise, returns NULL if there is no page table page corresponding to the
+ * specified virtual address.
*/
static __inline vm_page_t
-pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va)
+pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
{
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- return (vm_radix_lookup(&pmap->pm_root, pmap_pde_pindex(va)));
-}
-
-/*
- * Removes the specified page table page from the specified pmap's collection
- * of idle page table pages. The specified page table page must be a member of
- * the pmap's collection.
- */
-static __inline void
-pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte)
-{
-
- PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- vm_radix_remove(&pmap->pm_root, mpte->pindex);
+ return (vm_radix_remove(&pmap->pm_root, pmap_pde_pindex(va)));
}
/*
@@ -3460,10 +3447,8 @@ pmap_demote_pde_locked(pmap_t pmap, pd_e
oldpde = *pde;
KASSERT((oldpde & (PG_PS | PG_V)) == (PG_PS | PG_V),
("pmap_demote_pde: oldpde is missing PG_PS and/or PG_V"));
- if ((oldpde & PG_A) != 0 && (mpte = pmap_lookup_pt_page(pmap, va)) !=
- NULL)
- pmap_remove_pt_page(pmap, mpte);
- else {
+ if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==
+ NULL) {
KASSERT((oldpde & PG_W) == 0,
("pmap_demote_pde: page table page for a wired mapping"
" is missing"));
@@ -3577,11 +3562,10 @@ pmap_remove_kernel_pde(pmap_t pmap, pd_e
KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- mpte = pmap_lookup_pt_page(pmap, va);
+ mpte = pmap_remove_pt_page(pmap, va);
if (mpte == NULL)
panic("pmap_remove_kernel_pde: Missing pt page.");
- pmap_remove_pt_page(pmap, mpte);
mptepa = VM_PAGE_TO_PHYS(mpte);
newpde = mptepa | X86_PG_M | X86_PG_A | X86_PG_RW | X86_PG_V;
@@ -3668,9 +3652,8 @@ pmap_remove_pde(pmap_t pmap, pd_entry_t
if (pmap == kernel_pmap) {
pmap_remove_kernel_pde(pmap, pdq, sva);
} else {
- mpte = pmap_lookup_pt_page(pmap, sva);
+ mpte = pmap_remove_pt_page(pmap, sva);
if (mpte != NULL) {
- pmap_remove_pt_page(pmap, mpte);
pmap_resident_count_dec(pmap, 1);
KASSERT(mpte->wire_count == NPTEPG,
("pmap_remove_pde: pte page wire count error"));
@@ -5533,9 +5516,8 @@ pmap_remove_pages(pmap_t pmap)
TAILQ_EMPTY(&mt->md.pv_list))
vm_page_aflag_clear(mt, PGA_WRITEABLE);
}
- mpte = pmap_lookup_pt_page(pmap, pv->pv_va);
+ mpte = pmap_remove_pt_page(pmap, pv->pv_va);
if (mpte != NULL) {
- pmap_remove_pt_page(pmap, mpte);
pmap_resident_count_dec(pmap, 1);
KASSERT(mpte->wire_count == NPTEPG,
("pmap_remove_pages: pte page wire count error"));
Modified: stable/11/sys/arm64/arm64/pmap.c
==============================================================================
--- stable/11/sys/arm64/arm64/pmap.c Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/arm64/arm64/pmap.c Tue May 23 07:27:30 2017 (r318716)
@@ -2514,29 +2514,17 @@ pmap_insert_pt_page(pmap_t pmap, vm_page
}
/*
- * Looks for a page table page mapping the specified virtual address in the
- * specified pmap's collection of idle page table pages. Returns NULL if there
- * is no page table page corresponding to the specified virtual address.
+ * Removes the page table page mapping the specified virtual address from the
+ * specified pmap's collection of idle page table pages, and returns it.
+ * Otherwise, returns NULL if there is no page table page corresponding to the
+ * specified virtual address.
*/
static __inline vm_page_t
-pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va)
+pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
{
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- return (vm_radix_lookup(&pmap->pm_root, pmap_l2_pindex(va)));
-}
-
-/*
- * Removes the specified page table page from the specified pmap's collection
- * of idle page table pages. The specified page table page must be a member of
- * the pmap's collection.
- */
-static __inline void
-pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte)
-{
-
- PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- vm_radix_remove(&pmap->pm_root, mpte->pindex);
+ return (vm_radix_remove(&pmap->pm_root, pmap_l2_pindex(va)));
}
/*
@@ -3605,10 +3593,9 @@ pmap_remove_pages(pmap_t pmap)
TAILQ_EMPTY(&mt->md.pv_list))
vm_page_aflag_clear(mt, PGA_WRITEABLE);
}
- ml3 = pmap_lookup_pt_page(pmap,
+ ml3 = pmap_remove_pt_page(pmap,
pv->pv_va);
if (ml3 != NULL) {
- pmap_remove_pt_page(pmap, ml3);
pmap_resident_count_dec(pmap,1);
KASSERT(ml3->wire_count == NL3PG,
("pmap_remove_pages: l3 page wire count error"));
@@ -4381,9 +4368,7 @@ pmap_demote_l2_locked(pmap_t pmap, pt_en
return (NULL);
}
- if ((ml3 = pmap_lookup_pt_page(pmap, va)) != NULL) {
- pmap_remove_pt_page(pmap, ml3);
- } else {
+ if ((ml3 = pmap_remove_pt_page(pmap, va)) == NULL) {
ml3 = vm_page_alloc(NULL, pmap_l2_pindex(va),
(VIRT_IN_DMAP(va) ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) |
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED);
Modified: stable/11/sys/cddl/compat/opensolaris/sys/vnode.h
==============================================================================
--- stable/11/sys/cddl/compat/opensolaris/sys/vnode.h Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/cddl/compat/opensolaris/sys/vnode.h Tue May 23 07:27:30 2017 (r318716)
@@ -75,8 +75,7 @@ vn_is_readonly(vnode_t *vp)
#define vn_mountedvfs(vp) ((vp)->v_mountedhere)
#define vn_has_cached_data(vp) \
((vp)->v_object != NULL && \
- ((vp)->v_object->resident_page_count > 0 || \
- !vm_object_cache_is_empty((vp)->v_object)))
+ (vp)->v_object->resident_page_count > 0)
#define vn_exists(vp) do { } while (0)
#define vn_invalid(vp) do { } while (0)
#define vn_renamepath(tdvp, svp, tnm, lentnm) do { } while (0)
Modified: stable/11/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
==============================================================================
--- stable/11/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Tue May 23 07:27:30 2017 (r318716)
@@ -426,10 +426,6 @@ page_busy(vnode_t *vp, int64_t start, in
continue;
}
vm_page_sbusy(pp);
- } else if (pp == NULL) {
- pp = vm_page_alloc(obj, OFF_TO_IDX(start),
- VM_ALLOC_SYSTEM | VM_ALLOC_IFCACHED |
- VM_ALLOC_SBUSY);
} else {
ASSERT(pp != NULL && !pp->valid);
pp = NULL;
Modified: stable/11/sys/fs/tmpfs/tmpfs_subr.c
==============================================================================
--- stable/11/sys/fs/tmpfs/tmpfs_subr.c Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/fs/tmpfs/tmpfs_subr.c Tue May 23 07:27:30 2017 (r318716)
@@ -1401,12 +1401,9 @@ retry:
VM_WAIT;
VM_OBJECT_WLOCK(uobj);
goto retry;
- } else if (m->valid != VM_PAGE_BITS_ALL)
- rv = vm_pager_get_pages(uobj, &m, 1,
- NULL, NULL);
- else
- /* A cached page was reactivated. */
- rv = VM_PAGER_OK;
+ }
+ rv = vm_pager_get_pages(uobj, &m, 1, NULL,
+ NULL);
vm_page_lock(m);
if (rv == VM_PAGER_OK) {
vm_page_deactivate(m);
Modified: stable/11/sys/i386/i386/pmap.c
==============================================================================
--- stable/11/sys/i386/i386/pmap.c Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/i386/i386/pmap.c Tue May 23 07:27:30 2017 (r318716)
@@ -306,7 +306,6 @@ static boolean_t pmap_is_modified_pvh(st
static boolean_t pmap_is_referenced_pvh(struct md_page *pvh);
static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
static void pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde);
-static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va);
static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);
static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
@@ -316,7 +315,7 @@ static void pmap_remove_pde(pmap_t pmap,
struct spglist *free);
static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
struct spglist *free);
-static void pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte);
+static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
static void pmap_remove_page(struct pmap *pmap, vm_offset_t va,
struct spglist *free);
static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
@@ -1727,29 +1726,17 @@ pmap_insert_pt_page(pmap_t pmap, vm_page
}
/*
- * Looks for a page table page mapping the specified virtual address in the
- * specified pmap's collection of idle page table pages. Returns NULL if there
- * is no page table page corresponding to the specified virtual address.
+ * Removes the page table page mapping the specified virtual address from the
+ * specified pmap's collection of idle page table pages, and returns it.
+ * Otherwise, returns NULL if there is no page table page corresponding to the
+ * specified virtual address.
*/
static __inline vm_page_t
-pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va)
+pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
{
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- return (vm_radix_lookup(&pmap->pm_root, va >> PDRSHIFT));
-}
-
-/*
- * Removes the specified page table page from the specified pmap's collection
- * of idle page table pages. The specified page table page must be a member of
- * the pmap's collection.
- */
-static __inline void
-pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte)
-{
-
- PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- vm_radix_remove(&pmap->pm_root, mpte->pindex);
+ return (vm_radix_remove(&pmap->pm_root, va >> PDRSHIFT));
}
/*
@@ -2645,10 +2632,8 @@ pmap_demote_pde(pmap_t pmap, pd_entry_t
oldpde = *pde;
KASSERT((oldpde & (PG_PS | PG_V)) == (PG_PS | PG_V),
("pmap_demote_pde: oldpde is missing PG_PS and/or PG_V"));
- if ((oldpde & PG_A) != 0 && (mpte = pmap_lookup_pt_page(pmap, va)) !=
- NULL)
- pmap_remove_pt_page(pmap, mpte);
- else {
+ if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==
+ NULL) {
KASSERT((oldpde & PG_W) == 0,
("pmap_demote_pde: page table page for a wired mapping"
" is missing"));
@@ -2786,11 +2771,10 @@ pmap_remove_kernel_pde(pmap_t pmap, pd_e
vm_page_t mpte;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- mpte = pmap_lookup_pt_page(pmap, va);
+ mpte = pmap_remove_pt_page(pmap, va);
if (mpte == NULL)
panic("pmap_remove_kernel_pde: Missing pt page.");
- pmap_remove_pt_page(pmap, mpte);
mptepa = VM_PAGE_TO_PHYS(mpte);
newpde = mptepa | PG_M | PG_A | PG_RW | PG_V;
@@ -2872,9 +2856,8 @@ pmap_remove_pde(pmap_t pmap, pd_entry_t
if (pmap == kernel_pmap) {
pmap_remove_kernel_pde(pmap, pdq, sva);
} else {
- mpte = pmap_lookup_pt_page(pmap, sva);
+ mpte = pmap_remove_pt_page(pmap, sva);
if (mpte != NULL) {
- pmap_remove_pt_page(pmap, mpte);
pmap->pm_stats.resident_count--;
KASSERT(mpte->wire_count == NPTEPG,
("pmap_remove_pde: pte page wire count error"));
@@ -4616,9 +4599,8 @@ pmap_remove_pages(pmap_t pmap)
if (TAILQ_EMPTY(&mt->md.pv_list))
vm_page_aflag_clear(mt, PGA_WRITEABLE);
}
- mpte = pmap_lookup_pt_page(pmap, pv->pv_va);
+ mpte = pmap_remove_pt_page(pmap, pv->pv_va);
if (mpte != NULL) {
- pmap_remove_pt_page(pmap, mpte);
pmap->pm_stats.resident_count--;
KASSERT(mpte->wire_count == NPTEPG,
("pmap_remove_pages: pte page wire count error"));
Modified: stable/11/sys/kern/kern_exec.c
==============================================================================
--- stable/11/sys/kern/kern_exec.c Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/kern/kern_exec.c Tue May 23 07:27:30 2017 (r318716)
@@ -1006,7 +1006,7 @@ exec_map_first_page(imgp)
break;
} else {
ma[i] = vm_page_alloc(object, i,
- VM_ALLOC_NORMAL | VM_ALLOC_IFNOTCACHED);
+ VM_ALLOC_NORMAL);
if (ma[i] == NULL)
break;
}
Modified: stable/11/sys/kern/uipc_shm.c
==============================================================================
--- stable/11/sys/kern/uipc_shm.c Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/kern/uipc_shm.c Tue May 23 07:27:30 2017 (r318716)
@@ -455,12 +455,9 @@ retry:
VM_WAIT;
VM_OBJECT_WLOCK(object);
goto retry;
- } else if (m->valid != VM_PAGE_BITS_ALL)
- rv = vm_pager_get_pages(object, &m, 1,
- NULL, NULL);
- else
- /* A cached page was reactivated. */
- rv = VM_PAGER_OK;
+ }
+ rv = vm_pager_get_pages(object, &m, 1, NULL,
+ NULL);
vm_page_lock(m);
if (rv == VM_PAGER_OK) {
vm_page_deactivate(m);
Modified: stable/11/sys/sys/vmmeter.h
==============================================================================
--- stable/11/sys/sys/vmmeter.h Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/sys/vmmeter.h Tue May 23 07:27:30 2017 (r318716)
@@ -75,9 +75,10 @@ struct vmmeter {
u_int v_vnodepgsin; /* (p) vnode_pager pages paged in */
u_int v_vnodepgsout; /* (p) vnode pager pages paged out */
u_int v_intrans; /* (p) intransit blocking page faults */
- u_int v_reactivated; /* (f) pages reactivated from free list */
+ u_int v_reactivated; /* (p) pages reactivated by the pagedaemon */
u_int v_pdwakeups; /* (p) times daemon has awaken from sleep */
u_int v_pdpages; /* (p) pages analyzed by daemon */
+ u_int v_pdshortfalls; /* (p) page reclamation shortfalls */
u_int v_tcached; /* (p) total pages cached */
u_int v_dfree; /* (p) pages freed by daemon */
@@ -96,6 +97,7 @@ struct vmmeter {
u_int v_active_count; /* (q) pages active */
u_int v_inactive_target; /* (c) pages desired inactive */
u_int v_inactive_count; /* (q) pages inactive */
+ u_int v_laundry_count; /* (q) pages eligible for laundering */
u_int v_cache_count; /* (f) pages on cache queue */
u_int v_pageout_free_min; /* (c) min pages reserved for kernel */
u_int v_interrupt_free_min; /* (c) reserved pages for int code */
@@ -111,7 +113,6 @@ struct vmmeter {
u_int v_vforkpages; /* (p) VM pages affected by vfork() */
u_int v_rforkpages; /* (p) VM pages affected by rfork() */
u_int v_kthreadpages; /* (p) VM pages affected by fork() by kernel */
- u_int v_spare[2];
};
#ifdef _KERNEL
@@ -184,6 +185,25 @@ vm_paging_needed(void)
(u_int)vm_pageout_wakeup_thresh);
}
+/*
+ * Return the number of pages we need to launder.
+ * A positive number indicates that we have a shortfall of clean pages.
+ */
+static inline int
+vm_laundry_target(void)
+{
+
+ return (vm_paging_target());
+}
+
+/*
+ * Obtain the value of a per-CPU counter.
+ */
+#define VM_METER_PCPU_CNT(member) \
+ vm_meter_cnt(__offsetof(struct vmmeter, member))
+
+u_int vm_meter_cnt(size_t);
+
#endif
/* systemwide totals computed every five seconds */
Modified: stable/11/sys/vm/_vm_radix.h
==============================================================================
--- stable/11/sys/vm/_vm_radix.h Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/vm/_vm_radix.h Tue May 23 07:27:30 2017 (r318716)
@@ -36,12 +36,8 @@
*/
struct vm_radix {
uintptr_t rt_root;
- uint8_t rt_flags;
};
-#define RT_INSERT_INPROG 0x01
-#define RT_TRIE_MODIFIED 0x02
-
#ifdef _KERNEL
static __inline boolean_t
Modified: stable/11/sys/vm/swap_pager.c
==============================================================================
--- stable/11/sys/vm/swap_pager.c Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/vm/swap_pager.c Tue May 23 07:27:30 2017 (r318716)
@@ -1126,7 +1126,7 @@ swap_pager_getpages(vm_object_t object,
if (shift != 0) {
for (i = 1; i <= shift; i++) {
p = vm_page_alloc(object, m[0]->pindex - i,
- VM_ALLOC_NORMAL | VM_ALLOC_IFNOTCACHED);
+ VM_ALLOC_NORMAL);
if (p == NULL) {
/* Shift allocated pages to the left. */
for (j = 0; j < i - 1; j++)
@@ -1144,8 +1144,7 @@ swap_pager_getpages(vm_object_t object,
if (rahead != NULL) {
for (i = 0; i < *rahead; i++) {
p = vm_page_alloc(object,
- m[reqcount - 1]->pindex + i + 1,
- VM_ALLOC_NORMAL | VM_ALLOC_IFNOTCACHED);
+ m[reqcount - 1]->pindex + i + 1, VM_ALLOC_NORMAL);
if (p == NULL)
break;
bp->b_pages[shift + reqcount + i] = p;
@@ -1549,17 +1548,18 @@ swp_pager_async_iodone(struct buf *bp)
* For write success, clear the dirty
* status, then finish the I/O ( which decrements the
* busy count and possibly wakes waiter's up ).
+ * A page is only written to swap after a period of
+ * inactivity. Therefore, we do not expect it to be
+ * reused.
*/
KASSERT(!pmap_page_is_write_mapped(m),
("swp_pager_async_iodone: page %p is not write"
" protected", m));
vm_page_undirty(m);
+ vm_page_lock(m);
+ vm_page_deactivate_noreuse(m);
+ vm_page_unlock(m);
vm_page_sunbusy(m);
- if (vm_page_count_severe()) {
- vm_page_lock(m);
- vm_page_try_to_cache(m);
- vm_page_unlock(m);
- }
}
}
@@ -1635,12 +1635,15 @@ swap_pager_isswapped(vm_object_t object,
/*
* SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
*
- * This routine dissociates the page at the given index within a
- * swap block from its backing store, paging it in if necessary.
- * If the page is paged in, it is placed in the inactive queue,
- * since it had its backing store ripped out from under it.
- * We also attempt to swap in all other pages in the swap block,
- * we only guarantee that the one at the specified index is
+ * This routine dissociates the page at the given index within an object
+ * from its backing store, paging it in if it does not reside in memory.
+ * If the page is paged in, it is marked dirty and placed in the laundry
+ * queue. The page is marked dirty because it no longer has backing
+ * store. It is placed in the laundry queue because it has not been
+ * accessed recently. Otherwise, it would already reside in memory.
+ *
+ * We also attempt to swap in all other pages in the swap block.
+ * However, we only guarantee that the one at the specified index is
* paged in.
*
* XXX - The code to page the whole block in doesn't work, so we
@@ -1669,7 +1672,7 @@ swp_pager_force_pagein(vm_object_t objec
vm_object_pip_wakeup(object);
vm_page_dirty(m);
vm_page_lock(m);
- vm_page_deactivate(m);
+ vm_page_launder(m);
vm_page_unlock(m);
vm_page_xunbusy(m);
vm_pager_page_unswapped(m);
Modified: stable/11/sys/vm/vm_fault.c
==============================================================================
--- stable/11/sys/vm/vm_fault.c Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/vm/vm_fault.c Tue May 23 07:27:30 2017 (r318716)
@@ -485,11 +485,12 @@ int
vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
int fault_flags, vm_page_t *m_hold)
{
- vm_prot_t prot;
- vm_object_t next_object;
struct faultstate fs;
struct vnode *vp;
+ vm_object_t next_object, retry_object;
vm_offset_t e_end, e_start;
+ vm_pindex_t retry_pindex;
+ vm_prot_t prot, retry_prot;
int ahead, alloc_req, behind, cluster_offset, error, era, faultcount;
int locked, nera, result, rv;
u_char behavior;
@@ -755,8 +756,7 @@ RetryFault:;
unlock_and_deallocate(&fs);
VM_WAITPFAULT;
goto RetryFault;
- } else if (fs.m->valid == VM_PAGE_BITS_ALL)
- break;
+ }
}
readrest:
@@ -1143,10 +1143,6 @@ readrest:
* lookup.
*/
if (!fs.lookup_still_valid) {
- vm_object_t retry_object;
- vm_pindex_t retry_pindex;
- vm_prot_t retry_prot;
-
if (!vm_map_trylock_read(fs.map)) {
release_page(&fs);
unlock_and_deallocate(&fs);
Modified: stable/11/sys/vm/vm_map.c
==============================================================================
--- stable/11/sys/vm/vm_map.c Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/vm/vm_map.c Tue May 23 07:27:30 2017 (r318716)
@@ -1858,9 +1858,7 @@ vm_map_submap(
* limited number of page mappings are created at the low-end of the
* specified address range. (For this purpose, a superpage mapping
* counts as one page mapping.) Otherwise, all resident pages within
- * the specified address range are mapped. Because these mappings are
- * being created speculatively, cached pages are not reactivated and
- * mapped.
+ * the specified address range are mapped.
*/
static void
vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot,
Modified: stable/11/sys/vm/vm_meter.c
==============================================================================
--- stable/11/sys/vm/vm_meter.c Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/vm/vm_meter.c Tue May 23 07:27:30 2017 (r318716)
@@ -209,29 +209,37 @@ vmtotal(SYSCTL_HANDLER_ARGS)
}
/*
- * vcnt() - accumulate statistics from all cpus and the global cnt
- * structure.
+ * vm_meter_cnt() - accumulate statistics from all cpus and the global cnt
+ * structure.
*
* The vmmeter structure is now per-cpu as well as global. Those
* statistics which can be kept on a per-cpu basis (to avoid cache
* stalls between cpus) can be moved to the per-cpu vmmeter. Remaining
* statistics, such as v_free_reserved, are left in the global
* structure.
- *
- * (sysctl_oid *oidp, void *arg1, int arg2, struct sysctl_req *req)
*/
-static int
-vcnt(SYSCTL_HANDLER_ARGS)
+u_int
+vm_meter_cnt(size_t offset)
{
- int count = *(int *)arg1;
- int offset = (char *)arg1 - (char *)&vm_cnt;
+ struct pcpu *pcpu;
+ u_int count;
int i;
+ count = *(u_int *)((char *)&vm_cnt + offset);
CPU_FOREACH(i) {
- struct pcpu *pcpu = pcpu_find(i);
- count += *(int *)((char *)&pcpu->pc_cnt + offset);
+ pcpu = pcpu_find(i);
+ count += *(u_int *)((char *)&pcpu->pc_cnt + offset);
}
- return (SYSCTL_OUT(req, &count, sizeof(int)));
+ return (count);
+}
+
+static int
+cnt_sysctl(SYSCTL_HANDLER_ARGS)
+{
+ u_int count;
+
+ count = vm_meter_cnt((char *)arg1 - (char *)&vm_cnt);
+ return (SYSCTL_OUT(req, &count, sizeof(count)));
}
SYSCTL_PROC(_vm, VM_TOTAL, vmtotal, CTLTYPE_OPAQUE|CTLFLAG_RD|CTLFLAG_MPSAFE,
@@ -246,8 +254,8 @@ SYSCTL_NODE(_vm_stats, OID_AUTO, misc, C
#define VM_STATS(parent, var, descr) \
SYSCTL_PROC(parent, OID_AUTO, var, \
- CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, &vm_cnt.var, 0, vcnt, \
- "IU", descr)
+ CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, &vm_cnt.var, 0, \
+ cnt_sysctl, "IU", descr)
#define VM_STATS_VM(var, descr) VM_STATS(_vm_stats_vm, var, descr)
#define VM_STATS_SYS(var, descr) VM_STATS(_vm_stats_sys, var, descr)
@@ -271,9 +279,10 @@ VM_STATS_VM(v_vnodeout, "Vnode pager pag
VM_STATS_VM(v_vnodepgsin, "Vnode pages paged in");
VM_STATS_VM(v_vnodepgsout, "Vnode pages paged out");
VM_STATS_VM(v_intrans, "In transit page faults");
-VM_STATS_VM(v_reactivated, "Pages reactivated from free list");
+VM_STATS_VM(v_reactivated, "Pages reactivated by pagedaemon");
VM_STATS_VM(v_pdwakeups, "Pagedaemon wakeups");
VM_STATS_VM(v_pdpages, "Pages analyzed by pagedaemon");
+VM_STATS_VM(v_pdshortfalls, "Page reclamation shortfalls");
VM_STATS_VM(v_tcached, "Total pages cached");
VM_STATS_VM(v_dfree, "Pages freed by pagedaemon");
VM_STATS_VM(v_pfree, "Pages freed by exiting processes");
@@ -288,6 +297,7 @@ VM_STATS_VM(v_wire_count, "Wired pages")
VM_STATS_VM(v_active_count, "Active pages");
VM_STATS_VM(v_inactive_target, "Desired inactive pages");
VM_STATS_VM(v_inactive_count, "Inactive pages");
+VM_STATS_VM(v_laundry_count, "Pages eligible for laundering");
VM_STATS_VM(v_cache_count, "Pages on cache queue");
VM_STATS_VM(v_pageout_free_min, "Min pages reserved for kernel");
VM_STATS_VM(v_interrupt_free_min, "Reserved pages for interrupt code");
Modified: stable/11/sys/vm/vm_mmap.c
==============================================================================
--- stable/11/sys/vm/vm_mmap.c Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/vm/vm_mmap.c Tue May 23 07:27:30 2017 (r318716)
@@ -849,9 +849,6 @@ RestartScan:
pindex = OFF_TO_IDX(current->offset +
(addr - current->start));
m = vm_page_lookup(object, pindex);
- if (m == NULL &&
- vm_page_is_cached(object, pindex))
- mincoreinfo = MINCORE_INCORE;
if (m != NULL && m->valid == 0)
m = NULL;
if (m != NULL)
Modified: stable/11/sys/vm/vm_object.c
==============================================================================
--- stable/11/sys/vm/vm_object.c Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/vm/vm_object.c Tue May 23 07:27:30 2017 (r318716)
@@ -178,9 +178,6 @@ vm_object_zdtor(void *mem, int size, voi
("object %p has reservations",
object));
#endif
- KASSERT(vm_object_cache_is_empty(object),
- ("object %p has cached pages",
- object));
KASSERT(object->paging_in_progress == 0,
("object %p paging_in_progress = %d",
object, object->paging_in_progress));
@@ -208,12 +205,9 @@ vm_object_zinit(void *mem, int size, int
object->type = OBJT_DEAD;
object->ref_count = 0;
object->rtree.rt_root = 0;
- object->rtree.rt_flags = 0;
object->paging_in_progress = 0;
object->resident_page_count = 0;
object->shadow_count = 0;
- object->cache.rt_root = 0;
- object->cache.rt_flags = 0;
mtx_lock(&vm_object_list_mtx);
TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
@@ -792,8 +786,6 @@ vm_object_terminate(vm_object_t object)
if (__predict_false(!LIST_EMPTY(&object->rvq)))
vm_reserv_break_all(object);
#endif
- if (__predict_false(!vm_object_cache_is_empty(object)))
- vm_page_cache_free(object, 0, 0);
KASSERT(object->cred == NULL || object->type == OBJT_DEFAULT ||
object->type == OBJT_SWAP,
@@ -1135,13 +1127,6 @@ shadowlookup:
} else if ((tobject->flags & OBJ_UNMANAGED) != 0)
goto unlock_tobject;
m = vm_page_lookup(tobject, tpindex);
- if (m == NULL && advise == MADV_WILLNEED) {
- /*
- * If the page is cached, reactivate it.
- */
- m = vm_page_alloc(tobject, tpindex, VM_ALLOC_IFCACHED |
- VM_ALLOC_NOBUSY);
- }
if (m == NULL) {
/*
* There may be swap even if there is no backing page
@@ -1371,7 +1356,7 @@ retry:
goto retry;
}
- /* vm_page_rename() will handle dirty and cache. */
+ /* vm_page_rename() will dirty the page. */
if (vm_page_rename(m, new_object, idx)) {
VM_OBJECT_WUNLOCK(new_object);
VM_OBJECT_WUNLOCK(orig_object);
@@ -1406,19 +1391,6 @@ retry:
swap_pager_copy(orig_object, new_object, offidxstart, 0);
TAILQ_FOREACH(m, &new_object->memq, listq)
vm_page_xunbusy(m);
-
- /*
- * Transfer any cached pages from orig_object to new_object.
- * If swap_pager_copy() found swapped out pages within the
- * specified range of orig_object, then it changed
- * new_object's type to OBJT_SWAP when it transferred those
- * pages to new_object. Otherwise, new_object's type
- * should still be OBJT_DEFAULT and orig_object should not
- * contain any cached pages within the specified range.
- */
- if (__predict_false(!vm_object_cache_is_empty(orig_object)))
- vm_page_cache_transfer(orig_object, offidxstart,
- new_object);
}
VM_OBJECT_WUNLOCK(orig_object);
VM_OBJECT_WUNLOCK(new_object);
@@ -1471,6 +1443,13 @@ vm_object_scan_all_shadowed(vm_object_t
backing_object = object->backing_object;
+ /*
+ * Initial conditions:
+ *
+ * We do not want to have to test for the existence of swap
+ * pages in the backing object. XXX but with the new swapper this
+ * would be pretty easy to do.
+ */
if (backing_object->type != OBJT_DEFAULT &&
backing_object->type != OBJT_SWAP)
return (false);
@@ -1622,8 +1601,7 @@ vm_object_collapse_scan(vm_object_t obje
* backing object to the main object.
*
* If the page was mapped to a process, it can remain mapped
- * through the rename. vm_page_rename() will handle dirty and
- * cache.
+ * through the rename. vm_page_rename() will dirty the page.
*/
if (vm_page_rename(p, object, new_pindex)) {
next = vm_object_collapse_scan_wait(object, NULL, next,
@@ -1758,13 +1736,6 @@ vm_object_collapse(vm_object_t object)
backing_object,
object,
OFF_TO_IDX(object->backing_object_offset), TRUE);
-
- /*
- * Free any cached pages from backing_object.
- */
- if (__predict_false(
- !vm_object_cache_is_empty(backing_object)))
- vm_page_cache_free(backing_object, 0, 0);
}
/*
* Object now shadows whatever backing_object did.
@@ -1893,7 +1864,7 @@ vm_object_page_remove(vm_object_t object
(options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED,
("vm_object_page_remove: illegal options for object %p", object));
if (object->resident_page_count == 0)
- goto skipmemq;
+ return;
vm_object_pip_add(object, 1);
again:
p = vm_page_find_least(object, start);
@@ -1950,9 +1921,6 @@ next:
vm_page_unlock(p);
}
vm_object_pip_wakeup(object);
-skipmemq:
- if (__predict_false(!vm_object_cache_is_empty(object)))
- vm_page_cache_free(object, start, end);
}
/*
@@ -2333,9 +2301,9 @@ sysctl_vm_object_list(SYSCTL_HANDLER_ARG
* sysctl is only meant to give an
* approximation of the system anyway.
*/
- if (m->queue == PQ_ACTIVE)
+ if (vm_page_active(m))
kvo.kvo_active++;
- else if (m->queue == PQ_INACTIVE)
+ else if (vm_page_inactive(m))
kvo.kvo_inactive++;
}
Modified: stable/11/sys/vm/vm_object.h
==============================================================================
--- stable/11/sys/vm/vm_object.h Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/vm/vm_object.h Tue May 23 07:27:30 2017 (r318716)
@@ -79,17 +79,6 @@
*
* vm_object_t Virtual memory object.
*
- * The root of cached pages pool is protected by both the per-object lock
- * and the free pages queue mutex.
- * On insert in the cache radix trie, the per-object lock is expected
- * to be already held and the free pages queue mutex will be
- * acquired during the operation too.
- * On remove and lookup from the cache radix trie, only the free
- * pages queue mutex is expected to be locked.
- * These rules allow for reliably checking for the presence of cached
- * pages with only the per-object lock held, thereby reducing contention
- * for the free pages queue mutex.
- *
* List of locks
* (c) const until freed
* (o) per-object lock
@@ -118,7 +107,6 @@ struct vm_object {
vm_ooffset_t backing_object_offset;/* Offset in backing object */
TAILQ_ENTRY(vm_object) pager_object_list; /* list of all objects of this pager type */
LIST_HEAD(, vm_reserv) rvq; /* list of reservations */
- struct vm_radix cache; /* (o + f) root of the cache page radix trie */
void *handle;
union {
/*
@@ -306,13 +294,6 @@ void vm_object_pip_wakeup(vm_object_t ob
void vm_object_pip_wakeupn(vm_object_t object, short i);
void vm_object_pip_wait(vm_object_t object, char *waitid);
-static __inline boolean_t
-vm_object_cache_is_empty(vm_object_t object)
-{
-
- return (vm_radix_is_empty(&object->cache));
-}
-
void umtx_shm_object_init(vm_object_t object);
void umtx_shm_object_terminated(vm_object_t object);
extern int umtx_shm_vnobj_persistent;
Modified: stable/11/sys/vm/vm_page.c
==============================================================================
--- stable/11/sys/vm/vm_page.c Tue May 23 07:26:45 2017 (r318715)
+++ stable/11/sys/vm/vm_page.c Tue May 23 07:27:30 2017 (r318716)
@@ -155,8 +155,7 @@ static int vm_pageout_pages_needed;
static uma_zone_t fakepg_zone;
-static struct vnode *vm_page_alloc_init(vm_page_t m);
-static void vm_page_cache_turn_free(vm_page_t m);
+static void vm_page_alloc_check(vm_page_t m);
static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
static void vm_page_enqueue(uint8_t queue, vm_page_t m);
static void vm_page_free_wakeup(void);
@@ -391,6 +390,10 @@ vm_page_domain_init(struct vm_domain *vm
"vm active pagequeue";
*__DECONST(u_int **, &vmd->vmd_pagequeues[PQ_ACTIVE].pq_vcnt) =
&vm_cnt.v_active_count;
+ *__DECONST(char **, &vmd->vmd_pagequeues[PQ_LAUNDRY].pq_name) =
+ "vm laundry pagequeue";
+ *__DECONST(int **, &vmd->vmd_pagequeues[PQ_LAUNDRY].pq_vcnt) =
+ &vm_cnt.v_laundry_count;
vmd->vmd_page_count = 0;
vmd->vmd_free_count = 0;
vmd->vmd_segs = 0;
@@ -1136,9 +1139,7 @@ void
vm_page_dirty_KBI(vm_page_t m)
{
- /* These assertions refer to this operation by its public name. */
- KASSERT((m->flags & PG_CACHED) == 0,
- ("vm_page_dirty: page in cache!"));
+ /* Refer to this operation by its public name. */
KASSERT(m->valid == VM_PAGE_BITS_ALL,
("vm_page_dirty: page is invalid!"));
m->dirty = VM_PAGE_BITS_ALL;
@@ -1262,9 +1263,8 @@ vm_page_insert_radixdone(vm_page_t m, vm
/*
* vm_page_remove:
*
- * Removes the given mem entry from the object/offset-page
- * table and the object page list, but do not invalidate/terminate
- * the backing store.
+ * Removes the specified page from its containing object, but does not
+ * invalidate any backing storage.
*
* The object must be locked. The page must be locked if it is managed.
*/
@@ -1272,6 +1272,7 @@ void
vm_page_remove(vm_page_t m)
{
vm_object_t object;
+ vm_page_t mrem;
if ((m->oflags & VPO_UNMANAGED) == 0)
vm_page_assert_locked(m);
@@ -1280,11 +1281,12 @@ vm_page_remove(vm_page_t m)
VM_OBJECT_ASSERT_WLOCKED(object);
if (vm_page_xbusied(m))
vm_page_xunbusy_maybelocked(m);
+ mrem = vm_radix_remove(&object->rtree, m->pindex);
+ KASSERT(mrem == m, ("removed page %p, expected page %p", mrem, m));
/*
* Now remove from the object's list of backed pages.
*/
- vm_radix_remove(&object->rtree, m->pindex);
TAILQ_REMOVE(&object->memq, m, listq);
/*
@@ -1433,9 +1435,7 @@ vm_page_replace(vm_page_t mnew, vm_objec
*
* Note: we *always* dirty the page. It is necessary both for the
* fact that we moved it, and because we may be invalidating
- * swap. If the page is on the cache, we have to deactivate it
- * or vm_page_dirty() will panic. Dirty pages are not allowed
- * on the cache.
+ * swap.
*
* The objects must be locked.
*/
@@ -1481,142 +1481,6 @@ vm_page_rename(vm_page_t m, vm_object_t
}
/*
- * Convert all of the given object's cached pages that have a
- * pindex within the given range into free pages. If the value
- * zero is given for "end", then the range's upper bound is
- * infinity. If the given object is backed by a vnode and it
- * transitions from having one or more cached pages to none, the
- * vnode's hold count is reduced.
- */
-void
-vm_page_cache_free(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
-{
- vm_page_t m;
- boolean_t empty;
-
- mtx_lock(&vm_page_queue_free_mtx);
- if (__predict_false(vm_radix_is_empty(&object->cache))) {
- mtx_unlock(&vm_page_queue_free_mtx);
- return;
- }
- while ((m = vm_radix_lookup_ge(&object->cache, start)) != NULL) {
- if (end != 0 && m->pindex >= end)
- break;
- vm_radix_remove(&object->cache, m->pindex);
- vm_page_cache_turn_free(m);
- }
- empty = vm_radix_is_empty(&object->cache);
- mtx_unlock(&vm_page_queue_free_mtx);
- if (object->type == OBJT_VNODE && empty)
- vdrop(object->handle);
-}
-
-/*
- * Returns the cached page that is associated with the given
- * object and offset. If, however, none exists, returns NULL.
- *
- * The free page queue must be locked.
- */
-static inline vm_page_t
-vm_page_cache_lookup(vm_object_t object, vm_pindex_t pindex)
-{
-
- mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
- return (vm_radix_lookup(&object->cache, pindex));
-}
-
-/*
- * Remove the given cached page from its containing object's
- * collection of cached pages.
- *
- * The free page queue must be locked.
- */
-static void
-vm_page_cache_remove(vm_page_t m)
-{
-
- mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
- KASSERT((m->flags & PG_CACHED) != 0,
- ("vm_page_cache_remove: page %p is not cached", m));
- vm_radix_remove(&m->object->cache, m->pindex);
- m->object = NULL;
- vm_cnt.v_cache_count--;
-}
-
-/*
- * Transfer all of the cached pages with offset greater than or
- * equal to 'offidxstart' from the original object's cache to the
- * new object's cache. However, any cached pages with offset
- * greater than or equal to the new object's size are kept in the
- * original object. Initially, the new object's cache must be
- * empty. Offset 'offidxstart' in the original object must
- * correspond to offset zero in the new object.
- *
- * The new object must be locked.
- */
-void
-vm_page_cache_transfer(vm_object_t orig_object, vm_pindex_t offidxstart,
- vm_object_t new_object)
-{
- vm_page_t m;
-
- /*
- * Insertion into an object's collection of cached pages
- * requires the object to be locked. In contrast, removal does
- * not.
- */
- VM_OBJECT_ASSERT_WLOCKED(new_object);
- KASSERT(vm_radix_is_empty(&new_object->cache),
- ("vm_page_cache_transfer: object %p has cached pages",
- new_object));
- mtx_lock(&vm_page_queue_free_mtx);
- while ((m = vm_radix_lookup_ge(&orig_object->cache,
- offidxstart)) != NULL) {
- /*
- * Transfer all of the pages with offset greater than or
- * equal to 'offidxstart' from the original object's
- * cache to the new object's cache.
- */
- if ((m->pindex - offidxstart) >= new_object->size)
- break;
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-stable-11
mailing list