git: 9cc73397534e - main - vm_page: use iterators in page allocaction
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Mon, 14 Apr 2025 07:10:36 UTC
The branch main has been updated by dougm:
URL: https://cgit.FreeBSD.org/src/commit/?id=9cc73397534ee5373593f5946abe0b00c1d2b657
commit 9cc73397534ee5373593f5946abe0b00c1d2b657
Author: Doug Moore <dougm@FreeBSD.org>
AuthorDate: 2025-04-14 07:05:21 +0000
Commit: Doug Moore <dougm@FreeBSD.org>
CommitDate: 2025-04-14 07:05:21 +0000
vm_page: use iterators in page allocaction
Change vm_page_alloc_after() and vm_page_alloc_domain_after() to take
a page iterator argument, to allow for faster insertion into the radix
trie of pages. Where a caller of one of those functions doesn't
already have a page iterator to use, define one. Where vm_page_grab()
is invoked in a loop, invoke a new version with an iterator argument,
and pass the same iterator in each call. Make sure that
vm_page_alloc_after() and vm_page_alloc_domain_after() return with the
iterator in a valid state, even when the object lock was briefly lost
and regained.
Reviewed by: markj
Differential Revision: https://reviews.freebsd.org/D49688
---
sys/vm/phys_pager.c | 6 +-
sys/vm/swap_pager.c | 5 +-
sys/vm/vm_fault.c | 14 +++--
sys/vm/vm_glue.c | 43 +++++++-------
sys/vm/vm_kern.c | 6 +-
sys/vm/vm_object.c | 10 ++--
sys/vm/vm_page.c | 165 ++++++++++++++++++++++++++--------------------------
sys/vm/vm_page.h | 9 ++-
8 files changed, 136 insertions(+), 122 deletions(-)
diff --git a/sys/vm/phys_pager.c b/sys/vm/phys_pager.c
index 6ee5076f9a91..ee39221402fa 100644
--- a/sys/vm/phys_pager.c
+++ b/sys/vm/phys_pager.c
@@ -33,6 +33,7 @@
#include <sys/proc.h>
#include <sys/mutex.h>
#include <sys/mman.h>
+#include <sys/pctrie.h>
#include <sys/rwlock.h>
#include <sys/sysctl.h>
#include <sys/user.h>
@@ -230,10 +231,12 @@ default_phys_pager_populate(vm_object_t object, vm_pindex_t pidx,
int fault_type __unused, vm_prot_t max_prot __unused, vm_pindex_t *first,
vm_pindex_t *last)
{
+ struct pctrie_iter pages;
vm_page_t m;
vm_pindex_t base, end, i;
int ahead;
+ VM_OBJECT_ASSERT_WLOCKED(object);
base = rounddown(pidx, phys_pager_cluster);
end = base + phys_pager_cluster - 1;
if (end >= object->size)
@@ -244,10 +247,11 @@ default_phys_pager_populate(vm_object_t object, vm_pindex_t pidx,
end = *last;
*first = base;
*last = end;
+ vm_page_iter_init(&pages, object);
for (i = base; i <= end; i++) {
ahead = MIN(end - i, PHYSALLOC);
- m = vm_page_grab(object, i,
+ m = vm_page_grab_iter(object, &pages, i,
VM_ALLOC_NORMAL | VM_ALLOC_COUNT(ahead));
if (!vm_page_all_valid(m))
vm_page_zero_invalid(m, TRUE);
diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c
index 7b83ae4bfb7b..38203e4cd0af 100644
--- a/sys/vm/swap_pager.c
+++ b/sys/vm/swap_pager.c
@@ -1935,9 +1935,10 @@ swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object,
if (!vm_page_busy_acquire(m, VM_ALLOC_WAITFAIL))
break;
} else {
- m = vm_radix_iter_lookup_le(&pages,
+ m = vm_radix_iter_lookup_lt(&pages,
blks.index + i);
- m = vm_page_alloc_after(object, blks.index + i,
+ m = vm_page_alloc_after(
+ object, &pages, blks.index + i,
VM_ALLOC_NORMAL | VM_ALLOC_WAITFAIL, m);
if (m == NULL)
break;
diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index 0bd3a8207c4a..e1db7ac8b524 100644
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -1292,7 +1292,7 @@ vm_fault_allocate(struct faultstate *fs, struct pctrie_iter *pages)
vm_fault_unlock_and_deallocate(fs);
return (FAULT_FAILURE);
}
- fs->m = vm_page_alloc_after(fs->object, fs->pindex,
+ fs->m = vm_page_alloc_after(fs->object, pages, fs->pindex,
P_KILLED(curproc) ? VM_ALLOC_SYSTEM : 0,
vm_radix_iter_lookup_lt(pages, fs->pindex));
}
@@ -2100,6 +2100,7 @@ vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map __unused,
vm_map_entry_t dst_entry, vm_map_entry_t src_entry,
vm_ooffset_t *fork_charge)
{
+ struct pctrie_iter pages;
vm_object_t backing_object, dst_object, object, src_object;
vm_pindex_t dst_pindex, pindex, src_pindex;
vm_prot_t access, prot;
@@ -2176,6 +2177,7 @@ vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map __unused,
* with the source object, all of its pages must be dirtied,
* regardless of whether they can be written.
*/
+ vm_page_iter_init(&pages, dst_object);
mpred = (src_object == dst_object) ?
vm_page_mpred(src_object, src_pindex) : NULL;
for (vaddr = dst_entry->start, dst_pindex = 0;
@@ -2220,14 +2222,15 @@ again:
*/
pindex = (src_object == dst_object ? src_pindex : 0) +
dst_pindex;
- dst_m = vm_page_alloc_after(dst_object, pindex,
+ dst_m = vm_page_alloc_after(dst_object, &pages, pindex,
VM_ALLOC_NORMAL, mpred);
if (dst_m == NULL) {
VM_OBJECT_WUNLOCK(dst_object);
VM_OBJECT_RUNLOCK(object);
vm_wait(dst_object);
VM_OBJECT_WLOCK(dst_object);
- mpred = vm_page_mpred(dst_object, pindex);
+ pctrie_iter_reset(&pages);
+ mpred = vm_radix_iter_lookup_lt(&pages, pindex);
goto again;
}
@@ -2249,8 +2252,11 @@ again:
VM_OBJECT_RUNLOCK(object);
} else {
dst_m = src_m;
- if (vm_page_busy_acquire(dst_m, VM_ALLOC_WAITFAIL) == 0)
+ if (vm_page_busy_acquire(
+ dst_m, VM_ALLOC_WAITFAIL) == 0) {
+ pctrie_iter_reset(&pages);
goto again;
+ }
if (dst_m->pindex >= dst_object->size) {
/*
* We are upgrading. Index can occur
diff --git a/sys/vm/vm_glue.c b/sys/vm/vm_glue.c
index 0090904785ab..c441b8b3155f 100644
--- a/sys/vm/vm_glue.c
+++ b/sys/vm/vm_glue.c
@@ -98,6 +98,7 @@
#include <vm/vm_pagequeue.h>
#include <vm/vm_object.h>
#include <vm/vm_kern.h>
+#include <vm/vm_radix.h>
#include <vm/vm_extern.h>
#include <vm/vm_pager.h>
#include <vm/vm_phys.h>
@@ -611,40 +612,36 @@ static int
vm_thread_stack_back(vm_offset_t ks, vm_page_t ma[], int npages, int req_class,
int domain)
{
+ struct pctrie_iter pages;
vm_object_t obj = vm_thread_kstack_size_to_obj(npages);
vm_pindex_t pindex;
- vm_page_t m;
+ vm_page_t m, mpred;
int n;
pindex = vm_kstack_pindex(ks, npages);
+ vm_page_iter_init(&pages, obj);
VM_OBJECT_WLOCK(obj);
- for (n = 0; n < npages;) {
- m = vm_page_grab(obj, pindex + n,
+ for (n = 0; n < npages; ma[n++] = m) {
+ m = vm_page_grab_iter(obj, &pages, pindex + n,
VM_ALLOC_NOCREAT | VM_ALLOC_WIRED);
- if (m == NULL) {
- m = n > 0 ? ma[n - 1] : vm_page_mpred(obj, pindex);
- m = vm_page_alloc_domain_after(obj, pindex + n, domain,
- req_class | VM_ALLOC_WIRED, m);
+ if (m != NULL)
+ continue;
+ mpred = (n > 0) ? ma[n - 1] :
+ vm_radix_iter_lookup_lt(&pages, pindex);
+ m = vm_page_alloc_domain_after(obj, &pages, pindex + n,
+ domain, req_class | VM_ALLOC_WIRED, mpred);
+ if (m != NULL)
+ continue;
+ for (int i = 0; i < n; i++) {
+ m = ma[i];
+ (void)vm_page_unwire_noq(m);
+ vm_page_free(m);
}
- if (m == NULL)
- break;
- ma[n++] = m;
+ break;
}
- if (n < npages)
- goto cleanup;
VM_OBJECT_WUNLOCK(obj);
-
- return (0);
-cleanup:
- for (int i = 0; i < n; i++) {
- m = ma[i];
- (void)vm_page_unwire_noq(m);
- vm_page_free(m);
- }
- VM_OBJECT_WUNLOCK(obj);
-
- return (ENOMEM);
+ return (n < npages ? ENOMEM : 0);
}
static vm_object_t
diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c
index 8945061aabdc..2aa5588a7588 100644
--- a/sys/vm/vm_kern.c
+++ b/sys/vm/vm_kern.c
@@ -530,6 +530,7 @@ int
kmem_back_domain(int domain, vm_object_t object, vm_offset_t addr,
vm_size_t size, int flags)
{
+ struct pctrie_iter pages;
vm_offset_t offset, i;
vm_page_t m, mpred;
vm_prot_t prot;
@@ -546,11 +547,12 @@ kmem_back_domain(int domain, vm_object_t object, vm_offset_t addr,
prot = (flags & M_EXEC) != 0 ? VM_PROT_ALL : VM_PROT_RW;
i = 0;
+ vm_page_iter_init(&pages, object);
VM_OBJECT_WLOCK(object);
retry:
- mpred = vm_radix_lookup_le(&object->rtree, atop(offset + i));
+ mpred = vm_radix_iter_lookup_lt(&pages, atop(offset + i));
for (; i < size; i += PAGE_SIZE, mpred = m) {
- m = vm_page_alloc_domain_after(object, atop(offset + i),
+ m = vm_page_alloc_domain_after(object, &pages, atop(offset + i),
domain, pflags, mpred);
/*
diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 689ad7499f5c..19e585745a80 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -2275,19 +2275,21 @@ void
vm_object_prepare_buf_pages(vm_object_t object, vm_page_t *ma_dst, int count,
int *rbehind, int *rahead, vm_page_t *ma_src)
{
+ struct pctrie_iter pages;
vm_pindex_t pindex;
vm_page_t m, mpred, msucc;
+ vm_page_iter_init(&pages, object);
VM_OBJECT_ASSERT_LOCKED(object);
if (*rbehind != 0) {
m = ma_src[0];
pindex = m->pindex;
- mpred = TAILQ_PREV(m, pglist, listq);
+ mpred = vm_radix_iter_lookup_lt(&pages, pindex);
*rbehind = MIN(*rbehind,
pindex - (mpred != NULL ? mpred->pindex + 1 : 0));
/* Stepping backward from pindex, mpred doesn't change. */
for (int i = 0; i < *rbehind; i++) {
- m = vm_page_alloc_after(object, pindex - i - 1,
+ m = vm_page_alloc_after(object, &pages, pindex - i - 1,
VM_ALLOC_NORMAL, mpred);
if (m == NULL) {
/* Shift the array. */
@@ -2305,12 +2307,12 @@ vm_object_prepare_buf_pages(vm_object_t object, vm_page_t *ma_dst, int count,
if (*rahead != 0) {
m = ma_src[count - 1];
pindex = m->pindex + 1;
- msucc = TAILQ_NEXT(m, listq);
+ msucc = vm_radix_iter_lookup_ge(&pages, pindex);
*rahead = MIN(*rahead,
(msucc != NULL ? msucc->pindex : object->size) - pindex);
mpred = m;
for (int i = 0; i < *rahead; i++) {
- m = vm_page_alloc_after(object, pindex + i,
+ m = vm_page_alloc_after(object, &pages, pindex + i,
VM_ALLOC_NORMAL, mpred);
if (m == NULL) {
*rahead = i;
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index f1a891ff6955..65d278caf3fd 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -171,8 +171,6 @@ static void vm_page_enqueue(vm_page_t m, uint8_t queue);
static bool vm_page_free_prep(vm_page_t m);
static void vm_page_free_toq(vm_page_t m);
static void vm_page_init(void *dummy);
-static int vm_page_insert_after(vm_page_t m, vm_object_t object,
- vm_pindex_t pindex, vm_page_t mpred);
static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,
vm_page_t mpred);
static void vm_page_mvqueue(vm_page_t m, const uint8_t queue,
@@ -1473,18 +1471,17 @@ vm_page_dirty_KBI(vm_page_t m)
/*
* Insert the given page into the given object at the given pindex. mpred is
- * used for memq linkage. From vm_page_insert, lookup is true, mpred is
- * initially NULL, and this procedure looks it up. From vm_page_insert_after
- * and vm_page_iter_insert, lookup is false and mpred is known to the caller
- * to be valid, and may be NULL if this will be the page with the lowest
- * pindex.
+ * used for memq linkage. From vm_page_insert, iter is false, mpred is
+ * initially NULL, and this procedure looks it up. From vm_page_iter_insert,
+ * iter is true and mpred is known to the caller to be valid, and may be NULL if
+ * this will be the page with the lowest pindex.
*
* The procedure is marked __always_inline to suggest to the compiler to
* eliminate the lookup parameter and the associated alternate branch.
*/
static __always_inline int
vm_page_insert_lookup(vm_page_t m, vm_object_t object, vm_pindex_t pindex,
- struct pctrie_iter *pages, bool iter, vm_page_t mpred, bool lookup)
+ struct pctrie_iter *pages, bool iter, vm_page_t mpred)
{
int error;
@@ -1503,13 +1500,10 @@ vm_page_insert_lookup(vm_page_t m, vm_object_t object, vm_pindex_t pindex,
* Add this page to the object's radix tree, and look up mpred if
* needed.
*/
- if (iter) {
- KASSERT(!lookup, ("%s: cannot lookup mpred", __func__));
+ if (iter)
error = vm_radix_iter_insert(pages, m);
- } else if (lookup)
- error = vm_radix_insert_lookup_lt(&object->rtree, m, &mpred);
else
- error = vm_radix_insert(&object->rtree, m);
+ error = vm_radix_insert_lookup_lt(&object->rtree, m, &mpred);
if (__predict_false(error != 0)) {
m->object = NULL;
m->pindex = 0;
@@ -1535,26 +1529,7 @@ vm_page_insert_lookup(vm_page_t m, vm_object_t object, vm_pindex_t pindex,
int
vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
{
- return (vm_page_insert_lookup(m, object, pindex, NULL, false, NULL,
- true));
-}
-
-/*
- * vm_page_insert_after:
- *
- * Inserts the page "m" into the specified object at offset "pindex".
- *
- * The page "mpred" must immediately precede the offset "pindex" within
- * the specified object.
- *
- * The object must be locked.
- */
-static int
-vm_page_insert_after(vm_page_t m, vm_object_t object, vm_pindex_t pindex,
- vm_page_t mpred)
-{
- return (vm_page_insert_lookup(m, object, pindex, NULL, false, mpred,
- false));
+ return (vm_page_insert_lookup(m, object, pindex, NULL, false, NULL));
}
/*
@@ -1573,8 +1548,7 @@ static int
vm_page_iter_insert(struct pctrie_iter *pages, vm_page_t m, vm_object_t object,
vm_pindex_t pindex, vm_page_t mpred)
{
- return (vm_page_insert_lookup(m, object, pindex, pages, true, mpred,
- false));
+ return (vm_page_insert_lookup(m, object, pindex, pages, true, mpred));
}
/*
@@ -2125,8 +2099,10 @@ vm_page_mpred(vm_object_t object, vm_pindex_t pindex)
vm_page_t
vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
{
+ struct pctrie_iter pages;
- return (vm_page_alloc_after(object, pindex, req,
+ vm_page_iter_init(&pages, object);
+ return (vm_page_alloc_after(object, &pages, pindex, req,
vm_page_mpred(object, pindex)));
}
@@ -2137,8 +2113,8 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
* page index, or NULL if no such page exists.
*/
vm_page_t
-vm_page_alloc_after(vm_object_t object, vm_pindex_t pindex,
- int req, vm_page_t mpred)
+vm_page_alloc_after(vm_object_t object, struct pctrie_iter *pages,
+ vm_pindex_t pindex, int req, vm_page_t mpred)
{
struct vm_domainset_iter di;
vm_page_t m;
@@ -2146,8 +2122,8 @@ vm_page_alloc_after(vm_object_t object, vm_pindex_t pindex,
vm_domainset_iter_page_init(&di, object, pindex, &domain, &req);
do {
- m = vm_page_alloc_domain_after(object, pindex, domain, req,
- mpred);
+ m = vm_page_alloc_domain_after(object, pages, pindex, domain,
+ req, mpred);
if (m != NULL)
break;
} while (vm_domainset_iter_page(&di, object, &domain) == 0);
@@ -2209,8 +2185,8 @@ vm_domain_allocate(struct vm_domain *vmd, int req, int npages)
}
vm_page_t
-vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain,
- int req, vm_page_t mpred)
+vm_page_alloc_domain_after(vm_object_t object, struct pctrie_iter *pages,
+ vm_pindex_t pindex, int domain, int req, vm_page_t mpred)
{
struct vm_domain *vmd;
vm_page_t m;
@@ -2279,6 +2255,7 @@ again:
/*
* Not allocatable, give up.
*/
+ pctrie_iter_reset(pages);
if (vm_domain_alloc_fail(vmd, object, req))
goto again;
return (NULL);
@@ -2315,7 +2292,7 @@ found:
}
m->a.act_count = 0;
- if (vm_page_insert_after(m, object, pindex, mpred)) {
+ if (vm_page_insert_lookup(m, object, pindex, pages, true, mpred)) {
if (req & VM_ALLOC_WIRED) {
vm_wire_sub(1);
m->ref_count = 0;
@@ -2328,6 +2305,7 @@ found:
if (req & VM_ALLOC_WAITFAIL) {
VM_OBJECT_WUNLOCK(object);
vm_radix_wait();
+ pctrie_iter_reset(pages);
VM_OBJECT_WLOCK(object);
}
return (NULL);
@@ -2471,7 +2449,7 @@ vm_page_alloc_contig_domain(vm_object_t object, vm_pindex_t pindex, int domain,
KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero"));
vm_page_iter_init(&pages, object);
- mpred = vm_radix_iter_lookup_le(&pages, pindex);
+ mpred = vm_radix_iter_lookup_lt(&pages, pindex);
KASSERT(mpred == NULL || mpred->pindex != pindex,
("vm_page_alloc_contig: pindex already allocated"));
for (;;) {
@@ -4875,32 +4853,32 @@ vm_page_grab_lookup(struct pctrie_iter *pages, vm_object_t object,
}
/*
- * Grab a page. Keep on waiting, as long as the page exists in the object. If
- * the page doesn't exist, first allocate it and then conditionally zero it.
+ * Grab a page. Use an iterator parameter. Keep on waiting, as long as the page
+ * exists in the object. If the page doesn't exist, first allocate it and then
+ * conditionally zero it.
*
* The object must be locked on entry. This routine may sleep. The lock will,
* however, be released and reacquired if the routine sleeps.
*/
vm_page_t
-vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
+vm_page_grab_iter(vm_object_t object, struct pctrie_iter *pages,
+ vm_pindex_t pindex, int allocflags)
{
- struct pctrie_iter pages;
vm_page_t m, mpred;
bool found;
VM_OBJECT_ASSERT_WLOCKED(object);
vm_page_grab_check(allocflags);
- vm_page_iter_init(&pages, object);
while ((m = vm_page_grab_lookup(
- &pages, object, pindex, allocflags, &found)) == NULL) {
+ pages, object, pindex, allocflags, &found)) == NULL) {
if ((allocflags & VM_ALLOC_NOCREAT) != 0)
return (NULL);
if (found &&
(allocflags & (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL)) != 0)
return (NULL);
- mpred = vm_radix_iter_lookup_le(&pages, pindex);
- m = vm_page_alloc_after(object, pindex,
+ mpred = vm_radix_iter_lookup_lt(pages, pindex);
+ m = vm_page_alloc_after(object, pages, pindex,
vm_page_grab_pflags(allocflags), mpred);
if (m != NULL) {
if ((allocflags & VM_ALLOC_ZERO) != 0 &&
@@ -4911,13 +4889,29 @@ vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
if ((allocflags &
(VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL)) != 0)
return (NULL);
- pctrie_iter_reset(&pages);
}
vm_page_grab_release(m, allocflags);
return (m);
}
+/*
+ * Grab a page. Keep on waiting, as long as the page exists in the object. If
+ * the page doesn't exist, first allocate it and then conditionally zero it.
+ *
+ * The object must be locked on entry. This routine may sleep. The lock will,
+ * however, be released and reacquired if the routine sleeps.
+ */
+vm_page_t
+vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
+{
+ struct pctrie_iter pages;
+
+ VM_OBJECT_ASSERT_WLOCKED(object);
+ vm_page_iter_init(&pages, object);
+ return (vm_page_grab_iter(object, &pages, pindex, allocflags));
+}
+
/*
* Attempt to validate a page, locklessly acquiring it if necessary, given a
* (object, pindex) tuple and either an invalided page or NULL. The resulting
@@ -4998,9 +4992,11 @@ vm_page_grab_unlocked(vm_object_t object, vm_pindex_t pindex, int allocflags)
* will neither be wired nor busy regardless of allocflags.
*/
int
-vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, int allocflags)
+vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex,
+ int allocflags)
{
- vm_page_t m;
+ struct pctrie_iter pages;
+ vm_page_t m, mpred;
vm_page_t ma[VM_INITIAL_PAGEIN];
int after, i, pflags, rv;
@@ -5014,9 +5010,10 @@ vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, int al
pflags = allocflags & ~(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY |
VM_ALLOC_WIRED | VM_ALLOC_IGN_SBUSY);
pflags |= VM_ALLOC_WAITFAIL;
+ vm_page_iter_init(&pages, object);
retrylookup:
- if ((m = vm_page_lookup(object, pindex)) != NULL) {
+ if ((m = vm_radix_iter_lookup(&pages, pindex)) != NULL) {
/*
* If the page is fully valid it can only become invalid
* with the object lock held. If it is not valid it can
@@ -5030,6 +5027,7 @@ retrylookup:
vm_page_all_valid(m) ? allocflags : 0)) {
(void)vm_page_grab_sleep(object, m, pindex, "pgrbwt",
allocflags, true);
+ pctrie_iter_reset(&pages);
goto retrylookup;
}
if (vm_page_all_valid(m))
@@ -5042,12 +5040,16 @@ retrylookup:
} else if ((allocflags & VM_ALLOC_NOCREAT) != 0) {
*mp = NULL;
return (VM_PAGER_FAIL);
- } else if ((m = vm_page_alloc(object, pindex, pflags)) == NULL) {
- if (!vm_pager_can_alloc_page(object, pindex)) {
- *mp = NULL;
- return (VM_PAGER_AGAIN);
+ } else {
+ mpred = vm_radix_iter_lookup_lt(&pages, pindex);
+ m = vm_page_alloc_after(object, &pages, pindex, pflags, mpred);
+ if (m == NULL) {
+ if (!vm_pager_can_alloc_page(object, pindex)) {
+ *mp = NULL;
+ return (VM_PAGER_AGAIN);
+ }
+ goto retrylookup;
}
- goto retrylookup;
}
vm_page_assert_xbusied(m);
@@ -5055,18 +5057,17 @@ retrylookup:
after = MIN(after, VM_INITIAL_PAGEIN);
after = MIN(after, allocflags >> VM_ALLOC_COUNT_SHIFT);
after = MAX(after, 1);
- ma[0] = m;
+ ma[0] = mpred = m;
for (i = 1; i < after; i++) {
- if ((ma[i] = vm_page_next(ma[i - 1])) != NULL) {
- if (vm_page_any_valid(ma[i]) ||
- !vm_page_tryxbusy(ma[i]))
- break;
- } else {
- ma[i] = vm_page_alloc_after(object,
- m->pindex + i, VM_ALLOC_NORMAL, ma[i - 1]);
- if (ma[i] == NULL)
+ m = vm_radix_iter_lookup(&pages, pindex + i);
+ if (m == NULL) {
+ m = vm_page_alloc_after(object, &pages,
+ pindex + i, VM_ALLOC_NORMAL, mpred);
+ if (m == NULL)
break;
- }
+ } else if (vm_page_any_valid(m) || !vm_page_tryxbusy(m))
+ break;
+ mpred = ma[i] = m;
}
after = i;
vm_object_pip_add(object, after);
@@ -5131,8 +5132,8 @@ vm_page_grab_zero_partial(vm_object_t object, vm_pindex_t pindex, int base,
&pages, object, pindex, allocflags, &found)) == NULL) {
if (!vm_pager_has_page(object, pindex, NULL, NULL))
return (0);
- mpred = vm_radix_iter_lookup_le(&pages, pindex);
- m = vm_page_alloc_after(object, pindex,
+ mpred = vm_radix_iter_lookup_lt(&pages, pindex);
+ m = vm_page_alloc_after(object, &pages, pindex,
vm_page_grab_pflags(allocflags), mpred);
if (m != NULL) {
vm_object_pip_add(object, 1);
@@ -5154,7 +5155,6 @@ vm_page_grab_zero_partial(vm_object_t object, vm_pindex_t pindex, int base,
vm_page_launder(m);
break;
}
- pctrie_iter_reset(&pages);
}
pmap_zero_page_area(m, base, end - base);
@@ -5244,6 +5244,7 @@ int
vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
vm_page_t *ma, int count)
{
+ struct pctrie_iter pages;
vm_page_t m, mpred;
int pflags;
int i;
@@ -5257,25 +5258,24 @@ vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
pflags = vm_page_grab_pflags(allocflags);
i = 0;
+ vm_page_iter_init(&pages, object);
retrylookup:
- m = vm_page_mpred(object, pindex + i);
- if (m == NULL || m->pindex != pindex + i) {
- mpred = m;
- m = NULL;
- } else
- mpred = TAILQ_PREV(m, pglist, listq);
+ mpred = vm_radix_iter_lookup_lt(&pages, pindex + i);
for (; i < count; i++) {
+ m = vm_radix_iter_lookup(&pages, pindex + i);
if (m != NULL) {
if (!vm_page_tryacquire(m, allocflags)) {
if (vm_page_grab_sleep(object, m, pindex + i,
- "grbmaw", allocflags, true))
+ "grbmaw", allocflags, true)) {
+ pctrie_iter_reset(&pages);
goto retrylookup;
+ }
break;
}
} else {
if ((allocflags & VM_ALLOC_NOCREAT) != 0)
break;
- m = vm_page_alloc_after(object, pindex + i,
+ m = vm_page_alloc_after(object, &pages, pindex + i,
pflags | VM_ALLOC_COUNT(count - i), mpred);
if (m == NULL) {
if ((allocflags & (VM_ALLOC_NOWAIT |
@@ -5292,7 +5292,6 @@ retrylookup:
}
vm_page_grab_release(m, allocflags);
ma[i] = mpred = m;
- m = vm_page_next(m);
}
return (i);
}
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 1cac80c6a8ef..0a40b42ffb62 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -608,9 +608,10 @@ void vm_page_activate (vm_page_t);
void vm_page_advise(vm_page_t m, int advice);
vm_page_t vm_page_mpred(vm_object_t, vm_pindex_t);
vm_page_t vm_page_alloc(vm_object_t, vm_pindex_t, int);
-vm_page_t vm_page_alloc_after(vm_object_t, vm_pindex_t, int, vm_page_t);
-vm_page_t vm_page_alloc_domain_after(vm_object_t, vm_pindex_t, int, int,
- vm_page_t);
+vm_page_t vm_page_alloc_after(vm_object_t, struct pctrie_iter *, vm_pindex_t,
+ int, vm_page_t);
+vm_page_t vm_page_alloc_domain_after(vm_object_t, struct pctrie_iter *,
+ vm_pindex_t, int, int, vm_page_t);
vm_page_t vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
vm_paddr_t boundary, vm_memattr_t memattr);
@@ -631,6 +632,8 @@ bool vm_page_blacklist_add(vm_paddr_t pa, bool verbose);
int vm_page_grab_zero_partial(vm_object_t object, vm_pindex_t pindex, int base,
int end);
vm_page_t vm_page_grab(vm_object_t, vm_pindex_t, int);
+vm_page_t vm_page_grab_iter(vm_object_t object, struct pctrie_iter *pages, vm_pindex_t pindex,
+ int allocflags);
vm_page_t vm_page_grab_unlocked(vm_object_t, vm_pindex_t, int);
int vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
vm_page_t *ma, int count);