git: f06cbfc5592e - main - vm_fault: use iterator for allocation

The branch main has been updated by dougm:

URL: https://cgit.FreeBSD.org/src/commit/?id=f06cbfc5592e58104fe01f3f458928ee09be24fc

commit f06cbfc5592e58104fe01f3f458928ee09be24fc
Author:     Doug Moore <dougm@FreeBSD.org>
AuthorDate: 2025-04-10 07:19:06 +0000
Commit:     Doug Moore <dougm@FreeBSD.org>
CommitDate: 2025-04-10 07:19:06 +0000

    vm_fault: use iterator for allocation
    
    Pass an iterator to vm_fault_allocate, so that it can invoke
    vm_page_alloc_after. In vm_fault_object, use an iterator for a page
    lookup so that the search can benefit the allocation in
    vm_fault_allocate. In vm_fault, define an iterator, initialize it only
    when it might be used, and pass it on to vm_fault_allocate.
    
    Reviewed by:    kib
    Differential Revision:  https://reviews.freebsd.org/D49741
---
 sys/vm/vm_fault.c | 18 ++++++++++++------
 sys/vm/vm_radix.h | 13 +++++++++++++
 2 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index 48a7a47e4c59..81631b672040 100644
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -105,6 +105,7 @@
 #include <vm/vm_pageout.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_pager.h>
+#include <vm/vm_radix.h>
 #include <vm/vm_extern.h>
 #include <vm/vm_reserv.h>
 
@@ -1230,7 +1231,7 @@ vm_fault_allocate_oom(struct faultstate *fs)
  * Allocate a page directly or via the object populate method.
  */
 static enum fault_status
-vm_fault_allocate(struct faultstate *fs)
+vm_fault_allocate(struct faultstate *fs, struct pctrie_iter *pages)
 {
 	struct domainset *dset;
 	enum fault_status res;
@@ -1291,8 +1292,9 @@ vm_fault_allocate(struct faultstate *fs)
 			vm_fault_unlock_and_deallocate(fs);
 			return (FAULT_FAILURE);
 		}
-		fs->m = vm_page_alloc(fs->object, fs->pindex,
-		    P_KILLED(curproc) ? VM_ALLOC_SYSTEM : 0);
+		fs->m = vm_page_alloc_after(fs->object, fs->pindex,
+		    P_KILLED(curproc) ? VM_ALLOC_SYSTEM : 0,
+		    vm_radix_iter_lookup_lt(pages, fs->pindex));
 	}
 	if (fs->m == NULL) {
 		if (vm_fault_allocate_oom(fs))
@@ -1459,6 +1461,7 @@ vm_fault_busy_sleep(struct faultstate *fs)
 static enum fault_status
 vm_fault_object(struct faultstate *fs, int *behindp, int *aheadp)
 {
+	struct pctrie_iter pages;
 	enum fault_status res;
 	bool dead;
 
@@ -1484,7 +1487,8 @@ vm_fault_object(struct faultstate *fs, int *behindp, int *aheadp)
 	/*
 	 * See if the page is resident.
 	 */
-	fs->m = vm_page_lookup(fs->object, fs->pindex);
+	vm_page_iter_init(&pages, fs->object);
+	fs->m = vm_radix_iter_lookup(&pages, fs->pindex);
 	if (fs->m != NULL) {
 		if (!vm_page_tryxbusy(fs->m)) {
 			vm_fault_busy_sleep(fs);
@@ -1514,7 +1518,7 @@ vm_fault_object(struct faultstate *fs, int *behindp, int *aheadp)
 			vm_fault_unlock_and_deallocate(fs);
 			return (FAULT_RESTART);
 		}
-		res = vm_fault_allocate(fs);
+		res = vm_fault_allocate(fs, &pages);
 		if (res != FAULT_CONTINUE)
 			return (res);
 	}
@@ -1549,6 +1553,7 @@ int
 vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
     int fault_flags, vm_page_t *m_hold)
 {
+	struct pctrie_iter pages;
 	struct faultstate fs;
 	int ahead, behind, faultcount, rv;
 	enum fault_status res;
@@ -1603,6 +1608,7 @@ RetryFault:
 		}
 		VM_OBJECT_ASSERT_WLOCKED(fs.first_object);
 	} else {
+		vm_page_iter_init(&pages, fs.first_object);
 		VM_OBJECT_WLOCK(fs.first_object);
 	}
 
@@ -1627,7 +1633,7 @@ RetryFault:
 	fs.pindex = fs.first_pindex;
 
 	if ((fs.entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) != 0) {
-		res = vm_fault_allocate(&fs);
+		res = vm_fault_allocate(&fs, &pages);
 		switch (res) {
 		case FAULT_RESTART:
 			goto RetryFault;
diff --git a/sys/vm/vm_radix.h b/sys/vm/vm_radix.h
index 561e36f2e164..e9ee3e7f3911 100644
--- a/sys/vm/vm_radix.h
+++ b/sys/vm/vm_radix.h
@@ -270,6 +270,19 @@ vm_radix_iter_lookup_le(struct pctrie_iter *pages, vm_pindex_t index)
 	return (VM_RADIX_PCTRIE_ITER_LOOKUP_LE(pages, index));
 }
 
+/*
+ * Initialize an iterator pointing to the page with the greatest pindex that is
+ * less than to the specified pindex, or NULL if there are no such
+ * pages.  Return the page.
+ *
+ * Requires that access be externally synchronized by a lock.
+ */
+static __inline vm_page_t
+vm_radix_iter_lookup_lt(struct pctrie_iter *pages, vm_pindex_t index)
+{
+	return (index == 0 ? NULL : vm_radix_iter_lookup_le(pages, index - 1));
+}
+
 /*
  * Update the iterator to point to the page with the pindex that is one greater
  * than the current pindex, or NULL if there is no such page.  Return the page.