git: c296ac7e0f1c - main - vm: Optimize page rename

The branch main has been updated by alc:

URL: https://cgit.FreeBSD.org/src/commit/?id=c296ac7e0f1c2fc9bc8bcab0177afb123ce6993a

commit c296ac7e0f1c2fc9bc8bcab0177afb123ce6993a
Author:     Alan Cox <alc@FreeBSD.org>
AuthorDate: 2024-11-27 08:32:07 +0000
Commit:     Alan Cox <alc@FreeBSD.org>
CommitDate: 2024-11-30 08:59:15 +0000

    vm: Optimize page rename
    
    Rename vm_page_rename() to vm_page_iter_rename() to reflect its
    reimplementation using iterators, and pass the page to this function
    rather than spending clock cycles looking it up.  Change its return
    value from 0/1 to a bool.
    
    Reviewed by:    dougm, markj
    Differential Revision:  https://reviews.freebsd.org/D47829
---
 sys/vm/vm_object.c | 10 ++++++----
 sys/vm/vm_page.c   | 28 ++++++++++++++++------------
 sys/vm/vm_page.h   |  3 ++-
 sys/vm/vm_reserv.c |  4 ++--
 4 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 84981d7cc7cd..ff95469749b7 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -1604,8 +1604,9 @@ retry:
 			continue;
 		}
 
-		/* vm_page_rename() will dirty the page. */
-		if (vm_page_rename(&pages, new_object, m->pindex - offidxstart)) {
+		/* vm_page_iter_rename() will dirty the page. */
+		if (!vm_page_iter_rename(&pages, m, new_object, m->pindex -
+		    offidxstart)) {
 			vm_page_xunbusy(m);
 			VM_OBJECT_WUNLOCK(new_object);
 			VM_OBJECT_WUNLOCK(orig_object);
@@ -1789,9 +1790,10 @@ vm_object_collapse_scan(vm_object_t object)
 		 * backing object to the main object.
 		 *
 		 * If the page was mapped to a process, it can remain mapped
-		 * through the rename.  vm_page_rename() will dirty the page.
+		 * through the rename.  vm_page_iter_rename() will dirty the
+		 * page.
 		 */
-		if (vm_page_rename(&pages, object, new_pindex)) {
+		if (!vm_page_iter_rename(&pages, p, object, new_pindex)) {
 			vm_page_xunbusy(p);
 			next = vm_object_collapse_scan_wait(&pages, object,
 			    NULL);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index a37619c7743e..8a23f900e987 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -2065,10 +2065,14 @@ vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex,
 }
 
 /*
- *	vm_page_rename:
+ *	vm_page_iter_rename:
  *
- *	Move the current page, as identified by iterator, from its current
- *	object to the specified target object/offset.
+ *	Tries to move the specified page from its current object to a new object
+ *	and pindex, using the given iterator to remove the page from its current
+ *	object.  Returns true if the move was successful, and false if the move
+ *	was aborted due to a failed memory allocation.
+ *
+ *	Panics if a page already resides in the new object at the new pindex.
  *
  *	Note: swap associated with the page must be invalidated by the move.  We
  *	      have to do this for several reasons:  (1) we aren't freeing the
@@ -2082,18 +2086,18 @@ vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex,
  *
  *	The objects must be locked.
  */
-int
-vm_page_rename(struct pctrie_iter *pages,
+bool
+vm_page_iter_rename(struct pctrie_iter *old_pages, vm_page_t m,
     vm_object_t new_object, vm_pindex_t new_pindex)
 {
-	vm_page_t m, mpred;
+	vm_page_t mpred;
 	vm_pindex_t opidx;
 
+	KASSERT((m->ref_count & VPRC_OBJREF) != 0,
+	    ("%s: page %p is missing object ref", __func__, m));
+	VM_OBJECT_ASSERT_WLOCKED(m->object);
 	VM_OBJECT_ASSERT_WLOCKED(new_object);
 
-	m = vm_radix_iter_page(pages);
-	KASSERT(m->ref_count != 0, ("vm_page_rename: page %p has no refs", m));
-
 	/*
 	 * Create a custom version of vm_page_insert() which does not depend
 	 * by m_prev and can cheat on the implementation aspects of the
@@ -2103,7 +2107,7 @@ vm_page_rename(struct pctrie_iter *pages,
 	m->pindex = new_pindex;
 	if (vm_radix_insert_lookup_lt(&new_object->rtree, m, &mpred) != 0) {
 		m->pindex = opidx;
-		return (1);
+		return (false);
 	}
 
 	/*
@@ -2111,7 +2115,7 @@ vm_page_rename(struct pctrie_iter *pages,
 	 * the listq iterator is tainted.
 	 */
 	m->pindex = opidx;
-	vm_radix_iter_remove(pages);
+	vm_radix_iter_remove(old_pages);
 	vm_page_remove_radixdone(m);
 
 	/* Return back to the new pindex to complete vm_page_insert(). */
@@ -2121,7 +2125,7 @@ vm_page_rename(struct pctrie_iter *pages,
 	vm_page_insert_radixdone(m, new_object, mpred);
 	vm_page_dirty(m);
 	vm_pager_page_inserted(new_object, m);
-	return (0);
+	return (true);
 }
 
 /*
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index f641942efb87..29344eadbf05 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -655,6 +655,8 @@ void vm_page_iter_init(struct pctrie_iter *, vm_object_t);
 void vm_page_iter_limit_init(struct pctrie_iter *, vm_object_t, vm_pindex_t);
 vm_page_t vm_page_iter_lookup(struct pctrie_iter *, vm_pindex_t);
 bool vm_page_iter_remove(struct pctrie_iter *pages);
+bool vm_page_iter_rename(struct pctrie_iter *old_pages, vm_page_t m,
+    vm_object_t new_object, vm_pindex_t new_pindex);
 void vm_page_launder(vm_page_t m);
 vm_page_t vm_page_lookup(vm_object_t, vm_pindex_t);
 vm_page_t vm_page_lookup_unlocked(vm_object_t, vm_pindex_t);
@@ -682,7 +684,6 @@ void vm_page_release_locked(vm_page_t m, int flags);
 vm_page_t vm_page_relookup(vm_object_t, vm_pindex_t);
 bool vm_page_remove(vm_page_t);
 bool vm_page_remove_xbusy(vm_page_t);
-int vm_page_rename(struct pctrie_iter *, vm_object_t, vm_pindex_t);
 void vm_page_replace(vm_page_t mnew, vm_object_t object,
     vm_pindex_t pindex, vm_page_t mold);
 int vm_page_sbusied(vm_page_t m);
diff --git a/sys/vm/vm_reserv.c b/sys/vm/vm_reserv.c
index aa20eca129fb..ad513962d50d 100644
--- a/sys/vm/vm_reserv.c
+++ b/sys/vm/vm_reserv.c
@@ -678,7 +678,7 @@ vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t pindex, int domain,
 		if (pa < low || pa + size > high ||
 		    !vm_addr_ok(pa, size, alignment, boundary))
 			goto out;
-		/* Handle vm_page_rename(m, new_object, ...). */
+		/* Handle vm_page_iter_rename(..., m, new_object, ...). */
 		if (!bit_ntest(rv->popmap, index, index + npages - 1, 0))
 			goto out;
 		if (!vm_domain_allocate(vmd, req, npages))
@@ -847,7 +847,7 @@ vm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex, int domain,
 		vm_reserv_lock(rv);
 		/* Handle reclaim race. */
 		if (rv->object != object ||
-		    /* Handle vm_page_rename(m, new_object, ...). */
+		    /* Handle vm_page_iter_rename(..., m, new_object, ...). */
 		    bit_test(rv->popmap, index)) {
 			m = NULL;
 			goto out;