From nobody Sat Nov 30 17:35:20 2024 X-Original-To: dev-commits-src-all@mlmmj.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mlmmj.nyi.freebsd.org (Postfix) with ESMTP id 4Y0xyd2drrz5fMxD; Sat, 30 Nov 2024 17:35:21 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R10" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4Y0xyd03mXz45L5; Sat, 30 Nov 2024 17:35:21 +0000 (UTC) (envelope-from git@FreeBSD.org) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=freebsd.org; s=dkim; t=1732988121; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=QtGpj47a+NL5zfEDsedZvgJK3Qj88xIBVCmPtcXG/CM=; b=W5bFyx9nUTZgmWvRXmLGTSgNHFJiW3hbw0NTukPqjSCrE+V89B7o+hbyyg+/9TBcuhuoKA xuk21vTLR4dSD7DMBhvlX58n87vaimz7zDuWD1NpaJl7JW+4dKkjhmg6fKLd6T/aL0aEdR 94cvn7438hr0I69NttFTanZ5ZU+pnY+O1rO76uEiCg68/YryYn95Ur9Xk6EpR2AkJ0p+Q2 7BNOoRsPaj4DglJiNcOzYXlkvdukGGXvhg37sAcec0z7l2X0uMIBL47foRdA3M//y/b7U/ SoSLxhzjVXgzIlzx9Mk0SVr9o7Oui7AON0e6ubv4e0TMp8fMOxMMhhRI3tZwhw== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=freebsd.org; s=dkim; t=1732988121; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=QtGpj47a+NL5zfEDsedZvgJK3Qj88xIBVCmPtcXG/CM=; b=gO8sf7vZurDl0X4DwzgvUV7gc5TfhibyPhnj19O+Bh/zfVFGq9+5fZmm7k+IyhYFeXtV9X VBaKmCwtnydtsf6/rIMSKl/AZ4JxmKXK9cso3ICe3vM9c1IMCensLzcXtbxX6kogbYpw7X 58evF4nze1E1YnADOOjtF60OqBmqwQevr1WBgsQJPPo2OZUVI0ac4U2o9T5z5/0UK5ypTh JZQznoqzvbOO8GNH1E+C9+oEzOT8v3uo53oAPzByStMh67HVwxYJMwCZxSh1mvdF6PZw4c hMrH/5vfPsE/YB2+1Zd2hMpGa3JzMs5y+mSnwAYsdmg/w6k8oz1KJUrVTcGZyg== ARC-Authentication-Results: i=1; mx1.freebsd.org; none ARC-Seal: i=1; s=dkim; d=freebsd.org; t=1732988121; a=rsa-sha256; cv=none; b=GQhkx5aM6YH4J8RAB7r1xCdel4BdX0AlUo3jepHbLlslbbIrShg9rKEtNlCMBkdAVWLe5G Zt00p5SPUoiPe+M4n2mYO63z9J6Utz66y+Qjk+1e6xJbPnEi6eUJ7YXqfuufd79gw1hsSt OgKrUzzPtrV+yBgzvUAfZ2JfnTdue4O1T8z/1tkoJGebPsxk1I28VXA0O/CJrr1ACzsq7z Buplf2R8gzxc3AymDx90DrbVGLDIBJefN8p5lbwcGxAySTGCwdqZf7NCWJoLKT8rHswjTD U7fP5cIr4NfcJhMZIErnq11ZkC0hSq1Ic9mjO1KZBM/dRwUlTpaRp+/3HKyvkA== Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 4Y0xyc6myZzHfK; Sat, 30 Nov 2024 17:35:20 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.18.1/8.18.1) with ESMTP id 4AUHZKqW028956; Sat, 30 Nov 2024 17:35:20 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.18.1/8.18.1/Submit) id 4AUHZKpo028953; Sat, 30 Nov 2024 17:35:20 GMT (envelope-from git) Date: Sat, 30 Nov 2024 17:35:20 GMT Message-Id: <202411301735.4AUHZKpo028953@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org From: Alan Cox Subject: git: c296ac7e0f1c - main - vm: Optimize page rename List-Id: Commit messages for all branches of the src repository List-Archive: https://lists.freebsd.org/archives/dev-commits-src-all List-Help: List-Post: List-Subscribe: List-Unsubscribe: X-BeenThere: dev-commits-src-all@freebsd.org Sender: owner-dev-commits-src-all@FreeBSD.org MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: alc X-Git-Repository: src X-Git-Refname: refs/heads/main X-Git-Reftype: branch X-Git-Commit: c296ac7e0f1c2fc9bc8bcab0177afb123ce6993a Auto-Submitted: auto-generated The branch main has been updated by alc: URL: https://cgit.FreeBSD.org/src/commit/?id=c296ac7e0f1c2fc9bc8bcab0177afb123ce6993a commit c296ac7e0f1c2fc9bc8bcab0177afb123ce6993a Author: Alan Cox AuthorDate: 2024-11-27 08:32:07 +0000 Commit: Alan Cox CommitDate: 2024-11-30 08:59:15 +0000 vm: Optimize page rename Rename vm_page_rename() to vm_page_iter_rename() to reflect its reimplementation using iterators, and pass the page to this function rather than spending clock cycles looking it up. Change its return value from 0/1 to a bool. Reviewed by: dougm, markj Differential Revision: https://reviews.freebsd.org/D47829 --- sys/vm/vm_object.c | 10 ++++++---- sys/vm/vm_page.c | 28 ++++++++++++++++------------ sys/vm/vm_page.h | 3 ++- sys/vm/vm_reserv.c | 4 ++-- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index 84981d7cc7cd..ff95469749b7 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -1604,8 +1604,9 @@ retry: continue; } - /* vm_page_rename() will dirty the page. */ - if (vm_page_rename(&pages, new_object, m->pindex - offidxstart)) { + /* vm_page_iter_rename() will dirty the page. */ + if (!vm_page_iter_rename(&pages, m, new_object, m->pindex - + offidxstart)) { vm_page_xunbusy(m); VM_OBJECT_WUNLOCK(new_object); VM_OBJECT_WUNLOCK(orig_object); @@ -1789,9 +1790,10 @@ vm_object_collapse_scan(vm_object_t object) * backing object to the main object. * * If the page was mapped to a process, it can remain mapped - * through the rename. vm_page_rename() will dirty the page. + * through the rename. vm_page_iter_rename() will dirty the + * page. */ - if (vm_page_rename(&pages, object, new_pindex)) { + if (!vm_page_iter_rename(&pages, p, object, new_pindex)) { vm_page_xunbusy(p); next = vm_object_collapse_scan_wait(&pages, object, NULL); diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index a37619c7743e..8a23f900e987 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -2065,10 +2065,14 @@ vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex, } /* - * vm_page_rename: + * vm_page_iter_rename: * - * Move the current page, as identified by iterator, from its current - * object to the specified target object/offset. + * Tries to move the specified page from its current object to a new object + * and pindex, using the given iterator to remove the page from its current + * object. Returns true if the move was successful, and false if the move + * was aborted due to a failed memory allocation. + * + * Panics if a page already resides in the new object at the new pindex. * * Note: swap associated with the page must be invalidated by the move. We * have to do this for several reasons: (1) we aren't freeing the @@ -2082,18 +2086,18 @@ vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex, * * The objects must be locked. */ -int -vm_page_rename(struct pctrie_iter *pages, +bool +vm_page_iter_rename(struct pctrie_iter *old_pages, vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex) { - vm_page_t m, mpred; + vm_page_t mpred; vm_pindex_t opidx; + KASSERT((m->ref_count & VPRC_OBJREF) != 0, + ("%s: page %p is missing object ref", __func__, m)); + VM_OBJECT_ASSERT_WLOCKED(m->object); VM_OBJECT_ASSERT_WLOCKED(new_object); - m = vm_radix_iter_page(pages); - KASSERT(m->ref_count != 0, ("vm_page_rename: page %p has no refs", m)); - /* * Create a custom version of vm_page_insert() which does not depend * by m_prev and can cheat on the implementation aspects of the @@ -2103,7 +2107,7 @@ vm_page_rename(struct pctrie_iter *pages, m->pindex = new_pindex; if (vm_radix_insert_lookup_lt(&new_object->rtree, m, &mpred) != 0) { m->pindex = opidx; - return (1); + return (false); } /* @@ -2111,7 +2115,7 @@ vm_page_rename(struct pctrie_iter *pages, * the listq iterator is tainted. */ m->pindex = opidx; - vm_radix_iter_remove(pages); + vm_radix_iter_remove(old_pages); vm_page_remove_radixdone(m); /* Return back to the new pindex to complete vm_page_insert(). */ @@ -2121,7 +2125,7 @@ vm_page_rename(struct pctrie_iter *pages, vm_page_insert_radixdone(m, new_object, mpred); vm_page_dirty(m); vm_pager_page_inserted(new_object, m); - return (0); + return (true); } /* diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index f641942efb87..29344eadbf05 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -655,6 +655,8 @@ void vm_page_iter_init(struct pctrie_iter *, vm_object_t); void vm_page_iter_limit_init(struct pctrie_iter *, vm_object_t, vm_pindex_t); vm_page_t vm_page_iter_lookup(struct pctrie_iter *, vm_pindex_t); bool vm_page_iter_remove(struct pctrie_iter *pages); +bool vm_page_iter_rename(struct pctrie_iter *old_pages, vm_page_t m, + vm_object_t new_object, vm_pindex_t new_pindex); void vm_page_launder(vm_page_t m); vm_page_t vm_page_lookup(vm_object_t, vm_pindex_t); vm_page_t vm_page_lookup_unlocked(vm_object_t, vm_pindex_t); @@ -682,7 +684,6 @@ void vm_page_release_locked(vm_page_t m, int flags); vm_page_t vm_page_relookup(vm_object_t, vm_pindex_t); bool vm_page_remove(vm_page_t); bool vm_page_remove_xbusy(vm_page_t); -int vm_page_rename(struct pctrie_iter *, vm_object_t, vm_pindex_t); void vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex, vm_page_t mold); int vm_page_sbusied(vm_page_t m); diff --git a/sys/vm/vm_reserv.c b/sys/vm/vm_reserv.c index aa20eca129fb..ad513962d50d 100644 --- a/sys/vm/vm_reserv.c +++ b/sys/vm/vm_reserv.c @@ -678,7 +678,7 @@ vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t pindex, int domain, if (pa < low || pa + size > high || !vm_addr_ok(pa, size, alignment, boundary)) goto out; - /* Handle vm_page_rename(m, new_object, ...). */ + /* Handle vm_page_iter_rename(..., m, new_object, ...). */ if (!bit_ntest(rv->popmap, index, index + npages - 1, 0)) goto out; if (!vm_domain_allocate(vmd, req, npages)) @@ -847,7 +847,7 @@ vm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex, int domain, vm_reserv_lock(rv); /* Handle reclaim race. */ if (rv->object != object || - /* Handle vm_page_rename(m, new_object, ...). */ + /* Handle vm_page_iter_rename(..., m, new_object, ...). */ bit_test(rv->popmap, index)) { m = NULL; goto out;