From nobody Mon Sep 09 21:46:34 2024 X-Original-To: dev-commits-src-main@mlmmj.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mlmmj.nyi.freebsd.org (Postfix) with ESMTP id 4X2gQM5c8sz5V9F0; Mon, 09 Sep 2024 21:46:35 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R11" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4X2gQM2LRhz3xy1; Mon, 9 Sep 2024 21:46:35 +0000 (UTC) (envelope-from git@FreeBSD.org) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=freebsd.org; s=dkim; t=1725918395; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=K23Ua4itbmge6LkSd22VGVE0FyvRVFGMMl6tAeho4lM=; b=fMBIwkyToaeD2a3y93pTVqe73dCsnCN8BdX1GuSD7BSl/OSAI7o2SRHVGA8RnG1pJ8gkFV id9iMvQ0dZtVik1VxkUDISpj4Q6eSfYT8erbIZ8tMJnHIeClDAItvb++lLwU/n3Q32vNUT 3roNM9b+ma9RcOcVAsZK/UpquMtGQvReEJlqP/DsnC3dv7q//Ye6rtUHNSDsqVVz39MNNw 0L5JYtkn5Ei7d+9dfQOCjhVogNDwE6PPj0J8GT+yLtQINPxlzXl4GzS5KT1ew30U5qfRuP Z4V90X7caFBY/19EtYDJl9elUODDYq8dOr0pGGkG339ag267b5nSeS3WRIjH4g== ARC-Seal: i=1; s=dkim; d=freebsd.org; t=1725918395; a=rsa-sha256; cv=none; b=aPK8FG1V7Um4kx2Qi3mr/41aSJUTpfpkSj/83ZRrzXfss2Ag5zYSCG2U4IMaApQ7oRFw6M yJMKpqlrOjlp/c81yTzYS+tl4Z3tHCZzLM2pPr2sB6jJZ4UNkRxJD2T6WrxdzkImErCZpj 4lqG0cXXWL9Dod4pYWJvDf5uzAy6Oleli6ND5qDATgntkTKTPAdZIvb9YsGTDU5oPdXli0 iugu0MmZhur3y7f+y+TsprdhtIDhaufbcR/ZAFVZeb6DGAmuK4tLlcjY6H9bQrEdmo+gko kzt++slC+/hiblEEBdRn0zIbxW2kKxTmWPaw6U+3BsDYpKAVR8kI8/h6sNafvA== ARC-Authentication-Results: i=1; mx1.freebsd.org; none ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=freebsd.org; s=dkim; t=1725918395; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=K23Ua4itbmge6LkSd22VGVE0FyvRVFGMMl6tAeho4lM=; b=Mg5G0vq7sQT1yWseN5eXUv6uKi74kIOkcI6ItAg+/lyqK/Nk8UMP0f/zqGUxbdYfsxy6ne 87s+Hq5E2HJHyn4nbehMeitM0YwRnY3XelBJvoVg3BbV2y+DN4MQzyakL7sM/EsPLyWdv8 AX3i1EficBzgF2qnOu/JJ/+lN03TCONlRYExENsWSph5fQgIJLxoiRxmt3CLbg9ZnziZBK VAxXrJO+lBLyiaARboEJL0+AajQ2mNnxunCWiMxl8YmdnDSif9ezmSoQnahvyJDkd3ODgx ruA3Czho9mLQc16J9SyCqrxzerSR9ua2anXb1xqdP3PaJNXV//hpnRPoI+DNMQ== Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 4X2gQM10dyzKXL; Mon, 9 Sep 2024 21:46:35 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.18.1/8.18.1) with ESMTP id 489LkY8s023070; Mon, 9 Sep 2024 21:46:34 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.18.1/8.18.1/Submit) id 489LkYBL023066; Mon, 9 Sep 2024 21:46:34 GMT (envelope-from git) Date: Mon, 9 Sep 2024 21:46:34 GMT Message-Id: <202409092146.489LkYBL023066@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org From: Doug Moore Subject: git: 942647057295 - main - swap_pager: clean up pctrie usage List-Id: Commit messages for the main branch of the src repository List-Archive: https://lists.freebsd.org/archives/dev-commits-src-main List-Help: List-Post: List-Subscribe: List-Unsubscribe: X-BeenThere: dev-commits-src-main@freebsd.org Sender: owner-dev-commits-src-main@FreeBSD.org MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: dougm X-Git-Repository: src X-Git-Refname: refs/heads/main X-Git-Reftype: branch X-Git-Commit: 942647057295cd4720372dd85bcf6d1b52fbf38e Auto-Submitted: auto-generated The branch main has been updated by dougm: URL: https://cgit.FreeBSD.org/src/commit/?id=942647057295cd4720372dd85bcf6d1b52fbf38e commit 942647057295cd4720372dd85bcf6d1b52fbf38e Author: Doug Moore AuthorDate: 2024-09-09 21:44:13 +0000 Commit: Doug Moore CommitDate: 2024-09-09 21:44:13 +0000 swap_pager: clean up pctrie usage Define wrapper functions for the pctrie operations specific to swap_pager, to hide some verbose details. Separate the meta_transfer and meta_free functions into separate functions. Reviewed by: kib (previous version) Tested by: pho (previous version) Differential Revision: https://reviews.freebsd.org/D46315 --- sys/vm/swap_pager.c | 231 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 142 insertions(+), 89 deletions(-) diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c index 59d947c71279..c7a9f16a2953 100644 --- a/sys/vm/swap_pager.c +++ b/sys/vm/swap_pager.c @@ -491,7 +491,7 @@ static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t, static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t, vm_size_t *); static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst, - vm_pindex_t pindex, vm_pindex_t count, vm_size_t *freed); + vm_pindex_t pindex, vm_pindex_t count); static void swp_pager_meta_free_all(vm_object_t); static daddr_t swp_pager_meta_lookup(vm_object_t, vm_pindex_t); @@ -531,6 +531,59 @@ swblk_trie_free(struct pctrie *ptree, void *node) PCTRIE_DEFINE(SWAP, swblk, p, swblk_trie_alloc, swblk_trie_free); +static struct swblk * +swblk_lookup(vm_object_t object, vm_pindex_t pindex) +{ + return (SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, + rounddown(pindex, SWAP_META_PAGES))); +} + +static struct swblk * +swblk_start(vm_object_t object, vm_pindex_t pindex) +{ + return (SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, + rounddown(pindex, SWAP_META_PAGES))); +} + +static struct swblk * +swblk_next(vm_object_t object, struct swblk *sb) +{ + return (swblk_start(object, sb->p + SWAP_META_PAGES)); +} + +static struct swblk * +swblk_start_limit(vm_object_t object, vm_pindex_t pindex, vm_pindex_t limit) +{ + struct swblk *sb = swblk_start(object, pindex); + if (sb != NULL && sb->p < limit) + return (sb); + return (NULL); +} + +static struct swblk * +swblk_next_limit(vm_object_t object, struct swblk *sb, vm_pindex_t limit) +{ + return (swblk_start_limit(object, sb->p + SWAP_META_PAGES, limit)); +} + +static void +swblk_lookup_remove(vm_object_t object, struct swblk *sb) +{ + SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); +} + +static int +swblk_lookup_insert(vm_object_t object, struct swblk *sb) +{ + return (SWAP_PCTRIE_INSERT(&object->un_pager.swp.swp_blks, sb)); +} + +static bool +swblk_is_empty(vm_object_t object) +{ + return (pctrie_is_empty(&object->un_pager.swp.swp_blks)); +} + /* * SWP_SIZECHECK() - update swap_pager_full indication * @@ -1084,8 +1137,7 @@ swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject, /* * Transfer source to destination. */ - swp_pager_meta_transfer(srcobject, dstobject, offset, dstobject->size, - NULL); + swp_pager_meta_transfer(srcobject, dstobject, offset, dstobject->size); /* * Free left over swap blocks in source. @@ -1218,8 +1270,7 @@ swap_pager_unswapped(vm_page_t m) } swap_pager_unswapped_acct(m); - sb = SWAP_PCTRIE_LOOKUP(&m->object->un_pager.swp.swp_blks, - rounddown(m->pindex, SWAP_META_PAGES)); + sb = swblk_lookup(m->object, m->pindex); if (sb == NULL) return; range.start = sb->d[m->pindex % SWAP_META_PAGES]; @@ -1777,18 +1828,17 @@ u_long swap_pager_swapped_pages(vm_object_t object) { struct swblk *sb; - vm_pindex_t pi; u_long res; int i; VM_OBJECT_ASSERT_LOCKED(object); - if (pctrie_is_empty(&object->un_pager.swp.swp_blks)) + if (swblk_is_empty(object)) return (0); - for (res = 0, pi = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( - &object->un_pager.swp.swp_blks, pi)) != NULL; - pi = sb->p + SWAP_META_PAGES) { + res = 0; + for (sb = swblk_start(object, 0); sb != NULL; + sb = swblk_next(object, sb)) { for (i = 0; i < SWAP_META_PAGES; i++) { if (sb->d[i] != SWAPBLK_NONE) res++; @@ -1835,16 +1885,14 @@ swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object) if (i == SWAP_META_PAGES) { pi = sb->p + SWAP_META_PAGES; if (sb_empty) { - SWAP_PCTRIE_REMOVE( - &object->un_pager.swp.swp_blks, sb->p); + swblk_lookup_remove(object, sb); uma_zfree(swblk_zone, sb); } i = 0; } if (i == 0) { - sb = SWAP_PCTRIE_LOOKUP_GE( - &object->un_pager.swp.swp_blks, pi); + sb = swblk_start(object, pi); if (sb == NULL) break; sb_empty = true; @@ -2020,7 +2068,7 @@ swp_pager_free_empty_swblk(vm_object_t object, struct swblk *sb) { if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) { - SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); + swblk_lookup_remove(object, sb); uma_zfree(swblk_zone, sb); } } @@ -2050,7 +2098,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk, VM_OBJECT_ASSERT_WLOCKED(object); rdpi = rounddown(pindex, SWAP_META_PAGES); - sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi); + sb = swblk_lookup(object, rdpi); if (sb == NULL) { if (swapblk == SWAPBLK_NONE) return (SWAPBLK_NONE); @@ -2079,8 +2127,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk, } else uma_zwait(swblk_zone); VM_OBJECT_WLOCK(object); - sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, - rdpi); + sb = swblk_lookup(object, rdpi); if (sb != NULL) /* * Somebody swapped out a nearby page, @@ -2090,8 +2137,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk, goto allocated; } for (;;) { - error = SWAP_PCTRIE_INSERT( - &object->un_pager.swp.swp_blks, sb); + error = swblk_lookup_insert(object, sb); if (error == 0) { if (atomic_cmpset_int(&swpctrie_zone_exhausted, 1, 0)) @@ -2113,8 +2159,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk, } else uma_zwait(swpctrie_zone); VM_OBJECT_WLOCK(object); - sb1 = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, - rdpi); + sb1 = swblk_lookup(object, rdpi); if (sb1 != NULL) { uma_zfree(swblk_zone, sb); sb = sb1; @@ -2142,53 +2187,42 @@ allocated: } /* - * SWP_PAGER_META_TRANSFER() - free a range of blocks in the srcobject's swap - * metadata, or transfer it into dstobject. + * SWP_PAGER_META_TRANSFER() - transfer a range of blocks in the srcobject's + * swap metadata into dstobject. * * This routine will free swap metadata structures as they are cleaned * out. */ static void swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject, - vm_pindex_t pindex, vm_pindex_t count, vm_size_t *moved) + vm_pindex_t pindex, vm_pindex_t count) { struct page_range range; struct swblk *sb; daddr_t blk; - vm_page_t m; vm_pindex_t offset, last; - vm_size_t mc; int i, limit, start; VM_OBJECT_ASSERT_WLOCKED(srcobject); - MPASS(moved == NULL || dstobject == NULL); + VM_OBJECT_ASSERT_WLOCKED(dstobject); - mc = 0; - m = NULL; - if (count == 0 || pctrie_is_empty(&srcobject->un_pager.swp.swp_blks)) - goto out; + if (count == 0 || swblk_is_empty(srcobject)) + return; swp_pager_init_freerange(&range); offset = pindex; last = pindex + count; - for (;;) { - sb = SWAP_PCTRIE_LOOKUP_GE(&srcobject->un_pager.swp.swp_blks, - rounddown(pindex, SWAP_META_PAGES)); - if (sb == NULL || sb->p >= last) - break; - start = pindex > sb->p ? pindex - sb->p : 0; - limit = last - sb->p < SWAP_META_PAGES ? last - sb->p : - SWAP_META_PAGES; + sb = swblk_start_limit(srcobject, pindex, last); + start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0; + for (; sb != NULL; + sb = swblk_start_limit(srcobject, pindex, last), start = 0) { + limit = MIN(last - sb->p, SWAP_META_PAGES); for (i = start; i < limit; i++) { - blk = sb->d[i]; - if (blk == SWAPBLK_NONE) + if (sb->d[i] == SWAPBLK_NONE) continue; - if (dstobject == NULL || - (blk = swp_pager_meta_build(dstobject, - sb->p + i - offset, blk, true), - blk != sb->d[i] && blk != SWAPBLK_NONE)) - swp_pager_update_freerange(&range, sb->d[i]); - else if (blk == sb->d[i]) { + blk = swp_pager_meta_build(dstobject, + sb->p + i - offset, sb->d[i], true); + if (blk == sb->d[i]) { /* * Destination has no swapblk and is not * resident, so transfer source. @@ -2197,30 +2231,20 @@ swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject, */ VM_OBJECT_WUNLOCK(srcobject); swp_pager_meta_build(dstobject, - sb->p + i - offset, blk, false); + sb->p + i - offset, sb->d[i], false); VM_OBJECT_WLOCK(srcobject); - } - if (moved != NULL) { - m = (m != NULL && m->pindex == sb->p + i - 1) ? - vm_page_next(m) : - vm_page_lookup(srcobject, sb->p + i); - if (m == NULL || vm_page_none_valid(m)) - mc++; - } + } else if (blk != SWAPBLK_NONE) + swp_pager_update_freerange(&range, sb->d[i]); sb->d[i] = SWAPBLK_NONE; } pindex = sb->p + SWAP_META_PAGES; if (swp_pager_swblk_empty(sb, 0, start) && swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) { - SWAP_PCTRIE_REMOVE(&srcobject->un_pager.swp.swp_blks, - sb->p); + swblk_lookup_remove(srcobject, sb); uma_zfree(swblk_zone, sb); } } swp_pager_freeswapspace(&range); -out: - if (moved != NULL) - *moved = mc; } /* @@ -2237,7 +2261,51 @@ static void swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count, vm_size_t *freed) { - swp_pager_meta_transfer(object, NULL, pindex, count, freed); + struct page_range range; + struct swblk *sb; + vm_page_t m; + vm_pindex_t last; + vm_size_t fc; + int i, limit, start; + + VM_OBJECT_ASSERT_WLOCKED(object); + + fc = 0; + m = NULL; + if (count == 0 || swblk_is_empty(object)) + goto out; + + swp_pager_init_freerange(&range); + last = pindex + count; + sb = swblk_start_limit(object, pindex, last); + start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0; + for (; sb != NULL; + sb = swblk_start_limit(object, pindex, last), start = 0) { + limit = MIN(last - sb->p, SWAP_META_PAGES); + for (i = start; i < limit; i++) { + if (sb->d[i] == SWAPBLK_NONE) + continue; + swp_pager_update_freerange(&range, sb->d[i]); + if (freed != NULL) { + m = (m != NULL && m->pindex == sb->p + i - 1) ? + vm_page_next(m) : + vm_page_lookup(object, sb->p + i); + if (m == NULL || vm_page_none_valid(m)) + fc++; + } + sb->d[i] = SWAPBLK_NONE; + } + pindex = sb->p + SWAP_META_PAGES; + if (swp_pager_swblk_empty(sb, 0, start) && + swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) { + swblk_lookup_remove(object, sb); + uma_zfree(swblk_zone, sb); + } + } + swp_pager_freeswapspace(&range); +out: + if (freed != NULL) + *freed = fc; } static void @@ -2296,19 +2364,16 @@ swp_pager_meta_lookup(vm_object_t object, vm_pindex_t pindex) KASSERT((object->flags & OBJ_SWAP) != 0, ("Lookup object not swappable")); - sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, - rounddown(pindex, SWAP_META_PAGES)); + sb = swblk_lookup(object, pindex); if (sb == NULL) return (SWAPBLK_NONE); return (sb->d[pindex % SWAP_META_PAGES]); } /* - * Returns the least page index which is greater than or equal to the - * parameter pindex and for which there is a swap block allocated. - * Returns object's size if the object's type is not swap or if there - * are no allocated swap blocks for the object after the requested - * pindex. + * Returns the least page index which is greater than or equal to the parameter + * pindex and for which there is a swap block allocated. Returns OBJ_MAX_SIZE + * if are no allocated swap blocks for the object after the requested pindex. */ vm_pindex_t swap_pager_find_least(vm_object_t object, vm_pindex_t pindex) @@ -2316,24 +2381,15 @@ swap_pager_find_least(vm_object_t object, vm_pindex_t pindex) struct swblk *sb; int i; - VM_OBJECT_ASSERT_LOCKED(object); - MPASS((object->flags & OBJ_SWAP) != 0); - - if (pctrie_is_empty(&object->un_pager.swp.swp_blks)) - return (object->size); - sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, - rounddown(pindex, SWAP_META_PAGES)); - if (sb == NULL) - return (object->size); + if ((sb = swblk_start(object, pindex)) == NULL) + return (OBJ_MAX_SIZE); if (sb->p < pindex) { for (i = pindex % SWAP_META_PAGES; i < SWAP_META_PAGES; i++) { if (sb->d[i] != SWAPBLK_NONE) return (sb->p + i); } - sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, - roundup(pindex, SWAP_META_PAGES)); - if (sb == NULL) - return (object->size); + if ((sb = swblk_next(object, sb)) == NULL) + return (OBJ_MAX_SIZE); } for (i = 0; i < SWAP_META_PAGES; i++) { if (sb->d[i] != SWAPBLK_NONE) @@ -2345,7 +2401,7 @@ swap_pager_find_least(vm_object_t object, vm_pindex_t pindex) * doesn't map any blocks. */ MPASS(0); - return (object->size); + return (OBJ_MAX_SIZE); } /* @@ -2797,11 +2853,8 @@ vmspace_swap_count(struct vmspace *vmspace) goto unlock; pi = OFF_TO_IDX(cur->offset); e = pi + OFF_TO_IDX(cur->end - cur->start); - for (;; pi = sb->p + SWAP_META_PAGES) { - sb = SWAP_PCTRIE_LOOKUP_GE( - &object->un_pager.swp.swp_blks, pi); - if (sb == NULL || sb->p >= e) - break; + for (sb = swblk_start_limit(object, pi, e); + sb != NULL; sb = swblk_next_limit(object, sb, e)) { for (i = 0; i < SWAP_META_PAGES; i++) { if (sb->p + i < e && sb->d[i] != SWAPBLK_NONE)