git: 7becd87c988c - main - vm_grab: use iterator for grab lookup
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 01 Apr 2025 06:10:55 UTC
The branch main has been updated by dougm: URL: https://cgit.FreeBSD.org/src/commit/?id=7becd87c988c090bdc41a177bf06ad2894f79a5b commit 7becd87c988c090bdc41a177bf06ad2894f79a5b Author: Doug Moore <dougm@FreeBSD.org> AuthorDate: 2025-04-01 06:09:34 +0000 Commit: Doug Moore <dougm@FreeBSD.org> CommitDate: 2025-04-01 06:09:34 +0000 vm_grab: use iterator for grab lookup When a page lookup fails in a vm_page_grab operation, use the results of the failed search, stored in an iterator, to begin the search for a predecessor to use in a call to vm_page_alloc_after(), to avoid doing that search from the root in vm_page_alloc(), as happens now. Reviewed by: kib Differential Revision: https://reviews.freebsd.org/D49371 --- sys/vm/vm_page.c | 145 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 89 insertions(+), 56 deletions(-) diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 4d9a57544487..f351f60f833c 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -4823,46 +4823,70 @@ vm_page_grab_pflags(int allocflags) } /* - * Grab a page, waiting until we are waken up due to the page - * changing state. We keep on waiting, if the page continues - * to be in the object. If the page doesn't exist, first allocate it - * and then conditionally zero it. + * Grab a page, waiting until we are woken up due to the page changing state. + * We keep on waiting, if the page continues to be in the object, unless + * allocflags forbid waiting. * - * This routine may sleep. + * The object must be locked on entry. This routine may sleep. The lock will, + * however, be released and reacquired if the routine sleeps. * - * The object must be locked on entry. The lock will, however, be released - * and reacquired if the routine sleeps. + * Return a grabbed page, or NULL. Set *found if a page was found, whether or + * not it was grabbed. + */ +static inline vm_page_t +vm_page_grab_lookup(struct pctrie_iter *pages, vm_object_t object, + vm_pindex_t pindex, int allocflags, bool *found) +{ + vm_page_t m; + + while ((*found = (m = vm_radix_iter_lookup(pages, pindex)) != NULL) && + !vm_page_tryacquire(m, allocflags)) { + if (!vm_page_grab_sleep(object, m, pindex, "pgrbwt", + allocflags, true)) + return (NULL); + pctrie_iter_reset(pages); + } + return (m); +} + +/* + * Grab a page. Keep on waiting, as long as the page exists in the object. If + * the page doesn't exist, first allocate it and then conditionally zero it. + * + * The object must be locked on entry. This routine may sleep. The lock will, + * however, be released and reacquired if the routine sleeps. */ vm_page_t vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags) { - vm_page_t m; + struct pctrie_iter pages; + vm_page_t m, mpred; + bool found; VM_OBJECT_ASSERT_WLOCKED(object); vm_page_grab_check(allocflags); -retrylookup: - if ((m = vm_page_lookup(object, pindex)) != NULL) { - if (!vm_page_tryacquire(m, allocflags)) { - if (vm_page_grab_sleep(object, m, pindex, "pgrbwt", - allocflags, true)) - goto retrylookup; + vm_page_iter_init(&pages, object); + while ((m = vm_page_grab_lookup( + &pages, object, pindex, allocflags, &found)) == NULL) { + if ((allocflags & VM_ALLOC_NOCREAT) != 0) + return (NULL); + if (found && + (allocflags & (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL)) != 0) return (NULL); + mpred = vm_radix_iter_lookup_le(&pages, pindex); + m = vm_page_alloc_after(object, pindex, + vm_page_grab_pflags(allocflags), mpred); + if (m != NULL) { + if ((allocflags & VM_ALLOC_ZERO) != 0 && + (m->flags & PG_ZERO) == 0) + pmap_zero_page(m); + break; } - goto out; - } - if ((allocflags & VM_ALLOC_NOCREAT) != 0) - return (NULL); - m = vm_page_alloc(object, pindex, vm_page_grab_pflags(allocflags)); - if (m == NULL) { - if ((allocflags & (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL)) != 0) + if ((allocflags & + (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL)) != 0) return (NULL); - goto retrylookup; } - if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0) - pmap_zero_page(m); - -out: vm_page_grab_release(m, allocflags); return (m); @@ -5054,48 +5078,57 @@ out: } /* - * Fill a partial page with zeroes. The object write lock is held on entry and - * exit, but may be temporarily released. + * Grab a page. Keep on waiting, as long as the page exists in the object. If + * the page doesn't exist, and the pager has it, allocate it and zero part of + * it. + * + * The object must be locked on entry. This routine may sleep. The lock will, + * however, be released and reacquired if the routine sleeps. */ int vm_page_grab_zero_partial(vm_object_t object, vm_pindex_t pindex, int base, int end) { - vm_page_t m; - int rv; + struct pctrie_iter pages; + vm_page_t m, mpred; + int allocflags, rv; + bool found; VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(base >= 0, ("%s: base %d", __func__, base)); KASSERT(end - base <= PAGE_SIZE, ("%s: base %d end %d", __func__, base, end)); -retry: - m = vm_page_grab(object, pindex, VM_ALLOC_NOCREAT); - if (m != NULL) { - MPASS(vm_page_all_valid(m)); - } else if (vm_pager_has_page(object, pindex, NULL, NULL)) { - m = vm_page_alloc(object, pindex, - VM_ALLOC_NORMAL | VM_ALLOC_WAITFAIL); - if (m == NULL) - goto retry; - vm_object_pip_add(object, 1); - VM_OBJECT_WUNLOCK(object); - rv = vm_pager_get_pages(object, &m, 1, NULL, NULL); - VM_OBJECT_WLOCK(object); - vm_object_pip_wakeup(object); - if (rv != VM_PAGER_OK) { - vm_page_free(m); - return (EIO); - } + allocflags = VM_ALLOC_NOCREAT | VM_ALLOC_NORMAL | VM_ALLOC_WAITFAIL; + vm_page_iter_init(&pages, object); + while ((m = vm_page_grab_lookup( + &pages, object, pindex, allocflags, &found)) == NULL) { + if (!vm_pager_has_page(object, pindex, NULL, NULL)) + return (0); + mpred = vm_radix_iter_lookup_le(&pages, pindex); + m = vm_page_alloc_after(object, pindex, + vm_page_grab_pflags(allocflags), mpred); + if (m != NULL) { + vm_object_pip_add(object, 1); + VM_OBJECT_WUNLOCK(object); + rv = vm_pager_get_pages(object, &m, 1, NULL, NULL); + VM_OBJECT_WLOCK(object); + vm_object_pip_wakeup(object); + if (rv != VM_PAGER_OK) { + vm_page_free(m); + return (EIO); + } - /* - * Since the page was not resident, and therefore not recently - * accessed, immediately enqueue it for asynchronous laundering. - * The current operation is not regarded as an access. - */ - vm_page_launder(m); - } else - return (0); + /* + * Since the page was not resident, and therefore not + * recently accessed, immediately enqueue it for + * asynchronous laundering. The current operation is + * not regarded as an access. + */ + vm_page_launder(m); + break; + } + } pmap_zero_page_area(m, base, end - base); KASSERT(vm_page_all_valid(m), ("%s: page %p is invalid", __func__, m));