git: a8693e89e3e4 - main - vm: Introduce vm_page_alloc_nofree_domain

The branch main has been updated by bnovkov:

URL: https://cgit.FreeBSD.org/src/commit/?id=a8693e89e3e4a04efd02901cc93bb6148e3e40d6

commit a8693e89e3e4a04efd02901cc93bb6148e3e40d6
Author:     Bojan Novković <bnovkov@FreeBSD.org>
AuthorDate: 2024-07-14 13:14:22 +0000
Commit:     Bojan Novković <bnovkov@FreeBSD.org>
CommitDate: 2024-07-30 15:38:24 +0000

    vm: Introduce vm_page_alloc_nofree_domain
    
    This patch adds a reservation-aware bump allocator intended for
    allocating NOFREE pages. The main goal of this change is to reduce the
    long-term fragmentation issues caused by pages that are never freed during runtime.
    
    The `vm_page_alloc_nofree_domain` routine hands out 0-order pages from
    a preallocated superpage. Once an active NOFREE superpage fills up, the
    routine will try to allocate a new one and discard the old one.
    This routine will get invoked whenever VM_ALLOC_NOFREE is passed to
    vm_page_alloc_noobj or vm_page_alloc.
    
    Differential Revision:  https://reviews.freebsd.org/D45863
    Reviewed by:    alc, kib, markj
    Tested by:      alc
---
 sys/vm/vm_page.c      | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++
 sys/vm/vm_pagequeue.h |  4 ++++
 2 files changed, 66 insertions(+)

diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 3b6b88e4eb32..ff9df7f4a9fc 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -163,6 +163,7 @@ SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING | CTLFLAG_RD |
 static uma_zone_t fakepg_zone;
 
 static void vm_page_alloc_check(vm_page_t m);
+static vm_page_t vm_page_alloc_nofree_domain(int domain, int req);
 static bool _vm_page_busy_sleep(vm_object_t obj, vm_page_t m,
     vm_pindex_t pindex, const char *wmesg, int allocflags, bool locked);
 static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
@@ -2099,6 +2100,11 @@ vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain,
 	if (!vm_pager_can_alloc_page(object, pindex))
 		return (NULL);
 again:
+	if (__predict_false((req & VM_ALLOC_NOFREE) != 0)) {
+		m = vm_page_alloc_nofree_domain(domain, req);
+		if (m != NULL)
+			goto found;
+	}
 #if VM_NRESERVLEVEL > 0
 	/*
 	 * Can we allocate the page from a reservation?
@@ -2430,6 +2436,12 @@ vm_page_alloc_noobj_domain(int domain, int req)
 	    ((req & VM_ALLOC_NOFREE) != 0 ? PG_NOFREE : 0);
 	vmd = VM_DOMAIN(domain);
 again:
+	if (__predict_false((req & VM_ALLOC_NOFREE) != 0)) {
+		m = vm_page_alloc_nofree_domain(domain, req);
+		if (m != NULL)
+			goto found;
+	}
+
 	if (vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone != NULL) {
 		m = uma_zalloc(vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone,
 		    M_NOWAIT | M_NOVM);
@@ -2480,6 +2492,56 @@ found:
 	return (m);
 }
 
+#if VM_NRESERVLEVEL > 1
+#define	VM_NOFREE_IMPORT_ORDER	(VM_LEVEL_1_ORDER + VM_LEVEL_0_ORDER)
+#elif VM_NRESERVLEVEL > 0
+#define	VM_NOFREE_IMPORT_ORDER	VM_LEVEL_0_ORDER
+#else
+#define	VM_NOFREE_IMPORT_ORDER	8
+#endif
+
+/*
+ * Allocate a single NOFREE page.
+ *
+ * This routine hands out NOFREE pages from higher-order
+ * physical memory blocks in order to reduce memory fragmentation.
+ * When a NOFREE for a given domain chunk is used up,
+ * the routine will try to fetch a new one from the freelists
+ * and discard the old one.
+ */
+static vm_page_t
+vm_page_alloc_nofree_domain(int domain, int req)
+{
+	vm_page_t m;
+	struct vm_domain *vmd;
+	struct vm_nofreeq *nqp;
+
+	KASSERT((req & VM_ALLOC_NOFREE) != 0, ("invalid request %#x", req));
+
+	vmd = VM_DOMAIN(domain);
+	nqp = &vmd->vmd_nofreeq;
+	vm_domain_free_lock(vmd);
+	if (nqp->offs >= (1 << VM_NOFREE_IMPORT_ORDER) || nqp->ma == NULL) {
+		if (!vm_domain_allocate(vmd, req,
+		    1 << VM_NOFREE_IMPORT_ORDER)) {
+			vm_domain_free_unlock(vmd);
+			return (NULL);
+		}
+		nqp->ma = vm_phys_alloc_pages(domain, VM_FREEPOOL_DEFAULT,
+		    VM_LEVEL_0_ORDER);
+		if (nqp->ma == NULL) {
+			vm_domain_freecnt_inc(vmd, 1 << VM_NOFREE_IMPORT_ORDER);
+			vm_domain_free_unlock(vmd);
+			return (NULL);
+		}
+		nqp->offs = 0;
+	}
+	m = &nqp->ma[nqp->offs++];
+	vm_domain_free_unlock(vmd);
+
+	return (m);
+}
+
 vm_page_t
 vm_page_alloc_noobj(int req)
 {
diff --git a/sys/vm/vm_pagequeue.h b/sys/vm/vm_pagequeue.h
index 7e133ec947b4..86863a0a6400 100644
--- a/sys/vm/vm_pagequeue.h
+++ b/sys/vm/vm_pagequeue.h
@@ -246,6 +246,10 @@ struct vm_domain {
 	u_int vmd_domain;		/* (c) Domain number. */
 	u_int vmd_page_count;		/* (c) Total page count. */
 	long vmd_segs;			/* (c) bitmask of the segments */
+	struct vm_nofreeq {
+		vm_page_t ma;
+		int offs;
+	} vmd_nofreeq;			/* (f) NOFREE page bump allocator. */
 	u_int __aligned(CACHE_LINE_SIZE) vmd_free_count; /* (a,f) free page count */
 	u_int vmd_pageout_deficit;	/* (a) Estimated number of pages deficit */
 	uint8_t vmd_pad[CACHE_LINE_SIZE - (sizeof(u_int) * 2)];