svn commit: r262279 - projects/numa/sys/vm
Jeff Roberson
jeff at FreeBSD.org
Fri Feb 21 05:31:36 UTC 2014
Author: jeff
Date: Fri Feb 21 05:31:34 2014
New Revision: 262279
URL: http://svnweb.freebsd.org/changeset/base/262279
Log:
- Add the start of a domain selection policy system
Added:
projects/numa/sys/vm/_vm_domain.h
projects/numa/sys/vm/vm_domain.h
Modified:
projects/numa/sys/vm/vm_object.c
projects/numa/sys/vm/vm_object.h
projects/numa/sys/vm/vm_page.c
projects/numa/sys/vm/vm_phys.c
Added: projects/numa/sys/vm/_vm_domain.h
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/numa/sys/vm/_vm_domain.h Fri Feb 21 05:31:34 2014 (r262279)
@@ -0,0 +1,78 @@
+/*-
+ * Copyright (c) 2014, Jeffrey Roberson <jeff at freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice unmodified, this list of conditions, and the following
+ * disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VM__DOMAIN_H_
+#define _VM__DOMAIN_H_
+
+#include <sys/_bitset.h>
+#include <sys/bitset.h>
+
+#ifdef _KERNEL
+#define VM_DOMAIN_SETSIZE MAXMEMDOM
+#endif
+
+#define VM_DOMAIN_MAXSIZE 64
+
+#ifndef VM_DOMAIN_SETSIZE
+#define VM_DOMAIN_SETSIZE VM_DOMAIN_MAXSIZE
+#endif
+
+#define _NVM_DOMAINBITS _BITSET_BITS
+#define _NVM_DOMAINWORDS __bitset_words(VM_DOMAIN_SETSIZE)
+
+BITSET_DEFINE(_vm_domainset, VM_DOMAIN_SETSIZE);
+typedef struct _vm_domainset vm_domainset_t;
+
+#define VM_DOMAIN_FSET BITSET_FSET(_NVM_DOMAINWORDS)
+#define VM_DOMAIN_T_INITIALIZER BITSET_T_INITIALIZER
+#define VM_DOMAIN_SETBUFSIZ ((2 + sizeof(long) * 2) * _NVM_DOMAINWORDS)
+
+#ifdef _KERNEL
+
+/*
+ * Valid memory domain (NUMA) policy values.
+ */
+enum vm_domain_policy {
+ ROUNDROBIN, /* Select between any in the set. */
+ FIRSTTOUCH /* Select the current domain. */
+};
+
+/*
+ * The select structure encapsulate domain allocation strategy with
+ * allocator information.
+ */
+struct vm_domain_select {
+ vm_domainset_t ds_mask; /* bitmask of valid domains. */
+ enum vm_domain_policy ds_policy; /* Allocation policy. */
+ int ds_cursor; /* Allocation cursor. */
+ int ds_count; /* Domains in policy. */
+};
+
+#endif /* _KERNEL */
+
+#endif /* !_VM__DOMAIN_H_ */
Added: projects/numa/sys/vm/vm_domain.h
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/numa/sys/vm/vm_domain.h Fri Feb 21 05:31:34 2014 (r262279)
@@ -0,0 +1,110 @@
+/*-
+ * Copyright (c) 2014, Jeffrey Roberson <jeff at freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice unmodified, this list of conditions, and the following
+ * disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VM_DOMAIN_H_
+#define _VM_DOMAIN_H_
+
+#include <vm/_vm_domain.h>
+#include <sys/bitset.h>
+
+#define VM_DOMAIN_CLR(n, p) BIT_CLR(VM_DOMAIN_SETSIZE, n, p)
+#define VM_DOMAIN_COPY(f, t) BIT_COPY(VM_DOMAIN_SETSIZE, f, t)
+#define VM_DOMAIN_ISSET(n, p) BIT_ISSET(VM_DOMAIN_SETSIZE, n, p)
+#define VM_DOMAIN_SET(n, p) BIT_SET(VM_DOMAIN_SETSIZE, n, p)
+#define VM_DOMAIN_ZERO(p) BIT_ZERO(VM_DOMAIN_SETSIZE, p)
+#define VM_DOMAIN_FILL(p) BIT_FILL(VM_DOMAIN_SETSIZE, p)
+#define VM_DOMAIN_SETOF(n, p) BIT_SETOF(VM_DOMAIN_SETSIZE, n, p)
+#define VM_DOMAIN_EMPTY(p) BIT_EMPTY(VM_DOMAIN_SETSIZE, p)
+#define VM_DOMAIN_ISFULLSET(p) BIT_ISFULLSET(VM_DOMAIN_SETSIZE, p)
+#define VM_DOMAIN_SUBSET(p, c) BIT_SUBSET(VM_DOMAIN_SETSIZE, p, c)
+#define VM_DOMAIN_OVERLAP(p, c) BIT_OVERLAP(VM_DOMAIN_SETSIZE, p, c)
+#define VM_DOMAIN_CMP(p, c) BIT_CMP(VM_DOMAIN_SETSIZE, p, c)
+#define VM_DOMAIN_OR(d, s) BIT_OR(VM_DOMAIN_SETSIZE, d, s)
+#define VM_DOMAIN_AND(d, s) BIT_AND(VM_DOMAIN_SETSIZE, d, s)
+#define VM_DOMAIN_NAND(d, s) BIT_NAND(VM_DOMAIN_SETSIZE, d, s)
+#define VM_DOMAIN_CLR_ATOMIC(n, p) BIT_CLR_ATOMIC(VM_DOMAIN_SETSIZE, n, p)
+#define VM_DOMAIN_SET_ATOMIC(n, p) BIT_SET_ATOMIC(VM_DOMAIN_SETSIZE, n, p)
+#define VM_DOMAIN_AND_ATOMIC(n, p) BIT_AND_ATOMIC(VM_DOMAIN_SETSIZE, n, p)
+#define VM_DOMAIN_OR_ATOMIC(d, s) BIT_OR_ATOMIC(VM_DOMAIN_SETSIZE, d, s)
+#define VM_DOMAIN_COPY_STORE_REL(f, t) BIT_COPY_STORE_REL(VM_DOMAIN_SETSIZE, f, t)
+#define VM_DOMAIN_FFS(p) BIT_FFS(VM_DOMAIN_SETSIZE, p)
+
+#ifdef _KERNEL
+
+/*
+ * Domain sets.
+ */
+extern vm_domainset_t vm_alldomains; /* All domains. */
+extern vm_domainset_t vm_domset[MAXMEMDOM]; /* Specific domain bitmask. */
+extern int vm_ndomains;
+
+/*
+ * Domain allocation selectors.
+ */
+extern struct vm_domain_select vm_sel_def; /* default */
+extern struct vm_domain_select vm_sel_rr; /* round-robin */
+extern struct vm_domain_select vm_sel_ft; /* first-touch */
+extern struct vm_domain_select vm_sel_dom[MAXMEMDOM]; /* specific domain */
+
+static inline int
+vm_domain_select_first(struct vm_domain_select *sel)
+{
+ int domain;
+
+ switch (sel->ds_policy) {
+ case FIRSTTOUCH:
+ domain = PCPU_GET(domain);
+ if (VM_DOMAIN_ISSET(domain, &sel->ds_mask))
+ break;
+ /* FALLTHROUGH */
+ case ROUNDROBIN:
+ domain = atomic_fetchadd_int(&sel->ds_cursor, 1) % vm_ndomains;
+ while (!VM_DOMAIN_ISSET(domain, &sel->ds_mask))
+ domain = (domain + 1) % vm_ndomains;
+ }
+ return domain;
+}
+
+static inline int
+vm_domain_select_next(struct vm_domain_select *sel, int domain)
+{
+
+ switch (sel->ds_policy) {
+ case FIRSTTOUCH:
+ /* FALLTHROUGH */
+ case ROUNDROBIN:
+ do {
+ domain = (domain + 1) % vm_ndomains;
+ } while (!VM_DOMAIN_ISSET(domain, &sel->ds_mask));
+ }
+ return domain;
+}
+
+#endif /* _KERNEL */
+
+#endif /* !_VM_DOMAIN_H_ */
Modified: projects/numa/sys/vm/vm_object.c
==============================================================================
--- projects/numa/sys/vm/vm_object.c Fri Feb 21 05:17:30 2014 (r262278)
+++ projects/numa/sys/vm/vm_object.c Fri Feb 21 05:31:34 2014 (r262279)
@@ -86,6 +86,7 @@ __FBSDID("$FreeBSD$");
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/pmap.h>
+#include <vm/vm_domain.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
@@ -243,6 +244,9 @@ _vm_object_allocate(objtype_t type, vm_p
panic("_vm_object_allocate: type %d is undefined", type);
}
object->size = size;
+#if MAXMEMDOM > 1
+ object->selector = vm_sel_def;
+#endif
object->generation = 1;
object->ref_count = 1;
object->memattr = VM_MEMATTR_DEFAULT;
@@ -254,7 +258,6 @@ _vm_object_allocate(objtype_t type, vm_p
#if VM_NRESERVLEVEL > 0
LIST_INIT(&object->rvq);
#endif
-
mtx_lock(&vm_object_list_mtx);
TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
mtx_unlock(&vm_object_list_mtx);
@@ -1254,6 +1257,9 @@ vm_object_shadow(
result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) &
((1 << (VM_NFREEORDER - 1)) - 1);
#endif
+#if MAXMEMDOM > 0
+ result->selector = source->selector;
+#endif
VM_OBJECT_WUNLOCK(source);
}
@@ -1295,6 +1301,9 @@ vm_object_split(vm_map_entry_t entry)
* into a swap object.
*/
new_object = vm_object_allocate(OBJT_DEFAULT, size);
+#if MAXMEMDOM > 0
+ new_object->selector = orig_object->selector;
+#endif
/*
* At this point, the new object is still private, so the order in
@@ -2200,24 +2209,6 @@ vm_object_set_writeable_dirty(vm_object_
vm_object_set_flag(object, OBJ_MIGHTBEDIRTY);
}
-int
-vm_object_domain(vm_object_t object)
-{
-#if MAXMEMDOM > 1
- static volatile unsigned int noobj_domain;
-
- if (object == NULL)
- return (atomic_fetchadd_int(&noobj_domain, 1) % vm_ndomains);
-
- object->domain = (object->domain + 1) % vm_ndomains;
-
- return (object->domain);
-#else
- return (0);
-#endif
- return (0);
-}
-
#include "opt_ddb.h"
#ifdef DDB
#include <sys/kernel.h>
Modified: projects/numa/sys/vm/vm_object.h
==============================================================================
--- projects/numa/sys/vm/vm_object.h Fri Feb 21 05:17:30 2014 (r262278)
+++ projects/numa/sys/vm/vm_object.h Fri Feb 21 05:31:34 2014 (r262279)
@@ -73,6 +73,7 @@
#include <sys/_rwlock.h>
#include <vm/_vm_radix.h>
+#include <vm/_vm_domain.h>
/*
* Types defined:
@@ -106,7 +107,7 @@ struct vm_object {
struct vm_radix rtree; /* root of the resident page radix trie*/
vm_pindex_t size; /* Object size */
#if MAXMEMDOM > 1
- int domain; /* Last selected NUMA domain. */
+ struct vm_domain_select selector; /* NUMA domain policy. */
#endif
int generation; /* generation ID */
int ref_count; /* How many refs?? */
@@ -293,7 +294,6 @@ void vm_object_shadow (vm_object_t *, vm
void vm_object_split(vm_map_entry_t);
boolean_t vm_object_sync(vm_object_t, vm_ooffset_t, vm_size_t, boolean_t,
boolean_t);
-int vm_object_domain(vm_object_t);
#endif /* _KERNEL */
Modified: projects/numa/sys/vm/vm_page.c
==============================================================================
--- projects/numa/sys/vm/vm_page.c Fri Feb 21 05:17:30 2014 (r262278)
+++ projects/numa/sys/vm/vm_page.c Fri Feb 21 05:31:34 2014 (r262279)
@@ -104,6 +104,7 @@ __FBSDID("$FreeBSD$");
#include <vm/vm.h>
#include <vm/pmap.h>
#include <vm/vm_param.h>
+#include <vm/vm_domain.h>
#include <vm/vm_kern.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
@@ -1447,9 +1448,26 @@ vm_page_is_cached(vm_object_t object, vm
vm_page_t
vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
{
+#if MAXMEMDOM > 1
+ struct vm_domain_select *sel;
+ vm_page_t m;
+ int i, dom;
+
+ if (object == NULL)
+ sel = &vm_sel_def;
+ else
+ sel = &object->selector;
- return vm_page_alloc_domain(object, pindex, vm_object_domain(object),
- req);
+ for (i = 0, dom = vm_domain_select_first(sel);
+ i < sel->ds_count; i++, dom = vm_domain_select_next(sel, dom)) {
+ if ((m = vm_page_alloc_domain(object, pindex, dom,
+ req)) != NULL)
+ return (m);
+ }
+ return (NULL);
+#else
+ return vm_page_alloc_domain(object, pindex, 0, req);
+#endif
}
vm_page_t
@@ -1706,9 +1724,28 @@ vm_page_alloc_contig(vm_object_t object,
u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
vm_paddr_t boundary, vm_memattr_t memattr)
{
+#if MAXMEMDOM > 1
+ struct vm_domain_select *sel;
+ vm_page_t m;
+ int i, dom;
+
+ if (object == NULL)
+ sel = &vm_sel_def;
+ else
+ sel = &object->selector;
+
+ for (i = 0, dom = vm_domain_select_first(sel);
+ i < sel->ds_count; i++, dom = vm_domain_select_next(sel, dom)) {
+ if ((m = vm_page_alloc_contig_domain(object, pindex,
+ dom, req, npages, low, high,
+ alignment, boundary, memattr)) != NULL)
+ return (m);
+ }
+ return (NULL);
+#else
return vm_page_alloc_contig_domain(object, pindex,
- vm_object_domain(object), req, npages, low, high,
- alignment, boundary, memattr);
+ 0, req, npages, low, high, alignment, boundary, memattr);
+#endif
}
vm_page_t
Modified: projects/numa/sys/vm/vm_phys.c
==============================================================================
--- projects/numa/sys/vm/vm_phys.c Fri Feb 21 05:17:30 2014 (r262278)
+++ projects/numa/sys/vm/vm_phys.c Fri Feb 21 05:31:34 2014 (r262279)
@@ -60,6 +60,7 @@ __FBSDID("$FreeBSD$");
#include <vm/vm.h>
#include <vm/vm_param.h>
+#include <vm/vm_domain.h>
#include <vm/vm_kern.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
@@ -71,6 +72,12 @@ _Static_assert(sizeof(long) * NBBY >= VM
struct mem_affinity *mem_affinity;
int vm_ndomains = 1;
+vm_domainset_t vm_alldomains;
+vm_domainset_t vm_domset[MAXMEMDOM];
+struct vm_domain_select vm_sel_def;
+struct vm_domain_select vm_sel_rr;
+struct vm_domain_select vm_sel_ft;
+struct vm_domain_select vm_sel_dom[MAXMEMDOM];
struct vm_phys_seg vm_phys_segs[VM_PHYSSEG_MAX];
int vm_phys_nsegs;
@@ -327,6 +334,11 @@ vm_phys_init(void)
VM_FREELIST_DEFAULT);
}
for (dom = 0; dom < vm_ndomains; dom++) {
+ VM_DOMAIN_SET(dom, &vm_alldomains);
+ VM_DOMAIN_SET(dom, &vm_domset[dom]);
+ vm_sel_dom[dom].ds_mask = vm_domset[dom];
+ vm_sel_dom[dom].ds_policy = ROUNDROBIN;
+ vm_sel_dom[dom].ds_count = 1;
for (flind = 0; flind < vm_nfreelists; flind++) {
for (pind = 0; pind < VM_NFREEPOOL; pind++) {
fl = vm_phys_free_queues[dom][flind][pind];
@@ -335,6 +347,15 @@ vm_phys_init(void)
}
}
}
+ vm_sel_def.ds_mask = vm_alldomains;
+ vm_sel_def.ds_policy = ROUNDROBIN;
+ vm_sel_def.ds_count = vm_ndomains;
+ vm_sel_rr.ds_mask = vm_alldomains;
+ vm_sel_rr.ds_policy = ROUNDROBIN;
+ vm_sel_rr.ds_count = vm_ndomains;
+ vm_sel_ft.ds_mask = vm_alldomains;
+ vm_sel_ft.ds_policy = FIRSTTOUCH;
+ vm_sel_ft.ds_count = vm_ndomains;
mtx_init(&vm_phys_fictitious_reg_mtx, "vmfctr", NULL, MTX_DEF);
}
More information about the svn-src-projects
mailing list