git: 38e3125d6df9 - main - device_pager: user iterators to free device pages
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Thu, 21 Nov 2024 21:50:53 UTC
The branch main has been updated by dougm:
URL: https://cgit.FreeBSD.org/src/commit/?id=38e3125d6df98919983ed800471673ff0e5c2337
commit 38e3125d6df98919983ed800471673ff0e5c2337
Author: Doug Moore <dougm@FreeBSD.org>
AuthorDate: 2024-11-21 21:49:30 +0000
Commit: Doug Moore <dougm@FreeBSD.org>
CommitDate: 2024-11-21 21:49:30 +0000
device_pager: user iterators to free device pages
Change cdev_mgtdev_page_free_page to take an iterator, rather than an
object and page, so that removing the page from the object radix tree
can take advantage of locality with iterators. Define a
general-purpose function to free all pages, which can be used in
several places.
Reviewed by: kib
Differential Revision: https://reviews.freebsd.org/D47692
---
sys/arm/nvidia/drm2/tegra_bo.c | 6 +++--
sys/compat/linuxkpi/common/src/linux_page.c | 18 ++-----------
sys/dev/drm2/ttm/ttm_bo_vm.c | 20 +++------------
sys/dev/xen/gntdev/gntdev.c | 13 +---------
sys/dev/xen/privcmd/privcmd.c | 14 +---------
sys/vm/device_pager.c | 40 +++++++++++++++++++++--------
sys/vm/vm_pager.h | 3 ++-
7 files changed, 43 insertions(+), 71 deletions(-)
diff --git a/sys/arm/nvidia/drm2/tegra_bo.c b/sys/arm/nvidia/drm2/tegra_bo.c
index 346118b78c2b..e1a6a425be4d 100644
--- a/sys/arm/nvidia/drm2/tegra_bo.c
+++ b/sys/arm/nvidia/drm2/tegra_bo.c
@@ -47,6 +47,7 @@
static void
tegra_bo_destruct(struct tegra_bo *bo)
{
+ struct pctrie_iter pages;
vm_page_t m;
size_t size;
int i;
@@ -58,11 +59,12 @@ tegra_bo_destruct(struct tegra_bo *bo)
if (bo->vbase != 0)
pmap_qremove(bo->vbase, bo->npages);
+ vm_page_iter_init(&pages, bo->cdev_pager);
VM_OBJECT_WLOCK(bo->cdev_pager);
for (i = 0; i < bo->npages; i++) {
- m = bo->m[i];
+ m = vm_page_iter_lookup(&pages, i);
vm_page_busy_acquire(m, 0);
- cdev_mgtdev_pager_free_page(bo->cdev_pager, m);
+ cdev_mgtdev_pager_free_page(&pages);
m->flags &= ~PG_FICTITIOUS;
vm_page_unwire_noq(m);
vm_page_free(m);
diff --git a/sys/compat/linuxkpi/common/src/linux_page.c b/sys/compat/linuxkpi/common/src/linux_page.c
index 25243382f9ea..6ca926e89174 100644
--- a/sys/compat/linuxkpi/common/src/linux_page.c
+++ b/sys/compat/linuxkpi/common/src/linux_page.c
@@ -418,27 +418,13 @@ lkpi_io_mapping_map_user(struct io_mapping *iomap,
*/
void
lkpi_unmap_mapping_range(void *obj, loff_t const holebegin __unused,
- loff_t const holelen, int even_cows __unused)
+ loff_t const holelen __unused, int even_cows __unused)
{
vm_object_t devobj;
- vm_page_t page;
- int i, page_count;
devobj = cdev_pager_lookup(obj);
if (devobj != NULL) {
- page_count = OFF_TO_IDX(holelen);
-
- VM_OBJECT_WLOCK(devobj);
-retry:
- for (i = 0; i < page_count; i++) {
- page = vm_page_lookup(devobj, i);
- if (page == NULL)
- continue;
- if (!vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL))
- goto retry;
- cdev_mgtdev_pager_free_page(devobj, page);
- }
- VM_OBJECT_WUNLOCK(devobj);
+ cdev_mgtdev_pager_free_pages(devobj);
vm_object_deallocate(devobj);
}
}
diff --git a/sys/dev/drm2/ttm/ttm_bo_vm.c b/sys/dev/drm2/ttm/ttm_bo_vm.c
index e543b8dfb993..38575fc8a74b 100644
--- a/sys/dev/drm2/ttm/ttm_bo_vm.c
+++ b/sys/dev/drm2/ttm/ttm_bo_vm.c
@@ -361,26 +361,12 @@ void
ttm_bo_release_mmap(struct ttm_buffer_object *bo)
{
vm_object_t vm_obj;
- vm_page_t m;
- int i;
vm_obj = cdev_pager_lookup(bo);
- if (vm_obj == NULL)
- return;
-
- VM_OBJECT_WLOCK(vm_obj);
-retry:
- for (i = 0; i < bo->num_pages; i++) {
- m = vm_page_lookup(vm_obj, i);
- if (m == NULL)
- continue;
- if (vm_page_busy_acquire(m, VM_ALLOC_WAITFAIL) == 0)
- goto retry;
- cdev_mgtdev_pager_free_page(vm_obj, m);
+ if (vm_obj != NULL) {
+ cdev_mgtdev_pager_free_pages(vm_obj);
+ vm_object_deallocate(vm_obj);
}
- VM_OBJECT_WUNLOCK(vm_obj);
-
- vm_object_deallocate(vm_obj);
}
#if 0
diff --git a/sys/dev/xen/gntdev/gntdev.c b/sys/dev/xen/gntdev/gntdev.c
index 49f8aefad62e..e3bc1ecf35ab 100644
--- a/sys/dev/xen/gntdev/gntdev.c
+++ b/sys/dev/xen/gntdev/gntdev.c
@@ -563,7 +563,6 @@ notify_unmap_cleanup(struct gntdev_gmap *gmap)
{
uint32_t i;
int error, count;
- vm_page_t m;
struct gnttab_unmap_grant_ref *unmap_ops;
unmap_ops = malloc(sizeof(struct gnttab_unmap_grant_ref) * gmap->count,
@@ -592,17 +591,7 @@ notify_unmap_cleanup(struct gntdev_gmap *gmap)
}
/* Free the pages. */
- VM_OBJECT_WLOCK(gmap->map->mem);
-retry:
- for (i = 0; i < gmap->count; i++) {
- m = vm_page_lookup(gmap->map->mem, i);
- if (m == NULL)
- continue;
- if (vm_page_busy_acquire(m, VM_ALLOC_WAITFAIL) == 0)
- goto retry;
- cdev_mgtdev_pager_free_page(gmap->map->mem, m);
- }
- VM_OBJECT_WUNLOCK(gmap->map->mem);
+ cdev_mgtdev_pager_free_pages(gmap->map->mem);
/* Perform unmap hypercall. */
error = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref,
diff --git a/sys/dev/xen/privcmd/privcmd.c b/sys/dev/xen/privcmd/privcmd.c
index c04ac287183b..922d24b39432 100644
--- a/sys/dev/xen/privcmd/privcmd.c
+++ b/sys/dev/xen/privcmd/privcmd.c
@@ -120,25 +120,13 @@ privcmd_pg_dtor(void *handle)
struct privcmd_map *map = handle;
int error __diagused;
vm_size_t i;
- vm_page_t m;
/*
* Remove the mappings from the used pages. This will remove the
* underlying p2m bindings in Xen second stage translation.
*/
if (map->mapped == true) {
- VM_OBJECT_WLOCK(map->mem);
-retry:
- for (i = 0; i < map->size; i++) {
- m = vm_page_lookup(map->mem, i);
- if (m == NULL)
- continue;
- if (vm_page_busy_acquire(m, VM_ALLOC_WAITFAIL) == 0)
- goto retry;
- cdev_mgtdev_pager_free_page(map->mem, m);
- }
- VM_OBJECT_WUNLOCK(map->mem);
-
+ cdev_mgtdev_pager_free_pages(map->mem);
for (i = 0; i < map->size; i++) {
rm.gpfn = atop(map->phys_base_addr) + i;
HYPERVISOR_memory_op(XENMEM_remove_from_physmap, &rm);
diff --git a/sys/vm/device_pager.c b/sys/vm/device_pager.c
index a5fffbf629bd..55bc948a4c44 100644
--- a/sys/vm/device_pager.c
+++ b/sys/vm/device_pager.c
@@ -52,6 +52,7 @@
#include <vm/vm_page.h>
#include <vm/vm_pager.h>
#include <vm/vm_phys.h>
+#include <vm/vm_radix.h>
#include <vm/uma.h>
static void dev_pager_init(void);
@@ -262,9 +263,13 @@ void
cdev_pager_free_page(vm_object_t object, vm_page_t m)
{
- if (object->type == OBJT_MGTDEVICE)
- cdev_mgtdev_pager_free_page(object, m);
- else if (object->type == OBJT_DEVICE)
+ if (object->type == OBJT_MGTDEVICE) {
+ struct pctrie_iter pages;
+
+ vm_page_iter_init(&pages, object);
+ vm_page_iter_lookup(&pages, m->pindex);
+ cdev_mgtdev_pager_free_page(&pages);
+ } else if (object->type == OBJT_DEVICE)
dev_pager_free_page(object, m);
else
KASSERT(false,
@@ -272,15 +277,30 @@ cdev_pager_free_page(vm_object_t object, vm_page_t m)
}
void
-cdev_mgtdev_pager_free_page(vm_object_t object, vm_page_t m)
+cdev_mgtdev_pager_free_page(struct pctrie_iter *pages)
{
+ pmap_remove_all(vm_radix_iter_page(pages));
+ vm_page_iter_remove(pages);
+}
- VM_OBJECT_ASSERT_WLOCKED(object);
- KASSERT((object->type == OBJT_MGTDEVICE &&
- (m->oflags & VPO_UNMANAGED) == 0),
- ("Unmanaged device or page obj %p m %p", object, m));
- pmap_remove_all(m);
- (void)vm_page_remove(m);
+void
+cdev_mgtdev_pager_free_pages(vm_object_t object)
+{
+ struct pctrie_iter pages;
+ vm_page_t m;
+
+ vm_page_iter_init(&pages, object);
+ VM_OBJECT_WLOCK(object);
+retry:
+ for (m = vm_page_iter_lookup_ge(&pages, 0); m != NULL;
+ m = vm_radix_iter_step(&pages)) {
+ if (!vm_page_busy_acquire(m, VM_ALLOC_WAITFAIL)) {
+ pctrie_iter_reset(&pages);
+ goto retry;
+ }
+ cdev_mgtdev_pager_free_page(&pages);
+ }
+ VM_OBJECT_WUNLOCK(object);
}
static void
diff --git a/sys/vm/vm_pager.h b/sys/vm/vm_pager.h
index d30bf349e411..0958d82d6968 100644
--- a/sys/vm/vm_pager.h
+++ b/sys/vm/vm_pager.h
@@ -300,7 +300,8 @@ vm_object_t cdev_pager_allocate(void *handle, enum obj_type tp,
vm_ooffset_t foff, struct ucred *cred);
vm_object_t cdev_pager_lookup(void *handle);
void cdev_pager_free_page(vm_object_t object, vm_page_t m);
-void cdev_mgtdev_pager_free_page(vm_object_t object, vm_page_t m);
+void cdev_mgtdev_pager_free_page(struct pctrie_iter *pages);
+void cdev_mgtdev_pager_free_pages(vm_object_t object);
struct phys_pager_ops {
int (*phys_pg_getpages)(vm_object_t vm_obj, vm_page_t *m, int count,