svn commit: r261807 - in projects/sendfile/sys: kern sys ufs/ffs vm
Gleb Smirnoff
glebius at FreeBSD.org
Wed Feb 12 19:51:15 UTC 2014
Author: glebius
Date: Wed Feb 12 19:51:12 2014
New Revision: 261807
URL: http://svnweb.freebsd.org/changeset/base/261807
Log:
Add new VOP vop_getpages_async(), that is like vop_getpages() but doesn't
block on I/O, just returns. The callback function supplied will be executed
with the argument supplied after I/O is done and pages are populated and
valid.
Implement the VOP for the generic vnode pager, tearing the
vnode_pager_generic_getpages() into two parts.
Sponsored by: Netflix
Sponsored by: Nginx, Inc.
Modified:
projects/sendfile/sys/kern/vfs_default.c
projects/sendfile/sys/kern/vnode_if.src
projects/sendfile/sys/sys/vnode.h
projects/sendfile/sys/ufs/ffs/ffs_vnops.c
projects/sendfile/sys/vm/vm_pager.h
projects/sendfile/sys/vm/vnode_pager.c
projects/sendfile/sys/vm/vnode_pager.h
Modified: projects/sendfile/sys/kern/vfs_default.c
==============================================================================
--- projects/sendfile/sys/kern/vfs_default.c Wed Feb 12 19:26:08 2014 (r261806)
+++ projects/sendfile/sys/kern/vfs_default.c Wed Feb 12 19:51:12 2014 (r261807)
@@ -111,6 +111,7 @@ struct vop_vector default_vnodeops = {
.vop_close = VOP_NULL,
.vop_fsync = VOP_NULL,
.vop_getpages = vop_stdgetpages,
+ .vop_getpages_async = vop_stdgetpages_async,
.vop_getwritemount = vop_stdgetwritemount,
.vop_inactive = VOP_NULL,
.vop_ioctl = VOP_ENOTTY,
@@ -726,7 +727,16 @@ vop_stdgetpages(ap)
{
return vnode_pager_generic_getpages(ap->a_vp, ap->a_m,
- ap->a_count, ap->a_reqpage);
+ ap->a_count, ap->a_reqpage, NULL, NULL);
+}
+
+/* XXX Needs good comment and a manpage. */
+int
+vop_stdgetpages_async(struct vop_getpages_async_args *ap)
+{
+
+ return vnode_pager_generic_getpages(ap->a_vp, ap->a_m,
+ ap->a_count, ap->a_reqpage, ap->a_vop_getpages_iodone, ap->a_arg);
}
int
Modified: projects/sendfile/sys/kern/vnode_if.src
==============================================================================
--- projects/sendfile/sys/kern/vnode_if.src Wed Feb 12 19:26:08 2014 (r261806)
+++ projects/sendfile/sys/kern/vnode_if.src Wed Feb 12 19:51:12 2014 (r261807)
@@ -477,6 +477,19 @@ vop_getpages {
};
+%% getpages_async vp L L L
+
+vop_getpages_async {
+ IN struct vnode *vp;
+ IN vm_page_t *m;
+ IN int count;
+ IN int reqpage;
+ IN vm_ooffset_t offset;
+ IN void (*vop_getpages_iodone)(void *);
+ IN void *arg;
+};
+
+
%% putpages vp L L L
vop_putpages {
Modified: projects/sendfile/sys/sys/vnode.h
==============================================================================
--- projects/sendfile/sys/sys/vnode.h Wed Feb 12 19:26:08 2014 (r261806)
+++ projects/sendfile/sys/sys/vnode.h Wed Feb 12 19:51:12 2014 (r261807)
@@ -719,6 +719,7 @@ int vop_stdbmap(struct vop_bmap_args *);
int vop_stdfsync(struct vop_fsync_args *);
int vop_stdgetwritemount(struct vop_getwritemount_args *);
int vop_stdgetpages(struct vop_getpages_args *);
+int vop_stdgetpages_async(struct vop_getpages_async_args *);
int vop_stdinactive(struct vop_inactive_args *);
int vop_stdislocked(struct vop_islocked_args *);
int vop_stdkqfilter(struct vop_kqfilter_args *);
Modified: projects/sendfile/sys/ufs/ffs/ffs_vnops.c
==============================================================================
--- projects/sendfile/sys/ufs/ffs/ffs_vnops.c Wed Feb 12 19:26:08 2014 (r261806)
+++ projects/sendfile/sys/ufs/ffs/ffs_vnops.c Wed Feb 12 19:51:12 2014 (r261807)
@@ -105,6 +105,7 @@ extern int ffs_rawread(struct vnode *vp,
static vop_fsync_t ffs_fsync;
static vop_lock1_t ffs_lock;
static vop_getpages_t ffs_getpages;
+static vop_getpages_async_t ffs_getpages_async;
static vop_read_t ffs_read;
static vop_write_t ffs_write;
static int ffs_extread(struct vnode *vp, struct uio *uio, int ioflag);
@@ -125,6 +126,7 @@ struct vop_vector ffs_vnodeops1 = {
.vop_default = &ufs_vnodeops,
.vop_fsync = ffs_fsync,
.vop_getpages = ffs_getpages,
+ .vop_getpages_async = ffs_getpages_async,
.vop_lock1 = ffs_lock,
.vop_read = ffs_read,
.vop_reallocblks = ffs_reallocblks,
@@ -839,18 +841,16 @@ ffs_write(ap)
}
/*
- * get page routine
+ * Get page routines.
*/
static int
-ffs_getpages(ap)
- struct vop_getpages_args *ap;
+ffs_getpages_checkvalid(vm_page_t *m, int count, int reqpage)
{
- int i;
vm_page_t mreq;
int pcount;
- pcount = round_page(ap->a_count) / PAGE_SIZE;
- mreq = ap->a_m[ap->a_reqpage];
+ pcount = round_page(count) / PAGE_SIZE;
+ mreq = m[reqpage];
/*
* if ANY DEV_BSIZE blocks are valid on a large filesystem block,
@@ -862,23 +862,47 @@ ffs_getpages(ap)
if (mreq->valid) {
if (mreq->valid != VM_PAGE_BITS_ALL)
vm_page_zero_invalid(mreq, TRUE);
- for (i = 0; i < pcount; i++) {
- if (i != ap->a_reqpage) {
- vm_page_lock(ap->a_m[i]);
- vm_page_free(ap->a_m[i]);
- vm_page_unlock(ap->a_m[i]);
+ for (int i = 0; i < pcount; i++) {
+ if (i != reqpage) {
+ vm_page_lock(m[i]);
+ vm_page_free(m[i]);
+ vm_page_unlock(m[i]);
}
}
VM_OBJECT_WUNLOCK(mreq->object);
- return VM_PAGER_OK;
+ return (VM_PAGER_OK);
}
VM_OBJECT_WUNLOCK(mreq->object);
- return vnode_pager_generic_getpages(ap->a_vp, ap->a_m,
- ap->a_count,
- ap->a_reqpage);
+ return (-1);
}
+static int
+ffs_getpages(struct vop_getpages_args *ap)
+{
+ int rv;
+
+ rv = ffs_getpages_checkvalid(ap->a_m, ap->a_count, ap->a_reqpage);
+ if (rv == VM_PAGER_OK)
+ return (rv);
+
+ return (vnode_pager_generic_getpages(ap->a_vp, ap->a_m, ap->a_count,
+ ap->a_reqpage, NULL, NULL));
+}
+
+static int
+ffs_getpages_async(struct vop_getpages_async_args *ap)
+{
+ int rv;
+
+ rv = ffs_getpages_checkvalid(ap->a_m, ap->a_count, ap->a_reqpage);
+ if (rv == VM_PAGER_OK) {
+ (ap->a_vop_getpages_iodone)(ap->a_arg);
+ return (rv);
+ }
+ return (vnode_pager_generic_getpages(ap->a_vp, ap->a_m, ap->a_count,
+ ap->a_reqpage, ap->a_vop_getpages_iodone, ap->a_arg));
+}
/*
* Extended attribute area reading.
Modified: projects/sendfile/sys/vm/vm_pager.h
==============================================================================
--- projects/sendfile/sys/vm/vm_pager.h Wed Feb 12 19:26:08 2014 (r261806)
+++ projects/sendfile/sys/vm/vm_pager.h Wed Feb 12 19:51:12 2014 (r261807)
@@ -51,18 +51,21 @@ typedef vm_object_t pgo_alloc_t(void *,
struct ucred *);
typedef void pgo_dealloc_t(vm_object_t);
typedef int pgo_getpages_t(vm_object_t, vm_page_t *, int, int);
+typedef int pgo_getpages_async_t(vm_object_t, vm_page_t *, int, int,
+ void(*)(void *), void *);
typedef void pgo_putpages_t(vm_object_t, vm_page_t *, int, int, int *);
typedef boolean_t pgo_haspage_t(vm_object_t, vm_pindex_t, int *, int *);
typedef void pgo_pageunswapped_t(vm_page_t);
struct pagerops {
- pgo_init_t *pgo_init; /* Initialize pager. */
- pgo_alloc_t *pgo_alloc; /* Allocate pager. */
- pgo_dealloc_t *pgo_dealloc; /* Disassociate. */
- pgo_getpages_t *pgo_getpages; /* Get (read) page. */
- pgo_putpages_t *pgo_putpages; /* Put (write) page. */
- pgo_haspage_t *pgo_haspage; /* Does pager have page? */
- pgo_pageunswapped_t *pgo_pageunswapped;
+ pgo_init_t *pgo_init; /* Initialize pager. */
+ pgo_alloc_t *pgo_alloc; /* Allocate pager. */
+ pgo_dealloc_t *pgo_dealloc; /* Disassociate. */
+ pgo_getpages_t *pgo_getpages; /* Get (read) page. */
+ pgo_getpages_async_t *pgo_getpages_async; /* Get page asyncly. */
+ pgo_putpages_t *pgo_putpages; /* Put (write) page. */
+ pgo_haspage_t *pgo_haspage; /* Query page. */
+ pgo_pageunswapped_t *pgo_pageunswapped;
};
extern struct pagerops defaultpagerops;
@@ -103,6 +106,8 @@ vm_object_t vm_pager_allocate(objtype_t,
void vm_pager_bufferinit(void);
void vm_pager_deallocate(vm_object_t);
static __inline int vm_pager_get_pages(vm_object_t, vm_page_t *, int, int);
+static __inline int vm_pager_get_pages_async(vm_object_t, vm_page_t *, int,
+ int, void(*)(void *), void *);
static __inline boolean_t vm_pager_has_page(vm_object_t, vm_pindex_t, int *, int *);
void vm_pager_init(void);
vm_object_t vm_pager_object_lookup(struct pagerlst *, void *);
@@ -131,6 +136,27 @@ vm_pager_get_pages(
return (r);
}
+static __inline int
+vm_pager_get_pages_async(vm_object_t object, vm_page_t *m, int count,
+ int reqpage, void (*iodone)(void *), void *arg)
+{
+ int r;
+
+ VM_OBJECT_ASSERT_WLOCKED(object);
+
+ if (*pagertab[object->type]->pgo_getpages_async == NULL) {
+ /* Emulate async operation. */
+ r = vm_pager_get_pages(object, m, count, reqpage);
+ VM_OBJECT_WUNLOCK(object);
+ (iodone)(arg);
+ VM_OBJECT_WLOCK(object);
+ } else
+ r = (*pagertab[object->type]->pgo_getpages_async)(object, m,
+ count, reqpage, iodone, arg);
+
+ return (r);
+}
+
static __inline void
vm_pager_put_pages(
vm_object_t object,
Modified: projects/sendfile/sys/vm/vnode_pager.c
==============================================================================
--- projects/sendfile/sys/vm/vnode_pager.c Wed Feb 12 19:26:08 2014 (r261806)
+++ projects/sendfile/sys/vm/vnode_pager.c Wed Feb 12 19:51:12 2014 (r261807)
@@ -83,6 +83,8 @@ static int vnode_pager_input_smlfs(vm_ob
static int vnode_pager_input_old(vm_object_t object, vm_page_t m);
static void vnode_pager_dealloc(vm_object_t);
static int vnode_pager_getpages(vm_object_t, vm_page_t *, int, int);
+static int vnode_pager_getpages_async(vm_object_t, vm_page_t *, int, int,
+ void(*)(void *), void *);
static void vnode_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
static boolean_t vnode_pager_haspage(vm_object_t, vm_pindex_t, int *, int *);
static vm_object_t vnode_pager_alloc(void *, vm_ooffset_t, vm_prot_t,
@@ -92,6 +94,7 @@ struct pagerops vnodepagerops = {
.pgo_alloc = vnode_pager_alloc,
.pgo_dealloc = vnode_pager_dealloc,
.pgo_getpages = vnode_pager_getpages,
+ .pgo_getpages_async = vnode_pager_getpages_async,
.pgo_putpages = vnode_pager_putpages,
.pgo_haspage = vnode_pager_haspage,
};
@@ -664,17 +667,51 @@ vnode_pager_getpages(vm_object_t object,
return rtval;
}
+static int
+vnode_pager_getpages_async(vm_object_t object, vm_page_t *m, int count,
+ int reqpage, void (*iodone)(void *), void *arg)
+{
+ int rtval;
+ struct vnode *vp;
+ int bytes = count * PAGE_SIZE;
+
+ vp = object->handle;
+ VM_OBJECT_WUNLOCK(object);
+ rtval = VOP_GETPAGES_ASYNC(vp, m, bytes, reqpage, 0, iodone, arg);
+ KASSERT(rtval != EOPNOTSUPP,
+ ("vnode_pager: FS getpages_async not implemented\n"));
+ VM_OBJECT_WLOCK(object);
+ return rtval;
+}
+
+struct getpages_softc {
+ vm_page_t *m;
+ struct buf *bp;
+ vm_object_t object;
+ vm_offset_t kva;
+ off_t foff;
+ int size;
+ int count;
+ int unmapped;
+ int reqpage;
+ void (*iodone)(void *);
+ void *arg;
+};
+
+int vnode_pager_generic_getpages_done(struct getpages_softc *);
+void vnode_pager_generic_getpages_done_async(struct buf *);
+
/*
* This is now called from local media FS's to operate against their
* own vnodes if they fail to implement VOP_GETPAGES.
*/
int
vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount,
- int reqpage)
+ int reqpage, void (*iodone)(void *), void *arg)
{
vm_object_t object;
vm_offset_t kva;
- off_t foff, tfoff, nextoff;
+ off_t foff;
int i, j, size, bsize, first;
daddr_t firstaddr, reqblock;
struct bufobj *bo;
@@ -684,6 +721,7 @@ vnode_pager_generic_getpages(struct vnod
struct mount *mp;
int count;
int error;
+ int unmapped;
object = vp->v_object;
count = bytecount / PAGE_SIZE;
@@ -891,8 +929,8 @@ vnode_pager_generic_getpages(struct vnod
* requires mapped buffers.
*/
mp = vp->v_mount;
- if (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0 &&
- unmapped_buf_allowed) {
+ unmapped = (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS));
+ if (unmapped && unmapped_buf_allowed) {
bp->b_data = unmapped_buf;
bp->b_kvabase = unmapped_buf;
bp->b_offset = 0;
@@ -905,7 +943,6 @@ vnode_pager_generic_getpages(struct vnod
/* build a minimal buffer header */
bp->b_iocmd = BIO_READ;
- bp->b_iodone = bdone;
KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred"));
KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred"));
bp->b_rcred = crhold(curthread->td_ucred);
@@ -923,9 +960,89 @@ vnode_pager_generic_getpages(struct vnod
/* do the input */
bp->b_iooffset = dbtob(bp->b_blkno);
- bstrategy(bp);
- bwait(bp, PVM, "vnread");
+ if (iodone) { /* async */
+ struct getpages_softc *sc;
+
+ sc = malloc(sizeof(*sc), M_TEMP, M_WAITOK);
+
+ sc->m = m;
+ sc->bp = bp;
+ sc->object = object;
+ sc->foff = foff;
+ sc->size = size;
+ sc->count = count;
+ sc->unmapped = unmapped;
+ sc->reqpage = reqpage;
+ sc->kva = kva;
+
+ sc->iodone = iodone;
+ sc->arg = arg;
+
+ bp->b_iodone = vnode_pager_generic_getpages_done_async;
+ bp->b_caller1 = sc;
+ BUF_KERNPROC(bp);
+ bstrategy(bp);
+ /* Good bye! */
+ } else {
+ struct getpages_softc sc;
+
+ sc.m = m;
+ sc.bp = bp;
+ sc.object = object;
+ sc.foff = foff;
+ sc.size = size;
+ sc.count = count;
+ sc.unmapped = unmapped;
+ sc.reqpage = reqpage;
+ sc.kva = kva;
+
+ bp->b_iodone = bdone;
+ bstrategy(bp);
+ bwait(bp, PVM, "vnread");
+ error = vnode_pager_generic_getpages_done(&sc);
+ }
+
+ return (error ? VM_PAGER_ERROR : VM_PAGER_OK);
+}
+
+void
+vnode_pager_generic_getpages_done_async(struct buf *bp)
+{
+ struct getpages_softc *sc = bp->b_caller1;
+ int error;
+
+ error = vnode_pager_generic_getpages_done(sc);
+ if (error)
+ printf("zhopa %d\n", error);
+
+ vm_page_xunbusy(sc->m[sc->reqpage]);
+
+ sc->iodone(sc->arg);
+
+ free(sc, M_TEMP);
+}
+
+int
+vnode_pager_generic_getpages_done(struct getpages_softc *sc)
+{
+ vm_object_t object;
+ vm_offset_t kva;
+ vm_page_t *m;
+ struct buf *bp;
+ off_t foff, tfoff, nextoff;
+ int i, size, count, unmapped, reqpage;
+ int error = 0;
+
+ m = sc->m;
+ bp = sc->bp;
+ object = sc->object;
+ foff = sc->foff;
+ size = sc->size;
+ count = sc->count;
+ unmapped = sc->unmapped;
+ reqpage = sc->reqpage;
+ kva = sc->kva;
if ((bp->b_ioflags & BIO_ERROR) != 0)
error = EIO;
@@ -939,7 +1056,7 @@ vnode_pager_generic_getpages(struct vnod
}
if ((bp->b_flags & B_UNMAPPED) == 0)
pmap_qremove(kva, count);
- if (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0) {
+ if (unmapped) {
bp->b_data = (caddr_t)kva;
bp->b_kvabase = (caddr_t)kva;
bp->b_flags &= ~B_UNMAPPED;
@@ -995,7 +1112,8 @@ vnode_pager_generic_getpages(struct vnod
if (error) {
printf("vnode_pager_getpages: I/O read error\n");
}
- return (error ? VM_PAGER_ERROR : VM_PAGER_OK);
+
+ return (error);
}
/*
Modified: projects/sendfile/sys/vm/vnode_pager.h
==============================================================================
--- projects/sendfile/sys/vm/vnode_pager.h Wed Feb 12 19:26:08 2014 (r261806)
+++ projects/sendfile/sys/vm/vnode_pager.h Wed Feb 12 19:51:12 2014 (r261807)
@@ -41,7 +41,7 @@
#ifdef _KERNEL
int vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m,
- int count, int reqpage);
+ int count, int reqpage, void (*iodone)(void *), void *arg);
int vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *m,
int count, boolean_t sync,
int *rtvals);
More information about the svn-src-projects
mailing list