svn commit: r324530 - in projects/pnfs-planb-server-stable11/sys/fs: nfs nfsclient nfsserver
Rick Macklem
rmacklem at FreeBSD.org
Wed Oct 11 13:33:13 UTC 2017
Author: rmacklem
Date: Wed Oct 11 13:33:11 2017
New Revision: 324530
URL: https://svnweb.freebsd.org/changeset/base/324530
Log:
Update the stable11 pNFS client/server.
The main changes are:
- Make the pNFS client use a taskqueue to do writes and commits concurrently.
- Convert the pNFS server to using this taskqueue instead of kproc_create().
Modified:
projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonkrpc.c
projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonport.c
projects/pnfs-planb-server-stable11/sys/fs/nfs/nfsclstate.h
projects/pnfs-planb-server-stable11/sys/fs/nfsclient/nfs_clrpcops.c
projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdport.c
projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdserv.c
Modified: projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonkrpc.c
==============================================================================
--- projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonkrpc.c Wed Oct 11 13:20:24 2017 (r324529)
+++ projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonkrpc.c Wed Oct 11 13:33:11 2017 (r324530)
@@ -1119,9 +1119,29 @@ nfsmout:
int
newnfs_nmcancelreqs(struct nfsmount *nmp)
{
+ struct nfsclds *dsp;
+ struct __rpc_client *cl;
if (nmp->nm_sockreq.nr_client != NULL)
CLNT_CLOSE(nmp->nm_sockreq.nr_client);
+lookformore:
+ NFSLOCKMNT(nmp);
+ TAILQ_FOREACH(dsp, &nmp->nm_sess, nfsclds_list) {
+ NFSLOCKDS(dsp);
+ if (dsp != TAILQ_FIRST(&nmp->nm_sess) &&
+ (dsp->nfsclds_flags & NFSCLDS_CLOSED) == 0 &&
+ dsp->nfsclds_sockp != NULL &&
+ dsp->nfsclds_sockp->nr_client != NULL) {
+ dsp->nfsclds_flags |= NFSCLDS_CLOSED;
+ cl = dsp->nfsclds_sockp->nr_client;
+ NFSUNLOCKDS(dsp);
+ NFSUNLOCKMNT(nmp);
+ CLNT_CLOSE(cl);
+ goto lookformore;
+ }
+ NFSUNLOCKDS(dsp);
+ }
+ NFSUNLOCKMNT(nmp);
return (0);
}
Modified: projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonport.c
==============================================================================
--- projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonport.c Wed Oct 11 13:20:24 2017 (r324529)
+++ projects/pnfs-planb-server-stable11/sys/fs/nfs/nfs_commonport.c Wed Oct 11 13:33:11 2017 (r324530)
@@ -40,7 +40,9 @@ __FBSDID("$FreeBSD$");
* to this BSD variant.
*/
#include <fs/nfs/nfsport.h>
+#include <sys/smp.h>
#include <sys/sysctl.h>
+#include <sys/taskqueue.h>
#include <vm/vm.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
@@ -74,6 +76,8 @@ struct nfsdevicehead nfsrv_devidhead;
void (*nfsd_call_servertimer)(void) = NULL;
void (*ncl_call_invalcaches)(struct vnode *) = NULL;
+int nfs_pnfsio(task_fn_t *, void *);
+
static int nfs_realign_test;
static int nfs_realign_count;
static struct ext_nfsstats oldnfsstats;
@@ -90,6 +94,9 @@ SYSCTL_INT(_vfs_nfs, OID_AUTO, debuglevel, CTLFLAG_RW,
0, "Debug level for NFS client");
SYSCTL_INT(_vfs_nfs, OID_AUTO, userhashsize, CTLFLAG_RDTUN, &nfsrv_lughashsize,
0, "Size of hash tables for uid/name mapping");
+int nfs_pnfsiothreads = 0;
+SYSCTL_INT(_vfs_nfs, OID_AUTO, pnfsiothreads, CTLFLAG_RW, &nfs_pnfsiothreads,
+ 0, "Number of pNFS mirror I/O threads");
/*
* Defines for malloc
@@ -676,6 +683,50 @@ nfs_supportsnfsv4acls(struct vnode *vp)
if (error == 0 && retval != 0)
return (1);
return (0);
+}
+
+/*
+ * These are the first fields of all the context structures passed into
+ * nfs_pnfsio().
+ */
+struct pnfsio {
+ int done;
+ int inprog;
+ struct task tsk;
+};
+
+/*
+ * Do a mirror I/O on a pNFS thread.
+ */
+int
+nfs_pnfsio(task_fn_t *func, void *context)
+{
+ struct pnfsio *pio;
+ int ret;
+ static struct taskqueue *pnfsioq = NULL;
+
+ pio = (struct pnfsio *)context;
+ if (pnfsioq == NULL) {
+ if (nfs_pnfsiothreads == 0)
+ nfs_pnfsiothreads = mp_ncpus * 8;
+ pnfsioq = taskqueue_create("pnfsioq", M_WAITOK,
+ taskqueue_thread_enqueue, &pnfsioq);
+ if (pnfsioq == NULL)
+ return (ENOMEM);
+ ret = taskqueue_start_threads(&pnfsioq, nfs_pnfsiothreads,
+ 0, "pnfsiot");
+ if (ret != 0) {
+ taskqueue_free(pnfsioq);
+ pnfsioq = NULL;
+ return (ret);
+ }
+ }
+ pio->inprog = 1;
+ TASK_INIT(&pio->tsk, 0, func, context);
+ ret = taskqueue_enqueue(pnfsioq, &pio->tsk);
+ if (ret != 0)
+ pio->inprog = 0;
+ return (ret);
}
extern int (*nfsd_call_nfscommon)(struct thread *, struct nfssvc_args *);
Modified: projects/pnfs-planb-server-stable11/sys/fs/nfs/nfsclstate.h
==============================================================================
--- projects/pnfs-planb-server-stable11/sys/fs/nfs/nfsclstate.h Wed Oct 11 13:20:24 2017 (r324529)
+++ projects/pnfs-planb-server-stable11/sys/fs/nfs/nfsclstate.h Wed Oct 11 13:33:11 2017 (r324530)
@@ -91,6 +91,7 @@ struct nfsclds {
#define NFSCLDS_HASWRITEVERF 0x0001
#define NFSCLDS_MDS 0x0002
#define NFSCLDS_DS 0x0004
+#define NFSCLDS_CLOSED 0x0008
struct nfsclclient {
LIST_ENTRY(nfsclclient) nfsc_list;
Modified: projects/pnfs-planb-server-stable11/sys/fs/nfsclient/nfs_clrpcops.c
==============================================================================
--- projects/pnfs-planb-server-stable11/sys/fs/nfsclient/nfs_clrpcops.c Wed Oct 11 13:20:24 2017 (r324529)
+++ projects/pnfs-planb-server-stable11/sys/fs/nfsclient/nfs_clrpcops.c Wed Oct 11 13:33:11 2017 (r324530)
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
#include <fs/nfs/nfsport.h>
#include <sys/sysctl.h>
+#include <sys/taskqueue.h>
SYSCTL_DECL(_vfs_nfs);
@@ -64,6 +65,7 @@ extern nfstype nfsv34_type[9];
extern int nfsrv_useacl;
extern char nfsv4_callbackaddr[INET6_ADDRSTRLEN];
extern int nfscl_debuglevel;
+extern int nfs_pnfsiothreads;
NFSCLSTATEMUTEX;
int nfstest_outofseq = 0;
int nfscl_assumeposixlocks = 1;
@@ -87,6 +89,30 @@ enum nfsclds_state {
NFSDSP_NOTFOUND = 2,
};
+/*
+ * Do a write RPC on a DS data file, using this structure for the arguments,
+ * so that this function can be executed by a separate kernel process.
+ */
+struct nfsclwritedsdorpc {
+ int done;
+ int inprog;
+ struct task tsk;
+ struct vnode *vp;
+ int iomode;
+ int must_commit;
+ nfsv4stateid_t *stateidp;
+ struct nfsclds *dsp;
+ uint64_t off;
+ int len;
+ struct nfsfh *fhp;
+ struct mbuf *m;
+ int vers;
+ int minorvers;
+ struct ucred *cred;
+ NFSPROC_T *p;
+ int err;
+};
+
static int nfsrpc_setattrrpc(vnode_t , struct vattr *, nfsv4stateid_t *,
struct ucred *, NFSPROC_T *, struct nfsvattr *, int *, void *);
static int nfsrpc_readrpc(vnode_t , struct uio *, struct ucred *,
@@ -119,7 +145,7 @@ static int nfscl_doflayoutio(vnode_t, struct uio *, in
static int nfscl_dofflayoutio(vnode_t, struct uio *, int *, int *, int *,
nfsv4stateid_t *, int, struct nfscldevinfo *, struct nfscllayout *,
struct nfsclflayout *, uint64_t, uint64_t, int, int, struct mbuf *,
- struct ucred *, NFSPROC_T *);
+ struct nfsclwritedsdorpc *, struct ucred *, NFSPROC_T *);
static struct mbuf *nfsm_copym(struct mbuf *, int, int);
static int nfsrpc_readds(vnode_t, struct uio *, nfsv4stateid_t *, int *,
struct nfsclds *, uint64_t, int, struct nfsfh *, int, int, int,
@@ -127,11 +153,17 @@ static int nfsrpc_readds(vnode_t, struct uio *, nfsv4s
static int nfsrpc_writeds(vnode_t, struct uio *, int *, int *,
nfsv4stateid_t *, struct nfsclds *, uint64_t, int,
struct nfsfh *, int, int, int, int, struct ucred *, NFSPROC_T *);
+static int nfsio_writedsmir(vnode_t, int *, int *, nfsv4stateid_t *,
+ struct nfsclds *, uint64_t, int, struct nfsfh *, struct mbuf *, int, int,
+ struct nfsclwritedsdorpc *, struct ucred *, NFSPROC_T *);
static int nfsrpc_writedsmir(vnode_t, int *, int *, nfsv4stateid_t *,
struct nfsclds *, uint64_t, int, struct nfsfh *, struct mbuf *, int, int,
struct ucred *, NFSPROC_T *);
static enum nfsclds_state nfscl_getsameserver(struct nfsmount *,
struct nfsclds *, struct nfsclds **);
+static int nfsio_commitds(vnode_t, uint64_t, int, struct nfsclds *,
+ struct nfsfh *, int, int, struct nfsclwritedsdorpc *, struct ucred *,
+ NFSPROC_T *);
static int nfsrpc_commitds(vnode_t, uint64_t, int, struct nfsclds *,
struct nfsfh *, int, int, struct ucred *, NFSPROC_T *);
static void nfsrv_setuplayoutget(struct nfsrv_descript *, int, uint64_t,
@@ -163,6 +195,8 @@ static int nfsrpc_layoutgetres(struct nfsmount *, vnod
int, nfsv4stateid_t *, int, uint32_t *, struct nfscllayout **,
struct nfsclflayouthead *, int, int, int *, struct ucred *, NFSPROC_T *);
+int nfs_pnfsio(task_fn_t *, void *);
+
/*
* nfs null call from vfs.
*/
@@ -5539,10 +5573,11 @@ nfscl_doiods(vnode_t vp, struct uio *uiop, int *iomode
struct nfscldevinfo *dip;
struct nfsclflayout *rflp;
struct mbuf *m;
+ struct nfsclwritedsdorpc *drpc, *tdrpc;
nfsv4stateid_t stateid;
struct ucred *newcred;
uint64_t lastbyte, len, off, oresid, xfer;
- int eof, error, firstmirror, i, iolaymode, mirrorcnt, recalled;
+ int eof, error, firstmirror, i, iolaymode, mirrorcnt, recalled, timo;
void *lckp;
uint8_t *dev;
void *iovbase;
@@ -5622,6 +5657,7 @@ nfscl_doiods(vnode_t vp, struct uio *uiop, int *iomode
* do all mirrors.
*/
m = NULL;
+ drpc = NULL;
firstmirror = 0;
mirrorcnt = 1;
if ((layp->nfsly_flags & NFSLY_FLEXFILE) != 0 &&
@@ -5629,17 +5665,24 @@ nfscl_doiods(vnode_t vp, struct uio *uiop, int *iomode
if (rwaccess == NFSV4OPEN_ACCESSREAD) {
firstmirror = arc4random() % mirrorcnt;
mirrorcnt = firstmirror + 1;
- } else if (docommit == 0) {
- /*
- * Save values, so uiop can be rolled
- * back upon a write error.
- */
- offs = uiop->uio_offset;
- resid = uiop->uio_resid;
- iovbase = uiop->uio_iov->iov_base;
- iovlen = uiop->uio_iov->iov_len;
- m = nfsm_uiombuflist(uiop, len, NULL,
- NULL);
+ } else {
+ if (docommit == 0) {
+ /*
+ * Save values, so uiop can be
+ * rolled back upon a write
+ * error.
+ */
+ offs = uiop->uio_offset;
+ resid = uiop->uio_resid;
+ iovbase =
+ uiop->uio_iov->iov_base;
+ iovlen = uiop->uio_iov->iov_len;
+ m = nfsm_uiombuflist(uiop, len,
+ NULL, NULL);
+ }
+ tdrpc = drpc = malloc(sizeof(*drpc) *
+ (mirrorcnt - 1), M_TEMP, M_WAITOK |
+ M_ZERO);
}
}
for (i = firstmirror; i < mirrorcnt && error == 0; i++){
@@ -5656,8 +5699,8 @@ nfscl_doiods(vnode_t vp, struct uio *uiop, int *iomode
uiop, iomode, must_commit,
&eof, &stateid, rwaccess,
dip, layp, rflp, off, xfer,
- i, docommit, m, newcred,
- p);
+ i, docommit, m, tdrpc,
+ newcred, p);
else
error = nfscl_doflayoutio(vp,
uiop, iomode, must_commit,
@@ -5667,9 +5710,27 @@ nfscl_doiods(vnode_t vp, struct uio *uiop, int *iomode
nfscl_reldevinfo(dip);
} else
error = EIO;
+ tdrpc++;
}
if (m != NULL)
m_freem(m);
+ tdrpc = drpc;
+ timo = hz / 50; /* Wait for 20msec. */
+ if (timo < 1)
+ timo = 1;
+ for (i = firstmirror; i < mirrorcnt - 1 &&
+ tdrpc != NULL; i++, tdrpc++) {
+ /*
+ * For the unused drpc entries, both inprog and
+ * err == 0, so this loop won't break.
+ */
+ while (tdrpc->inprog != 0 && tdrpc->done == 0)
+ tsleep(&tdrpc->tsk, PVFS, "clrpcio",
+ timo);
+ if (error == 0 && tdrpc->err != 0)
+ error = tdrpc->err;
+ }
+ free(drpc, M_TEMP);
if (error == 0) {
if (mirrorcnt > 1 && rwaccess ==
NFSV4OPEN_ACCESSWRITE && docommit == 0) {
@@ -5893,8 +5954,8 @@ static int
nfscl_dofflayoutio(vnode_t vp, struct uio *uiop, int *iomode, int *must_commit,
int *eofp, nfsv4stateid_t *stateidp, int rwflag, struct nfscldevinfo *dp,
struct nfscllayout *lyp, struct nfsclflayout *flp, uint64_t off,
- uint64_t len, int mirror, int docommit, struct mbuf *mp, struct ucred *cred,
- NFSPROC_T *p)
+ uint64_t len, int mirror, int docommit, struct mbuf *mp,
+ struct nfsclwritedsdorpc *drpc, struct ucred *cred, NFSPROC_T *p)
{
uint64_t transfer, xfer;
int error, rel_off;
@@ -5935,13 +5996,23 @@ nfscl_dofflayoutio(vnode_t vp, struct uio *uiop, int *
else
xfer = len;
if (docommit != 0) {
- if (error == 0)
- error = nfsrpc_commitds(vp, off, xfer, *dspp,
- fhp, dp->nfsdi_vers, dp->nfsdi_minorvers,
- tcred, p);
- NFSCL_DEBUG(4, "aft nfsrpc_commitds=%d\n", error);
if (error == 0) {
/*
+ * Do last mirrored DS commit with this thread.
+ */
+ if (mirror < flp->nfsfl_mirrorcnt - 1)
+ error = nfsio_commitds(vp, off, xfer,
+ *dspp, fhp, dp->nfsdi_vers,
+ dp->nfsdi_minorvers, drpc, tcred,
+ p);
+ else
+ error = nfsrpc_commitds(vp, off, xfer,
+ *dspp, fhp, dp->nfsdi_vers,
+ dp->nfsdi_minorvers, tcred, p);
+ }
+ NFSCL_DEBUG(4, "aft nfsio_commitds=%d\n", error);
+ if (error == 0) {
+ /*
* Set both eof and uio_resid = 0 to end any
* loops.
*/
@@ -5971,11 +6042,22 @@ nfscl_dofflayoutio(vnode_t vp, struct uio *uiop, int *
m = nfsm_copym(mp, rel_off, xfer);
NFSCL_DEBUG(4, "mcopy reloff=%d xfer=%jd\n",
rel_off, (uintmax_t)xfer);
- error = nfsrpc_writedsmir(vp, iomode,
- must_commit, stateidp, *dspp, off, xfer,
- fhp, m, dp->nfsdi_vers, dp->nfsdi_minorvers,
- tcred, p);
- NFSCL_DEBUG(4, "nfsrpc_writedsmir=%d\n", error);
+ /*
+ * Do last write to a mirrored DS with this
+ * thread.
+ */
+ if (mirror < flp->nfsfl_mirrorcnt - 1)
+ error = nfsio_writedsmir(vp, iomode,
+ must_commit, stateidp, *dspp, off,
+ xfer, fhp, m, dp->nfsdi_vers,
+ dp->nfsdi_minorvers, drpc, tcred,
+ p);
+ else
+ error = nfsrpc_writedsmir(vp, iomode,
+ must_commit, stateidp, *dspp, off,
+ xfer, fhp, m, dp->nfsdi_vers,
+ dp->nfsdi_minorvers, tcred, p);
+ NFSCL_DEBUG(4, "nfsio_writedsmir=%d\n", error);
}
}
NFSCL_DEBUG(4, "aft read/writeds=%d\n", error);
@@ -6302,6 +6384,62 @@ nfsmout:
}
/*
+ * Start up the thread that will execute nfsrpc_writedsmir().
+ */
+static void
+start_writedsmir(void *arg, int pending)
+{
+ struct nfsclwritedsdorpc *drpc;
+
+ drpc = (struct nfsclwritedsdorpc *)arg;
+ drpc->err = nfsrpc_writedsmir(drpc->vp, &drpc->iomode,
+ &drpc->must_commit, drpc->stateidp, drpc->dsp, drpc->off, drpc->len,
+ drpc->fhp, drpc->m, drpc->vers, drpc->minorvers, drpc->cred,
+ drpc->p);
+ drpc->done = 1;
+ NFSCL_DEBUG(4, "start_writedsmir: err=%d\n", drpc->err);
+}
+
+/*
+ * Set up the write DS mirror call for the pNFS I/O thread.
+ */
+static int
+nfsio_writedsmir(vnode_t vp, int *iomode, int *must_commit,
+ nfsv4stateid_t *stateidp, struct nfsclds *dsp, uint64_t off, int len,
+ struct nfsfh *fhp, struct mbuf *m, int vers, int minorvers,
+ struct nfsclwritedsdorpc *drpc, struct ucred *cred, NFSPROC_T *p)
+{
+ int error, ret;
+
+ error = 0;
+ drpc->done = 0;
+ drpc->vp = vp;
+ drpc->iomode = *iomode;
+ drpc->must_commit = *must_commit;
+ drpc->stateidp = stateidp;
+ drpc->dsp = dsp;
+ drpc->off = off;
+ drpc->len = len;
+ drpc->fhp = fhp;
+ drpc->m = m;
+ drpc->vers = vers;
+ drpc->minorvers = minorvers;
+ drpc->cred = cred;
+ drpc->p = p;
+ drpc->inprog = 0;
+ ret = EIO;
+ if (nfs_pnfsiothreads > 0) {
+ ret = nfs_pnfsio(start_writedsmir, drpc);
+ NFSCL_DEBUG(4, "nfsio_writedsmir: nfs_pnfsio=%d\n", ret);
+ }
+ if (ret != 0)
+ error = nfsrpc_writedsmir(vp, iomode, must_commit, stateidp,
+ dsp, off, len, fhp, m, vers, minorvers, cred, p);
+ NFSCL_DEBUG(4, "nfsio_writedsmir: error=%d\n", error);
+ return (error);
+}
+
+/*
* Free up the nfsclds structure.
*/
void
@@ -6427,6 +6565,56 @@ nfsmout:
if (error == 0 && nd->nd_repstat != 0)
error = nd->nd_repstat;
mbuf_freem(nd->nd_mrep);
+ return (error);
+}
+
+/*
+ * Start up the thread that will execute nfsrpc_commitds().
+ */
+static void
+start_commitds(void *arg, int pending)
+{
+ struct nfsclwritedsdorpc *drpc;
+
+ drpc = (struct nfsclwritedsdorpc *)arg;
+ drpc->err = nfsrpc_commitds(drpc->vp, drpc->off, drpc->len,
+ drpc->dsp, drpc->fhp, drpc->vers, drpc->minorvers, drpc->cred,
+ drpc->p);
+ drpc->done = 1;
+ NFSCL_DEBUG(4, "start_commitds: err=%d\n", drpc->err);
+}
+
+/*
+ * Set up the commit DS mirror call for the pNFS I/O thread.
+ */
+static int
+nfsio_commitds(vnode_t vp, uint64_t offset, int cnt, struct nfsclds *dsp,
+ struct nfsfh *fhp, int vers, int minorvers,
+ struct nfsclwritedsdorpc *drpc, struct ucred *cred, NFSPROC_T *p)
+{
+ int error, ret;
+
+ error = 0;
+ drpc->done = 0;
+ drpc->vp = vp;
+ drpc->off = offset;
+ drpc->len = cnt;
+ drpc->dsp = dsp;
+ drpc->fhp = fhp;
+ drpc->vers = vers;
+ drpc->minorvers = minorvers;
+ drpc->cred = cred;
+ drpc->p = p;
+ drpc->inprog = 0;
+ ret = EIO;
+ if (nfs_pnfsiothreads > 0) {
+ ret = nfs_pnfsio(start_commitds, drpc);
+ NFSCL_DEBUG(4, "nfsio_commitds: nfs_pnfsio=%d\n", ret);
+ }
+ if (ret != 0)
+ error = nfsrpc_commitds(vp, offset, cnt, dsp, fhp, vers,
+ minorvers, cred, p);
+ NFSCL_DEBUG(4, "nfsio_commitds: error=%d\n", error);
return (error);
}
Modified: projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdport.c
==============================================================================
--- projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdport.c Wed Oct 11 13:20:24 2017 (r324529)
+++ projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdport.c Wed Oct 11 13:33:11 2017 (r324530)
@@ -72,6 +72,7 @@ extern struct mtx nfsrv_dsrmlock_mtx;
extern struct mtx nfsrv_dwrpclock_mtx;
extern struct mtx nfsrv_dsrpclock_mtx;
extern struct mtx nfsrv_darpclock_mtx;
+extern int nfs_pnfsiothreads;
struct vfsoptlist nfsv4root_opt, nfsv4root_newopt;
NFSDLOCKMUTEX;
NFSSTATESPINLOCK;
@@ -127,6 +128,8 @@ static int nfsrv_putfhname(fhandle_t *, char *);
static int nfsrv_pnfslookupds(struct vnode *, struct pnfsdsfile *,
struct vnode *, NFSPROC_T *);
+int nfs_pnfsio(task_fn_t *, void *);
+
SYSCTL_NODE(_vfs, OID_AUTO, nfsd, CTLFLAG_RW, 0, "NFS server");
SYSCTL_INT(_vfs_nfsd, OID_AUTO, mirrormnt, CTLFLAG_RW,
&nfsrv_enable_crossmntpt, 0, "Enable nfsd to cross mount points");
@@ -4419,6 +4422,9 @@ nfsmout:
* so that this function can be executed by a separate kernel process.
*/
struct nfsrvwritedsdorpc {
+ int done;
+ int inprog;
+ struct task tsk;
fhandle_t fh;
off_t off;
int len;
@@ -4426,7 +4432,6 @@ struct nfsrvwritedsdorpc {
struct ucred *cred;
NFSPROC_T *p;
struct mbuf *m;
- int haskproc;
int err;
};
@@ -4543,18 +4548,15 @@ nfsmout:
* Start up the thread that will execute nfsrv_writedsdorpc().
*/
static void
-start_writedsdorpc(void *arg)
+start_writedsdorpc(void *arg, int pending)
{
struct nfsrvwritedsdorpc *drpc;
drpc = (struct nfsrvwritedsdorpc *)arg;
drpc->err = nfsrv_writedsdorpc(drpc->nmp, &drpc->fh, drpc->off,
drpc->len, NULL, drpc->m, drpc->cred, drpc->p);
- NFSDWRPCLOCK();
- drpc->haskproc = 0;
- wakeup(drpc);
- NFSDWRPCUNLOCK();
- kproc_exit(0);
+ drpc->done = 1;
+ NFSD_DEBUG(4, "start_writedsdorpc: err=%d\n", drpc->err);
}
static int
@@ -4565,7 +4567,7 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s
struct nfsrvwritedsdorpc *drpc, *tdrpc;
struct nfsvattr na;
struct mbuf *m;
- int error, haskproc, i, offs, ret;
+ int error, i, offs, ret, timo;
NFSD_DEBUG(4, "in nfsrv_writedsrpc\n");
KASSERT(*mpp != NULL, ("nfsrv_writedsrpc: NULL mbuf chain"));
@@ -4582,23 +4584,25 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s
* Do the write RPC for every DS, using a separate kernel process
* for every DS except the last one.
*/
- haskproc = 0;
error = 0;
for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
+ tdrpc->done = 0;
tdrpc->fh = *fhp;
tdrpc->off = off;
tdrpc->len = len;
tdrpc->nmp = *nmpp;
tdrpc->cred = cred;
tdrpc->p = p;
+ tdrpc->inprog = 0;
+ tdrpc->err = 0;
tdrpc->m = m_copym(*mpp, offs, NFSM_RNDUP(len), M_WAITOK);
- tdrpc->haskproc = 1;
- ret = kproc_create(start_writedsdorpc, (void *)tdrpc, NULL, 0,
- 0, "nfsdpw");
- if (ret == 0)
- haskproc = 1;
- else {
- tdrpc->haskproc = 0;
+ ret = EIO;
+ if (nfs_pnfsiothreads > 0) {
+ ret = nfs_pnfsio(start_writedsdorpc, tdrpc);
+ NFSD_DEBUG(4, "nfsrv_writedsrpc: nfs_pnfsio=%d\n",
+ ret);
+ }
+ if (ret != 0) {
ret = nfsrv_writedsdorpc(*nmpp, fhp, off, len, NULL,
tdrpc->m, cred, p);
if (error == 0 && ret != 0)
@@ -4615,17 +4619,16 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s
error = nfsrv_setextattr(vp, &na, p);
NFSD_DEBUG(4, "nfsrv_writedsrpc: aft setextat=%d\n",
error);
- if (haskproc != 0) {
- /* Wait for kernel proc(s) to complete. */
- NFSDWRPCLOCK();
- for (tdrpc = drpc, i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
- while (tdrpc->haskproc != 0)
- mtx_sleep(tdrpc, NFSDWRPCLOCKMUTEXPTR, PVFS,
- "nfspw", 0);
- if (error == 0 && tdrpc->err != 0)
- error = tdrpc->err;
- }
- NFSDWRPCUNLOCK();
+ tdrpc = drpc;
+ timo = hz / 50; /* Wait for 20msec. */
+ if (timo < 1)
+ timo = 1;
+ for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
+ /* Wait for RPCs on separate threads to complete. */
+ while (tdrpc->inprog != 0 && tdrpc->done == 0)
+ tsleep(&tdrpc->tsk, PVFS, "srvwrds", timo);
+ if (error == 0 && tdrpc->err != 0)
+ error = tdrpc->err;
}
free(drpc, M_TEMP);
return (error);
@@ -4715,6 +4718,9 @@ nfsmout:
}
struct nfsrvsetattrdsdorpc {
+ int done;
+ int inprog;
+ struct task tsk;
fhandle_t fh;
struct nfsmount *nmp;
struct vnode *vp;
@@ -4722,7 +4728,6 @@ struct nfsrvsetattrdsdorpc {
NFSPROC_T *p;
struct nfsvattr na;
struct nfsvattr dsna;
- int haskproc;
int err;
};
@@ -4730,18 +4735,14 @@ struct nfsrvsetattrdsdorpc {
* Start up the thread that will execute nfsrv_setattrdsdorpc().
*/
static void
-start_setattrdsdorpc(void *arg)
+start_setattrdsdorpc(void *arg, int pending)
{
struct nfsrvsetattrdsdorpc *drpc;
drpc = (struct nfsrvsetattrdsdorpc *)arg;
drpc->err = nfsrv_setattrdsdorpc(&drpc->fh, drpc->cred, drpc->p,
drpc->vp, drpc->nmp, &drpc->na, &drpc->dsna);
- NFSDSRPCLOCK();
- drpc->haskproc = 0;
- wakeup(drpc);
- NFSDSRPCUNLOCK();
- kproc_exit(0);
+ drpc->done = 1;
}
static int
@@ -4751,7 +4752,7 @@ nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred,
{
struct nfsrvsetattrdsdorpc *drpc, *tdrpc;
struct nfsvattr na;
- int error, haskproc, i, ret;
+ int error, i, ret, timo;
NFSD_DEBUG(4, "in nfsrv_setattrdsrpc\n");
drpc = NULL;
@@ -4763,22 +4764,24 @@ nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred,
* Do the setattr RPC for every DS, using a separate kernel process
* for every DS except the last one.
*/
- haskproc = 0;
error = 0;
for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
+ tdrpc->done = 0;
+ tdrpc->inprog = 0;
tdrpc->fh = *fhp;
tdrpc->nmp = *nmpp;
tdrpc->vp = vp;
tdrpc->cred = cred;
tdrpc->p = p;
tdrpc->na = *nap;
- tdrpc->haskproc = 1;
- ret = kproc_create(start_setattrdsdorpc, (void *)tdrpc, NULL, 0,
- 0, "nfsdps");
- if (ret == 0)
- haskproc = 1;
- else {
- tdrpc->haskproc = 0;
+ tdrpc->err = 0;
+ ret = EIO;
+ if (nfs_pnfsiothreads > 0) {
+ ret = nfs_pnfsio(start_setattrdsdorpc, tdrpc);
+ NFSD_DEBUG(4, "nfsrv_setattrdsrpc: nfs_pnfsio=%d\n",
+ ret);
+ }
+ if (ret != 0) {
ret = nfsrv_setattrdsdorpc(fhp, cred, p, vp, *nmpp, nap,
&na);
if (error == 0 && ret != 0)
@@ -4793,17 +4796,16 @@ nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred,
if (error == 0)
error = nfsrv_setextattr(vp, &na, p);
NFSD_DEBUG(4, "nfsrv_setattrdsrpc: aft setextat=%d\n", error);
- if (haskproc != 0) {
- /* Wait for kernel proc(s) to complete. */
- NFSDSRPCLOCK();
- for (tdrpc = drpc, i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
- while (tdrpc->haskproc != 0)
- mtx_sleep(tdrpc, NFSDSRPCLOCKMUTEXPTR, PVFS,
- "nfsps", 0);
- if (error == 0 && tdrpc->err != 0)
- error = tdrpc->err;
- }
- NFSDSRPCUNLOCK();
+ tdrpc = drpc;
+ timo = hz / 50; /* Wait for 20msec. */
+ if (timo < 1)
+ timo = 1;
+ for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
+ /* Wait for RPCs on separate threads to complete. */
+ while (tdrpc->inprog != 0 && tdrpc->done == 0)
+ tsleep(&tdrpc->tsk, PVFS, "srvsads", timo);
+ if (error == 0 && tdrpc->err != 0)
+ error = tdrpc->err;
}
free(drpc, M_TEMP);
return (error);
@@ -4860,13 +4862,15 @@ nfsrv_setacldsdorpc(fhandle_t *fhp, struct ucred *cred
}
struct nfsrvsetacldsdorpc {
+ int done;
+ int inprog;
+ struct task tsk;
fhandle_t fh;
struct nfsmount *nmp;
struct vnode *vp;
struct ucred *cred;
NFSPROC_T *p;
struct acl *aclp;
- int haskproc;
int err;
};
@@ -4874,18 +4878,14 @@ struct nfsrvsetacldsdorpc {
* Start up the thread that will execute nfsrv_setacldsdorpc().
*/
static void
-start_setacldsdorpc(void *arg)
+start_setacldsdorpc(void *arg, int pending)
{
struct nfsrvsetacldsdorpc *drpc;
drpc = (struct nfsrvsetacldsdorpc *)arg;
drpc->err = nfsrv_setacldsdorpc(&drpc->fh, drpc->cred, drpc->p,
drpc->vp, drpc->nmp, drpc->aclp);
- NFSDARPCLOCK();
- drpc->haskproc = 0;
- wakeup(drpc);
- NFSDARPCUNLOCK();
- kproc_exit(0);
+ drpc->done = 1;
}
static int
@@ -4893,7 +4893,7 @@ nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred,
struct vnode *vp, struct nfsmount **nmpp, int mirrorcnt, struct acl *aclp)
{
struct nfsrvsetacldsdorpc *drpc, *tdrpc;
- int error, haskproc, i, ret;
+ int error, i, ret, timo;
NFSD_DEBUG(4, "in nfsrv_setacldsrpc\n");
drpc = NULL;
@@ -4905,22 +4905,24 @@ nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred,
* Do the setattr RPC for every DS, using a separate kernel process
* for every DS except the last one.
*/
- haskproc = 0;
error = 0;
for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
+ tdrpc->done = 0;
+ tdrpc->inprog = 0;
tdrpc->fh = *fhp;
tdrpc->nmp = *nmpp;
tdrpc->vp = vp;
tdrpc->cred = cred;
tdrpc->p = p;
tdrpc->aclp = aclp;
- tdrpc->haskproc = 1;
- ret = kproc_create(start_setacldsdorpc, (void *)tdrpc, NULL, 0,
- 0, "nfsdpa");
- if (ret == 0)
- haskproc = 1;
- else {
- tdrpc->haskproc = 0;
+ tdrpc->err = 0;
+ ret = EIO;
+ if (nfs_pnfsiothreads > 0) {
+ ret = nfs_pnfsio(start_setacldsdorpc, tdrpc);
+ NFSD_DEBUG(4, "nfsrv_setacldsrpc: nfs_pnfsio=%d\n",
+ ret);
+ }
+ if (ret != 0) {
ret = nfsrv_setacldsdorpc(fhp, cred, p, vp, *nmpp,
aclp);
if (error == 0 && ret != 0)
@@ -4933,17 +4935,16 @@ nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred,
if (error == 0 && ret != 0)
error = ret;
NFSD_DEBUG(4, "nfsrv_setacldsrpc: aft setextat=%d\n", error);
- if (haskproc != 0) {
- /* Wait for kernel proc(s) to complete. */
- NFSDARPCLOCK();
- for (tdrpc = drpc, i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
- while (tdrpc->haskproc != 0)
- mtx_sleep(tdrpc, NFSDARPCLOCKMUTEXPTR, PVFS,
- "nfspa", 0);
- if (error == 0 && tdrpc->err != 0)
- error = tdrpc->err;
- }
- NFSDARPCUNLOCK();
+ tdrpc = drpc;
+ timo = hz / 50; /* Wait for 20msec. */
+ if (timo < 1)
+ timo = 1;
+ for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
+ /* Wait for RPCs on separate threads to complete. */
+ while (tdrpc->inprog != 0 && tdrpc->done == 0)
+ tsleep(&tdrpc->tsk, PVFS, "srvacds", timo);
+ if (error == 0 && tdrpc->err != 0)
+ error = tdrpc->err;
}
free(drpc, M_TEMP);
return (error);
Modified: projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdserv.c
==============================================================================
--- projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdserv.c Wed Oct 11 13:20:24 2017 (r324529)
+++ projects/pnfs-planb-server-stable11/sys/fs/nfsserver/nfs_nfsdserv.c Wed Oct 11 13:33:11 2017 (r324530)
@@ -1000,7 +1000,7 @@ nfsrvd_write(struct nfsrv_descript *nd, __unused int i
nd->nd_md, nd->nd_dpos, nd->nd_cred, p);
error = nfsm_advance(nd, NFSM_RNDUP(retlen), -1);
if (error)
- panic("nfsrv_write mbuf");
+ goto nfsmout;
}
if (nd->nd_flag & ND_NFSV4)
aftat_ret = 0;
More information about the svn-src-projects
mailing list