svn commit: r358765 - projects/nfs-over-tls/sys/rpc
Rick Macklem
rmacklem at FreeBSD.org
Sun Mar 8 18:37:07 UTC 2020
Author: rmacklem
Date: Sun Mar 8 18:37:04 2020
New Revision: 358765
URL: https://svnweb.freebsd.org/changeset/base/358765
Log:
Make a bunch of changes to the kernel RPC so that it can handle reception
of ext_pgs mbuf lists.
jhb@ thinks this will be be needed for certain cases of the KERN_TLS.
There are also some changes for handling of flags passed down from
the rpctlssd daemon that indicate the results of client certificate
validation.
One of these flags, RPCTLS_FLAGS_DISABLE, causes all RPCs on the
connection to fail with AUTH_REJECTEDCRED.
The others will be used by future commits to the NFS server code
to check against new export flags.
There are also changes in rpcsec_tls and the nfs code to make this
support work.
Modified:
projects/nfs-over-tls/sys/rpc/clnt.h
projects/nfs-over-tls/sys/rpc/clnt_bck.c
projects/nfs-over-tls/sys/rpc/clnt_vc.c
projects/nfs-over-tls/sys/rpc/rpc_generic.c
projects/nfs-over-tls/sys/rpc/rpcsec_tls.h
projects/nfs-over-tls/sys/rpc/svc.c
projects/nfs-over-tls/sys/rpc/svc.h
projects/nfs-over-tls/sys/rpc/svc_auth.c
projects/nfs-over-tls/sys/rpc/svc_vc.c
Modified: projects/nfs-over-tls/sys/rpc/clnt.h
==============================================================================
--- projects/nfs-over-tls/sys/rpc/clnt.h Sun Mar 8 18:24:15 2020 (r358764)
+++ projects/nfs-over-tls/sys/rpc/clnt.h Sun Mar 8 18:37:04 2020 (r358765)
@@ -120,6 +120,7 @@ struct rpc_callextra {
void *rc_feedback_arg; /* argument for callback */
struct rpc_timers *rc_timers; /* optional RTT timers */
struct rpc_err rc_err; /* detailed call status */
+ u_int rc_mbufoffs; /* Offset in resultsp mbuf */
};
#endif
Modified: projects/nfs-over-tls/sys/rpc/clnt_bck.c
==============================================================================
--- projects/nfs-over-tls/sys/rpc/clnt_bck.c Sun Mar 8 18:24:15 2020 (r358764)
+++ projects/nfs-over-tls/sys/rpc/clnt_bck.c Sun Mar 8 18:37:04 2020 (r358765)
@@ -61,8 +61,11 @@ __FBSDID("$FreeBSD$");
* connection provided by the client to the server.
*/
+#include "opt_kern_tls.h"
+
#include <sys/param.h>
#include <sys/systm.h>
+#include <sys/ktls.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
@@ -84,7 +87,12 @@ __FBSDID("$FreeBSD$");
#include <rpc/rpc.h>
#include <rpc/rpc_com.h>
#include <rpc/krpc.h>
+#include <rpc/rpcsec_tls.h>
+#ifdef KERN_TLS
+extern u_int ktls_maxlen;
+#endif
+
struct cmessage {
struct cmsghdr cmsg;
struct cmsgcred cmcred;
@@ -203,7 +211,8 @@ clnt_bck_call(
uint32_t xid;
struct mbuf *mreq = NULL, *results;
struct ct_request *cr;
- int error;
+ int error, maxextsiz;
+ uint32_t junk;
cr = malloc(sizeof(struct ct_request), M_RPC, M_WAITOK);
@@ -296,6 +305,18 @@ call_again:
TAILQ_INSERT_TAIL(&ct->ct_pending, cr, cr_link);
mtx_unlock(&ct->ct_lock);
+ /* For RPC-over-TLS, copy mrep to a chain of ext_pgs. */
+ if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0) {
+ /*
+ * Copy the mbuf chain to a chain of
+ * ext_pgs mbuf(s) as required by KERN_TLS.
+ */
+ maxextsiz = TLS_MAX_MSG_SIZE_V10_2;
+#ifdef KERN_TLS
+ maxextsiz = min(maxextsiz, ktls_maxlen);
+#endif
+ mreq = _rpc_copym_into_ext_pgs(mreq, maxextsiz);
+ }
/*
* sosend consumes mreq.
*/
@@ -403,7 +424,9 @@ got_reply:
ext->rc_feedback(FEEDBACK_OK, proc, ext->rc_feedback_arg);
xdrmbuf_create(&xdrs, cr->cr_mrep, XDR_DECODE);
- ok = xdr_replymsg(&xdrs, &reply_msg);
+ ok = xdr_uint32_t(&xdrs, &junk);
+ if (ok)
+ ok = xdr_replymsg(&xdrs, &reply_msg);
cr->cr_mrep = NULL;
if (ok) {
@@ -422,6 +445,14 @@ got_reply:
} else {
KASSERT(results,
("auth validated but no result"));
+ if (ext) {
+ if ((results->m_flags & M_NOMAP) !=
+ 0)
+ ext->rc_mbufoffs =
+ xdrs.x_handy;
+ else
+ ext->rc_mbufoffs = 0;
+ }
*resultsp = results;
}
} /* end successful completion */
Modified: projects/nfs-over-tls/sys/rpc/clnt_vc.c
==============================================================================
--- projects/nfs-over-tls/sys/rpc/clnt_vc.c Sun Mar 8 18:24:15 2020 (r358764)
+++ projects/nfs-over-tls/sys/rpc/clnt_vc.c Sun Mar 8 18:37:04 2020 (r358765)
@@ -84,6 +84,7 @@ __FBSDID("$FreeBSD$");
#include <rpc/rpc.h>
#include <rpc/rpc_com.h>
#include <rpc/krpc.h>
+#include <rpc/rpcsec_tls.h>
#ifdef KERN_TLS
extern u_int ktls_maxlen;
@@ -532,6 +533,19 @@ got_reply:
if (ext && ext->rc_feedback)
ext->rc_feedback(FEEDBACK_OK, proc, ext->rc_feedback_arg);
+#ifdef notnow
+{ struct mbuf *m, *m2;
+int txxxx;
+if (cr->cr_mrep != NULL) {
+txxxx = m_length(cr->cr_mrep, NULL);
+if (txxxx > 0) {
+m = mb_copym_ext_pgs(cr->cr_mrep, txxxx, 16384, M_WAITOK,
+ false, mb_free_mext_pgs, &m2);
+m2 = cr->cr_mrep;
+cr->cr_mrep = m;
+m_freem(m2);
+} } }
+#endif
xdrmbuf_create(&xdrs, cr->cr_mrep, XDR_DECODE);
ok = xdr_replymsg(&xdrs, &reply_msg);
cr->cr_mrep = NULL;
@@ -553,6 +567,14 @@ got_reply:
} else {
KASSERT(results,
("auth validated but no result"));
+ if (ext) {
+ if ((results->m_flags & M_NOMAP) !=
+ 0)
+ ext->rc_mbufoffs =
+ xdrs.x_handy;
+ else
+ ext->rc_mbufoffs = 0;
+ }
*resultsp = results;
}
} /* end successful completion */
@@ -749,8 +771,9 @@ clnt_vc_control(CLIENT *cl, u_int request, void *info)
if (ct->ct_backchannelxprt == NULL) {
xprt->xp_p2 = ct;
if (ct->ct_tls)
- xprt->xp_tls = TRUE;
+ xprt->xp_tls = RPCTLS_FLAGS_HANDSHAKE;
ct->ct_backchannelxprt = xprt;
+printf("backch tls=0x%x xprt=%p\n", xprt->xp_tls, xprt);
}
break;
@@ -1032,9 +1055,11 @@ clnt_vc_soupcall(struct socket *so, void *arg, int wai
ntohl(xid_plus_direction[1]);
/* Check message direction. */
if (xid_plus_direction[1] == CALL) {
+printf("Got backchannel callback\n");
/* This is a backchannel request. */
mtx_lock(&ct->ct_lock);
xprt = ct->ct_backchannelxprt;
+printf("backxprt=%p\n", xprt);
if (xprt == NULL) {
mtx_unlock(&ct->ct_lock);
/* Just throw it away. */
Modified: projects/nfs-over-tls/sys/rpc/rpc_generic.c
==============================================================================
--- projects/nfs-over-tls/sys/rpc/rpc_generic.c Sun Mar 8 18:24:15 2020 (r358764)
+++ projects/nfs-over-tls/sys/rpc/rpc_generic.c Sun Mar 8 18:37:04 2020 (r358765)
@@ -904,8 +904,7 @@ _rpc_copym_into_ext_pgs(struct mbuf *mp, int maxextsiz
tlen = mp->m_len;
m2 = mp;
for (m = mp->m_next; m != NULL; m = m->m_next) {
- if ((m->m_flags & (M_EXT | M_NOMAP)) ==
- (M_EXT | M_NOMAP))
+ if ((m->m_flags & M_NOMAP) != 0)
break;
tlen += m->m_len;
m2 = m;
Modified: projects/nfs-over-tls/sys/rpc/rpcsec_tls.h
==============================================================================
--- projects/nfs-over-tls/sys/rpc/rpcsec_tls.h Sun Mar 8 18:24:15 2020 (r358764)
+++ projects/nfs-over-tls/sys/rpc/rpcsec_tls.h Sun Mar 8 18:37:04 2020 (r358765)
@@ -35,10 +35,16 @@
#define RPCTLS_SYSC_CONNECT 2
#define RPCTLS_SYSC_SERVER 3
+/* Flag bits to indicate certificate results. */
+#define RPCTLS_FLAGS_HANDSHAKE 0x01
+#define RPCTLS_FLAGS_GOTCERT 0x02
+#define RPCTLS_FLAGS_SELFSIGNED 0x04
+#define RPCTLS_FLAGS_VERIFIED 0x08
+#define RPCTLS_FLAGS_DISABLED 0x10
+
#ifdef _KERNEL
/* Functions that perform upcalls to the rpctlsd daemon. */
enum clnt_stat rpctls_connect(CLIENT *newclient, struct socket *so);
-enum clnt_stat rpctls_server(struct socket *so);
/* String for AUTH_TLS reply verifier. */
#define RPCTLS_START_STRING "STARTTLS"
Modified: projects/nfs-over-tls/sys/rpc/svc.c
==============================================================================
--- projects/nfs-over-tls/sys/rpc/svc.c Sun Mar 8 18:24:15 2020 (r358764)
+++ projects/nfs-over-tls/sys/rpc/svc.c Sun Mar 8 18:37:04 2020 (r358765)
@@ -670,10 +670,13 @@ svc_sendreply_common(struct svc_req *rqstp, struct rpc
replay_setreply(xprt->xp_pool->sp_rcache,
rply, svc_getrpccaller(rqstp), body);
+printf("sendreply_common\n");
if (!SVCAUTH_WRAP(&rqstp->rq_auth, &body))
return (FALSE);
+printf("at SVC_REPLY\n");
ok = SVC_REPLY(xprt, rply, rqstp->rq_addr, body, &rqstp->rq_reply_seq);
+printf("aft SVC_REPLY ok=%d\n", ok);
if (rqstp->rq_addr) {
free(rqstp->rq_addr, M_SONAME);
rqstp->rq_addr = NULL;
@@ -814,6 +817,7 @@ svcerr_auth(struct svc_req *rqstp, enum auth_stat why)
if (xprt->xp_pool->sp_rcache)
replay_setreply(xprt->xp_pool->sp_rcache,
&rply, svc_getrpccaller(rqstp), NULL);
+printf("SVC SENDAUTHERR\n");
svc_sendreply_common(rqstp, &rply, NULL);
}
@@ -976,6 +980,7 @@ svc_getreq(SVCXPRT *xprt, struct svc_req **rqstp_ret)
* should not be dispatched to the
* application.
*/
+printf("AUTH FAILED=%d\n", why);
if (why != RPCSEC_GSS_NODISPATCH)
svcerr_auth(r, why);
goto call_done;
Modified: projects/nfs-over-tls/sys/rpc/svc.h
==============================================================================
--- projects/nfs-over-tls/sys/rpc/svc.h Sun Mar 8 18:24:15 2020 (r358764)
+++ projects/nfs-over-tls/sys/rpc/svc.h Sun Mar 8 18:37:04 2020 (r358765)
@@ -175,7 +175,9 @@ typedef struct __rpc_svcxprt {
int xp_upcallset; /* socket upcall is set up */
uint32_t xp_snd_cnt; /* # of bytes to send to socket */
uint32_t xp_snt_cnt; /* # of bytes sent to socket */
+ u_int xp_mbufoffs; /* Offset into ext_pgs mbuf */
bool_t xp_dontrcv; /* Do not receive on the socket */
+ uint32_t xp_tls; /* RPC-over-TLS on socket */
#else
int xp_fd;
u_short xp_port; /* associated port number */
Modified: projects/nfs-over-tls/sys/rpc/svc_auth.c
==============================================================================
--- projects/nfs-over-tls/sys/rpc/svc_auth.c Sun Mar 8 18:24:15 2020 (r358764)
+++ projects/nfs-over-tls/sys/rpc/svc_auth.c Sun Mar 8 18:37:04 2020 (r358765)
@@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$");
#include <sys/ucred.h>
#include <rpc/rpc.h>
+#include <rpc/rpcsec_tls.h>
static enum auth_stat (*_svcauth_rpcsec_gss)(struct svc_req *,
struct rpc_msg *) = NULL;
@@ -94,12 +95,18 @@ _authenticate(struct svc_req *rqst, struct rpc_msg *ms
dummy = _svcauth_null(rqst, msg);
return (dummy);
case AUTH_SYS:
+ if ((rqst->rq_xprt->xp_tls & RPCTLS_FLAGS_DISABLED) != 0)
+ return (AUTH_REJECTEDCRED);
dummy = _svcauth_unix(rqst, msg);
return (dummy);
case AUTH_SHORT:
+ if ((rqst->rq_xprt->xp_tls & RPCTLS_FLAGS_DISABLED) != 0)
+ return (AUTH_REJECTEDCRED);
dummy = _svcauth_short(rqst, msg);
return (dummy);
case RPCSEC_GSS:
+ if ((rqst->rq_xprt->xp_tls & RPCTLS_FLAGS_DISABLED) != 0)
+ return (AUTH_REJECTEDCRED);
if (!_svcauth_rpcsec_gss)
return (AUTH_REJECTEDCRED);
dummy = _svcauth_rpcsec_gss(rqst, msg);
Modified: projects/nfs-over-tls/sys/rpc/svc_vc.c
==============================================================================
--- projects/nfs-over-tls/sys/rpc/svc_vc.c Sun Mar 8 18:24:15 2020 (r358764)
+++ projects/nfs-over-tls/sys/rpc/svc_vc.c Sun Mar 8 18:37:04 2020 (r358765)
@@ -45,10 +45,13 @@ __FBSDID("$FreeBSD$");
* and a record/tcp stream.
*/
+#include "opt_kern_tls.h"
+
#include <sys/param.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/kernel.h>
+#include <sys/ktls.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/mutex.h>
@@ -66,12 +69,17 @@ __FBSDID("$FreeBSD$");
#include <netinet/tcp.h>
#include <rpc/rpc.h>
+#include <rpc/rpcsec_tls.h>
#include <rpc/krpc.h>
#include <rpc/rpc_com.h>
#include <security/mac/mac_framework.h>
+#ifdef KERN_TLS
+extern u_int ktls_maxlen;
+#endif
+
static bool_t svc_vc_rendezvous_recv(SVCXPRT *, struct rpc_msg *,
struct sockaddr **, struct mbuf **);
static enum xprt_stat svc_vc_rendezvous_stat(SVCXPRT *);
@@ -581,6 +589,30 @@ svc_vc_process_pending(SVCXPRT *xprt)
struct socket *so = xprt->xp_socket;
struct mbuf *m;
+{ struct mbuf *m1, *m2, *m3, *m4;
+ int txxxx;
+ m3 = cd->mpending;
+ m4 = NULL;
+ while (m3 != NULL && (m3->m_flags & M_NOMAP) != 0) {
+ m4 = m3;
+ m3 = m3->m_next;
+ }
+ if (m3 != NULL) {
+ txxxx = m_length(m3, NULL);
+ if (txxxx > 0) {
+ m1 = mb_copym_ext_pgs(m3, txxxx, 16384, M_WAITOK,
+ false, mb_free_mext_pgs, &m2);
+ if (m4 != NULL) {
+ m4->m_next = m1;
+ m_freem(m3);
+ } else {
+ m2 = cd->mpending;
+ cd->mpending = m1;
+ m_freem(m2);
+ }
+ }
+ }
+}
/*
* If cd->resid is non-zero, we have part of the
* record already, otherwise we are expecting a record
@@ -610,7 +642,7 @@ svc_vc_process_pending(SVCXPRT *xprt)
header = ntohl(header);
cd->eor = (header & 0x80000000) != 0;
cd->resid = header & 0x7fffffff;
- m_adj(cd->mpending, sizeof(uint32_t));
+ cd->resid += sizeof(uint32_t);
}
/*
@@ -623,10 +655,14 @@ svc_vc_process_pending(SVCXPRT *xprt)
while (cd->mpending && cd->resid) {
m = cd->mpending;
if (cd->mpending->m_next
- || cd->mpending->m_len > cd->resid)
- cd->mpending = m_split(cd->mpending,
- cd->resid, M_WAITOK);
- else
+ || cd->mpending->m_len > cd->resid) {
+ if ((cd->mpending->m_flags & M_NOMAP) != 0)
+ cd->mpending = mb_splitatpos_ext(
+ cd->mpending, cd->resid, M_WAITOK);
+ else
+ cd->mpending = m_split(cd->mpending,
+ cd->resid, M_WAITOK);
+ } else
cd->mpending = NULL;
if (cd->mreq)
m_last(cd->mreq)->m_next = m;
@@ -660,7 +696,7 @@ svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg,
struct socket* so = xprt->xp_socket;
XDR xdrs;
int error, rcvflag;
- uint32_t xid_plus_direction[2];
+ uint32_t xid_plus_direction[3], junk;
/*
* Serialise access to the socket and our own record parsing
@@ -691,15 +727,15 @@ svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg,
m_copydata(cd->mreq, 0,
sizeof(xid_plus_direction),
(char *)xid_plus_direction);
- xid_plus_direction[0] =
- ntohl(xid_plus_direction[0]);
xid_plus_direction[1] =
ntohl(xid_plus_direction[1]);
+ xid_plus_direction[2] =
+ ntohl(xid_plus_direction[2]);
/* Check message direction. */
- if (xid_plus_direction[1] == REPLY) {
+ if (xid_plus_direction[2] == REPLY) {
clnt_bck_svccall(xprt->xp_p2,
cd->mreq,
- xid_plus_direction[0]);
+ xid_plus_direction[1]);
cd->mreq = NULL;
continue;
}
@@ -719,13 +755,18 @@ svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg,
sx_xunlock(&xprt->xp_lock);
- if (! xdr_callmsg(&xdrs, msg)) {
+ if (! xdr_uint32_t(&xdrs, &junk) ||
+ ! xdr_callmsg(&xdrs, msg)) {
XDR_DESTROY(&xdrs);
return (FALSE);
}
*addrp = NULL;
*mp = xdrmbuf_getall(&xdrs);
+ if (((*mp)->m_flags & M_NOMAP) != 0)
+ xprt->xp_mbufoffs = xdrs.x_handy;
+ else
+ xprt->xp_mbufoffs = 0;
XDR_DESTROY(&xdrs);
return (TRUE);
@@ -827,13 +868,31 @@ svc_vc_backchannel_recv(SVCXPRT *xprt, struct rpc_msg
mtx_unlock(&ct->ct_lock);
sx_xunlock(&xprt->xp_lock);
+printf("recv backch m=%p\n", m);
+{ struct mbuf *m1, *m2;
+int txxxx;
+if (m != NULL) {
+txxxx = m_length(m, NULL);
+if (txxxx > 0) {
+m1 = mb_copym_ext_pgs(m, txxxx, 16384, M_WAITOK,
+ false, mb_free_mext_pgs, &m2);
+m2 = m;
+m = m1;
+m_freem(m2);
+} } }
xdrmbuf_create(&xdrs, m, XDR_DECODE);
if (! xdr_callmsg(&xdrs, msg)) {
+printf("recv backch callmsg failed\n");
XDR_DESTROY(&xdrs);
return (FALSE);
}
*addrp = NULL;
*mp = xdrmbuf_getall(&xdrs);
+ if (((*mp)->m_flags & M_NOMAP) != 0)
+ xprt->xp_mbufoffs = xdrs.x_handy;
+ else
+ xprt->xp_mbufoffs = 0;
+printf("backch offs=%d\n", xprt->xp_mbufoffs);
XDR_DESTROY(&xdrs);
return (TRUE);
}
@@ -845,7 +904,7 @@ svc_vc_reply(SVCXPRT *xprt, struct rpc_msg *msg,
XDR xdrs;
struct mbuf *mrep;
bool_t stat = TRUE;
- int error, len;
+ int error, len, maxextsiz;
/*
* Leave space for record mark.
@@ -875,7 +934,23 @@ svc_vc_reply(SVCXPRT *xprt, struct rpc_msg *msg,
len = mrep->m_pkthdr.len;
*mtod(mrep, uint32_t *) =
htonl(0x80000000 | (len - sizeof(uint32_t)));
+
+ /* For RPC-over-TLS, copy mrep to a chain of ext_pgs. */
+ if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0) {
+ /*
+ * Copy the mbuf chain to a chain of
+ * ext_pgs mbuf(s) as required by KERN_TLS.
+ */
+ maxextsiz = TLS_MAX_MSG_SIZE_V10_2;
+#ifdef KERN_TLS
+ maxextsiz = min(maxextsiz, ktls_maxlen);
+#endif
+ mrep = _rpc_copym_into_ext_pgs(mrep, maxextsiz);
+ }
atomic_add_32(&xprt->xp_snd_cnt, len);
+ /*
+ * sosend consumes mreq.
+ */
error = sosend(xprt->xp_socket, NULL, NULL, mrep, NULL,
0, curthread);
if (!error) {
@@ -902,7 +977,7 @@ svc_vc_backchannel_reply(SVCXPRT *xprt, struct rpc_msg
XDR xdrs;
struct mbuf *mrep;
bool_t stat = TRUE;
- int error;
+ int error, maxextsiz;
/*
* Leave space for record mark.
@@ -932,6 +1007,19 @@ svc_vc_backchannel_reply(SVCXPRT *xprt, struct rpc_msg
*mtod(mrep, uint32_t *) =
htonl(0x80000000 | (mrep->m_pkthdr.len
- sizeof(uint32_t)));
+
+ /* For RPC-over-TLS, copy mrep to a chain of ext_pgs. */
+ if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0) {
+ /*
+ * Copy the mbuf chain to a chain of
+ * ext_pgs mbuf(s) as required by KERN_TLS.
+ */
+ maxextsiz = TLS_MAX_MSG_SIZE_V10_2;
+#ifdef KERN_TLS
+ maxextsiz = min(maxextsiz, ktls_maxlen);
+#endif
+ mrep = _rpc_copym_into_ext_pgs(mrep, maxextsiz);
+ }
sx_xlock(&xprt->xp_lock);
ct = (struct ct_data *)xprt->xp_p2;
if (ct != NULL)
More information about the svn-src-projects
mailing list