svn commit: r216727 -
projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp
Jeff Roberson
jeff at FreeBSD.org
Mon Dec 27 00:12:58 UTC 2010
Author: jeff
Date: Mon Dec 27 00:12:57 2010
New Revision: 216727
URL: http://svn.freebsd.org/changeset/base/216727
Log:
- Implement missing OOB support modeled after TCP.
- Document and resolve some potential socket teardown races with rx. The
rx path proceeds under a separate lock and ssk->socket may become NULL
while it's operating.
Sponsored by: Isilon Systems, iX Systems, and Panasas.
Modified:
projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h
projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c
projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_rx.c
Modified: projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h
==============================================================================
--- projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h Sun Dec 26 23:19:16 2010 (r216726)
+++ projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h Mon Dec 27 00:12:57 2010 (r216727)
@@ -357,6 +357,7 @@ struct sdp_moderation {
int moder_time;
};
+/* These are flags fields. */
#define SDP_TIMEWAIT 0x0001 /* In ssk timewait state. */
#define SDP_DROPPED 0x0002 /* Socket has been dropped. */
#define SDP_SOCKREF 0x0004 /* Holding a sockref for close. */
@@ -364,8 +365,10 @@ struct sdp_moderation {
#define SDP_NEEDFIN 0x0010 /* Send a fin on the next tx. */
#define SDP_DREQWAIT 0x0020 /* Waiting on DREQ. */
#define SDP_HAVEOOB 0x0040 /* Have OOB data. */
-#define SDP_HADOOB 0x0080 /* Had OOB data. */
-#define SDP_DESTROY 0x0100 /* Being destroyed. */
+
+/* These are oobflags */
+#define SDP_HADOOB 0x0001 /* Had OOB data. */
+#define SDP_DESTROY 0x0002 /* Being destroyed. */
struct sdp_sock {
LIST_ENTRY(sdp_sock) list;
@@ -383,6 +386,7 @@ struct sdp_sock {
in_port_t fport;
in_addr_t faddr;
int flags;
+ int oobflags; /* protected by rx lock. */
int state;
int softerror;
int recv_bytes; /* Bytes per recv. buf including header */
Modified: projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c
==============================================================================
--- projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c Sun Dec 26 23:19:16 2010 (r216726)
+++ projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c Mon Dec 27 00:12:57 2010 (r216727)
@@ -1391,6 +1391,10 @@ sdp_rcvoob(struct socket *so, struct mbu
ssk = sdp_sk(so);
SDP_WLOCK(ssk);
+ if (!rx_ring_trylock(&ssk->rx_ring)) {
+ SDP_WUNLOCK(ssk);
+ return (ECONNRESET);
+ }
if (ssk->flags & (SDP_TIMEWAIT | SDP_DROPPED)) {
error = ECONNRESET;
goto out;
@@ -1398,20 +1402,20 @@ sdp_rcvoob(struct socket *so, struct mbu
if ((so->so_oobmark == 0 &&
(so->so_rcv.sb_state & SBS_RCVATMARK) == 0) ||
so->so_options & SO_OOBINLINE ||
- ssk->flags & SDP_HADOOB) {
+ ssk->oobflags & SDP_HADOOB) {
error = EINVAL;
goto out;
}
- if ((ssk->flags & SDP_HAVEOOB) == 0) {
+ if ((ssk->oobflags & SDP_HAVEOOB) == 0) {
error = EWOULDBLOCK;
goto out;
}
m->m_len = 1;
*mtod(m, caddr_t) = ssk->iobc;
if ((flags & MSG_PEEK) == 0)
- ssk->flags ^= (SDP_HAVEOOB | SDP_HADOOB);
-
+ ssk->oobflags ^= (SDP_HAVEOOB | SDP_HADOOB);
out:
+ rx_ring_unlock(&ssk->rx_ring);
SDP_WUNLOCK(ssk);
return (error);
}
@@ -1425,6 +1429,28 @@ sdp_sock_init(void *mem, int size, int f
return (0);
}
+void
+sdp_urg(struct sdp_sock *ssk, struct mbuf *mb)
+{
+ struct mbuf *m;
+ struct socket *so;
+
+ so = ssk->socket;
+ if (so == NULL)
+ return;
+
+ so->so_oobmark = so->so_rcv.sb_cc + mb->m_pkthdr.len - 1;
+ sohasoutofband(so);
+ ssk->oobflags &= ~(SDP_HAVEOOB | SDP_HADOOB);
+ if (!(so->so_options & SO_OOBINLINE)) {
+ for (m = mb; m->m_next != NULL; m = m->m_next);
+ ssk->iobc = *(mtod(m, char *) + m->m_len - 1);
+ ssk->oobflags |= SDP_HAVEOOB;
+ m->m_len--;
+ mb->m_pkthdr.len--;
+ }
+}
+
/*
* Notify a sdp socket of an asynchronous error.
*
Modified: projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_rx.c
==============================================================================
--- projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_rx.c Sun Dec 26 23:19:16 2010 (r216726)
+++ projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_rx.c Mon Dec 27 00:12:57 2010 (r216727)
@@ -166,7 +166,7 @@ sdp_post_recvs_needed(struct sdp_sock *s
int buffer_size = ssk->recv_bytes;
unsigned long max_bytes;
- if (!ssk->qp_active)
+ if (!ssk->qp_active || !ssk->socket)
return 0;
max_bytes = ssk->socket->so_snd.sb_mbmax * scale;
@@ -243,8 +243,8 @@ sdp_sock_queue_rcv_mb(struct socket *sk,
m_adj(mb, SDP_HEAD_SIZE);
SOCKBUF_LOCK(&sk->so_rcv);
-/* if (unlikely(h->flags & SDP_OOB_PRES))
- sdp_urg(ssk, mb); XXX */
+ if (unlikely(h->flags & SDP_OOB_PRES))
+ sdp_urg(ssk, mb);
sbappend_locked(&sk->so_rcv, mb);
sorwakeup_locked(sk);
return mb;
@@ -327,9 +327,10 @@ static int
sdp_process_rx_ctl_mb(struct sdp_sock *ssk, struct mbuf *mb)
{
struct sdp_bsdh *h;
- struct socket *sk = ssk->socket;
+ struct socket *sk;
SDP_WLOCK_ASSERT(ssk);
+ sk = ssk->socket;
h = mtod(mb, struct sdp_bsdh *);
switch (h->mid) {
case SDP_MID_DATA:
@@ -404,12 +405,23 @@ sdp_process_rx_ctl_mb(struct sdp_sock *s
static int
sdp_process_rx_mb(struct sdp_sock *ssk, struct mbuf *mb)
{
- struct socket *sk = ssk->socket;
+ struct socket *sk;
struct sdp_bsdh *h;
unsigned long mseq_ack;
int credits_before;
h = mtod(mb, struct sdp_bsdh *);
+ sk = ssk->socket;
+ /*
+ * If another thread is in so_pcbfree this may be partially torn
+ * down but no further synchronization is required as the destroying
+ * thread will wait for receive to shutdown before discarding the
+ * socket.
+ */
+ if (sk == NULL) {
+ m_freem(mb);
+ return 0;
+ }
SDPSTATS_HIST_LINEAR(credits_before_update, tx_credits(ssk));
@@ -481,7 +493,7 @@ sdp_process_rx_wc(struct sdp_sock *ssk,
return NULL;
if (unlikely(wc->status)) {
- if (ssk->qp_active) {
+ if (ssk->qp_active && sk) {
sdp_dbg(sk, "Recv completion with error. "
"Status %d, vendor: %d\n",
wc->status, wc->vendor_err);
@@ -524,7 +536,7 @@ sdp_bzcopy_write_space(struct sdp_sock *
{
struct socket *sk = ssk->socket;
- if (tx_credits(ssk) >= ssk->min_bufs)
+ if (tx_credits(ssk) >= ssk->min_bufs && sk)
sowwakeup(sk);
}
More information about the svn-src-projects
mailing list