svn commit: r218001 -
projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp
Jeff Roberson
jeff at FreeBSD.org
Fri Jan 28 02:10:31 UTC 2011
Author: jeff
Date: Fri Jan 28 02:10:30 2011
New Revision: 218001
URL: http://svn.freebsd.org/changeset/base/218001
Log:
- Improve the tx completion interrupt and polling logic.
- Don't call sdp_do_posts() immediately when the connection is
established. This causes a race with mlx4 cards but not mthca.
- Add disabled interrupt moderation as it did not help me but it may
help in other cases.
Sponsored by: Isilon Systems, iX Systems, and Panasas.
Modified:
projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h
projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_bcopy.c
projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_cma.c
projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c
projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_rx.c
projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_tx.c
Modified: projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h
==============================================================================
--- projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h Fri Jan 28 00:22:03 2011 (r218000)
+++ projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h Fri Jan 28 02:10:30 2011 (r218001)
@@ -316,7 +316,6 @@ struct sdp_tx_ring {
atomic_t tail;
struct ib_cq *cq;
- int una_seq;
atomic_t credits;
#define tx_credits(ssk) (atomic_read(&ssk->tx_ring.credits))
Modified: projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_bcopy.c
==============================================================================
--- projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_bcopy.c Fri Jan 28 00:22:03 2011 (r218000)
+++ projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_bcopy.c Fri Jan 28 02:10:30 2011 (r218001)
@@ -163,10 +163,10 @@ out:
void
sdp_post_sends(struct sdp_sock *ssk, int wait)
{
- /* TODO: nonagle? */
struct mbuf *mb;
int post_count = 0;
struct socket *sk;
+ int low;
sk = ssk->socket;
if (unlikely(!ssk->id)) {
@@ -177,7 +177,7 @@ sdp_post_sends(struct sdp_sock *ssk, int
}
return;
}
-
+again:
if (sdp_tx_ring_slots_left(ssk) < SDP_TX_SIZE / 2)
sdp_xmit_poll(ssk, 1);
@@ -242,8 +242,13 @@ sdp_post_sends(struct sdp_sock *ssk, int
sdp_post_send(ssk, mb);
post_count++;
}
- if (post_count)
- sdp_xmit_poll(ssk, 0);
+ low = (sdp_tx_ring_slots_left(ssk) <= SDP_MIN_TX_CREDITS);
+ if (post_count || low) {
+ if (low)
+ sdp_arm_tx_cq(ssk);
+ if (sdp_xmit_poll(ssk, low))
+ goto again;
+ }
return;
allocfail:
Modified: projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_cma.c
==============================================================================
--- projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_cma.c Fri Jan 28 00:22:03 2011 (r218000)
+++ projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_cma.c Fri Jan 28 02:10:30 2011 (r218001)
@@ -192,7 +192,7 @@ sdp_response_handler(struct socket *sk,
ssk = sdp_sk(sk);
SDP_WLOCK(ssk);
ssk->state = TCPS_ESTABLISHED;
-/* sdp_set_default_moderation(ssk); */
+ sdp_set_default_moderation(ssk);
if (ssk->flags & SDP_DROPPED) {
SDP_WUNLOCK(ssk);
return 0;
@@ -212,7 +212,6 @@ sdp_response_handler(struct socket *sk,
ssk->fport = dst_addr->sin_port;
ssk->faddr = dst_addr->sin_addr.s_addr;
soisconnected(sk);
- sdp_do_posts(ssk);
SDP_WUNLOCK(ssk);
return 0;
@@ -229,15 +228,13 @@ sdp_connected_handler(struct socket *sk,
SDP_WLOCK(ssk);
ssk->state = TCPS_ESTABLISHED;
-/* sdp_set_default_moderation(ssk); */
+ sdp_set_default_moderation(ssk);
if (sk->so_options & SO_KEEPALIVE)
sdp_start_keepalive_timer(sk);
- if ((ssk->flags & SDP_DROPPED) == 0) {
+ if ((ssk->flags & SDP_DROPPED) == 0)
soisconnected(sk);
- sdp_do_posts(ssk);
- }
SDP_WUNLOCK(ssk);
return 0;
}
Modified: projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c
==============================================================================
--- projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c Fri Jan 28 00:22:03 2011 (r218000)
+++ projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c Fri Jan 28 02:10:30 2011 (r218001)
@@ -1385,15 +1385,11 @@ deliver:
sbdrop_locked(sb, len);
/* Notify protocol that we drained some data. */
- if ((so->so_proto->pr_flags & PR_WANTRCVD) &&
- (((flags & MSG_WAITALL) && uio->uio_resid > 0) ||
- !(flags & MSG_SOCALLBCK))) {
- SOCKBUF_UNLOCK(sb);
- SDP_WLOCK(ssk);
- sdp_do_posts(ssk);
- SDP_WUNLOCK(ssk);
- SOCKBUF_LOCK(sb);
- }
+ SOCKBUF_UNLOCK(sb);
+ SDP_WLOCK(ssk);
+ sdp_do_posts(ssk);
+ SDP_WUNLOCK(ssk);
+ SOCKBUF_LOCK(sb);
}
/*
@@ -1699,6 +1695,18 @@ sdp_ctloutput(struct socket *so, struct
}
#undef SDP_WLOCK_RECHECK
+int sdp_mod_count = 0;
+int sdp_mod_usec = 0;
+
+void
+sdp_set_default_moderation(struct sdp_sock *ssk)
+{
+ if (sdp_mod_count <= 0 || sdp_mod_usec <= 0)
+ return;
+ ib_modify_cq(ssk->rx_ring.cq, sdp_mod_count, sdp_mod_usec);
+}
+
+
static void
sdp_dev_add(struct ib_device *device)
{
Modified: projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_rx.c
==============================================================================
--- projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_rx.c Fri Jan 28 00:22:03 2011 (r218000)
+++ projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_rx.c Fri Jan 28 02:10:30 2011 (r218001)
@@ -33,7 +33,7 @@
SDP_MODPARAM_INT(rcvbuf_initial_size, 32 * 1024,
"Receive buffer initial size in bytes.");
-SDP_MODPARAM_SINT(rcvbuf_scale, 0x10,
+SDP_MODPARAM_SINT(rcvbuf_scale, 0x8,
"Receive buffer size scale factor.");
/* Like tcp_fin - called when SDP_MID_DISCONNECT is received */
@@ -161,31 +161,31 @@ sdp_post_recv(struct sdp_sock *ssk)
static inline int
sdp_post_recvs_needed(struct sdp_sock *ssk)
{
- int scale = rcvbuf_scale;
- int buffer_size = ssk->recv_bytes;
+ unsigned long bytes_in_process;
unsigned long max_bytes;
+ int buffer_size;
+ int posted;
if (!ssk->qp_active || !ssk->socket)
return 0;
- max_bytes = ssk->socket->so_snd.sb_mbmax * scale;
- if (unlikely(rx_ring_posted(ssk) >= SDP_RX_SIZE))
+ posted = rx_ring_posted(ssk);
+ if (posted >= SDP_RX_SIZE)
return 0;
+ if (posted < SDP_MIN_TX_CREDITS)
+ return 1;
- if (likely(rx_ring_posted(ssk) >= SDP_MIN_TX_CREDITS)) {
- unsigned long bytes_in_process =
- (rx_ring_posted(ssk) - SDP_MIN_TX_CREDITS) *
- buffer_size;
- bytes_in_process += ssk->socket->so_rcv.sb_cc;
- if (bytes_in_process >= max_bytes) {
- sdp_prf(ssk->socket, NULL,
- "bytes_in_process:%ld > max_bytes:%ld",
- bytes_in_process, max_bytes);
- return 0;
- }
- }
+ buffer_size = ssk->recv_bytes;
+ max_bytes = max(ssk->socket->so_snd.sb_hiwat,
+ (1 + SDP_MIN_TX_CREDITS) * buffer_size);
+ max_bytes *= rcvbuf_scale;
+ /*
+ * Compute bytes in the receive queue and socket buffer.
+ */
+ bytes_in_process = (posted - SDP_MIN_TX_CREDITS) * buffer_size;
+ bytes_in_process += ssk->socket->so_rcv.sb_cc;
- return 1;
+ return bytes_in_process < max_bytes;
}
static inline void
Modified: projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_tx.c
==============================================================================
--- projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_tx.c Fri Jan 28 00:22:03 2011 (r218000)
+++ projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_tx.c Fri Jan 28 02:10:30 2011 (r218001)
@@ -134,7 +134,7 @@ sdp_post_send(struct sdp_sock *ssk, stru
sge->lkey = ssk->sdp_dev->mr->lkey;
}
tx_wr.next = NULL;
- tx_wr.wr_id = ring_head(ssk->tx_ring) | SDP_OP_SEND;
+ tx_wr.wr_id = mseq | SDP_OP_SEND;
tx_wr.sg_list = ibsge;
tx_wr.num_sge = i;
tx_wr.opcode = IB_WR_SEND;
@@ -178,11 +178,8 @@ sdp_send_completion(struct sdp_sock *ssk
dev = ssk->ib_device;
tx_req = &tx_ring->buffer[mseq & (SDP_TX_SIZE - 1)];
mb = tx_req->mb;
-
sdp_cleanup_sdp_buf(ssk, tx_req, DMA_TO_DEVICE);
- tx_ring->una_seq += mb->m_pkthdr.len - sizeof(struct sdp_bsdh);
-
#ifdef SDP_ZCOPY
/* TODO: AIO and real zcopy code; add their context support here */
if (BZCOPY_STATE(mb))
@@ -201,10 +198,6 @@ sdp_handle_send_comp(struct sdp_sock *ss
struct mbuf *mb = NULL;
struct sdp_bsdh *h;
- mb = sdp_send_completion(ssk, wc->wr_id);
- if (unlikely(!mb))
- return -1;
-
if (unlikely(wc->status)) {
if (wc->status != IB_WC_WR_FLUSH_ERR) {
sdp_prf(ssk->socket, mb, "Send completion with error. "
@@ -215,6 +208,10 @@ sdp_handle_send_comp(struct sdp_sock *ss
}
}
+ mb = sdp_send_completion(ssk, wc->wr_id);
+ if (unlikely(!mb))
+ return -1;
+
h = mtod(mb, struct sdp_bsdh *);
sdp_prf1(ssk->socket, mb, "tx completion. mseq:%d", ntohl(h->mseq));
sdp_dbg(ssk->socket, "tx completion. %p %d mseq:%d",
@@ -300,21 +297,10 @@ sdp_process_tx_cq(struct sdp_sock *ssk)
} while (n == SDP_NUM_WC);
if (wc_processed) {
- struct socket *sk = ssk->socket;
sdp_post_sends(ssk, M_DONTWAIT);
sdp_prf1(sk, NULL, "Waking sendmsg. inflight=%d",
(u32) tx_ring_posted(ssk));
sowwakeup(ssk->socket);
- /*
- * If there is no room in the tx queue we arm the tx cq
- * to force an interrupt.
- */
- if (tx_ring_posted(ssk) && sk->so_snd.sb_cc >=
- sk->so_snd.sb_mbmax - ssk->xmit_size_goal) {
- sdp_prf(ssk->socket, NULL, "pending tx - rearming");
- sdp_arm_tx_cq(ssk);
- }
-
}
return wc_processed;
More information about the svn-src-projects
mailing list