git: a5a965d75934 - main - cxgbe tom: Enable ULP_MODE_TCPDDP on demand
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Wed, 31 Jan 2024 00:42:13 UTC
The branch main has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=a5a965d75934ae809884f8613bcad156bb5d7148 commit a5a965d75934ae809884f8613bcad156bb5d7148 Author: John Baldwin <jhb@FreeBSD.org> AuthorDate: 2024-01-31 00:41:43 +0000 Commit: John Baldwin <jhb@FreeBSD.org> CommitDate: 2024-01-31 00:41:43 +0000 cxgbe tom: Enable ULP_MODE_TCPDDP on demand Most ULP modes in cxgbe's TOE are enabled on the fly when a protocol is needed (e.g. ULP_MODE_ISCSI is enabled by cxgbei when offloading a connection using iSCSI, and ULP_MODE_TLS is enabled when RX TLS keys are programmed for a TOE connection). The one exception to this is ULP_MODE_TCPDDP. Currently the cxgbe driver enables ULP_MODE_TCPDDP when a TOE connection is first created. However, since DDP connections cannot be converted to other connection types, this requires some special handling in the driver. For example, iSCSI daemons use the SO_NO_DDP socket option to ensure TOE connections use ULP_MODE_NONE so they can be converted to ULP_MODE_ISCSI. Similarly, using TLS receive offload (ULP_MODE_TLS) requires disabling TCP DDP for new connections by default. This commit changes cxgbe to instead switch a connection from ULP_MODE_NONE to ULP_MODE_TCPDDP when a connection first attempts to use TCP DDP via aio_read(2). This permits connections to always start as ULP_MODE_NONE and switch to a protocol-specific mode as needed. Reviewed by: np Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D43670 --- sys/dev/cxgbe/tom/t4_cpl_io.c | 11 +++--- sys/dev/cxgbe/tom/t4_ddp.c | 86 ++++++++++++++++++++++++++++++++++++++++++- sys/dev/cxgbe/tom/t4_tom.c | 17 +++------ sys/dev/cxgbe/tom/t4_tom.h | 1 - 4 files changed, 95 insertions(+), 20 deletions(-) diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c index 2f425c7b5c6d..4d61189f5fe3 100644 --- a/sys/dev/cxgbe/tom/t4_cpl_io.c +++ b/sys/dev/cxgbe/tom/t4_cpl_io.c @@ -1753,17 +1753,18 @@ do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) if (changed) { if (toep->ddp.flags & DDP_SC_REQ) toep->ddp.flags ^= DDP_ON | DDP_SC_REQ; - else { - KASSERT(cpl->ddp_off == 1, - ("%s: DDP switched on by itself.", - __func__)); - + else if (cpl->ddp_off == 1) { /* Fell out of DDP mode */ toep->ddp.flags &= ~DDP_ON; CTR1(KTR_CXGBE, "%s: fell out of DDP mode", __func__); insert_ddp_data(toep, ddp_placed); + } else { + /* + * Data was received while still + * ULP_MODE_NONE, just fall through. + */ } } diff --git a/sys/dev/cxgbe/tom/t4_ddp.c b/sys/dev/cxgbe/tom/t4_ddp.c index c50446d98e2b..89cb7269e218 100644 --- a/sys/dev/cxgbe/tom/t4_ddp.c +++ b/sys/dev/cxgbe/tom/t4_ddp.c @@ -192,7 +192,7 @@ free_ddp_buffer(struct tom_data *td, struct ddp_buffer *db) free_pageset(td, db->ps); } -void +static void ddp_init_toep(struct toepcb *toep) { @@ -810,6 +810,78 @@ do_rx_ddp_complete(struct sge_iq *iq, const struct rss_header *rss, return (0); } +static bool +set_ddp_ulp_mode(struct toepcb *toep) +{ + struct adapter *sc = toep->vi->adapter; + struct wrqe *wr; + struct work_request_hdr *wrh; + struct ulp_txpkt *ulpmc; + int fields, len; + + if (!sc->tt.ddp) + return (false); + + fields = 0; + + /* Overlay region including W_TCB_RX_DDP_FLAGS */ + fields += 3; + + /* W_TCB_ULP_TYPE */ + fields++; + +#ifdef USE_DDP_RX_FLOW_CONTROL + /* W_TCB_T_FLAGS */ + fields++; +#endif + + len = sizeof(*wrh) + fields * roundup2(LEN__SET_TCB_FIELD_ULP, 16); + KASSERT(len <= SGE_MAX_WR_LEN, + ("%s: WR with %d TCB field updates too large", __func__, fields)); + + wr = alloc_wrqe(len, toep->ctrlq); + if (wr == NULL) + return (false); + + CTR(KTR_CXGBE, "%s: tid %u", __func__, toep->tid); + + wrh = wrtod(wr); + INIT_ULPTX_WRH(wrh, len, 1, 0); /* atomic */ + ulpmc = (struct ulp_txpkt *)(wrh + 1); + + /* + * Words 26/27 are zero except for the DDP_OFF flag in + * W_TCB_RX_DDP_FLAGS (27). + */ + ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, 26, + 0xffffffffffffffff, (uint64_t)V_TF_DDP_OFF(1) << 32); + + /* Words 28/29 are zero. */ + ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, 28, + 0xffffffffffffffff, 0); + + /* Words 30/31 are zero. */ + ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, 30, + 0xffffffffffffffff, 0); + + /* Set the ULP mode to ULP_MODE_TCPDDP. */ + toep->params.ulp_mode = ULP_MODE_TCPDDP; + ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_ULP_TYPE, + V_TCB_ULP_TYPE(M_TCB_ULP_TYPE), + V_TCB_ULP_TYPE(ULP_MODE_TCPDDP)); + +#ifdef USE_DDP_RX_FLOW_CONTROL + /* Set TF_RX_FLOW_CONTROL_DDP. */ + ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_T_FLAGS, + V_TF_RX_FLOW_CONTROL_DDP(1), V_TF_RX_FLOW_CONTROL_DDP(1)); +#endif + + ddp_init_toep(toep); + + t4_wrq_tx(sc, wr); + return (true); +} + static void enable_ddp(struct adapter *sc, struct toepcb *toep) { @@ -2203,7 +2275,8 @@ t4_aio_cancel_queued(struct kaiocb *job) int t4_aio_queue_ddp(struct socket *so, struct kaiocb *job) { - struct tcpcb *tp = sototcpcb(so); + struct inpcb *inp = sotoinpcb(so); + struct tcpcb *tp = intotcpcb(inp); struct toepcb *toep = tp->t_toe; @@ -2211,6 +2284,15 @@ t4_aio_queue_ddp(struct socket *so, struct kaiocb *job) if (job->uaiocb.aio_lio_opcode != LIO_READ) return (EOPNOTSUPP); + INP_WLOCK(inp); + if (__predict_false(ulp_mode(toep) == ULP_MODE_NONE)) { + if (!set_ddp_ulp_mode(toep)) { + INP_WUNLOCK(inp); + return (EOPNOTSUPP); + } + } + INP_WUNLOCK(inp); + DDP_LOCK(toep); /* diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c index 77b6ba5d4032..9bfe6ec818e4 100644 --- a/sys/dev/cxgbe/tom/t4_tom.c +++ b/sys/dev/cxgbe/tom/t4_tom.c @@ -179,8 +179,7 @@ init_toepcb(struct vi_info *vi, struct toepcb *toep) toep->ctrlq = &sc->sge.ctrlq[pi->port_id]; tls_init_toep(toep); - if (ulp_mode(toep) == ULP_MODE_TCPDDP) - ddp_init_toep(toep); + MPASS(ulp_mode(toep) != ULP_MODE_TCPDDP); toep->flags |= TPF_INITIALIZED; @@ -1216,10 +1215,7 @@ calc_options2(struct vi_info *vi, struct conn_params *cp) opt2 |= V_RX_COALESCE(M_RX_COALESCE); opt2 |= V_RX_FC_DDP(0) | V_RX_FC_DISABLE(0); -#ifdef USE_DDP_RX_FLOW_CONTROL - if (cp->ulp_mode == ULP_MODE_TCPDDP) - opt2 |= F_RX_FC_DDP; -#endif + MPASS(cp->ulp_mode != ULP_MODE_TCPDDP); return (htobe32(opt2)); } @@ -1327,11 +1323,7 @@ init_conn_params(struct vi_info *vi , struct offload_settings *s, cp->tx_align = 0; /* ULP mode. */ - if (s->ddp > 0 || - (s->ddp < 0 && sc->tt.ddp && (so_options_get(so) & SO_NO_DDP) == 0)) - cp->ulp_mode = ULP_MODE_TCPDDP; - else - cp->ulp_mode = ULP_MODE_NONE; + cp->ulp_mode = ULP_MODE_NONE; /* Rx coalescing. */ if (s->rx_coalesce >= 0) @@ -1972,7 +1964,8 @@ t4_aio_queue_tom(struct socket *so, struct kaiocb *job) if (SOLISTENING(so)) return (EINVAL); - if (ulp_mode(toep) == ULP_MODE_TCPDDP) { + if (ulp_mode(toep) == ULP_MODE_TCPDDP || + ulp_mode(toep) == ULP_MODE_NONE) { error = t4_aio_queue_ddp(so, job); if (error != EOPNOTSUPP) return (error); diff --git a/sys/dev/cxgbe/tom/t4_tom.h b/sys/dev/cxgbe/tom/t4_tom.h index b492fe86ae3a..d74f3f908286 100644 --- a/sys/dev/cxgbe/tom/t4_tom.h +++ b/sys/dev/cxgbe/tom/t4_tom.h @@ -505,7 +505,6 @@ int t4_aio_queue_ddp(struct socket *, struct kaiocb *); void t4_ddp_mod_load(void); void t4_ddp_mod_unload(void); void ddp_assert_empty(struct toepcb *); -void ddp_init_toep(struct toepcb *); void ddp_uninit_toep(struct toepcb *); void ddp_queue_toep(struct toepcb *); void release_ddp_resources(struct toepcb *toep);