svn commit: r346852 - in stable/11/sys/dev/cxgbe: . tom
Navdeep Parhar
np at FreeBSD.org
Sun Apr 28 18:50:27 UTC 2019
Author: np
Date: Sun Apr 28 18:50:25 2019
New Revision: 346852
URL: https://svnweb.freebsd.org/changeset/base/346852
Log:
MFC r333114:
cxgbe(4): Use opaque cookies or tid range-checks to determine the
intended recipient of a CPL when it can't be determined solely from the
opcode. Retire the per-queue handlers for such CPLs in favor of the new
scheme.
Sponsored by: Chelsio Communications
Modified:
stable/11/sys/dev/cxgbe/adapter.h
stable/11/sys/dev/cxgbe/t4_main.c
stable/11/sys/dev/cxgbe/t4_sge.c
stable/11/sys/dev/cxgbe/tom/t4_cpl_io.c
stable/11/sys/dev/cxgbe/tom/t4_ddp.c
stable/11/sys/dev/cxgbe/tom/t4_tom.c
stable/11/sys/dev/cxgbe/tom/t4_tom.h
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/sys/dev/cxgbe/adapter.h
==============================================================================
--- stable/11/sys/dev/cxgbe/adapter.h Sun Apr 28 18:45:44 2019 (r346851)
+++ stable/11/sys/dev/cxgbe/adapter.h Sun Apr 28 18:50:25 2019 (r346852)
@@ -366,6 +366,19 @@ enum {
NM_BUSY = 2,
};
+enum {
+ CPL_COOKIE_RESERVED = 0,
+ CPL_COOKIE_FILTER,
+ CPL_COOKIE_DDP0,
+ CPL_COOKIE_DDP1,
+ CPL_COOKIE_TOM,
+ CPL_COOKIE_AVAILABLE1,
+ CPL_COOKIE_AVAILABLE2,
+ CPL_COOKIE_AVAILABLE3,
+
+ NUM_CPL_COOKIES = 8 /* Limited by M_COOKIE. Do not increase. */
+};
+
struct sge_iq;
struct rss_header;
typedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *,
@@ -380,8 +393,6 @@ struct sge_iq {
uint32_t flags;
volatile int state;
struct adapter *adapter;
- cpl_handler_t set_tcb_rpl;
- cpl_handler_t l2t_write_rpl;
struct iq_desc *desc; /* KVA of descriptor ring */
int8_t intr_pktc_idx; /* packet count threshold index */
uint8_t gen; /* generation bit */
@@ -1158,9 +1169,10 @@ int parse_pkt(struct adapter *, struct mbuf **);
void *start_wrq_wr(struct sge_wrq *, int, struct wrq_cookie *);
void commit_wrq_wr(struct sge_wrq *, void *, struct wrq_cookie *);
int tnl_cong(struct port_info *, int);
-int t4_register_an_handler(an_handler_t);
-int t4_register_fw_msg_handler(int, fw_msg_handler_t);
-int t4_register_cpl_handler(int, cpl_handler_t);
+void t4_register_an_handler(an_handler_t);
+void t4_register_fw_msg_handler(int, fw_msg_handler_t);
+void t4_register_cpl_handler(int, cpl_handler_t);
+void t4_register_shared_cpl_handler(int, cpl_handler_t, int);
/* t4_tracer.c */
struct t4_tracer;
Modified: stable/11/sys/dev/cxgbe/t4_main.c
==============================================================================
--- stable/11/sys/dev/cxgbe/t4_main.c Sun Apr 28 18:45:44 2019 (r346851)
+++ stable/11/sys/dev/cxgbe/t4_main.c Sun Apr 28 18:50:25 2019 (r346852)
@@ -694,8 +694,6 @@ static int del_filter(struct adapter *, struct t4_filt
static void clear_filter(struct filter_entry *);
static int set_filter_wr(struct adapter *, int);
static int del_filter_wr(struct adapter *, int);
-static int set_tcb_rpl(struct sge_iq *, const struct rss_header *,
- struct mbuf *);
static int get_sge_context(struct adapter *, struct t4_sge_context *);
static int load_fw(struct adapter *, struct t4_data *);
static int load_cfg(struct adapter *, struct t4_data *);
@@ -9703,22 +9701,6 @@ t4_filter_rpl(struct sge_iq *iq, const struct rss_head
}
static int
-set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
-{
-
- MPASS(iq->set_tcb_rpl != NULL);
- return (iq->set_tcb_rpl(iq, rss, m));
-}
-
-static int
-l2t_write_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
-{
-
- MPASS(iq->l2t_write_rpl != NULL);
- return (iq->l2t_write_rpl(iq, rss, m));
-}
-
-static int
get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
{
int rc;
@@ -11007,8 +10989,10 @@ mod_event(module_t mod, int cmd, void *arg)
sx_xlock(&mlu);
if (loaded++ == 0) {
t4_sge_modload();
- t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl);
- t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl);
+ t4_register_shared_cpl_handler(CPL_SET_TCB_RPL,
+ t4_filter_rpl, CPL_COOKIE_FILTER);
+ t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL,
+ do_l2t_write_rpl, CPL_COOKIE_FILTER);
t4_register_cpl_handler(CPL_TRACE_PKT, t4_trace_pkt);
t4_register_cpl_handler(CPL_T5_TRACE_PKT, t5_trace_pkt);
sx_init(&t4_list_lock, "T4/T5 adapters");
Modified: stable/11/sys/dev/cxgbe/t4_sge.c
==============================================================================
--- stable/11/sys/dev/cxgbe/t4_sge.c Sun Apr 28 18:45:44 2019 (r346851)
+++ stable/11/sys/dev/cxgbe/t4_sge.c Sun Apr 28 18:50:25 2019 (r346852)
@@ -296,98 +296,122 @@ static counter_u64_t extfree_rels;
an_handler_t t4_an_handler;
fw_msg_handler_t t4_fw_msg_handler[NUM_FW6_TYPES];
cpl_handler_t t4_cpl_handler[NUM_CPL_CMDS];
+cpl_handler_t set_tcb_rpl_handlers[NUM_CPL_COOKIES];
+cpl_handler_t l2t_write_rpl_handlers[NUM_CPL_COOKIES];
-
-static int
-an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
+void
+t4_register_an_handler(an_handler_t h)
{
+ uintptr_t *loc;
-#ifdef INVARIANTS
- panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
-#else
- log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
- __func__, iq, ctrl);
-#endif
- return (EDOOFUS);
+ MPASS(h == NULL || t4_an_handler == NULL);
+
+ loc = (uintptr_t *)&t4_an_handler;
+ atomic_store_rel_ptr(loc, (uintptr_t)h);
}
-int
-t4_register_an_handler(an_handler_t h)
+void
+t4_register_fw_msg_handler(int type, fw_msg_handler_t h)
{
- uintptr_t *loc, new;
+ uintptr_t *loc;
- new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
- loc = (uintptr_t *) &t4_an_handler;
- atomic_store_rel_ptr(loc, new);
+ MPASS(type < nitems(t4_fw_msg_handler));
+ MPASS(h == NULL || t4_fw_msg_handler[type] == NULL);
+ /*
+ * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
+ * handler dispatch table. Reject any attempt to install a handler for
+ * this subtype.
+ */
+ MPASS(type != FW_TYPE_RSSCPL);
+ MPASS(type != FW6_TYPE_RSSCPL);
- return (0);
+ loc = (uintptr_t *)&t4_fw_msg_handler[type];
+ atomic_store_rel_ptr(loc, (uintptr_t)h);
}
-static int
-fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
+void
+t4_register_cpl_handler(int opcode, cpl_handler_t h)
{
- const struct cpl_fw6_msg *cpl =
- __containerof(rpl, struct cpl_fw6_msg, data[0]);
+ uintptr_t *loc;
-#ifdef INVARIANTS
- panic("%s: fw_msg type %d", __func__, cpl->type);
-#else
- log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
-#endif
- return (EDOOFUS);
+ MPASS(opcode < nitems(t4_cpl_handler));
+ MPASS(h == NULL || t4_cpl_handler[opcode] == NULL);
+
+ loc = (uintptr_t *)&t4_cpl_handler[opcode];
+ atomic_store_rel_ptr(loc, (uintptr_t)h);
}
-int
-t4_register_fw_msg_handler(int type, fw_msg_handler_t h)
+static int
+set_tcb_rpl_handler(struct sge_iq *iq, const struct rss_header *rss,
+ struct mbuf *m)
{
- uintptr_t *loc, new;
+ const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1);
+ u_int tid;
+ int cookie;
- if (type >= nitems(t4_fw_msg_handler))
- return (EINVAL);
+ MPASS(m == NULL);
- /*
- * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
- * handler dispatch table. Reject any attempt to install a handler for
- * this subtype.
- */
- if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
- return (EINVAL);
+ tid = GET_TID(cpl);
+ if (is_ftid(iq->adapter, tid)) {
+ /*
+ * The return code for filter-write is put in the CPL cookie so
+ * we have to rely on the hardware tid (is_ftid) to determine
+ * that this is a response to a filter.
+ */
+ cookie = CPL_COOKIE_FILTER;
+ } else {
+ cookie = G_COOKIE(cpl->cookie);
+ }
+ MPASS(cookie > CPL_COOKIE_RESERVED);
+ MPASS(cookie < nitems(set_tcb_rpl_handlers));
- new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
- loc = (uintptr_t *) &t4_fw_msg_handler[type];
- atomic_store_rel_ptr(loc, new);
-
- return (0);
+ return (set_tcb_rpl_handlers[cookie](iq, rss, m));
}
static int
-cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
+l2t_write_rpl_handler(struct sge_iq *iq, const struct rss_header *rss,
+ struct mbuf *m)
{
+ const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
+ unsigned int cookie;
-#ifdef INVARIANTS
- panic("%s: opcode 0x%02x on iq %p with payload %p",
- __func__, rss->opcode, iq, m);
-#else
- log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
- __func__, rss->opcode, iq, m);
- m_freem(m);
-#endif
- return (EDOOFUS);
+ MPASS(m == NULL);
+
+ cookie = GET_TID(rpl) & F_SYNC_WR ? CPL_COOKIE_TOM : CPL_COOKIE_FILTER;
+ return (l2t_write_rpl_handlers[cookie](iq, rss, m));
}
-int
-t4_register_cpl_handler(int opcode, cpl_handler_t h)
+static void
+t4_init_shared_cpl_handlers(void)
{
- uintptr_t *loc, new;
- if (opcode >= nitems(t4_cpl_handler))
- return (EINVAL);
+ t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl_handler);
+ t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl_handler);
+}
- new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
- loc = (uintptr_t *) &t4_cpl_handler[opcode];
- atomic_store_rel_ptr(loc, new);
+void
+t4_register_shared_cpl_handler(int opcode, cpl_handler_t h, int cookie)
+{
+ uintptr_t *loc;
- return (0);
+ MPASS(opcode < nitems(t4_cpl_handler));
+ MPASS(cookie > CPL_COOKIE_RESERVED);
+ MPASS(cookie < NUM_CPL_COOKIES);
+ MPASS(t4_cpl_handler[opcode] != NULL);
+
+ switch (opcode) {
+ case CPL_SET_TCB_RPL:
+ loc = (uintptr_t *)&set_tcb_rpl_handlers[cookie];
+ break;
+ case CPL_L2T_WRITE_RPL:
+ loc = (uintptr_t *)&l2t_write_rpl_handlers[cookie];
+ break;
+ default:
+ MPASS(0);
+ return;
+ }
+ MPASS(h == NULL || *loc == (uintptr_t)NULL);
+ atomic_store_rel_ptr(loc, (uintptr_t)h);
}
/*
@@ -396,7 +420,6 @@ t4_register_cpl_handler(int opcode, cpl_handler_t h)
void
t4_sge_modload(void)
{
- int i;
if (fl_pktshift < 0 || fl_pktshift > 7) {
printf("Invalid hw.cxgbe.fl_pktshift value (%d),"
@@ -436,12 +459,7 @@ t4_sge_modload(void)
counter_u64_zero(extfree_refs);
counter_u64_zero(extfree_rels);
- t4_an_handler = an_not_handled;
- for (i = 0; i < nitems(t4_fw_msg_handler); i++)
- t4_fw_msg_handler[i] = fw_msg_not_handled;
- for (i = 0; i < nitems(t4_cpl_handler); i++)
- t4_cpl_handler[i] = cpl_not_handled;
-
+ t4_init_shared_cpl_handlers();
t4_register_cpl_handler(CPL_FW4_MSG, handle_fw_msg);
t4_register_cpl_handler(CPL_FW6_MSG, handle_fw_msg);
t4_register_cpl_handler(CPL_SGE_EGR_UPDATE, handle_sge_egr_update);
@@ -2903,11 +2921,8 @@ alloc_fwq(struct adapter *sc)
init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE);
if (sc->flags & IS_VF)
intr_idx = 0;
- else {
+ else
intr_idx = sc->intr_count > 1 ? 1 : 0;
- fwq->set_tcb_rpl = t4_filter_rpl;
- fwq->l2t_write_rpl = do_l2t_write_rpl;
- }
rc = alloc_iq_fl(&sc->port[0]->vi[0], fwq, NULL, intr_idx, -1);
if (rc != 0) {
device_printf(sc->dev,
Modified: stable/11/sys/dev/cxgbe/tom/t4_cpl_io.c
==============================================================================
--- stable/11/sys/dev/cxgbe/tom/t4_cpl_io.c Sun Apr 28 18:45:44 2019 (r346851)
+++ stable/11/sys/dev/cxgbe/tom/t4_cpl_io.c Sun Apr 28 18:50:25 2019 (r346852)
@@ -1912,44 +1912,6 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header
return (0);
}
-int
-do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
-{
- struct adapter *sc = iq->adapter;
- const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1);
- unsigned int tid = GET_TID(cpl);
- struct toepcb *toep;
-#ifdef INVARIANTS
- unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
-#endif
-
- KASSERT(opcode == CPL_SET_TCB_RPL,
- ("%s: unexpected opcode 0x%x", __func__, opcode));
- KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
- MPASS(iq != &sc->sge.fwq);
-
- toep = lookup_tid(sc, tid);
- if (toep->ulp_mode == ULP_MODE_TCPDDP) {
- handle_ddp_tcb_rpl(toep, cpl);
- return (0);
- }
-
- /*
- * TOM and/or other ULPs don't request replies for CPL_SET_TCB or
- * CPL_SET_TCB_FIELD requests. This can easily change and when it does
- * the dispatch code will go here.
- */
-#ifdef INVARIANTS
- panic("%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p", __func__,
- tid, iq);
-#else
- log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n",
- __func__, tid, iq);
-#endif
-
- return (0);
-}
-
void
t4_set_tcb_field(struct adapter *sc, struct sge_wrq *wrq, struct toepcb *toep,
uint16_t word, uint64_t mask, uint64_t val, int reply, int cookie)
@@ -1959,6 +1921,9 @@ t4_set_tcb_field(struct adapter *sc, struct sge_wrq *w
struct ofld_tx_sdesc *txsd;
MPASS((cookie & ~M_COOKIE) == 0);
+ if (reply) {
+ MPASS(cookie != CPL_COOKIE_RESERVED);
+ }
wr = alloc_wrqe(sizeof(*req), wrq);
if (wr == NULL) {
Modified: stable/11/sys/dev/cxgbe/tom/t4_ddp.c
==============================================================================
--- stable/11/sys/dev/cxgbe/tom/t4_ddp.c Sun Apr 28 18:45:44 2019 (r346851)
+++ stable/11/sys/dev/cxgbe/tom/t4_ddp.c Sun Apr 28 18:50:25 2019 (r346852)
@@ -631,11 +631,17 @@ enum {
DDP_BUF1_INVALIDATED
};
-void
-handle_ddp_tcb_rpl(struct toepcb *toep, const struct cpl_set_tcb_rpl *cpl)
+CTASSERT(DDP_BUF0_INVALIDATED == CPL_COOKIE_DDP0);
+
+static int
+do_ddp_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
+ struct adapter *sc = iq->adapter;
+ const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1);
+ unsigned int tid = GET_TID(cpl);
unsigned int db_idx;
- struct inpcb *inp = toep->inp;
+ struct toepcb *toep;
+ struct inpcb *inp;
struct ddp_buffer *db;
struct kaiocb *job;
long copied;
@@ -643,6 +649,8 @@ handle_ddp_tcb_rpl(struct toepcb *toep, const struct c
if (cpl->status != CPL_ERR_NONE)
panic("XXX: tcp_rpl failed: %d", cpl->status);
+ toep = lookup_tid(sc, tid);
+ inp = toep->inp;
switch (cpl->cookie) {
case V_WORD(W_TCB_RX_DDP_FLAGS) | V_COOKIE(DDP_BUF0_INVALIDATED):
case V_WORD(W_TCB_RX_DDP_FLAGS) | V_COOKIE(DDP_BUF1_INVALIDATED):
@@ -650,6 +658,7 @@ handle_ddp_tcb_rpl(struct toepcb *toep, const struct c
* XXX: This duplicates a lot of code with handle_ddp_data().
*/
db_idx = G_COOKIE(cpl->cookie) - DDP_BUF0_INVALIDATED;
+ MPASS(db_idx < nitems(toep->ddp.db));
INP_WLOCK(inp);
DDP_LOCK(toep);
db = &toep->ddp.db[db_idx];
@@ -694,6 +703,8 @@ handle_ddp_tcb_rpl(struct toepcb *toep, const struct c
panic("XXX: unknown tcb_rpl offset %#x, cookie %#x",
G_WORD(cpl->cookie), G_COOKIE(cpl->cookie));
}
+
+ return (0);
}
void
@@ -1946,6 +1957,10 @@ void
t4_ddp_mod_load(void)
{
+ t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, do_ddp_tcb_rpl,
+ CPL_COOKIE_DDP0);
+ t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, do_ddp_tcb_rpl,
+ CPL_COOKIE_DDP1);
t4_register_cpl_handler(CPL_RX_DATA_DDP, do_rx_data_ddp);
t4_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_rx_ddp_complete);
TAILQ_INIT(&ddp_orphan_pagesets);
Modified: stable/11/sys/dev/cxgbe/tom/t4_tom.c
==============================================================================
--- stable/11/sys/dev/cxgbe/tom/t4_tom.c Sun Apr 28 18:45:44 2019 (r346851)
+++ stable/11/sys/dev/cxgbe/tom/t4_tom.c Sun Apr 28 18:50:25 2019 (r346852)
@@ -1326,8 +1326,7 @@ t4_tom_activate(struct adapter *sc)
struct tom_data *td;
struct toedev *tod;
struct vi_info *vi;
- struct sge_ofld_rxq *ofld_rxq;
- int i, j, rc, v;
+ int i, rc, v;
ASSERT_SYNCHRONIZED_OP(sc);
@@ -1392,10 +1391,6 @@ t4_tom_activate(struct adapter *sc)
for_each_port(sc, i) {
for_each_vi(sc->port[i], v, vi) {
TOEDEV(vi->ifp) = &td->tod;
- for_each_ofld_rxq(vi, j, ofld_rxq) {
- ofld_rxq->iq.set_tcb_rpl = do_set_tcb_rpl;
- ofld_rxq->iq.l2t_write_rpl = do_l2t_write_rpl2;
- }
}
}
@@ -1498,6 +1493,8 @@ t4_tom_mod_load(void)
struct protosw *tcp_protosw, *tcp6_protosw;
/* CPL handlers */
+ t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL, do_l2t_write_rpl2,
+ CPL_COOKIE_TOM);
t4_init_connect_cpl_handlers();
t4_init_listen_cpl_handlers();
t4_init_cpl_io_handlers();
@@ -1562,6 +1559,7 @@ t4_tom_mod_unload(void)
t4_uninit_connect_cpl_handlers();
t4_uninit_listen_cpl_handlers();
t4_uninit_cpl_io_handlers();
+ t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL, NULL, CPL_COOKIE_TOM);
return (0);
}
Modified: stable/11/sys/dev/cxgbe/tom/t4_tom.h
==============================================================================
--- stable/11/sys/dev/cxgbe/tom/t4_tom.h Sun Apr 28 18:45:44 2019 (r346851)
+++ stable/11/sys/dev/cxgbe/tom/t4_tom.h Sun Apr 28 18:50:25 2019 (r346852)
@@ -386,7 +386,6 @@ void t4_set_tcb_field(struct adapter *, struct sge_wrq
uint16_t, uint64_t, uint64_t, int, int);
void t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop);
void t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop);
-int do_set_tcb_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *);
/* t4_ddp.c */
int t4_init_ppod_region(struct ppod_region *, struct t4_range *, u_int,
@@ -412,7 +411,6 @@ void ddp_queue_toep(struct toepcb *);
void release_ddp_resources(struct toepcb *toep);
void handle_ddp_close(struct toepcb *, struct tcpcb *, uint32_t);
void handle_ddp_indicate(struct toepcb *);
-void handle_ddp_tcb_rpl(struct toepcb *, const struct cpl_set_tcb_rpl *);
void insert_ddp_data(struct toepcb *, uint32_t);
const struct offload_settings *lookup_offload_policy(struct adapter *, int,
struct mbuf *, uint16_t, struct inpcb *);
More information about the svn-src-all
mailing list