svn commit: r279897 - in projects/cxl_iscsi/sys/dev/cxgbe: . cxgbei tom
Navdeep Parhar
np at FreeBSD.org
Thu Mar 12 01:30:38 UTC 2015
Author: np
Date: Thu Mar 12 01:30:36 2015
New Revision: 279897
URL: https://svnweb.freebsd.org/changeset/base/279897
Log:
First round of code cleanup and reorganization. This is mostly code on
the slow path. Changes to the fast path will follow later.
- Use the standard ULD registration and activation mechanism offered by
if_cxgbe. This eliminates all the code that managed the list of
offload_device structures. This simplifies the CPL dispatch too by
eliminating t4tom_cpl_handler_register_flag and associated code.
- Remove all unused or write-only fields from various structures
(iscsi_socket, offload_device, cxgbei_ulp2_ddp_info, ulp_iscsi_info)
- Eliminate the two line wrappers around malloc/free. While here,
switch to using M_CXGBE for all allocations.
- Simplify the page size settings in the chip (for iSCSI). This ULD
"owns" these settings so it should simply write the values that it
wants to the A_ULP_RX_ISCSI_TAGMASK and A_ULP_RX_ISCSI_PSZ registers.
This eliminates the globals ddp_page_order[], ddp_page_shift[],
page_idx and all related code.
- Maintain the per-adapter ULD state in one data structure instead of
two. This consolidates struct offload_device and struct
cxgbei_ulp2_ddp_info and into struct cxgbei_data, which is stored in
adapter->iscsi_softc.
- Leave socket->so_emuldata alone, it exists for a different purpose
(which is definitely not iSCSI). Store the per-socket offload state
in a new field in struct icl_conn instead. (The new field exists only
in this project branch and hasn't been reviewed for inclusion into
head yet).
- Switch to the system version of mbufq.
- Tidy up the CPL/callback dispatch from if_cxgbe/t4_tom into cxgbei.
The tid/toepcb is always available (t4_tom has looked it up) so it's a
waste of time looking it up again.
Deleted:
projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/mbufq.h
Modified:
projects/cxl_iscsi/sys/dev/cxgbe/adapter.h
projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.c
projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.h
projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.c
projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.h
projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
projects/cxl_iscsi/sys/dev/cxgbe/tom/t4_cpl_io.c
projects/cxl_iscsi/sys/dev/cxgbe/tom/t4_ddp.c
projects/cxl_iscsi/sys/dev/cxgbe/tom/t4_tom.h
Modified: projects/cxl_iscsi/sys/dev/cxgbe/adapter.h
==============================================================================
--- projects/cxl_iscsi/sys/dev/cxgbe/adapter.h Thu Mar 12 01:05:54 2015 (r279896)
+++ projects/cxl_iscsi/sys/dev/cxgbe/adapter.h Thu Mar 12 01:30:36 2015 (r279897)
@@ -750,7 +750,7 @@ struct adapter {
void *tom_softc; /* (struct tom_data *) */
struct tom_tunables tt;
void *iwarp_softc; /* (struct c4iw_dev *) */
- void *iscsi_softc;
+ void *iscsi_softc; /* (struct cxgbei_data *) */
#endif
struct l2t_data *l2t; /* L2 table */
struct tid_info tids;
Modified: projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.c
==============================================================================
--- projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.c Thu Mar 12 01:05:54 2015 (r279896)
+++ projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.c Thu Mar 12 01:30:36 2015 (r279897)
@@ -135,17 +135,10 @@ find_ulp_mbuf_cb(struct mbuf *m)
* The location of the pagepod entry is encoded into ddp tag which is used as
* the base for ITT/TTT.
*/
-#define T4_DDP
-#ifdef T4_DDP
+
/*
* functions to program the pagepod in h/w
*/
-static void *
-t4_tdev2ddp(void *tdev)
-{
- struct adapter *sc = ((struct toedev *)tdev)->tod_softc;
- return (sc->iscsi_softc);
-}
static void inline
ppod_set(struct pagepod *ppod,
struct cxgbei_ulp2_pagepod_hdr *hdr,
@@ -193,27 +186,27 @@ ulp_mem_io_set_hdr(struct adapter *sc, i
#define PCIE_MEMWIN_MAX_NPPODS 16 /* 1024/PPOD_SIZE */
static int
-ppod_write_idata(struct cxgbei_ulp2_ddp_info *ddp,
+ppod_write_idata(struct cxgbei_data *ci,
struct cxgbei_ulp2_pagepod_hdr *hdr,
unsigned int idx, unsigned int npods,
struct cxgbei_ulp2_gather_list *gl,
unsigned int gl_pidx, struct toepcb *toep)
{
- unsigned int dlen = PPOD_SIZE * npods;
- unsigned int pm_addr = idx * PPOD_SIZE + ddp->llimit;
- unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) +
- sizeof(struct ulptx_idata) + dlen, 16);
+ u_int dlen = PPOD_SIZE * npods;
+ u_int pm_addr = idx * PPOD_SIZE + ci->llimit;
+ u_int wr_len = roundup(sizeof(struct ulp_mem_io) +
+ sizeof(struct ulptx_idata) + dlen, 16);
struct ulp_mem_io *req;
struct ulptx_idata *idata;
struct pagepod *ppod;
- unsigned int i;
+ u_int i;
struct wrqe *wr;
struct adapter *sc = toep->port->adapter;
wr = alloc_wrqe(wr_len, toep->ctrlq);
if (wr == NULL) {
- printf("%s: alloc wrqe failed\n", __func__);
- return ENOMEM;
+ CXGBE_UNIMPLEMENTED("ppod_write_idata: alloc_wrqe failure");
+ return (ENOMEM);
}
req = wrtod(wr);
@@ -233,25 +226,16 @@ ppod_write_idata(struct cxgbei_ulp2_ddp_
return 0;
}
-static int
-t4_ddp_set_map(struct cxgbei_ulp2_ddp_info *ddp,
- void *isockp, struct cxgbei_ulp2_pagepod_hdr *hdr,
- unsigned int idx, unsigned int npods,
- struct cxgbei_ulp2_gather_list *gl, int reply)
+int
+t4_ddp_set_map(struct cxgbei_data *ci, void *isockp,
+ struct cxgbei_ulp2_pagepod_hdr *hdr, u_int idx, u_int npods,
+ struct cxgbei_ulp2_gather_list *gl, int reply)
{
- iscsi_socket *isock = (iscsi_socket *)isockp;
- struct socket *sk;
- struct toepcb *toep;
- struct tcpcb *tp;
+ struct iscsi_socket *isock = (struct iscsi_socket *)isockp;
+ struct toepcb *toep = isock->toep;
int err;
unsigned int pidx = 0, w_npods = 0, cnt;
- if (isock == NULL)
- return EINVAL;
- sk = isock->sock;
- tp = so_sototcpcb(sk);
- toep = tp->t_toe;
-
/*
* on T4, if we use a mix of IMMD and DSGL with ULP_MEM_WRITE,
* the order would not be garanteed, so we will stick with IMMD
@@ -266,7 +250,7 @@ t4_ddp_set_map(struct cxgbei_ulp2_ddp_in
cnt = npods - w_npods;
if (cnt > ULPMEM_IDATA_MAX_NPPODS)
cnt = ULPMEM_IDATA_MAX_NPPODS;
- err = ppod_write_idata(ddp, hdr, idx, cnt, gl,
+ err = ppod_write_idata(ci, hdr, idx, cnt, gl,
pidx, toep);
if (err) {
printf("%s: ppod_write_idata failed\n", __func__);
@@ -276,126 +260,29 @@ t4_ddp_set_map(struct cxgbei_ulp2_ddp_in
return err;
}
-static void
-t4_ddp_clear_map(struct cxgbei_ulp2_ddp_info *ddp,
- struct cxgbei_ulp2_gather_list *gl,
- unsigned int tag, unsigned int idx, unsigned int npods,
- iscsi_socket *isock)
+void
+t4_ddp_clear_map(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl,
+ u_int tag, u_int idx, u_int npods, struct iscsi_socket *isock)
{
- struct socket *sk;
- struct toepcb *toep;
- struct tcpcb *tp;
+ struct toepcb *toep = isock->toep;
int err = -1;
-
- sk = isock->sock;
- tp = so_sototcpcb(sk);
- toep = tp->t_toe;
-
- /* send via immediate data */
- unsigned int pidx = 0;
- unsigned int w_npods = 0;
- unsigned int cnt;
+ u_int pidx = 0;
+ u_int w_npods = 0;
+ u_int cnt;
for (; w_npods < npods; idx += cnt, w_npods += cnt,
pidx += PPOD_PAGES) {
cnt = npods - w_npods;
if (cnt > ULPMEM_IDATA_MAX_NPPODS)
cnt = ULPMEM_IDATA_MAX_NPPODS;
- err = ppod_write_idata(ddp, NULL, idx, cnt, gl, 0, toep);
+ err = ppod_write_idata(ci, NULL, idx, cnt, gl, 0, toep);
if (err)
break;
}
}
-#endif
-
-/*
- * cxgbei device management
- * maintains a list of the cxgbei devices
- */
-typedef struct offload_device {
- SLIST_ENTRY(offload_device) link;
- unsigned char d_version;
- unsigned char d_tx_hdrlen; /* CPL_TX_DATA, < 256 */
- unsigned char d_ulp_rx_datagap; /* for coalesced iscsi msg */
- unsigned char filler;
-
- unsigned int d_flag;
- unsigned int d_payload_tmax;
- unsigned int d_payload_rmax;
-
- struct cxgbei_ulp2_tag_format d_tag_format;
- void *d_tdev;
- void *d_pdev;
- void* (*tdev2ddp)(void *tdev);
-}offload_device;
-
-SLIST_HEAD(, offload_device) odev_list;
-
-static void t4_unregister_cpl_handler_with_tom(struct adapter *sc);
-static offload_device *
-offload_device_new(void *tdev)
-{
- offload_device *odev = NULL;
- odev = malloc(sizeof(struct offload_device),
- M_CXGBE, M_NOWAIT | M_ZERO);
- if (odev) {
- odev->d_tdev = tdev;
- SLIST_INSERT_HEAD(&odev_list, odev, link);
- }
-
- return odev;
-}
-
-static offload_device *
-offload_device_find(struct toedev *tdev)
-{
- offload_device *odev = NULL;
-
- if (!SLIST_EMPTY(&odev_list)) {
- SLIST_FOREACH(odev, &odev_list, link) {
- if (odev->d_tdev == tdev)
- break;
- }
- }
- return odev;
-}
-
-static void
-cxgbei_odev_cleanup(offload_device *odev)
-{
- struct toedev *tdev = odev->d_tdev;
- struct adapter *sc = (struct adapter *)tdev->tod_softc;
-
- /* de-register ULP CPL handlers with TOM */
- t4_unregister_cpl_handler_with_tom(sc);
- if (odev->d_flag & ODEV_FLAG_ULP_DDP_ENABLED) {
- if (sc->iscsi_softc)
- cxgbei_ulp2_ddp_cleanup(
- (struct cxgbei_ulp2_ddp_info **)&sc->iscsi_softc);
- }
- return;
-}
-
-static void
-offload_device_remove()
-{
- offload_device *odev = NULL, *next = NULL;
-
- if (SLIST_EMPTY(&odev_list))
- return;
-
- for (odev = SLIST_FIRST(&odev_list); odev != NULL; odev = next) {
- SLIST_REMOVE(&odev_list, odev, offload_device, link);
- next = SLIST_NEXT(odev, link);
- cxgbei_odev_cleanup(odev);
- free(odev, M_CXGBE);
- }
-
- return;
-}
static int
-cxgbei_map_sg(cxgbei_sgl *sgl, struct ccb_scsiio *csio)
+cxgbei_map_sg(struct cxgbei_sgl *sgl, struct ccb_scsiio *csio)
{
unsigned int data_len = csio->dxfer_len;
unsigned int sgoffset = (uint64_t)csio->data_ptr & PAGE_MASK;
@@ -430,7 +317,7 @@ cxgbei_map_sg(cxgbei_sgl *sgl, struct cc
}
static int
-cxgbei_map_sg_tgt(cxgbei_sgl *sgl, union ctl_io *io)
+cxgbei_map_sg_tgt(struct cxgbei_sgl *sgl, union ctl_io *io)
{
unsigned int data_len, sgoffset, nsge;
unsigned char *sgaddr;
@@ -486,34 +373,19 @@ cxgbei_map_sg_tgt(cxgbei_sgl *sgl, union
}
static int
-t4_sk_ddp_tag_reserve(iscsi_socket *isock, unsigned int xferlen,
- cxgbei_sgl *sgl, unsigned int sgcnt,
- unsigned int *ddp_tag)
+t4_sk_ddp_tag_reserve(struct cxgbei_data *ci, struct iscsi_socket *isock,
+ u_int xferlen, struct cxgbei_sgl *sgl, u_int sgcnt, u_int *ddp_tag)
{
- offload_device *odev = isock->s_odev;
- struct toedev *tdev = odev->d_tdev;
struct cxgbei_ulp2_gather_list *gl;
int err = -EINVAL;
- struct adapter *sc = tdev->tod_softc;
- struct cxgbei_ulp2_ddp_info *ddp;
+ struct toepcb *toep = isock->toep;
- ddp = (struct cxgbei_ulp2_ddp_info *)sc->iscsi_softc;
- if (ddp == NULL)
- return ENOMEM;
-
- gl = cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(xferlen, sgl, sgcnt,
- odev->d_tdev, 0);
+ gl = cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(xferlen, sgl, sgcnt, ci, 0);
if (gl) {
- err = cxgbei_ulp2_ddp_tag_reserve(odev->tdev2ddp(tdev),
- isock,
- isock->s_tid,
- &odev->d_tag_format,
- ddp_tag, gl,
- 0, 0);
+ err = cxgbei_ulp2_ddp_tag_reserve(ci, isock, toep->tid,
+ &ci->tag_format, ddp_tag, gl, 0, 0);
if (err) {
- CTR1(KTR_CXGBE,
- "%s: ddp_tag_reserve failed\n", __func__);
- cxgbei_ulp2_ddp_release_gl(gl, odev->d_tdev);
+ cxgbei_ulp2_ddp_release_gl(ci, gl);
}
}
@@ -525,17 +397,17 @@ cxgbei_task_reserve_itt(struct icl_conn
struct ccb_scsiio *scmd, unsigned int *itt)
{
int xferlen = scmd->dxfer_len;
- cxgbei_task_data *tdata = NULL;
- cxgbei_sgl *sge = NULL;
- struct socket *so = ic->ic_socket;
- iscsi_socket *isock = (iscsi_socket *)(so)->so_emuldata;
+ struct cxgbei_task_data *tdata = NULL;
+ struct cxgbei_sgl *sge = NULL;
+ struct iscsi_socket *isock = ic->ic_ofld_prv0;
+ struct toepcb *toep = isock->toep;
+ struct adapter *sc = td_adapter(toep->td);
+ struct cxgbei_data *ci = sc->iscsi_softc;
int err = -1;
- offload_device *odev = isock->s_odev;
- tdata = (cxgbei_task_data *)*prv;
- if ((xferlen == 0) || (tdata == NULL)) {
+ tdata = (struct cxgbei_task_data *)*prv;
+ if (xferlen == 0 || tdata == NULL)
goto out;
- }
if (xferlen < DDP_THRESHOLD)
goto out;
@@ -551,10 +423,10 @@ cxgbei_task_reserve_itt(struct icl_conn
CTR3(KTR_CXGBE, "%s: *itt:0x%x sc_ddp_tag:0x%x\n",
__func__, *itt, tdata->sc_ddp_tag);
- if (cxgbei_ulp2_sw_tag_usable(&odev->d_tag_format,
+ if (cxgbei_ulp2_sw_tag_usable(&ci->tag_format,
tdata->sc_ddp_tag)) {
- err = t4_sk_ddp_tag_reserve(isock, scmd->dxfer_len, sge,
- tdata->nsge, &tdata->sc_ddp_tag);
+ err = t4_sk_ddp_tag_reserve(ci, isock, scmd->dxfer_len,
+ sge, tdata->nsge, &tdata->sc_ddp_tag);
} else {
CTR3(KTR_CXGBE,
"%s: itt:0x%x sc_ddp_tag:0x%x not usable\n",
@@ -564,7 +436,7 @@ cxgbei_task_reserve_itt(struct icl_conn
out:
if (err < 0)
tdata->sc_ddp_tag =
- cxgbei_ulp2_set_non_ddp_tag(&odev->d_tag_format, *itt);
+ cxgbei_ulp2_set_non_ddp_tag(&ci->tag_format, *itt);
return tdata->sc_ddp_tag;
}
@@ -573,15 +445,16 @@ static unsigned int
cxgbei_task_reserve_ttt(struct icl_conn *ic, void **prv, union ctl_io *io,
unsigned int *ttt)
{
- struct socket *so = ic->ic_socket;
- iscsi_socket *isock = (iscsi_socket *)(so)->so_emuldata;
- cxgbei_task_data *tdata = NULL;
- offload_device *odev = isock->s_odev;
+ struct iscsi_socket *isock = ic->ic_ofld_prv0;
+ struct toepcb *toep = isock->toep;
+ struct adapter *sc = td_adapter(toep->td);
+ struct cxgbei_data *ci = sc->iscsi_softc;
+ struct cxgbei_task_data *tdata = NULL;
int xferlen, err = -1;
- cxgbei_sgl *sge = NULL;
+ struct cxgbei_sgl *sge = NULL;
xferlen = (io->scsiio.kern_data_len - io->scsiio.ext_data_filled);
- tdata = (cxgbei_task_data *)*prv;
+ tdata = (struct cxgbei_task_data *)*prv;
if ((xferlen == 0) || (tdata == NULL))
goto out;
if (xferlen < DDP_THRESHOLD)
@@ -594,9 +467,9 @@ cxgbei_task_reserve_ttt(struct icl_conn
sge = tdata->sgl;
tdata->sc_ddp_tag = *ttt;
- if (cxgbei_ulp2_sw_tag_usable(&odev->d_tag_format, tdata->sc_ddp_tag)) {
- err = t4_sk_ddp_tag_reserve(isock, xferlen, sge, tdata->nsge,
- &tdata->sc_ddp_tag);
+ if (cxgbei_ulp2_sw_tag_usable(&ci->tag_format, tdata->sc_ddp_tag)) {
+ err = t4_sk_ddp_tag_reserve(ci, isock, xferlen, sge,
+ tdata->nsge, &tdata->sc_ddp_tag);
} else {
CTR2(KTR_CXGBE, "%s: sc_ddp_tag:0x%x not usable\n",
__func__, tdata->sc_ddp_tag);
@@ -604,102 +477,95 @@ cxgbei_task_reserve_ttt(struct icl_conn
out:
if (err < 0)
tdata->sc_ddp_tag =
- cxgbei_ulp2_set_non_ddp_tag(&odev->d_tag_format, *ttt);
+ cxgbei_ulp2_set_non_ddp_tag(&ci->tag_format, *ttt);
return tdata->sc_ddp_tag;
}
static int
-t4_sk_ddp_tag_release(iscsi_socket *isock, unsigned int ddp_tag)
+t4_sk_ddp_tag_release(struct iscsi_socket *isock, unsigned int ddp_tag)
{
- offload_device *odev = isock->s_odev;
- struct toedev *tdev = odev->d_tdev;
+ struct toepcb *toep = isock->toep;
+ struct adapter *sc = td_adapter(toep->td);
+ struct cxgbei_data *ci = sc->iscsi_softc;
- cxgbei_ulp2_ddp_tag_release(odev->tdev2ddp(tdev), ddp_tag, isock);
- return 0;
-}
-#ifdef T4_DDP
-static struct cxgbei_ulp2_ddp_info *
-t4_ddp_init(struct ifnet *dev, struct toedev *tdev)
-{
- struct cxgbei_ulp2_ddp_info *ddp;
- struct adapter *sc = tdev->tod_softc;
- struct ulp_iscsi_info uinfo;
-
- memset(&uinfo, 0, sizeof(struct ulp_iscsi_info));
- uinfo.llimit = sc->vres.iscsi.start;
- uinfo.ulimit = sc->vres.iscsi.start + sc->vres.iscsi.size - 1;
- uinfo.max_rxsz = uinfo.max_txsz =
- G_MAXRXDATA(t4_read_reg(sc, A_TP_PARA_REG2));
-
- if (sc->vres.iscsi.size == 0) {
- printf("%s: iSCSI capabilities not enabled.\n", __func__);
- return NULL;
- }
- printf("T4, ddp 0x%x ~ 0x%x, size %u, iolen %u, ulpddp:0x%p\n",
- uinfo.llimit, uinfo.ulimit, sc->vres.iscsi.size,
- uinfo.max_rxsz, sc->iscsi_softc);
-
- cxgbei_ulp2_ddp_init((void *)tdev,
- (struct cxgbei_ulp2_ddp_info **)&sc->iscsi_softc,
- &uinfo);
- ddp = (struct cxgbei_ulp2_ddp_info *)sc->iscsi_softc;
- if (ddp) {
- unsigned int pgsz_order[4];
- int i;
-
- for (i = 0; i < 4; i++)
- pgsz_order[i] = uinfo.pgsz_factor[i];
+ cxgbei_ulp2_ddp_tag_release(ci, ddp_tag, isock);
- t4_iscsi_init(dev, uinfo.tagmask, pgsz_order);
-
- ddp->ddp_set_map = t4_ddp_set_map;
- ddp->ddp_clear_map = t4_ddp_clear_map;
- }
- return ddp;
+ return (0);
}
-#endif
-static struct socket *
-cpl_find_sock(struct adapter *sc, unsigned int hwtid)
+static int
+cxgbei_ddp_init(struct adapter *sc, struct cxgbei_data *ci)
{
- struct socket *sk;
- struct toepcb *toep = lookup_tid(sc, hwtid);
- struct inpcb *inp = toep->inp;
+ int nppods, bits, max_sz, rc;
+ static const u_int pgsz_order[] = {0, 1, 2, 3};
- INP_WLOCK(inp);
- sk = inp->inp_socket;
- INP_WUNLOCK(inp);
- if (sk == NULL)
- CTR2(KTR_CXGBE,
- "%s: T4 CPL tid 0x%x, sk NULL.\n", __func__, hwtid);
- return sk;
+ MPASS(sc->vres.iscsi.size > 0);
+
+ ci->llimit = sc->vres.iscsi.start;
+ ci->ulimit = sc->vres.iscsi.start + sc->vres.iscsi.size - 1;
+ max_sz = G_MAXRXDATA(t4_read_reg(sc, A_TP_PARA_REG2));
+
+ nppods = sc->vres.iscsi.size >> IPPOD_SIZE_SHIFT;
+ if (nppods <= 1024)
+ return (ENXIO);
+
+ bits = fls(nppods);
+ if (bits > IPPOD_IDX_MAX_SIZE)
+ bits = IPPOD_IDX_MAX_SIZE;
+ nppods = (1 << (bits - 1)) - 1;
+
+ rc = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR,
+ BUS_SPACE_MAXADDR, NULL, NULL, UINT32_MAX , 8, BUS_SPACE_MAXSIZE,
+ BUS_DMA_ALLOCNOW, NULL, NULL, &ci->ulp_ddp_tag);
+ if (rc != 0) {
+ device_printf(sc->dev, "%s: failed to create DMA tag: %u.\n",
+ __func__, rc);
+ return (rc);
+ }
+
+ ci->colors = malloc(nppods * sizeof(char), M_CXGBE, M_NOWAIT | M_ZERO);
+ ci->gl_map = malloc(nppods * sizeof(struct cxgbei_ulp2_gather_list *),
+ M_CXGBE, M_NOWAIT | M_ZERO);
+ if (ci->colors == NULL || ci->gl_map == NULL) {
+ bus_dma_tag_destroy(ci->ulp_ddp_tag);
+ free(ci->colors, M_CXGBE);
+ free(ci->gl_map, M_CXGBE);
+ return (ENOMEM);
+ }
+
+ mtx_init(&ci->map_lock, "ddp lock", NULL, MTX_DEF | MTX_DUPOK);
+ ci->max_txsz = ci->max_rxsz = min(max_sz, ULP2_MAX_PKT_SIZE);
+ ci->nppods = nppods;
+ ci->idx_last = nppods;
+ ci->idx_bits = bits;
+ ci->idx_mask = (1 << bits) - 1;
+ ci->rsvd_tag_mask = (1 << (bits + IPPOD_IDX_SHIFT)) - 1;
+
+ ci->tag_format.sw_bits = bits;
+ ci->tag_format.rsvd_bits = bits;
+ ci->tag_format.rsvd_shift = IPPOD_IDX_SHIFT;
+ ci->tag_format.rsvd_mask = ci->idx_mask;
+
+ t4_iscsi_init(sc, ci->idx_mask << IPPOD_IDX_SHIFT, pgsz_order);
+
+ return (rc);
}
static void
-process_rx_iscsi_hdr(struct socket *sk, struct mbuf *m)
+process_rx_iscsi_hdr(struct toepcb *toep, struct mbuf *m)
{
- struct tcpcb *tp = so_sototcpcb(sk);
- struct toepcb *toep = tp->t_toe;
-
struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *);
struct ulp_mbuf_cb *cb, *lcb;
struct mbuf *lmbuf;
- unsigned char *byte;
- iscsi_socket *isock = (iscsi_socket *)(sk)->so_emuldata;
- unsigned int hlen, dlen, plen;
+ u_char *byte;
+ struct iscsi_socket *isock = toep->ulpcb;
+ struct tcpcb *tp = intotcpcb(toep->inp);
+ u_int hlen, dlen, plen;
- if (isock == NULL)
- goto err_out;
-
- if (toep == NULL)
- goto err_out;
- if ((m->m_flags & M_PKTHDR) == 0) {
- printf("%s: m:%p no M_PKTHDR can't allocate m_tag\n",
- __func__, m);
- goto err_out;
- }
+ MPASS(isock != NULL);
+ M_ASSERTPKTHDR(m);
- mtx_lock(&isock->iscsi_rcv_mbufq.lock);
+ mtx_lock(&isock->iscsi_rcvq_lock);
/* allocate m_tag to hold ulp info */
cb = get_ulp_mbuf_cb(m);
@@ -715,7 +581,6 @@ process_rx_iscsi_hdr(struct socket *sk,
/* figure out if this is the pdu header or data */
cb->ulp_mode = ULP_MODE_ISCSI;
if (isock->mbuf_ulp_lhdr == NULL) {
- iscsi_socket *isock = (iscsi_socket *)(sk)->so_emuldata;
isock->mbuf_ulp_lhdr = lmbuf = m;
lcb = cb;
@@ -761,15 +626,13 @@ process_rx_iscsi_hdr(struct socket *sk,
m->m_len += 4 - (m->m_len % 4);
}
}
- mbufq_tail(&isock->iscsi_rcv_mbufq, m);
- mtx_unlock(&isock->iscsi_rcv_mbufq.lock);
+ mbufq_enqueue(&isock->iscsi_rcvq, m);
+ mtx_unlock(&isock->iscsi_rcvq_lock);
return;
err_out1:
- mtx_unlock(&isock->iscsi_rcv_mbufq.lock);
-err_out:
+ mtx_unlock(&isock->iscsi_rcvq_lock);
m_freem(m);
- return;
}
/* hand over received PDU to iscsi_initiator */
@@ -787,7 +650,7 @@ iscsi_conn_receive_pdu(struct iscsi_sock
panic("%s: failed to alloc icl_pdu\n", __func__);
return;
}
- m = mbufq_peek(&isock->iscsi_rcv_mbufq);
+ m = mbufq_first(&isock->iscsi_rcvq);
if (m) {
cb = find_ulp_mbuf_cb(m);
if (cb == NULL) {
@@ -798,22 +661,22 @@ iscsi_conn_receive_pdu(struct iscsi_sock
goto err_out;
}
/* BHS */
- mbufq_dequeue(&isock->iscsi_rcv_mbufq);
+ mbufq_dequeue(&isock->iscsi_rcvq);
data_len = cb->ulp.iscsi.pdulen;
- CTR5(KTR_CXGBE, "%s: response:%p m:%p m_len:%d data_len:%d\n",
+ CTR5(KTR_CXGBE, "%s: response:%p m:%p m_len:%d data_len:%d",
__func__, response, m, m->m_len, data_len);
response->ip_bhs_mbuf = m;
response->ip_bhs = mtod(response->ip_bhs_mbuf, struct iscsi_bhs *);
/* data */
if (cb->flags & SBUF_ULP_FLAG_DATA_RCVD) {
- m = mbufq_peek(&isock->iscsi_rcv_mbufq);
+ m = mbufq_first(&isock->iscsi_rcvq);
if (m == NULL) {
CTR1(KTR_CXGBE, "%s:No Data\n", __func__);
goto err_out;
}
- mbufq_dequeue(&isock->iscsi_rcv_mbufq);
+ mbufq_dequeue(&isock->iscsi_rcvq);
response->ip_data_mbuf = m;
response->ip_data_len += response->ip_data_mbuf->m_len;
} else {
@@ -829,26 +692,22 @@ err_out:
}
static void
-process_rx_data_ddp(struct socket *sk, void *m)
+process_rx_data_ddp(struct toepcb *toep, const struct cpl_rx_data_ddp *cpl)
{
- struct cpl_rx_data_ddp *cpl = (struct cpl_rx_data_ddp *)m;
- struct tcpcb *tp = so_sototcpcb(sk);
- struct toepcb *toep = tp->t_toe;
- struct inpcb *inp = toep->inp;
struct mbuf *lmbuf;
struct ulp_mbuf_cb *lcb, *lcb1;
unsigned int val, pdulen;
- iscsi_socket *isock = (iscsi_socket *)(sk)->so_emuldata;
+ struct iscsi_socket *isock = toep->ulpcb;
+ struct inpcb *inp = toep->inp;
- if (isock == NULL)
- return;
+ MPASS(isock != NULL);
if (isock->mbuf_ulp_lhdr == NULL) {
panic("%s: tid 0x%x, rcv RX_DATA_DDP w/o pdu header.\n",
- __func__, toep->tid);
+ __func__, toep->tid);
return;
}
- mtx_lock(&isock->iscsi_rcv_mbufq.lock);
+ mtx_lock(&isock->iscsi_rcvq_lock);
lmbuf = isock->mbuf_ulp_lhdr;
if (lmbuf->m_nextpkt) {
lcb1 = find_ulp_mbuf_cb(lmbuf->m_nextpkt);
@@ -857,7 +716,7 @@ process_rx_data_ddp(struct socket *sk, v
lcb = find_ulp_mbuf_cb(isock->mbuf_ulp_lhdr);
if (lcb == NULL) {
CTR2(KTR_CXGBE, "%s: mtag NULL lmbuf :%p\n", __func__, lmbuf);
- mtx_unlock(&isock->iscsi_rcv_mbufq.lock);
+ mtx_unlock(&isock->iscsi_rcvq_lock);
return;
}
lcb->flags |= SBUF_ULP_FLAG_STATUS_RCVD;
@@ -914,251 +773,143 @@ process_rx_data_ddp(struct socket *sk, v
#endif
iscsi_conn_receive_pdu(isock);
- mtx_unlock(&isock->iscsi_rcv_mbufq.lock);
+ mtx_unlock(&isock->iscsi_rcvq_lock);
/* update rx credits */
INP_WLOCK(inp);
- SOCK_LOCK(sk);
+ /* XXXNP: does this want the so_rcv lock? (happens to be the same) */
+ SOCK_LOCK(inp->inp_socket);
toep->sb_cc += pdulen;
- SOCK_UNLOCK(sk);
- CTR4(KTR_CXGBE, "sk:%p sb_cc 0x%x, rcv_nxt 0x%x rcv_wnd:0x%lx.\n",
- sk, toep->sb_cc, tp->rcv_nxt, tp->rcv_wnd);
- t4_rcvd(&toep->td->tod, tp);
+ SOCK_UNLOCK(inp->inp_socket);
+ t4_rcvd(&toep->td->tod, intotcpcb(inp));
INP_WUNLOCK(inp);
return;
}
static void
-drop_fw_acked_ulp_data(struct socket *sk, struct toepcb *toep, int len)
+drop_fw_acked_ulp_data(struct toepcb *toep, int len)
{
struct mbuf *m, *next;
struct ulp_mbuf_cb *cb;
- iscsi_socket *isock = (iscsi_socket *)(sk)->so_emuldata;
struct icl_pdu *req;
+ struct iscsi_socket *isock = toep->ulpcb;
- if (len == 0 || (isock == NULL))
- return;
+ MPASS(len > 0);
- mtx_lock(&isock->ulp2_wrq.lock);
+ mtx_lock(&isock->ulp2_wrq_lock);
while (len > 0) {
m = mbufq_dequeue(&isock->ulp2_wrq);
- if(m == NULL) break;
+ MPASS(m != NULL); /* excess credits */
- for(next = m; next !=NULL; next = next->m_next)
+ for (next = m; next != NULL; next = next->m_next) {
+ MPASS(len >= next->m_len); /* excess credits */
len -= next->m_len;
+ }
cb = find_ulp_mbuf_cb(m);
-
- if (cb && isock && cb->pdu) {
+ if (cb && cb->pdu) {
req = (struct icl_pdu *)cb->pdu;
req->ip_bhs_mbuf = NULL;
icl_pdu_free(req);
}
m_freem(m);
}
- mtx_unlock(&isock->ulp2_wrq.lock);
- return;
-}
-
-static void
-process_fw4_ack(struct socket *sk, int *plen)
-{
- struct tcpcb *tp = so_sototcpcb(sk);
- struct toepcb *toep = tp->t_toe;
-
- drop_fw_acked_ulp_data(sk, toep, *plen);
-
- return;
-}
-
-static int
-do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
-{
- return 0;
+ mtx_unlock(&isock->ulp2_wrq_lock);
}
static int
do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
- struct socket *sk;
struct adapter *sc = iq->adapter;
- struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *);
- sk = cpl_find_sock(sc, GET_TID(cpl));
-
- if (sk == NULL)
- return CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE;
+ struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *); /* XXXNP */
+ u_int tid = GET_TID(cpl);
+ struct toepcb *toep = lookup_tid(sc, tid);
- process_rx_iscsi_hdr(sk, m);
- return 0;
-}
+ process_rx_iscsi_hdr(toep, m);
-static int
-do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
-{
- return 0;
+ return (0);
}
static int
do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
- struct socket *sk;
- struct adapter *sc;
- const struct cpl_rx_iscsi_ddp *cpl = (const void *)(rss + 1);
-
- if (iq == NULL)
- return 0;
- sc = iq->adapter;
- if (sc == NULL)
- return 0;
+ struct adapter *sc = iq->adapter;
+ const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1);
+ u_int tid = GET_TID(cpl);
+ struct toepcb *toep = lookup_tid(sc, tid);
- sk = cpl_find_sock(sc, GET_TID(cpl));
- if (sk == NULL)
- return CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE;
+ process_rx_data_ddp(toep, cpl);
- process_rx_data_ddp(sk, (void *)cpl);
- return 0;
+ return (0);
}
+
static int
-t4_ulp_mbuf_push(struct socket *so, struct mbuf *m)
+t4_ulp_mbuf_push(struct iscsi_socket *isock, struct mbuf *m)
{
- struct tcpcb *tp = so_sototcpcb(so);
- struct toepcb *toep = tp->t_toe;
- struct inpcb *inp = so_sotoinpcb(so);
- iscsi_socket *isock = (iscsi_socket *)(so)->so_emuldata;;
-
- if (isock == NULL) {
- m_freem(m);
- return EINVAL;
- }
+ struct toepcb *toep = isock->toep;
/* append mbuf to ULP queue */
- mtx_lock(&isock->ulp2_writeq.lock);
- mbufq_tail(&isock->ulp2_writeq, m);
- mtx_unlock(&isock->ulp2_writeq.lock);
+ mtx_lock(&isock->ulp2_writeq_lock);
+ mbufq_enqueue(&isock->ulp2_writeq, m);
+ mtx_unlock(&isock->ulp2_writeq_lock);
- INP_WLOCK(inp);
+ INP_WLOCK(toep->inp);
t4_ulp_push_frames(toep->td->tod.tod_softc, toep, 0);
- INP_WUNLOCK(inp);
- return 0;
+ INP_WUNLOCK(toep->inp);
+
+ return (0);
}
static struct mbuf *
-iscsi_queue_handler_callback(struct socket *sk, unsigned int cmd, int *qlen)
+get_writeq_len(struct toepcb *toep, int *qlen)
{
- iscsi_socket *isock;
- struct mbuf *m0 = NULL;
+ struct iscsi_socket *isock = toep->ulpcb;
- if (sk == NULL)
- return NULL;
- isock = (iscsi_socket *)(sk)->so_emuldata;
- if (isock == NULL)
- return NULL;
-
- switch (cmd) {
- case 0:/* PEEK */
- m0 = mbufq_peek(&isock->ulp2_writeq);
- break;
- case 1:/* QUEUE_LEN */
- *qlen = mbufq_len(&isock->ulp2_writeq);
- m0 = mbufq_peek(&isock->ulp2_writeq);
- break;
- case 2:/* DEQUEUE */
- mtx_lock(&isock->ulp2_writeq.lock);
- m0 = mbufq_dequeue(&isock->ulp2_writeq);
- mtx_unlock(&isock->ulp2_writeq.lock);
-
- mtx_lock(&isock->ulp2_wrq.lock);
- mbufq_tail(&isock->ulp2_wrq, m0);
- mtx_unlock(&isock->ulp2_wrq.lock);
-
- m0 = mbufq_peek(&isock->ulp2_writeq);
- break;
- }
- return m0;
+ *qlen = mbufq_len(&isock->ulp2_writeq);
+ return (mbufq_first(&isock->ulp2_writeq));
}
-static void
-iscsi_cpl_handler_callback(struct tom_data *td, struct socket *sk,
- void *m, unsigned int op)
+static struct mbuf *
+do_writeq_next(struct toepcb *toep)
{
- if ((sk == NULL) || (sk->so_emuldata == NULL))
- return;
+ struct iscsi_socket *isock = toep->ulpcb;
+ struct mbuf *m;
- switch (op) {
- case CPL_ISCSI_HDR:
- process_rx_iscsi_hdr(sk, m);
- break;
- case CPL_RX_DATA_DDP:
- process_rx_data_ddp(sk, m);
- break;
- case CPL_SET_TCB_RPL:
- break;
- case CPL_FW4_ACK:
- process_fw4_ack(sk, m);
- break;
- default:
- CTR2(KTR_CXGBE, "sk 0x%p, op 0x%x from TOM, NOT supported.\n",
- sk, op);
- break;
- }
+ mtx_lock(&isock->ulp2_writeq_lock);
+ m = mbufq_dequeue(&isock->ulp2_writeq);
+ mtx_unlock(&isock->ulp2_writeq_lock);
+
+ mtx_lock(&isock->ulp2_wrq_lock);
+ mbufq_enqueue(&isock->ulp2_wrq, m);
+ mtx_unlock(&isock->ulp2_wrq_lock);
+
+ return (mbufq_first(&isock->ulp2_writeq));
}
static void
t4_register_cpl_handler_with_tom(struct adapter *sc)
{
- t4tom_register_cpl_iscsi_callback(iscsi_cpl_handler_callback);
- t4tom_register_queue_iscsi_callback(iscsi_queue_handler_callback);
+
t4_register_cpl_handler(sc, CPL_ISCSI_HDR, do_rx_iscsi_hdr);
t4_register_cpl_handler(sc, CPL_ISCSI_DATA, do_rx_iscsi_hdr);
- t4tom_cpl_handler_register_flag |=
- 1 << TOM_CPL_ISCSI_HDR_REGISTERED_BIT;
-
- if (!t4tom_cpl_handler_registered(sc, CPL_SET_TCB_RPL)) {
- t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, do_set_tcb_rpl);
- t4tom_cpl_handler_register_flag |=
- 1 << TOM_CPL_SET_TCB_RPL_REGISTERED_BIT;
- CTR0(KTR_CXGBE, "register t4 cpl handler CPL_SET_TCB_RPL.\n");
- }
-
t4_register_cpl_handler(sc, CPL_RX_ISCSI_DDP, do_rx_iscsi_ddp);
-
- if (!t4tom_cpl_handler_registered(sc, CPL_RX_DATA_DDP)) {
- t4_register_cpl_handler(sc, CPL_RX_DATA_DDP, do_rx_data_ddp);
- t4tom_cpl_handler_register_flag |=
- 1 << TOM_CPL_RX_DATA_DDP_REGISTERED_BIT;
- CTR0(KTR_CXGBE, "register t4 cpl handler CPL_RX_DATA_DDP.\n");
- }
}
static void
t4_unregister_cpl_handler_with_tom(struct adapter *sc)
{
- /* de-register CPL handles */
- t4tom_register_cpl_iscsi_callback(NULL);
- t4tom_register_queue_iscsi_callback(NULL);
- if (t4tom_cpl_handler_register_flag &
- (1 << TOM_CPL_ISCSI_HDR_REGISTERED_BIT)) {
- t4_register_cpl_handler(sc, CPL_ISCSI_HDR, NULL);
- t4_register_cpl_handler(sc, CPL_ISCSI_DATA, NULL);
- }
- if (t4tom_cpl_handler_register_flag &
- (1 << TOM_CPL_SET_TCB_RPL_REGISTERED_BIT))
- t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, NULL);
+
+ t4_register_cpl_handler(sc, CPL_ISCSI_HDR, NULL);
+ t4_register_cpl_handler(sc, CPL_ISCSI_DATA, NULL);
t4_register_cpl_handler(sc, CPL_RX_ISCSI_DDP, NULL);
- if (t4tom_cpl_handler_register_flag &
- (1 << TOM_CPL_RX_DATA_DDP_REGISTERED_BIT))
- t4_register_cpl_handler(sc, CPL_RX_DATA_DDP, NULL);
}
static int
-send_set_tcb_field(struct socket *sk, u16 word, u64 mask, u64 val,
- int no_reply)
+send_set_tcb_field(struct toepcb * toep, uint16_t word, uint64_t mask,
+ uint64_t val, int no_reply)
{
struct wrqe *wr;
struct cpl_set_tcb_field *req;
- struct inpcb *inp = sotoinpcb(sk);
- struct tcpcb *tp = intotcpcb(inp);
- struct toepcb *toep = tp->t_toe;
wr = alloc_wrqe(sizeof(*req), toep->ctrlq);
if (wr == NULL)
@@ -1173,58 +924,23 @@ send_set_tcb_field(struct socket *sk, u1
req->val = htobe64(val);
t4_wrq_tx(toep->td->tod.tod_softc, wr);
- return 0;
+
+ return (0);
}
static int
-cxgbei_set_ulp_mode(struct socket *so, struct toepcb *toep,
- unsigned char hcrc, unsigned char dcrc)
+cxgbei_set_ulp_mode(struct toepcb *toep, u_char hcrc, u_char dcrc)
{
- int rv = 0, val = 0;
+ int val = 0;
- toep->ulp_mode = ULP_MODE_ISCSI;
if (hcrc)
val |= ULP_CRC_HEADER;
if (dcrc)
val |= ULP_CRC_DATA;
val <<= 4;
val |= ULP_MODE_ISCSI;
- rv = send_set_tcb_field(so, 0, 0xfff, val, 0);
- return rv;
-}
-static offload_device *
-add_cxgbei_dev(struct ifnet *dev, struct toedev *tdev)
-{
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-projects
mailing list