* [PATCH net-next 1/3] cxgb4: Add LRO infrastructure for cxgb4i driver
2016-01-06 12:29 [PATCH net-next 0/3] cxgb4/cxgb4i: add iscsi LRO support for chelsio t4/t5 adapters Hariprasad Shenai
@ 2016-01-06 12:29 ` Hariprasad Shenai
2016-01-06 12:29 ` [PATCH net-next 2/3] libcxgbi: add pdu parsing of LRO'ed skbs Hariprasad Shenai
2016-01-06 12:29 ` [PATCH net-next 3/3] cxgb4i: add iscsi LRO for chelsio t4/t5 adapters Hariprasad Shenai
2 siblings, 0 replies; 6+ messages in thread
From: Hariprasad Shenai @ 2016-01-06 12:29 UTC (permalink / raw)
To: netdev, linux-scsi
Cc: davem, JBottomley, martin.petersen, leedom, kxie, manojmalviya,
nirranjan, Hariprasad Shenai
Signed-off-by: Hariprasad Shenai <hariprasad@chelsio.com>
---
drivers/net/ethernet/chelsio/cxgb4/cxgb4.h | 15 +++++++++-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c | 40 ++++++++++++++++++-------
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 5 ++++
drivers/net/ethernet/chelsio/cxgb4/sge.c | 10 +++++--
4 files changed, 56 insertions(+), 14 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
index ec6e849..a89e64e 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
@@ -508,6 +508,16 @@ struct pkt_gl {
typedef int (*rspq_handler_t)(struct sge_rspq *q, const __be64 *rsp,
const struct pkt_gl *gl);
+typedef void (*rspq_flush_handler_t)(struct sge_rspq *q);
+
+/* LRO related declarations for ULD */
+struct t4_lro_mgr {
+#define MAX_LRO_SESSIONS 64
+ u8 lro_session_cnt; /* # of sessions to aggregate */
+ unsigned long lro_pkts; /* # of LRO super packets */
+ unsigned long lro_merged; /* # of wire packets merged by LRO */
+ struct sk_buff *skb_head, *skb_tail; /* list of aggregated sessions */
+};
struct sge_rspq { /* state for an SGE response queue */
struct napi_struct napi;
@@ -532,6 +542,8 @@ struct sge_rspq { /* state for an SGE response queue */
struct adapter *adap;
struct net_device *netdev; /* associated net device */
rspq_handler_t handler;
+ rspq_flush_handler_t flush_handler;
+ struct t4_lro_mgr lro_mgr;
#ifdef CONFIG_NET_RX_BUSY_POLL
#define CXGB_POLL_STATE_IDLE 0
#define CXGB_POLL_STATE_NAPI BIT(0) /* NAPI owns this poll */
@@ -1107,7 +1119,8 @@ int t4_mgmt_tx(struct adapter *adap, struct sk_buff *skb);
int t4_ofld_send(struct adapter *adap, struct sk_buff *skb);
int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
struct net_device *dev, int intr_idx,
- struct sge_fl *fl, rspq_handler_t hnd, int cong);
+ struct sge_fl *fl, rspq_handler_t hnd,
+ rspq_flush_handler_t flush_hnd, int cong);
int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq,
struct net_device *dev, struct netdev_queue *netdevq,
unsigned int iqid);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index b8a5fb0..68c2aae 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -640,6 +640,13 @@ out:
return 0;
}
+/* Flush the aggregated lro sessions */
+static void uldrx_flush_handler(struct sge_rspq *q)
+{
+ if (ulds[q->uld].lro_flush)
+ ulds[q->uld].lro_flush(&q->lro_mgr);
+}
+
/**
* uldrx_handler - response queue handler for ULD queues
* @q: the response queue that received the packet
@@ -653,6 +660,8 @@ static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
const struct pkt_gl *gl)
{
struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
+ unsigned int napi_id = q->napi.napi_id;
+ int ret;
/* FW can send CPLs encapsulated in a CPL_FW4_MSG.
*/
@@ -660,7 +669,15 @@ static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
rsp += 2;
- if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
+ if (q->flush_handler)
+ ret = ulds[q->uld].lro_rx_handler(q->adap->uld_handle[q->uld],
+ rsp, gl, &q->lro_mgr,
+ napi_id);
+ else
+ ret = ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld],
+ rsp, gl);
+
+ if (ret) {
rxq->stats.nomem++;
return -1;
}
@@ -960,7 +977,7 @@ static void enable_rx(struct adapter *adap)
static int alloc_ofld_rxqs(struct adapter *adap, struct sge_ofld_rxq *q,
unsigned int nq, unsigned int per_chan, int msi_idx,
- u16 *ids)
+ u16 *ids, bool lro)
{
int i, err;
@@ -970,7 +987,8 @@ static int alloc_ofld_rxqs(struct adapter *adap, struct sge_ofld_rxq *q,
err = t4_sge_alloc_rxq(adap, &q->rspq, false,
adap->port[i / per_chan],
msi_idx, q->fl.size ? &q->fl : NULL,
- uldrx_handler, 0);
+ uldrx_handler,
+ lro ? uldrx_flush_handler : NULL, 0);
if (err)
return err;
memset(&q->stats, 0, sizeof(q->stats));
@@ -1000,7 +1018,7 @@ static int setup_sge_queues(struct adapter *adap)
msi_idx = 1; /* vector 0 is for non-queue interrupts */
else {
err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
- NULL, NULL, -1);
+ NULL, NULL, NULL, -1);
if (err)
return err;
msi_idx = -((int)s->intrq.abs_id + 1);
@@ -1020,7 +1038,7 @@ static int setup_sge_queues(struct adapter *adap)
* new/deleted queues.
*/
err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
- msi_idx, NULL, fwevtq_handler, -1);
+ msi_idx, NULL, fwevtq_handler, NULL, -1);
if (err) {
freeout: t4_free_sge_resources(adap);
return err;
@@ -1037,7 +1055,7 @@ freeout: t4_free_sge_resources(adap);
msi_idx++;
err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
msi_idx, &q->fl,
- t4_ethrx_handler,
+ t4_ethrx_handler, NULL,
t4_get_mps_bg_map(adap,
pi->tx_chan));
if (err)
@@ -1063,18 +1081,18 @@ freeout: t4_free_sge_resources(adap);
goto freeout;
}
-#define ALLOC_OFLD_RXQS(firstq, nq, per_chan, ids) do { \
- err = alloc_ofld_rxqs(adap, firstq, nq, per_chan, msi_idx, ids); \
+#define ALLOC_OFLD_RXQS(firstq, nq, per_chan, ids, lro) do { \
+ err = alloc_ofld_rxqs(adap, firstq, nq, per_chan, msi_idx, ids, lro); \
if (err) \
goto freeout; \
if (msi_idx > 0) \
msi_idx += nq; \
} while (0)
- ALLOC_OFLD_RXQS(s->iscsirxq, s->iscsiqsets, j, s->iscsi_rxq);
- ALLOC_OFLD_RXQS(s->rdmarxq, s->rdmaqs, 1, s->rdma_rxq);
+ ALLOC_OFLD_RXQS(s->iscsirxq, s->iscsiqsets, j, s->iscsi_rxq, true);
+ ALLOC_OFLD_RXQS(s->rdmarxq, s->rdmaqs, 1, s->rdma_rxq, false);
j = s->rdmaciqs / adap->params.nports; /* rdmaq queues per channel */
- ALLOC_OFLD_RXQS(s->rdmaciq, s->rdmaciqs, j, s->rdma_ciq);
+ ALLOC_OFLD_RXQS(s->rdmaciq, s->rdmaciqs, j, s->rdma_ciq, false);
#undef ALLOC_OFLD_RXQS
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
index cf711d5..ef75eb3 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
@@ -212,6 +212,7 @@ struct l2t_data;
struct net_device;
struct pkt_gl;
struct tp_tcp_stats;
+struct t4_lro_mgr;
struct cxgb4_range {
unsigned int start;
@@ -283,6 +284,10 @@ struct cxgb4_uld_info {
const struct pkt_gl *gl);
int (*state_change)(void *handle, enum cxgb4_state new_state);
int (*control)(void *handle, enum cxgb4_control control, ...);
+ int (*lro_rx_handler)(void *handle, const __be64 *rsp,
+ const struct pkt_gl *gl,
+ struct t4_lro_mgr *lro_mgr, unsigned int napi_id);
+ void (*lro_flush)(struct t4_lro_mgr *);
};
int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c
index b4eb468..3590d9c 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c
@@ -2157,8 +2157,11 @@ static int process_responses(struct sge_rspq *q, int budget)
while (likely(budget_left)) {
rc = (void *)q->cur_desc + (q->iqe_len - sizeof(*rc));
- if (!is_new_response(rc, q))
+ if (!is_new_response(rc, q)) {
+ if (q->flush_handler)
+ q->flush_handler(q);
break;
+ }
dma_rmb();
rsp_type = RSPD_TYPE_G(rc->type_gen);
@@ -2544,7 +2547,8 @@ static void __iomem *bar2_address(struct adapter *adapter,
*/
int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
struct net_device *dev, int intr_idx,
- struct sge_fl *fl, rspq_handler_t hnd, int cong)
+ struct sge_fl *fl, rspq_handler_t hnd,
+ rspq_flush_handler_t flush_hnd, int cong)
{
int ret, flsz = 0;
struct fw_iq_cmd c;
@@ -2638,6 +2642,8 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
iq->size--; /* subtract status entry */
iq->netdev = dev;
iq->handler = hnd;
+ iq->flush_handler = flush_hnd;
+ memset(&iq->lro_mgr, 0, sizeof(struct t4_lro_mgr));
/* set offset to -1 to distinguish ingress queues without FL */
iq->offset = fl ? 0 : -1;
--
2.3.4
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH net-next 2/3] libcxgbi: add pdu parsing of LRO'ed skbs
2016-01-06 12:29 [PATCH net-next 0/3] cxgb4/cxgb4i: add iscsi LRO support for chelsio t4/t5 adapters Hariprasad Shenai
2016-01-06 12:29 ` [PATCH net-next 1/3] cxgb4: Add LRO infrastructure for cxgb4i driver Hariprasad Shenai
@ 2016-01-06 12:29 ` Hariprasad Shenai
2016-01-06 13:12 ` kbuild test robot
2016-01-06 12:29 ` [PATCH net-next 3/3] cxgb4i: add iscsi LRO for chelsio t4/t5 adapters Hariprasad Shenai
2 siblings, 1 reply; 6+ messages in thread
From: Hariprasad Shenai @ 2016-01-06 12:29 UTC (permalink / raw)
To: netdev, linux-scsi
Cc: davem, JBottomley, martin.petersen, leedom, kxie, manojmalviya,
nirranjan, Hariprasad Shenai
The skbs could contain both paritial pdus and multiple completed pdus.
Signed-off-by: Karen Xie <kxie@chelsio.com>
Signed-off-by: Manoj Malviya <manojmalviya@chelsio.com>
Signed-off-by: Hariprasad Shenai <hariprasad@chelsio.com>
---
drivers/scsi/cxgbi/libcxgbi.c | 415 +++++++++++++++++++++++++++++++++---------
drivers/scsi/cxgbi/libcxgbi.h | 4 +
2 files changed, 330 insertions(+), 89 deletions(-)
diff --git a/drivers/scsi/cxgbi/libcxgbi.c b/drivers/scsi/cxgbi/libcxgbi.c
index f3bb7af..dc9b470 100644
--- a/drivers/scsi/cxgbi/libcxgbi.c
+++ b/drivers/scsi/cxgbi/libcxgbi.c
@@ -35,6 +35,7 @@
static unsigned int dbg_level;
#include "libcxgbi.h"
+#include "cxgbi_lro.h"
#define DRV_MODULE_NAME "libcxgbi"
#define DRV_MODULE_DESC "Chelsio iSCSI driver library"
@@ -533,6 +534,7 @@ static void sock_put_port(struct cxgbi_sock *csk)
/*
* iscsi tcp connection
*/
+static void skb_lro_hold_done(struct cxgbi_sock *csk);
void cxgbi_sock_free_cpl_skbs(struct cxgbi_sock *csk)
{
if (csk->cpl_close) {
@@ -547,6 +549,11 @@ void cxgbi_sock_free_cpl_skbs(struct cxgbi_sock *csk)
kfree_skb(csk->cpl_abort_rpl);
csk->cpl_abort_rpl = NULL;
}
+ if (csk->skb_lro_hold) {
+ skb_lro_hold_done(csk);
+ kfree_skb(csk->skb_lro_hold);
+ csk->skb_lro_hold = NULL;
+ }
}
EXPORT_SYMBOL_GPL(cxgbi_sock_free_cpl_skbs);
@@ -1145,15 +1152,15 @@ static int cxgbi_sock_send_pdus(struct cxgbi_sock *csk, struct sk_buff *skb)
if (unlikely(skb_headroom(skb) < cdev->skb_tx_rsvd)) {
pr_err("csk 0x%p, skb head %u < %u.\n",
- csk, skb_headroom(skb), cdev->skb_tx_rsvd);
+ csk, skb_headroom(skb), cdev->skb_tx_rsvd);
err = -EINVAL;
goto out_err;
}
if (frags >= SKB_WR_LIST_SIZE) {
pr_err("csk 0x%p, frags %d, %u,%u >%u.\n",
- csk, skb_shinfo(skb)->nr_frags, skb->len,
- skb->data_len, (uint)(SKB_WR_LIST_SIZE));
+ csk, skb_shinfo(skb)->nr_frags, skb->len,
+ skb->data_len, (uint)(SKB_WR_LIST_SIZE));
err = -EINVAL;
goto out_err;
}
@@ -1776,10 +1783,8 @@ EXPORT_SYMBOL_GPL(cxgbi_conn_tx_open);
/*
* pdu receive, interact with libiscsi_tcp
*/
-static inline int read_pdu_skb(struct iscsi_conn *conn,
- struct sk_buff *skb,
- unsigned int offset,
- int offloaded)
+static inline int read_pdu_skb(struct iscsi_conn *conn, struct sk_buff *skb,
+ unsigned int offset, int offloaded)
{
int status = 0;
int bytes_read;
@@ -1817,7 +1822,8 @@ static inline int read_pdu_skb(struct iscsi_conn *conn,
}
}
-static int skb_read_pdu_bhs(struct iscsi_conn *conn, struct sk_buff *skb)
+static int skb_read_pdu_bhs(struct iscsi_conn *conn, struct sk_buff *skb,
+ unsigned int offset, unsigned long *flag)
{
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
@@ -1831,18 +1837,18 @@ static int skb_read_pdu_bhs(struct iscsi_conn *conn, struct sk_buff *skb)
return -EIO;
}
- if (conn->hdrdgst_en &&
- cxgbi_skcb_test_flag(skb, SKCBF_RX_HCRC_ERR)) {
+ if (conn->hdrdgst_en && test_bit(SKCBF_RX_HCRC_ERR, flag)) {
pr_info("conn 0x%p, skb 0x%p, hcrc.\n", conn, skb);
iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST);
return -EIO;
}
- return read_pdu_skb(conn, skb, 0, 0);
+ return read_pdu_skb(conn, skb, offset, 0);
}
static int skb_read_pdu_data(struct iscsi_conn *conn, struct sk_buff *lskb,
- struct sk_buff *skb, unsigned int offset)
+ struct sk_buff *skb, unsigned int offset,
+ unsigned long *flag)
{
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
bool offloaded = 0;
@@ -1850,12 +1856,11 @@ static int skb_read_pdu_data(struct iscsi_conn *conn, struct sk_buff *lskb,
log_debug(1 << CXGBI_DBG_PDU_RX,
"conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n",
- conn, skb, skb->len, cxgbi_skcb_flags(skb));
+ conn, skb, skb->len, *flag);
- if (conn->datadgst_en &&
- cxgbi_skcb_test_flag(lskb, SKCBF_RX_DCRC_ERR)) {
+ if (conn->datadgst_en && test_bit(SKCBF_RX_DCRC_ERR, flag)) {
pr_info("conn 0x%p, skb 0x%p, dcrc 0x%lx.\n",
- conn, lskb, cxgbi_skcb_flags(lskb));
+ conn, lskb, *flag);
iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
return -EIO;
}
@@ -1867,7 +1872,7 @@ static int skb_read_pdu_data(struct iscsi_conn *conn, struct sk_buff *lskb,
if (lskb == skb && conn->hdrdgst_en)
offset += ISCSI_DIGEST_SIZE;
- if (cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA_DDPD))
+ if (test_bit(SKCBF_RX_DATA_DDPD, flag))
offloaded = 1;
if (opcode == ISCSI_OP_SCSI_DATA_IN)
@@ -1905,11 +1910,304 @@ static void csk_return_rx_credits(struct cxgbi_sock *csk, int copied)
csk->rcv_wup += cdev->csk_send_rx_credits(csk, credits);
}
+static struct sk_buff *cxgbi_conn_rxq_get_skb(struct cxgbi_sock *csk)
+{
+ struct sk_buff *skb = skb_peek(&csk->receive_queue);
+
+ if (!skb)
+ return NULL;
+ if (!(cxgbi_skcb_test_flag(skb, SKCBF_RX_LRO)) &&
+ !(cxgbi_skcb_test_flag(skb, SKCBF_RX_STATUS))) {
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "skb 0x%p, NOT ready 0x%lx.\n",
+ skb, cxgbi_skcb_flags(skb));
+ return NULL;
+ }
+ __skb_unlink(skb, &csk->receive_queue);
+ return skb;
+}
+
+void cxgbi_lro_skb_dump(struct sk_buff *skb)
+{
+ struct skb_shared_info *ssi = skb_shinfo(skb);
+ struct cxgbi_rx_lro_cb *lro_cb = cxgbi_skb_rx_lro_cb(skb);
+ struct cxgbi_rx_pdu_cb *pdu_cb = cxgbi_skb_rx_pdu_cb(skb, 0);
+ int i;
+
+ pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
+ skb, skb->head, skb->data, skb->len, skb->data_len,
+ ssi->nr_frags);
+ pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
+ skb, lro_cb->csk, lro_cb->pdu_cnt, lro_cb->pdu_totallen);
+
+ for (i = 0; i < lro_cb->pdu_cnt; i++, pdu_cb++)
+ pr_info("0x%p, pdu %d,%u,0x%lx,seq 0x%x,dcrc 0x%x,frags %u.\n",
+ skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq,
+ pdu_cb->ddigest, pdu_cb->frags);
+ for (i = 0; i < ssi->nr_frags; i++)
+ pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
+ skb, i, ssi->frags[i].page_offset, ssi->frags[i].size);
+}
+EXPORT_SYMBOL_GPL(cxgbi_lro_skb_dump);
+
+static void skb_lro_hold_done(struct cxgbi_sock *csk)
+{
+ struct sk_buff *skb = csk->skb_lro_hold;
+ struct cxgbi_rx_lro_cb *lro_cb = cxgbi_skb_rx_lro_cb(skb);
+
+ if (lro_cb->flags & CXGBI_LRO_CB_USED) {
+ struct skb_shared_info *ssi = skb_shinfo(skb);
+ int i;
+
+ memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
+ for (i = 0; i < ssi->nr_frags; i++)
+ put_page(skb_frag_page(&ssi->frags[i]));
+ ssi->nr_frags = 0;
+ }
+}
+
+static void skb_lro_hold_merge(struct cxgbi_sock *csk, struct sk_buff *skb,
+ int pdu_idx)
+{
+ struct sk_buff *hskb = csk->skb_lro_hold;
+ struct skb_shared_info *hssi = skb_shinfo(hskb);
+ struct cxgbi_rx_lro_cb *hlro_cb = cxgbi_skb_rx_lro_cb(hskb);
+ struct cxgbi_rx_pdu_cb *hpdu_cb = cxgbi_skb_rx_pdu_cb(hskb, 0);
+ struct skb_shared_info *ssi = skb_shinfo(skb);
+ struct cxgbi_rx_pdu_cb *pdu_cb = cxgbi_skb_rx_pdu_cb(skb, pdu_idx);
+ int frag_idx = 0;
+ int hfrag_idx = 0;
+
+ /* the partial pdu is either 1st or last */
+ if (pdu_idx)
+ frag_idx = ssi->nr_frags - pdu_cb->frags;
+
+ if (cxgbi_rx_cb_test_flag(pdu_cb, SKCBF_RX_HDR)) {
+ unsigned int len = 0;
+
+ skb_lro_hold_done(csk);
+
+ hlro_cb->csk = csk;
+ hlro_cb->pdu_cnt = 1;
+ hlro_cb->flags = CXGBI_LRO_CB_USED;
+
+ hpdu_cb->flags = pdu_cb->flags;
+ hpdu_cb->seq = pdu_cb->seq;
+
+ memcpy(&hssi->frags[hfrag_idx], &ssi->frags[frag_idx],
+ sizeof(skb_frag_t));
+ get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
+ frag_idx++;
+ hfrag_idx++;
+ hssi->nr_frags = 1;
+ hpdu_cb->frags = 1;
+
+ len = hssi->frags[0].size;
+ hskb->len = len;
+ hskb->data_len = len;
+ hskb->truesize = len;
+ }
+
+ if (cxgbi_rx_cb_test_flag(pdu_cb, SKCBF_RX_DATA)) {
+ unsigned int len = 0;
+ int i, n;
+
+ hpdu_cb->flags |= pdu_cb->flags;
+
+ for (i = 1, n = hfrag_idx; n < pdu_cb->frags;
+ i++, frag_idx++, n++) {
+ memcpy(&hssi->frags[i], &ssi->frags[frag_idx],
+ sizeof(skb_frag_t));
+ get_page(skb_frag_page(&hssi->frags[i]));
+ len += hssi->frags[i].size;
+
+ hssi->nr_frags++;
+ hpdu_cb->frags++;
+ }
+ hskb->len += len;
+ hskb->data_len += len;
+ hskb->truesize += len;
+ }
+
+ if (cxgbi_rx_cb_test_flag(pdu_cb, SKCBF_RX_STATUS)) {
+ hpdu_cb->flags |= pdu_cb->flags;
+ hpdu_cb->ddigest = pdu_cb->ddigest;
+ hpdu_cb->pdulen = pdu_cb->pdulen;
+ hlro_cb->pdu_totallen = pdu_cb->pdulen;
+ }
+}
+
+static int rx_skb_lro_read_pdu(struct sk_buff *skb, struct iscsi_conn *conn,
+ unsigned int *off_p, int idx)
+{
+ struct cxgbi_rx_pdu_cb *pdu_cb = cxgbi_skb_rx_pdu_cb(skb, idx);
+ unsigned int offset = *off_p;
+ int err = 0;
+
+ err = skb_read_pdu_bhs(conn, skb, offset, &pdu_cb->flags);
+ if (err < 0) {
+ pr_err("conn 0x%p, bhs, skb 0x%p, pdu %d, offset %u.\n",
+ conn, skb, idx, offset);
+ cxgbi_lro_skb_dump(skb);
+ goto done;
+ }
+ offset += err;
+
+ err = skb_read_pdu_data(conn, skb, skb, offset, &pdu_cb->flags);
+ if (err < 0) {
+ pr_err("%s: conn 0x%p data, skb 0x%p, pdu %d.\n",
+ __func__, conn, skb, idx);
+ cxgbi_lro_skb_dump(skb);
+ goto done;
+ }
+ offset += err;
+ if (conn->hdrdgst_en)
+ offset += ISCSI_DIGEST_SIZE;
+
+done:
+ *off_p = offset;
+ return err;
+}
+
+static int rx_skb_lro(struct sk_buff *skb, struct cxgbi_device *cdev,
+ struct cxgbi_sock *csk, struct iscsi_conn *conn,
+ unsigned int *read)
+{
+ struct sk_buff *hskb = csk->skb_lro_hold;
+ struct cxgbi_rx_lro_cb *lro_cb = cxgbi_skb_rx_lro_cb(skb);
+ struct cxgbi_rx_pdu_cb *pdu_cb = cxgbi_skb_rx_pdu_cb(skb, 0);
+ int last = lro_cb->pdu_cnt - 1;
+ int i = 0;
+ int err = 0;
+ unsigned int offset = 0;
+
+ if (!cxgbi_rx_cb_test_flag(pdu_cb, SKCBF_RX_HDR)) {
+ /* there could be partial pdus spanning 2 lro skbs */
+ skb_lro_hold_merge(csk, skb, 0);
+
+ if (cxgbi_rx_cb_test_flag(pdu_cb, SKCBF_RX_STATUS)) {
+ unsigned int dummy = 0;
+
+ err = rx_skb_lro_read_pdu(hskb, conn, &dummy, 0);
+ if (err < 0)
+ goto done;
+
+ if (pdu_cb->frags) {
+ struct skb_shared_info *ssi = skb_shinfo(skb);
+ int k;
+
+ for (k = 0; k < pdu_cb->frags; k++)
+ offset += ssi->frags[k].size;
+ }
+ }
+ i = 1;
+ }
+
+ for (; i < last; i++, pdu_cb++) {
+ err = rx_skb_lro_read_pdu(skb, conn, &offset, i);
+ if (err < 0)
+ goto done;
+ }
+
+ if (i == last) {
+ pdu_cb = cxgbi_skb_rx_pdu_cb(skb, last);
+ if (!cxgbi_rx_cb_test_flag(pdu_cb, SKCBF_RX_STATUS)) {
+ skb_lro_hold_merge(csk, skb, last);
+ } else {
+ err = rx_skb_lro_read_pdu(skb, conn, &offset, last);
+ if (err < 0)
+ goto done;
+ }
+ }
+
+ *read += lro_cb->pdu_totallen;
+
+done:
+ __kfree_skb(skb);
+ return err;
+}
+
+static int rx_skb_coalesced(struct sk_buff *skb, struct cxgbi_device *cdev,
+ struct cxgbi_sock *csk, struct iscsi_conn *conn,
+ unsigned int *read)
+{
+ int err = 0;
+
+ *read += cxgbi_skcb_rx_pdulen(skb);
+
+ err = skb_read_pdu_bhs(conn, skb, 0, &cxgbi_skcb_flags(skb));
+ if (err < 0) {
+ pr_err("bhs, csk 0x%p, skb 0x%p,%u, f 0x%lx, plen %u.\n",
+ csk, skb, skb->len, cxgbi_skcb_flags(skb),
+ cxgbi_skcb_rx_pdulen(skb));
+ pr_info("bhs %*ph\n", 48, skb->data);
+ goto done;
+ }
+
+ err = skb_read_pdu_data(conn, skb, skb, err + cdev->skb_rx_extra,
+ &cxgbi_skcb_flags(skb));
+ if (err < 0) {
+ pr_err("data, csk 0x%p, skb 0x%p,%u, f 0x%lx, plen %u.\n",
+ csk, skb, skb->len, cxgbi_skcb_flags(skb),
+ cxgbi_skcb_rx_pdulen(skb));
+ pr_info("bhs %*ph\n", 48, skb->data);
+ }
+
+done:
+ __kfree_skb(skb);
+ return err;
+}
+
+static int rx_skb(struct sk_buff *skb, struct cxgbi_device *cdev,
+ struct cxgbi_sock *csk, struct iscsi_conn *conn,
+ unsigned int *read)
+{
+ int err = 0;
+
+ *read += cxgbi_skcb_rx_pdulen(skb);
+
+ err = skb_read_pdu_bhs(conn, skb, 0, &cxgbi_skcb_flags(skb));
+ if (err < 0) {
+ pr_err("bhs, csk 0x%p, skb 0x%p,%u, f 0x%lx, plen %u.\n",
+ csk, skb, skb->len, cxgbi_skcb_flags(skb),
+ cxgbi_skcb_rx_pdulen(skb));
+ pr_err("bhs %*ph\n", 48, skb->data);
+ goto done;
+ }
+
+ if (cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA)) {
+ struct sk_buff *dskb = skb_peek(&csk->receive_queue);
+
+ if (!dskb) {
+ pr_err("csk 0x%p, NO data.\n", csk);
+ err = -EAGAIN;
+ goto done;
+ }
+ __skb_unlink(dskb, &csk->receive_queue);
+
+ err = skb_read_pdu_data(conn, skb, dskb, 0,
+ &cxgbi_skcb_flags(skb));
+ if (err < 0) {
+ pr_err("data, csk 0x%p, 0x%p,%u, 0x%p,%u,0x%lx,%u.\n",
+ csk, skb, skb->len, dskb, dskb->len,
+ cxgbi_skcb_flags(dskb),
+ cxgbi_skcb_rx_pdulen(dskb));
+ pr_info("bhs %*ph\n", 48, skb->data);
+ }
+ __kfree_skb(dskb);
+ } else
+ err = skb_read_pdu_data(conn, skb, skb, 0,
+ &cxgbi_skcb_flags(skb));
+
+done:
+ __kfree_skb(skb);
+ return err;
+}
+
void cxgbi_conn_pdu_ready(struct cxgbi_sock *csk)
{
struct cxgbi_device *cdev = csk->cdev;
struct iscsi_conn *conn = csk->user_data;
- struct sk_buff *skb;
unsigned int read = 0;
int err = 0;
@@ -1925,83 +2223,22 @@ void cxgbi_conn_pdu_ready(struct cxgbi_sock *csk)
}
while (!err) {
- skb = skb_peek(&csk->receive_queue);
- if (!skb ||
- !(cxgbi_skcb_test_flag(skb, SKCBF_RX_STATUS))) {
- if (skb)
- log_debug(1 << CXGBI_DBG_PDU_RX,
- "skb 0x%p, NOT ready 0x%lx.\n",
- skb, cxgbi_skcb_flags(skb));
+ struct sk_buff *skb = cxgbi_conn_rxq_get_skb(csk);
+
+ if (!skb)
break;
- }
- __skb_unlink(skb, &csk->receive_queue);
- read += cxgbi_skcb_rx_pdulen(skb);
log_debug(1 << CXGBI_DBG_PDU_RX,
"csk 0x%p, skb 0x%p,%u,f 0x%lx, pdu len %u.\n",
csk, skb, skb->len, cxgbi_skcb_flags(skb),
cxgbi_skcb_rx_pdulen(skb));
- if (cxgbi_skcb_test_flag(skb, SKCBF_RX_COALESCED)) {
- err = skb_read_pdu_bhs(conn, skb);
- if (err < 0) {
- pr_err("coalesced bhs, csk 0x%p, skb 0x%p,%u, "
- "f 0x%lx, plen %u.\n",
- csk, skb, skb->len,
- cxgbi_skcb_flags(skb),
- cxgbi_skcb_rx_pdulen(skb));
- goto skb_done;
- }
- err = skb_read_pdu_data(conn, skb, skb,
- err + cdev->skb_rx_extra);
- if (err < 0)
- pr_err("coalesced data, csk 0x%p, skb 0x%p,%u, "
- "f 0x%lx, plen %u.\n",
- csk, skb, skb->len,
- cxgbi_skcb_flags(skb),
- cxgbi_skcb_rx_pdulen(skb));
- } else {
- err = skb_read_pdu_bhs(conn, skb);
- if (err < 0) {
- pr_err("bhs, csk 0x%p, skb 0x%p,%u, "
- "f 0x%lx, plen %u.\n",
- csk, skb, skb->len,
- cxgbi_skcb_flags(skb),
- cxgbi_skcb_rx_pdulen(skb));
- goto skb_done;
- }
-
- if (cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA)) {
- struct sk_buff *dskb;
-
- dskb = skb_peek(&csk->receive_queue);
- if (!dskb) {
- pr_err("csk 0x%p, skb 0x%p,%u, f 0x%lx,"
- " plen %u, NO data.\n",
- csk, skb, skb->len,
- cxgbi_skcb_flags(skb),
- cxgbi_skcb_rx_pdulen(skb));
- err = -EIO;
- goto skb_done;
- }
- __skb_unlink(dskb, &csk->receive_queue);
-
- err = skb_read_pdu_data(conn, skb, dskb, 0);
- if (err < 0)
- pr_err("data, csk 0x%p, skb 0x%p,%u, "
- "f 0x%lx, plen %u, dskb 0x%p,"
- "%u.\n",
- csk, skb, skb->len,
- cxgbi_skcb_flags(skb),
- cxgbi_skcb_rx_pdulen(skb),
- dskb, dskb->len);
- __kfree_skb(dskb);
- } else
- err = skb_read_pdu_data(conn, skb, skb, 0);
- }
-skb_done:
- __kfree_skb(skb);
-
+ if (cxgbi_skcb_test_flag(skb, SKCBF_RX_LRO))
+ err = rx_skb_lro(skb, cdev, csk, conn, &read);
+ else if (cxgbi_skcb_test_flag(skb, SKCBF_RX_COALESCED))
+ err = rx_skb_coalesced(skb, cdev, csk, conn, &read);
+ else
+ err = rx_skb(skb, cdev, csk, conn, &read);
if (err < 0)
break;
}
@@ -2014,7 +2251,7 @@ skb_done:
}
if (err < 0) {
- pr_info("csk 0x%p, 0x%p, rx failed %d, read %u.\n",
+ pr_info("csk 0x%p,0x%p, rx failed %d, read %u.\n",
csk, conn, err, read);
iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
}
diff --git a/drivers/scsi/cxgbi/libcxgbi.h b/drivers/scsi/cxgbi/libcxgbi.h
index 9842301..13ec7d7 100644
--- a/drivers/scsi/cxgbi/libcxgbi.h
+++ b/drivers/scsi/cxgbi/libcxgbi.h
@@ -208,6 +208,8 @@ struct cxgbi_sock {
struct sk_buff *cpl_abort_req;
struct sk_buff *cpl_abort_rpl;
struct sk_buff *skb_ulp_lhdr;
+ struct sk_buff *skb_lro; /* accumulated rx so far */
+ struct sk_buff *skb_lro_hold; /* holds a partial pdu */
spinlock_t lock;
struct kref refcnt;
unsigned int state;
@@ -279,6 +281,7 @@ struct cxgbi_skb_tx_cb {
enum cxgbi_skcb_flags {
SKCBF_TX_NEED_HDR, /* packet needs a header */
+ SKCBF_RX_LRO, /* received via lro */
SKCBF_RX_COALESCED, /* received whole pdu */
SKCBF_RX_HDR, /* received pdu header */
SKCBF_RX_DATA, /* received pdu payload */
@@ -752,4 +755,5 @@ void cxgbi_ddp_page_size_factor(int *);
void cxgbi_ddp_ppod_clear(struct cxgbi_pagepod *);
void cxgbi_ddp_ppod_set(struct cxgbi_pagepod *, struct cxgbi_pagepod_hdr *,
struct cxgbi_gather_list *, unsigned int);
+
#endif /*__LIBCXGBI_H__*/
--
2.3.4
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH net-next 3/3] cxgb4i: add iscsi LRO for chelsio t4/t5 adapters
2016-01-06 12:29 [PATCH net-next 0/3] cxgb4/cxgb4i: add iscsi LRO support for chelsio t4/t5 adapters Hariprasad Shenai
2016-01-06 12:29 ` [PATCH net-next 1/3] cxgb4: Add LRO infrastructure for cxgb4i driver Hariprasad Shenai
2016-01-06 12:29 ` [PATCH net-next 2/3] libcxgbi: add pdu parsing of LRO'ed skbs Hariprasad Shenai
@ 2016-01-06 12:29 ` Hariprasad Shenai
2016-01-06 13:32 ` kbuild test robot
2 siblings, 1 reply; 6+ messages in thread
From: Hariprasad Shenai @ 2016-01-06 12:29 UTC (permalink / raw)
To: netdev, linux-scsi
Cc: davem, JBottomley, martin.petersen, leedom, kxie, manojmalviya,
nirranjan, Hariprasad Shenai
Signed-off-by: Karen Xie <kxie@chelsio.com>
Signed-off-by: Manoj Malviya <manojmalviya@chelsio.com>
Signed-off-by: Hariprasad Shenai <hariprasad@chelsio.com>
---
drivers/scsi/cxgbi/cxgb4i/cxgb4i.c | 396 +++++++++++++++++++++++++++++++++++--
1 file changed, 382 insertions(+), 14 deletions(-)
diff --git a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
index 804806e..68e94e6 100644
--- a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
+++ b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
@@ -33,6 +33,7 @@
static unsigned int dbg_level;
#include "../libcxgbi.h"
+#include "../cxgbi_lro.h"
#define DRV_MODULE_NAME "cxgb4i"
#define DRV_MODULE_DESC "Chelsio T4/T5 iSCSI Driver"
@@ -81,13 +82,6 @@ static int t4_uld_rx_handler(void *, const __be64 *, const struct pkt_gl *);
static int t4_uld_state_change(void *, enum cxgb4_state state);
static inline int send_tx_flowc_wr(struct cxgbi_sock *);
-static const struct cxgb4_uld_info cxgb4i_uld_info = {
- .name = DRV_MODULE_NAME,
- .add = t4_uld_add,
- .rx_handler = t4_uld_rx_handler,
- .state_change = t4_uld_state_change,
-};
-
static struct scsi_host_template cxgb4i_host_template = {
.module = THIS_MODULE,
.name = DRV_MODULE_NAME,
@@ -1182,8 +1176,9 @@ static void do_rx_data_ddp(struct cxgbi_device *cdev,
}
log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
- "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p.\n",
- csk, csk->state, csk->flags, skb, status, csk->skb_ulp_lhdr);
+ "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p, len %u.\n",
+ csk, csk->state, csk->flags, skb, status, csk->skb_ulp_lhdr,
+ ntohs(rpl->len));
spin_lock_bh(&csk->lock);
@@ -1212,13 +1207,13 @@ static void do_rx_data_ddp(struct cxgbi_device *cdev,
csk->tid, ntohs(rpl->len), cxgbi_skcb_rx_pdulen(lskb));
if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) {
- pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, hcrc bad 0x%lx.\n",
- csk, lskb, status, cxgbi_skcb_flags(lskb));
+ pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, hcrc bad.\n",
+ csk, lskb, status);
cxgbi_skcb_set_flag(lskb, SKCBF_RX_HCRC_ERR);
}
if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) {
- pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, dcrc bad 0x%lx.\n",
- csk, lskb, status, cxgbi_skcb_flags(lskb));
+ pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, dcrc bad.\n",
+ csk, lskb, status);
cxgbi_skcb_set_flag(lskb, SKCBF_RX_DCRC_ERR);
}
if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) {
@@ -1311,6 +1306,12 @@ static int alloc_cpls(struct cxgbi_sock *csk)
0, GFP_KERNEL);
if (!csk->cpl_abort_rpl)
goto free_cpls;
+
+ csk->skb_lro_hold = alloc_skb(LRO_SKB_MIN_HEADROOM, GFP_KERNEL);
+ if (unlikely(!csk->skb_lro_hold))
+ goto free_cpls;
+ memset(csk->skb_lro_hold->data, 0, LRO_SKB_MIN_HEADROOM);
+
return 0;
free_cpls:
@@ -1539,7 +1540,7 @@ int cxgb4i_ofld_init(struct cxgbi_device *cdev)
cdev->csk_alloc_cpls = alloc_cpls;
cdev->csk_init_act_open = init_act_open;
- pr_info("cdev 0x%p, offload up, added.\n", cdev);
+ pr_info("cdev 0x%p, offload up, added, f 0x%x.\n", cdev, cdev->flags);
return 0;
}
@@ -1891,6 +1892,373 @@ static int t4_uld_state_change(void *handle, enum cxgb4_state state)
return 0;
}
+static void proc_ddp_status(unsigned int tid, struct cpl_rx_data_ddp *cpl,
+ struct cxgbi_rx_pdu_cb *pdu_cb)
+{
+ unsigned int status = ntohl(cpl->ddpvld);
+
+ cxgbi_rx_cb_set_flag(pdu_cb, SKCBF_RX_STATUS);
+
+ pdu_cb->ddigest = ntohl(cpl->ulp_crc);
+ pdu_cb->pdulen = ntohs(cpl->len);
+
+ if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) {
+ pr_info("tid 0x%x, status 0x%x, hcrc bad.\n", tid, status);
+ cxgbi_rx_cb_set_flag(pdu_cb, SKCBF_RX_HCRC_ERR);
+ }
+ if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) {
+ pr_info("tid 0x%x, status 0x%x, dcrc bad.\n", tid, status);
+ cxgbi_rx_cb_set_flag(pdu_cb, SKCBF_RX_DCRC_ERR);
+ }
+ if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) {
+ pr_info("tid 0x%x, status 0x%x, pad bad.\n", tid, status);
+ cxgbi_rx_cb_set_flag(pdu_cb, SKCBF_RX_PAD_ERR);
+ }
+ if ((status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT)) &&
+ !cxgbi_rx_cb_test_flag(pdu_cb, SKCBF_RX_DATA)) {
+ cxgbi_rx_cb_set_flag(pdu_cb, SKCBF_RX_DATA_DDPD);
+ }
+}
+
+static void lro_skb_add_packet_rsp(struct sk_buff *skb, u8 op,
+ const __be64 *rsp)
+{
+ struct cxgbi_rx_lro_cb *lro_cb = cxgbi_skb_rx_lro_cb(skb);
+ struct cxgbi_rx_pdu_cb *pdu_cb = cxgbi_skb_rx_pdu_cb(skb,
+ lro_cb->pdu_cnt);
+ struct cpl_rx_data_ddp *cpl = (struct cpl_rx_data_ddp *)(rsp + 1);
+
+ proc_ddp_status(lro_cb->csk->tid, cpl, pdu_cb);
+
+ lro_cb->pdu_totallen += pdu_cb->pdulen;
+ lro_cb->pdu_cnt++;
+}
+
+static void lro_skb_add_packet_gl(struct sk_buff *skb, u8 op,
+ const struct pkt_gl *gl)
+{
+ struct cxgbi_rx_lro_cb *lro_cb = cxgbi_skb_rx_lro_cb(skb);
+ struct cxgbi_rx_pdu_cb *pdu_cb = cxgbi_skb_rx_pdu_cb(skb,
+ lro_cb->pdu_cnt);
+ struct skb_shared_info *ssi = skb_shinfo(skb);
+ int i = ssi->nr_frags;
+ unsigned int offset;
+ unsigned int len;
+
+ if (op == CPL_ISCSI_HDR) {
+ struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)gl->va;
+
+ offset = sizeof(struct cpl_iscsi_hdr);
+ cxgbi_rx_cb_set_flag(pdu_cb, SKCBF_RX_HDR);
+
+ pdu_cb->seq = ntohl(cpl->seq);
+ len = ntohs(cpl->len);
+ } else {
+ struct cpl_rx_data *cpl = (struct cpl_rx_data *)gl->va;
+
+ offset = sizeof(struct cpl_rx_data);
+ cxgbi_rx_cb_set_flag(pdu_cb, SKCBF_RX_DATA);
+
+ len = ntohs(cpl->len);
+ }
+
+ memcpy(&ssi->frags[i], &gl->frags[0], gl->nfrags * sizeof(skb_frag_t));
+ ssi->frags[i].page_offset += offset;
+ ssi->frags[i].size -= offset;
+ ssi->nr_frags += gl->nfrags;
+ pdu_cb->frags += gl->nfrags;
+
+ skb->len += len;
+ skb->data_len += len;
+ skb->truesize += len;
+
+ for (i = 0; i < gl->nfrags; i++)
+ get_page(gl->frags[i].page);
+}
+
+static inline int cxgbi_sock_check_rx_state(struct cxgbi_sock *csk)
+{
+ if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u, bad state.\n",
+ csk, csk->state, csk->flags, csk->tid);
+ if (csk->state != CTP_ABORTING)
+ send_abort_req(csk);
+ return -1;
+ }
+ return 0;
+}
+
+static void do_rx_iscsi_lro(struct cxgbi_sock *csk, struct sk_buff *skb)
+{
+ struct cxgbi_rx_lro_cb *lro_cb = cxgbi_skb_rx_lro_cb(skb);
+ struct cxgbi_rx_pdu_cb *pdu_cb = cxgbi_skb_rx_pdu_cb(skb, 0);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
+ "%s: csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, %u.\n",
+ __func__, csk, csk->state, csk->flags, csk->tid, skb,
+ skb->len, skb->data_len);
+
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_LRO);
+
+ spin_lock_bh(&csk->lock);
+
+ if (cxgbi_sock_check_rx_state(csk) < 0)
+ goto discard;
+
+ if (cxgbi_rx_cb_test_flag(pdu_cb, SKCBF_RX_HDR) &&
+ pdu_cb->seq != csk->rcv_nxt) {
+ pr_info("%s, csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
+ __func__, csk, csk->tid, pdu_cb->seq, csk->rcv_nxt);
+ cxgbi_lro_skb_dump(skb);
+ goto abort_conn;
+ }
+
+ /* partial pdus */
+ if (!lro_cb->pdu_cnt) {
+ lro_cb->pdu_cnt = 1;
+ } else {
+ struct cxgbi_rx_pdu_cb *pdu_cb = cxgbi_skb_rx_pdu_cb(skb,
+ lro_cb->pdu_cnt);
+
+ if (!(cxgbi_rx_cb_test_flag(pdu_cb, SKCBF_RX_STATUS)) &&
+ pdu_cb->frags)
+ lro_cb->pdu_cnt++;
+ }
+
+ csk->rcv_nxt += lro_cb->pdu_totallen;
+
+ skb_reset_transport_header(skb);
+ __skb_queue_tail(&csk->receive_queue, skb);
+
+ cxgbi_conn_pdu_ready(csk);
+ spin_unlock_bh(&csk->lock);
+
+ return;
+
+abort_conn:
+ send_abort_req(csk);
+discard:
+ spin_unlock_bh(&csk->lock);
+ __kfree_skb(skb);
+}
+
+static struct sk_buff *lro_init_skb(struct cxgbi_sock *csk, u8 op,
+ const struct pkt_gl *gl, const __be64 *rsp)
+{
+ struct sk_buff *skb;
+ struct cxgbi_rx_lro_cb *lro_cb;
+
+ skb = alloc_skb(LRO_SKB_MAX_HEADROOM, GFP_ATOMIC);
+ if (unlikely(!skb))
+ return NULL;
+ memset(skb->data, 0, LRO_SKB_MAX_HEADROOM);
+
+ lro_cb = cxgbi_skb_rx_lro_cb(skb);
+ cxgbi_sock_get(csk);
+ lro_cb->csk = csk;
+
+ return skb;
+}
+
+static void lro_flush(struct t4_lro_mgr *lro_mgr, struct sk_buff *skb)
+{
+ struct cxgbi_rx_lro_cb *lro_cb = cxgbi_skb_rx_lro_cb(skb);
+ struct cxgbi_sock *csk = lro_cb->csk;
+
+ /* Update head pointer */
+ if (skb == lro_mgr->skb_head) {
+ if (!skb->next) {
+ /* Only skb in the list */
+ lro_mgr->skb_head = NULL;
+ lro_mgr->skb_tail = NULL;
+ } else {
+ lro_mgr->skb_head = skb->next;
+ }
+ } else if (skb == lro_mgr->skb_tail) {
+ /* Update tail pointer */
+ skb->prev->next = NULL;
+ lro_mgr->skb_tail = skb->prev;
+ } else {
+ skb->prev->next = skb->next;
+ skb->next->prev = skb->prev;
+ }
+
+ csk->skb_lro = NULL;
+ do_rx_iscsi_lro(csk, skb);
+ cxgbi_sock_put(csk);
+
+ lro_mgr->lro_pkts++;
+ lro_mgr->lro_session_cnt--;
+}
+
+static int lro_receive(struct cxgbi_sock *csk, u8 op, const __be64 *rsp,
+ const struct pkt_gl *gl, struct t4_lro_mgr *lro_mgr)
+{
+ struct sk_buff *skb;
+ struct cxgbi_rx_lro_cb *lro_cb;
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev);
+
+ if (!is_t5(lldi->adapter_type))
+ return -EOPNOTSUPP;
+
+ if (!csk) {
+ pr_err("%s: csk NULL, op 0x%x.\n", __func__, op);
+ goto out;
+ }
+
+ if (csk->skb_lro)
+ goto add_packet;
+
+start_lro:
+ /* Did we reach the hash size limit */
+ if (lro_mgr->lro_session_cnt >= MAX_LRO_SESSIONS)
+ goto out;
+
+ skb = lro_init_skb(csk, op, gl, rsp);
+ csk->skb_lro = skb;
+ if (unlikely(!skb))
+ goto out;
+ lro_mgr->lro_session_cnt++;
+
+ if (lro_mgr->skb_tail) {
+ lro_mgr->skb_tail->next = skb;
+ skb->prev = lro_mgr->skb_tail;
+ } else {
+ lro_mgr->skb_head = skb;
+ }
+ lro_mgr->skb_tail = skb;
+
+ /* continue to add the packet */
+add_packet:
+ skb = csk->skb_lro;
+ lro_cb = cxgbi_skb_rx_lro_cb(skb);
+
+ /* Check if this packet can be aggregated */
+ if ((gl && ((skb_shinfo(skb)->nr_frags + gl->nfrags) >= MAX_SKB_FRAGS ||
+ lro_cb->pdu_totallen >= LRO_FLUSH_TOTALLEN_MAX)) ||
+ /* lro_cb->pdu_cnt must be less than MAX_SKB_FRAGS */
+ lro_cb->pdu_cnt >= (MAX_SKB_FRAGS - 1)) {
+ lro_flush(lro_mgr, skb);
+ goto start_lro;
+ }
+
+ if (gl)
+ lro_skb_add_packet_gl(skb, op, gl);
+ else
+ lro_skb_add_packet_rsp(skb, op, rsp);
+ lro_mgr->lro_merged++;
+
+ return 0;
+
+out:
+ return -1;
+}
+
+static int t4_uld_rx_lro_handler(void *hndl, const __be64 *rsp,
+ const struct pkt_gl *gl,
+ struct t4_lro_mgr *lro_mgr,
+ unsigned int napi_id)
+{
+ struct cxgbi_device *cdev = hndl;
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct cpl_act_establish *rpl = NULL;
+ struct cxgbi_sock *csk = NULL;
+ unsigned int tid = 0;
+ struct sk_buff *skb;
+ unsigned int op = *(u8 *)rsp;
+
+ if (lro_mgr && (op != CPL_FW6_MSG) && (op != CPL_RX_PKT) &&
+ (op != CPL_ACT_OPEN_RPL)) {
+ /* Get the TID of this connection */
+ rpl = gl ? (struct cpl_act_establish *)gl->va :
+ (struct cpl_act_establish *)(rsp + 1);
+ tid = GET_TID(rpl);
+ csk = lookup_tid(lldi->tids, tid);
+ }
+
+ /*
+ * Flush the LROed skb on receiving any cpl other than FW4_ACK and
+ * CPL_ISCSI_XXX
+ */
+ if (csk && csk->skb_lro && op != CPL_FW6_MSG && op != CPL_ISCSI_HDR &&
+ op != CPL_ISCSI_DATA && op != CPL_RX_ISCSI_DDP) {
+ lro_flush(lro_mgr, csk->skb_lro);
+ }
+
+ if (!gl) {
+ unsigned int len;
+
+ if (op == CPL_RX_ISCSI_DDP) {
+ if (!lro_receive(csk, op, rsp, NULL, lro_mgr))
+ return 0;
+ }
+
+ len = 64 - sizeof(struct rsp_ctrl) - 8;
+ skb = alloc_wr(len, 0, GFP_ATOMIC);
+ if (!skb)
+ goto nomem;
+ skb_copy_to_linear_data(skb, &rsp[1], len);
+ } else {
+ if (unlikely(op != *(u8 *)gl->va)) {
+ pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n",
+ gl->va, be64_to_cpu(*rsp),
+ be64_to_cpu(*(u64 *)gl->va),
+ gl->tot_len);
+ return 0;
+ }
+
+ if (op == CPL_ISCSI_HDR || op == CPL_ISCSI_DATA) {
+ if (!lro_receive(csk, op, rsp, gl, lro_mgr))
+ return 0;
+ }
+
+ skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN);
+ if (unlikely(!skb))
+ goto nomem;
+ }
+
+ rpl = (struct cpl_act_establish *)skb->data;
+ op = rpl->ot.opcode;
+ log_debug(1 << CXGBI_DBG_TOE,
+ "cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n",
+ cdev, op, rpl->ot.opcode_tid, ntohl(rpl->ot.opcode_tid), skb);
+
+ if (op < NUM_CPL_CMDS && cxgb4i_cplhandlers[op]) {
+ cxgb4i_cplhandlers[op](cdev, skb);
+ } else {
+ pr_err("No handler for opcode 0x%x.\n", op);
+ __kfree_skb(skb);
+ }
+ return 0;
+nomem:
+ log_debug(1 << CXGBI_DBG_TOE, "OOM bailing out.\n");
+ return 1;
+}
+
+static void t4_uld_lro_flush_all(struct t4_lro_mgr *lro_mgr)
+{
+ struct sk_buff *skb = lro_mgr->skb_head;
+ struct sk_buff *temp;
+
+ while (skb) {
+ temp = skb->next;
+ lro_flush(lro_mgr, skb);
+ skb = temp;
+ }
+ lro_mgr->skb_head = NULL;
+ lro_mgr->skb_tail = NULL;
+}
+
+static const struct cxgb4_uld_info cxgb4i_uld_info = {
+ .name = DRV_MODULE_NAME,
+ .add = t4_uld_add,
+ .rx_handler = t4_uld_rx_handler,
+ .state_change = t4_uld_state_change,
+ .lro_rx_handler = t4_uld_rx_lro_handler,
+ .lro_flush = t4_uld_lro_flush_all,
+};
+
static int __init cxgb4i_init_module(void)
{
int rc;
--
2.3.4
^ permalink raw reply related [flat|nested] 6+ messages in thread