* [PATCH 1/4] RDMA/cxgb4: Fix race in fini path.
[not found] ` <20100723191221.15832.79704.stgit-T4OLL4TyM9aNDNWfRnPdfg@public.gmane.org>
@ 2010-07-23 19:12 ` Steve Wise
2010-07-23 19:12 ` [PATCH 2/4] RDMA/cxgb4: Use correct control txq Steve Wise
` (2 subsequent siblings)
3 siblings, 0 replies; 6+ messages in thread
From: Steve Wise @ 2010-07-23 19:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
There exists a race condition where the app disconnects, which initiates
an orderly close (via rdma_fini()) concurrently with an ingress abort
condition which initiates an abortive close operation. Since rdma_fini()
must be called without IRQs disabled, the fini can be called after the
QP has been transitioned to ERROR. This is ok, but we need to protect
against qp->ep getting NULLed.
Signed-off-by: Steve Wise <swise-7bPotxP6k4+P2YhJcF5u+vpXobYPEAuW@public.gmane.org>
---
drivers/infiniband/hw/cxgb4/qp.c | 14 +++++++-------
1 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c
index 7d87fe5..6d13f82 100644
--- a/drivers/infiniband/hw/cxgb4/qp.c
+++ b/drivers/infiniband/hw/cxgb4/qp.c
@@ -961,7 +961,8 @@ static void flush_qp(struct c4iw_qp *qhp, unsigned long *flag)
__flush_qp(qhp, rchp, schp, flag);
}
-static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp)
+static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
+ struct c4iw_ep *ep)
{
struct fw_ri_wr *wqe;
int ret;
@@ -969,12 +970,12 @@ static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp)
struct sk_buff *skb;
PDBG("%s qhp %p qid 0x%x tid %u\n", __func__, qhp, qhp->wq.sq.qid,
- qhp->ep->hwtid);
+ ep->hwtid);
skb = alloc_skb(sizeof *wqe, GFP_KERNEL | __GFP_NOFAIL);
if (!skb)
return -ENOMEM;
- set_wr_txq(skb, CPL_PRIORITY_DATA, qhp->ep->txq_idx);
+ set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
wqe = (struct fw_ri_wr *)__skb_put(skb, sizeof(*wqe));
memset(wqe, 0, sizeof *wqe);
@@ -982,7 +983,7 @@ static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp)
FW_WR_OP(FW_RI_INIT_WR) |
FW_WR_COMPL(1));
wqe->flowid_len16 = cpu_to_be32(
- FW_WR_FLOWID(qhp->ep->hwtid) |
+ FW_WR_FLOWID(ep->hwtid) |
FW_WR_LEN16(DIV_ROUND_UP(sizeof *wqe, 16)));
wqe->cookie = (u64)&wr_wait;
@@ -1212,17 +1213,16 @@ int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
case C4IW_QP_STATE_CLOSING:
BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2);
qhp->attr.state = C4IW_QP_STATE_CLOSING;
+ ep = qhp->ep;
if (!internal) {
abort = 0;
disconnect = 1;
- ep = qhp->ep;
c4iw_get_ep(&ep->com);
}
spin_unlock_irqrestore(&qhp->lock, flag);
- ret = rdma_fini(rhp, qhp);
+ ret = rdma_fini(rhp, qhp, ep);
spin_lock_irqsave(&qhp->lock, flag);
if (ret) {
- ep = qhp->ep;
c4iw_get_ep(&ep->com);
disconnect = abort = 1;
goto err;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 2/4] RDMA/cxgb4: Use correct control txq.
[not found] ` <20100723191221.15832.79704.stgit-T4OLL4TyM9aNDNWfRnPdfg@public.gmane.org>
2010-07-23 19:12 ` [PATCH 1/4] RDMA/cxgb4: Fix race in fini path Steve Wise
@ 2010-07-23 19:12 ` Steve Wise
2010-07-23 19:12 ` [PATCH 3/4] RDMA/cxgb4: Set/Reset the EP timer inside EP lock Steve Wise
2010-07-23 19:12 ` [PATCH 4/4] RDMA/cxgb4: Add timeouts when waiting for FW responses Steve Wise
3 siblings, 0 replies; 6+ messages in thread
From: Steve Wise @ 2010-07-23 19:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
There is only one control txq per tx channel. So use the port number
as the queue index when sending.
Signed-off-by: Steve Wise <swise-7bPotxP6k4+P2YhJcF5u+vpXobYPEAuW@public.gmane.org>
---
drivers/infiniband/hw/cxgb4/cm.c | 13 +++++++++----
drivers/infiniband/hw/cxgb4/iw_cxgb4.h | 1 +
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index 8d1f09b..6e480cd 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -473,7 +473,7 @@ static int send_connect(struct c4iw_ep *ep)
__func__);
return -ENOMEM;
}
- set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->txq_idx);
+ set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx);
cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx);
wscale = compute_wscale(rcv_win);
@@ -853,7 +853,7 @@ static int update_rx_credits(struct c4iw_ep *ep, u32 credits)
req->credit_dack = cpu_to_be32(credits | RX_FORCE_ACK(1) |
F_RX_DACK_CHANGE |
V_RX_DACK_MODE(dack_mode));
- set_wr_txq(skb, CPL_PRIORITY_ACK, ep->txq_idx);
+ set_wr_txq(skb, CPL_PRIORITY_ACK, ep->ctrlq_idx);
c4iw_ofld_send(&ep->com.dev->rdev, skb);
return credits;
}
@@ -1295,7 +1295,7 @@ static void accept_cr(struct c4iw_ep *ep, __be32 peer_ip, struct sk_buff *skb,
ep->hwtid));
rpl->opt0 = cpu_to_be64(opt0);
rpl->opt2 = cpu_to_be32(opt2);
- set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->txq_idx);
+ set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx);
c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
return;
@@ -1352,7 +1352,7 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
u16 rss_qid;
u32 mtu;
int step;
- int txq_idx;
+ int txq_idx, ctrlq_idx;
parent_ep = lookup_stid(t, stid);
PDBG("%s parent ep %p tid %u\n", __func__, parent_ep, hwtid);
@@ -1384,6 +1384,7 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1;
step = dev->rdev.lldi.ntxq / dev->rdev.lldi.nchan;
txq_idx = cxgb4_port_idx(pdev) * step;
+ ctrlq_idx = cxgb4_port_idx(pdev);
step = dev->rdev.lldi.nrxq / dev->rdev.lldi.nchan;
rss_qid = dev->rdev.lldi.rxq_ids[cxgb4_port_idx(pdev) * step];
dev_put(pdev);
@@ -1395,6 +1396,7 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
smac_idx = (cxgb4_port_viid(dst->neighbour->dev) & 0x7F) << 1;
step = dev->rdev.lldi.ntxq / dev->rdev.lldi.nchan;
txq_idx = cxgb4_port_idx(dst->neighbour->dev) * step;
+ ctrlq_idx = cxgb4_port_idx(dst->neighbour->dev);
step = dev->rdev.lldi.nrxq / dev->rdev.lldi.nchan;
rss_qid = dev->rdev.lldi.rxq_ids[
cxgb4_port_idx(dst->neighbour->dev) * step];
@@ -1434,6 +1436,7 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
child_ep->rss_qid = rss_qid;
child_ep->mtu = mtu;
child_ep->txq_idx = txq_idx;
+ child_ep->ctrlq_idx = ctrlq_idx;
PDBG("%s tx_chan %u smac_idx %u rss_qid %u\n", __func__,
tx_chan, smac_idx, rss_qid);
@@ -1965,6 +1968,7 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
ep->txq_idx = cxgb4_port_idx(pdev) * step;
step = ep->com.dev->rdev.lldi.nrxq /
ep->com.dev->rdev.lldi.nchan;
+ ep->ctrlq_idx = cxgb4_port_idx(pdev);
ep->rss_qid = ep->com.dev->rdev.lldi.rxq_ids[
cxgb4_port_idx(pdev) * step];
dev_put(pdev);
@@ -1979,6 +1983,7 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
step = ep->com.dev->rdev.lldi.ntxq /
ep->com.dev->rdev.lldi.nchan;
ep->txq_idx = cxgb4_port_idx(ep->dst->neighbour->dev) * step;
+ ep->ctrlq_idx = cxgb4_port_idx(ep->dst->neighbour->dev);
step = ep->com.dev->rdev.lldi.nrxq /
ep->com.dev->rdev.lldi.nchan;
ep->rss_qid = ep->com.dev->rdev.lldi.rxq_ids[
diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
index 277ab58..bda0af7 100644
--- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
+++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
@@ -619,6 +619,7 @@ struct c4iw_ep {
u16 plen;
u16 rss_qid;
u16 txq_idx;
+ u16 ctrlq_idx;
u8 tos;
};
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 3/4] RDMA/cxgb4: Set/Reset the EP timer inside EP lock.
[not found] ` <20100723191221.15832.79704.stgit-T4OLL4TyM9aNDNWfRnPdfg@public.gmane.org>
2010-07-23 19:12 ` [PATCH 1/4] RDMA/cxgb4: Fix race in fini path Steve Wise
2010-07-23 19:12 ` [PATCH 2/4] RDMA/cxgb4: Use correct control txq Steve Wise
@ 2010-07-23 19:12 ` Steve Wise
2010-07-23 19:12 ` [PATCH 4/4] RDMA/cxgb4: Add timeouts when waiting for FW responses Steve Wise
3 siblings, 0 replies; 6+ messages in thread
From: Steve Wise @ 2010-07-23 19:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Endpoint timer manipulation needs to be done inside the lock. Otherwise
we can get into a situation where a timer is stopped before it is started,
which hits the WARN_ON() in stop_ep_timer().
Signed-off-by: Steve Wise <swise-7bPotxP6k4+P2YhJcF5u+vpXobYPEAuW@public.gmane.org>
---
drivers/infiniband/hw/cxgb4/cm.c | 34 ++++++++--------------------------
1 files changed, 8 insertions(+), 26 deletions(-)
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index 6e480cd..bc3f9f0 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -1484,8 +1484,6 @@ static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb)
int closing = 0;
struct tid_info *t = dev->rdev.lldi.tids;
unsigned int tid = GET_TID(hdr);
- int start_timer = 0;
- int stop_timer = 0;
ep = lookup_tid(t, tid);
PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
@@ -1522,7 +1520,7 @@ static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb)
wake_up(&ep->com.waitq);
break;
case FPDU_MODE:
- start_timer = 1;
+ start_ep_timer(ep);
__state_set(&ep->com, CLOSING);
closing = 1;
peer_close_upcall(ep);
@@ -1535,7 +1533,7 @@ static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb)
disconnect = 0;
break;
case MORIBUND:
- stop_timer = 1;
+ stop_ep_timer(ep);
if (ep->com.cm_id && ep->com.qp) {
attrs.next_state = C4IW_QP_STATE_IDLE;
c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp,
@@ -1558,10 +1556,6 @@ static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb)
c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp,
C4IW_QP_ATTR_NEXT_STATE, &attrs, 1);
}
- if (start_timer)
- start_ep_timer(ep);
- if (stop_timer)
- stop_ep_timer(ep);
if (disconnect)
c4iw_ep_disconnect(ep, 0, GFP_KERNEL);
if (release)
@@ -1590,7 +1584,6 @@ static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb)
unsigned long flags;
struct tid_info *t = dev->rdev.lldi.tids;
unsigned int tid = GET_TID(req);
- int stop_timer = 0;
ep = lookup_tid(t, tid);
if (is_neg_adv_abort(req->status)) {
@@ -1605,10 +1598,10 @@ static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb)
case CONNECTING:
break;
case MPA_REQ_WAIT:
- stop_timer = 1;
+ stop_ep_timer(ep);
break;
case MPA_REQ_SENT:
- stop_timer = 1;
+ stop_ep_timer(ep);
connect_reply_upcall(ep, -ECONNRESET);
break;
case MPA_REP_SENT:
@@ -1632,7 +1625,7 @@ static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb)
break;
case MORIBUND:
case CLOSING:
- stop_timer = 1;
+ stop_ep_timer(ep);
/*FALLTHROUGH*/
case FPDU_MODE:
if (ep->com.cm_id && ep->com.qp) {
@@ -1678,8 +1671,6 @@ static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb)
rpl->cmd = CPL_ABORT_NO_RST;
c4iw_ofld_send(&ep->com.dev->rdev, rpl_skb);
out:
- if (stop_timer)
- stop_ep_timer(ep);
if (release)
release_ep_resources(ep);
return 0;
@@ -1694,7 +1685,6 @@ static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
int release = 0;
struct tid_info *t = dev->rdev.lldi.tids;
unsigned int tid = GET_TID(rpl);
- int stop_timer = 0;
ep = lookup_tid(t, tid);
@@ -1708,7 +1698,7 @@ static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
__state_set(&ep->com, MORIBUND);
break;
case MORIBUND:
- stop_timer = 1;
+ stop_ep_timer(ep);
if ((ep->com.cm_id) && (ep->com.qp)) {
attrs.next_state = C4IW_QP_STATE_IDLE;
c4iw_modify_qp(ep->com.qp->rhp,
@@ -1728,8 +1718,6 @@ static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
break;
}
spin_unlock_irqrestore(&ep->com.lock, flags);
- if (stop_timer)
- stop_ep_timer(ep);
if (release)
release_ep_resources(ep);
return 0;
@@ -2108,8 +2096,6 @@ int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp)
int close = 0;
int fatal = 0;
struct c4iw_rdev *rdev;
- int start_timer = 0;
- int stop_timer = 0;
spin_lock_irqsave(&ep->com.lock, flags);
@@ -2133,7 +2119,7 @@ int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp)
ep->com.state = ABORTING;
else {
ep->com.state = CLOSING;
- start_timer = 1;
+ start_ep_timer(ep);
}
set_bit(CLOSE_SENT, &ep->com.flags);
break;
@@ -2141,7 +2127,7 @@ int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp)
if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) {
close = 1;
if (abrupt) {
- stop_timer = 1;
+ stop_ep_timer(ep);
ep->com.state = ABORTING;
} else
ep->com.state = MORIBUND;
@@ -2159,10 +2145,6 @@ int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp)
}
spin_unlock_irqrestore(&ep->com.lock, flags);
- if (start_timer)
- start_ep_timer(ep);
- if (stop_timer)
- stop_ep_timer(ep);
if (close) {
if (abrupt)
ret = abort_connection(ep, NULL, gfp);
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 4/4] RDMA/cxgb4: Add timeouts when waiting for FW responses.
[not found] ` <20100723191221.15832.79704.stgit-T4OLL4TyM9aNDNWfRnPdfg@public.gmane.org>
` (2 preceding siblings ...)
2010-07-23 19:12 ` [PATCH 3/4] RDMA/cxgb4: Set/Reset the EP timer inside EP lock Steve Wise
@ 2010-07-23 19:12 ` Steve Wise
[not found] ` <20100723191242.15832.77852.stgit-T4OLL4TyM9aNDNWfRnPdfg@public.gmane.org>
3 siblings, 1 reply; 6+ messages in thread
From: Steve Wise @ 2010-07-23 19:12 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
Don't hang a host thread if the FW stops responding.
Signed-off-by: Steve Wise <swise-7bPotxP6k4+P2YhJcF5u+vpXobYPEAuW@public.gmane.org>
---
drivers/infiniband/hw/cxgb4/cm.c | 22 ++++++++++++++++++----
1 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index bc3f9f0..296aa2c 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -2050,8 +2050,15 @@ int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog)
goto fail3;
/* wait for pass_open_rpl */
- wait_event(ep->com.waitq, ep->com.rpl_done);
- err = ep->com.rpl_err;
+ wait_event_timeout(ep->com.waitq, ep->com.rpl_done, C4IW_WR_TO);
+ if (ep->com.rpl_done)
+ err = ep->com.rpl_err;
+ else {
+ printk(KERN_ERR MOD "Device %s not responding!\n",
+ pci_name(ep->com.dev->rdev.lldi.pdev));
+ ep->com.dev->rdev.flags = T4_FATAL_ERROR;
+ err = -EIO;
+ }
if (!err) {
cm_id->provider_data = ep;
goto out;
@@ -2080,10 +2087,17 @@ int c4iw_destroy_listen(struct iw_cm_id *cm_id)
err = listen_stop(ep);
if (err)
goto done;
- wait_event(ep->com.waitq, ep->com.rpl_done);
+ wait_event_timeout(ep->com.waitq, ep->com.rpl_done, C4IW_WR_TO);
+ if (ep->com.rpl_done)
+ err = ep->com.rpl_err;
+ else {
+ printk(KERN_ERR MOD "Device %s not responding!\n",
+ pci_name(ep->com.dev->rdev.lldi.pdev));
+ ep->com.dev->rdev.flags = T4_FATAL_ERROR;
+ err = -EIO;
+ }
cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET);
done:
- err = ep->com.rpl_err;
cm_id->rem_ref(cm_id);
c4iw_put_ep(&ep->com);
return err;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related [flat|nested] 6+ messages in thread