From: Bart Van Assche <bvanassche-HInyCGIudOg@public.gmane.org>
To: "linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org"
<linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org>
Cc: David Dillow <dillowda-1Heg1YXhbW8@public.gmane.org>,
Roland Dreier <roland-BHEL68pLQRGGvPXPguhicg@public.gmane.org>
Subject: [PATCH 09/19] ib_srp: Make srp_disconnect_target() wait for IB completions
Date: Fri, 26 Oct 2012 14:50:53 +0200 [thread overview]
Message-ID: <508A872D.1010600@acm.org> (raw)
In-Reply-To: <508A85BB.1000505-HInyCGIudOg@public.gmane.org>
Modify srp_disconnect_target() such that it waits until it is
sure that no new IB completions will be received anymore.
Cc: David Dillow <dillowda-1Heg1YXhbW8@public.gmane.org>
Cc: Roland Dreier <roland-BHEL68pLQRGGvPXPguhicg@public.gmane.org>
Signed-off-by: Bart Van Assche <bvanassche-HInyCGIudOg@public.gmane.org>
---
drivers/infiniband/ulp/srp/ib_srp.c | 104 ++++++++++++++++++++++++++++++-----
drivers/infiniband/ulp/srp/ib_srp.h | 6 ++
2 files changed, 95 insertions(+), 15 deletions(-)
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 8ae2070..58d19a2 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -40,7 +40,7 @@
#include <linux/parser.h>
#include <linux/random.h>
#include <linux/jiffies.h>
-
+#include <linux/delay.h>
#include <linux/atomic.h>
#include <scsi/scsi.h>
@@ -229,14 +229,16 @@ static int srp_create_target_ib(struct srp_target_port *target)
return -ENOMEM;
target->recv_cq = ib_create_cq(target->srp_host->srp_dev->dev,
- srp_recv_completion, NULL, target, SRP_RQ_SIZE, 0);
+ srp_recv_completion, NULL, target,
+ SRP_RQ_SIZE + 1, 0);
if (IS_ERR(target->recv_cq)) {
ret = PTR_ERR(target->recv_cq);
goto err;
}
target->send_cq = ib_create_cq(target->srp_host->srp_dev->dev,
- srp_send_completion, NULL, target, SRP_SQ_SIZE, 0);
+ srp_send_completion, NULL, target,
+ SRP_SQ_SIZE + 1, 0);
if (IS_ERR(target->send_cq)) {
ret = PTR_ERR(target->send_cq);
goto err_recv_cq;
@@ -245,8 +247,8 @@ static int srp_create_target_ib(struct srp_target_port *target)
ib_req_notify_cq(target->recv_cq, IB_CQ_NEXT_COMP);
init_attr->event_handler = srp_qp_event;
- init_attr->cap.max_send_wr = SRP_SQ_SIZE;
- init_attr->cap.max_recv_wr = SRP_RQ_SIZE;
+ init_attr->cap.max_send_wr = SRP_SQ_SIZE + 1;
+ init_attr->cap.max_recv_wr = SRP_RQ_SIZE + 1;
init_attr->cap.max_recv_sge = 1;
init_attr->cap.max_send_sge = 1;
init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
@@ -460,11 +462,69 @@ static bool srp_change_conn_state(struct srp_target_port *target,
return changed;
}
+static void srp_wait_last_recv_wqe(struct srp_target_port *target)
+{
+ static struct ib_recv_wr wr = {
+ .wr_id = SRP_LAST_RECV,
+ };
+ struct ib_recv_wr *bad_wr;
+ int ret;
+
+ if (target->last_recv_wqe)
+ return;
+
+ ret = ib_post_recv(target->qp, &wr, &bad_wr);
+ if (ret < 0) {
+ shost_printk(KERN_ERR, target->scsi_host,
+ "ib_post_recv() failed (%d)\n", ret);
+ return;
+ }
+
+ ret = wait_event_timeout(target->qp_wq, target->last_recv_wqe,
+ target->rq_tmo_jiffies);
+ WARN(ret <= 0, "Timeout while waiting for last recv WQE (ret = %d)\n",
+ ret);
+}
+
+static void srp_wait_last_send_wqe(struct srp_target_port *target)
+{
+ static struct ib_send_wr wr = {
+ .wr_id = SRP_LAST_SEND,
+ };
+ struct ib_send_wr *bad_wr;
+ unsigned long deadline = jiffies + target->rq_tmo_jiffies;
+ int ret;
+
+ if (target->last_send_wqe)
+ return;
+
+ ret = ib_post_send(target->qp, &wr, &bad_wr);
+ if (ret < 0) {
+ shost_printk(KERN_ERR, target->scsi_host,
+ "ib_post_send() failed (%d)\n", ret);
+ return;
+ }
+
+ while (!target->last_send_wqe && time_before(jiffies, deadline)) {
+ srp_send_completion(target->send_cq, target);
+ msleep(20);
+ }
+
+ WARN_ON(!target->last_send_wqe);
+}
+
static void srp_disconnect_target(struct srp_target_port *target)
{
+ static struct ib_qp_attr qp_attr = {
+ .qp_state = IB_QPS_ERR
+ };
+ int ret;
+
if (srp_change_conn_state(target, false)) {
/* XXX should send SRP_I_LOGOUT request */
+ BUG_ON(!target->cm_id);
+
init_completion(&target->done);
if (ib_send_cm_dreq(target->cm_id, NULL, 0)) {
shost_printk(KERN_DEBUG, target->scsi_host,
@@ -473,6 +533,20 @@ static void srp_disconnect_target(struct srp_target_port *target)
wait_for_completion(&target->done);
}
}
+
+ if (target->cm_id) {
+ ib_destroy_cm_id(target->cm_id);
+ target->cm_id = NULL;
+ }
+
+ if (target->qp) {
+ ret = ib_modify_qp(target->qp, &qp_attr, IB_QP_STATE);
+ WARN(ret != 0, "ib_modify_qp() failed: %d\n", ret);
+
+ srp_wait_last_recv_wqe(target);
+
+ srp_wait_last_send_wqe(target);
+ }
}
static void srp_free_req_data(struct srp_target_port *target)
@@ -516,7 +590,6 @@ static void srp_remove_target(struct srp_target_port *target)
srp_remove_host(target->scsi_host);
scsi_remove_host(target->scsi_host);
srp_disconnect_target(target);
- ib_destroy_cm_id(target->cm_id);
srp_free_target_ib(target);
srp_free_req_data(target);
scsi_host_put(target->scsi_host);
@@ -544,6 +617,8 @@ static int srp_connect_target(struct srp_target_port *target)
WARN_ON(target->connected);
target->qp_in_error = false;
+ target->last_recv_wqe = false;
+ target->last_send_wqe = false;
ret = srp_lookup_path(target);
if (ret)
@@ -679,7 +754,6 @@ static int srp_reconnect_target(struct srp_target_port *target)
{
struct Scsi_Host *shost = target->scsi_host;
struct ib_qp_attr qp_attr;
- struct ib_wc wc;
int i, ret;
if (target->state != SRP_TARGET_LIVE)
@@ -705,11 +779,6 @@ static int srp_reconnect_target(struct srp_target_port *target)
if (ret)
goto err;
- while (ib_poll_cq(target->recv_cq, 1, &wc) > 0)
- ; /* nothing */
- while (ib_poll_cq(target->send_cq, 1, &wc) > 0)
- ; /* nothing */
-
for (i = 0; i < SRP_CMD_SQ_SIZE; ++i) {
struct srp_request *req = &target->req_ring[i];
if (req->scmnd)
@@ -1293,7 +1362,7 @@ static void srp_handle_qp_err(enum ib_wc_status wc_status,
enum ib_wc_opcode wc_opcode,
struct srp_target_port *target)
{
- if (target->connected)
+ if (target->connected && !target->qp_in_error)
shost_printk(KERN_ERR, target->scsi_host,
PFX "failed %s status %d\n",
wc_opcode & IB_WC_RECV ? "receive" : "send",
@@ -1311,8 +1380,11 @@ static void srp_recv_completion(struct ib_cq *cq, void *target_ptr)
if (likely(wc.status == IB_WC_SUCCESS)) {
srp_handle_recv(target, &wc);
} else {
+ if (wc.wr_id == SRP_LAST_RECV) {
+ target->last_recv_wqe = true;
+ wake_up(&target->qp_wq);
+ }
srp_handle_qp_err(wc.status, wc.opcode, target);
- break;
}
}
}
@@ -1328,8 +1400,9 @@ static void srp_send_completion(struct ib_cq *cq, void *target_ptr)
iu = (struct srp_iu *) (uintptr_t) wc.wr_id;
list_add(&iu->list, &target->free_tx);
} else {
+ if (wc.wr_id == SRP_LAST_SEND)
+ target->last_send_wqe = true;
srp_handle_qp_err(wc.status, wc.opcode, target);
- break;
}
}
}
@@ -2260,6 +2333,7 @@ static ssize_t srp_create_target(struct device *dev,
spin_lock_init(&target->lock);
INIT_LIST_HEAD(&target->free_tx);
INIT_LIST_HEAD(&target->free_reqs);
+ init_waitqueue_head(&target->qp_wq);
for (i = 0; i < SRP_CMD_SQ_SIZE; ++i) {
struct srp_request *req = &target->req_ring[i];
diff --git a/drivers/infiniband/ulp/srp/ib_srp.h b/drivers/infiniband/ulp/srp/ib_srp.h
index de2d0b3..1b11117 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.h
+++ b/drivers/infiniband/ulp/srp/ib_srp.h
@@ -76,6 +76,9 @@ enum {
SRP_MAP_ALLOW_FMR = 0,
SRP_MAP_NO_FMR = 1,
+
+ SRP_LAST_RECV = 0,
+ SRP_LAST_SEND = 0,
};
enum srp_target_state {
@@ -180,6 +183,9 @@ struct srp_target_port {
struct completion done;
int status;
bool qp_in_error;
+ bool last_recv_wqe;
+ bool last_send_wqe;
+ wait_queue_head_t qp_wq;
struct completion tsk_mgmt_done;
u8 tsk_mgmt_status;
--
1.7.10.4
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
next prev parent reply other threads:[~2012-10-26 12:50 UTC|newest]
Thread overview: 34+ messages / expand[flat|nested] mbox.gz Atom feed top
2012-10-26 12:44 [PATCH 00/19, v5] Make ib_srp better suited for H.A. purposes Bart Van Assche
2012-10-26 12:52 ` [PATCH 11/19] srp_transport: Fix attribute registration Bart Van Assche
2012-10-26 12:54 ` [PATCH 13/19] srp_transport: Document sysfs attributes Bart Van Assche
2012-10-26 12:55 ` [PATCH 14/19] ib_srp: Allow SRP disconnect through sysfs Bart Van Assche
[not found] ` <508A85BB.1000505-HInyCGIudOg@public.gmane.org>
2012-10-26 12:45 ` [PATCH 01/19] ib_srp: Enlarge block layer timeout Bart Van Assche
2012-10-26 12:46 ` [PATCH 02/19] ib_srp: Eliminate state SRP_TARGET_CONNECTING Bart Van Assche
2012-10-26 12:46 ` [PATCH 03/19] ib_srp: Introduce srp_handle_qp_err() Bart Van Assche
2012-10-26 12:47 ` [PATCH 04/19] ib_srp: Suppress superfluous error messages Bart Van Assche
2012-10-26 12:48 ` [PATCH 05/19] ib_srp: Avoid that SCSI error handling causes trouble Bart Van Assche
2012-10-26 12:49 ` [PATCH 06/19] ib_srp: Introduce the helper function srp_remove_target() Bart Van Assche
2012-10-26 12:49 ` [PATCH 07/19] ib_srp: Eliminate state SRP_TARGET_DEAD Bart Van Assche
2012-10-26 12:50 ` [PATCH 08/19] ib_srp: Keep processing commands during host removal Bart Van Assche
2012-10-26 12:50 ` Bart Van Assche [this message]
2012-10-26 12:51 ` [PATCH 10/19] ib_srp: Document sysfs attributes Bart Van Assche
2012-10-26 12:53 ` [PATCH 12/19] srp_transport: Simplify attribute initialization code Bart Van Assche
2012-10-26 12:55 ` [PATCH 15/19] ib_srp: Maintain a single connection per I_T nexus Bart Van Assche
2012-10-26 12:56 ` [PATCH 16/19] srp_transport: Add transport layer error handling Bart Van Assche
2012-10-26 12:57 ` [PATCH 17/19] ib_srp: Add dev_loss_tmo support Bart Van Assche
2012-10-26 12:58 ` [PATCH 18/19] ib_srp: Remove SCSI devices upon port down event Bart Van Assche
[not found] ` <508A88D8.2050905-HInyCGIudOg@public.gmane.org>
2012-11-12 22:40 ` Or Gerlitz
[not found] ` <CAJZOPZL8mKU2MsrPPACvWjiA59aGnWDj0HNTQQNhbDrMsE0+Tg-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2012-11-13 8:59 ` Bart Van Assche
[not found] ` <50A20C03.9040607-HInyCGIudOg@public.gmane.org>
2012-11-13 20:54 ` Or Gerlitz
[not found] ` <CAJZOPZ+PiDQ6GYLkDO4MaPTDxLr2XDMn8q3gTaX-COx04PSegg-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2012-11-13 21:20 ` Bart Van Assche
[not found] ` <50A2B989.8000600-HInyCGIudOg@public.gmane.org>
2012-11-13 21:23 ` Or Gerlitz
[not found] ` <CAJZOPZLSPz7f99tj2w-79sPbibrHP3WZY_ct0Cq07Q1so54kFQ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2012-11-13 21:35 ` Bart Van Assche
2012-10-26 12:58 ` [PATCH 19/19] scsi_transport_srp: Fail I/O faster Bart Van Assche
2012-11-12 22:36 ` [PATCH 00/19, v5] Make ib_srp better suited for H.A. purposes Or Gerlitz
[not found] ` <CAJZOPZJPQkJ-kkW3ro9sRJXQJg_Yz_tjoJ1Rwb=XEePO3j_iJw-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2012-11-13 8:41 ` Bart Van Assche
[not found] ` <50A207D5.6060207-HInyCGIudOg@public.gmane.org>
2012-11-13 21:04 ` Or Gerlitz
[not found] ` <CAJZOPZJXdLRH9NPCt0snGNP8LKODO+phtV7uts6Vj-gxEEjpsw-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2012-11-13 21:30 ` Bart Van Assche
[not found] ` <50A2BC01.40609-HInyCGIudOg@public.gmane.org>
2012-11-13 21:41 ` Or Gerlitz
[not found] ` <CAJZOPZLQ8B9UGvGdM5LvA6r+XDARO5BXGoMmtdSH6+8EMyMaXw-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2012-11-13 22:35 ` Bart Van Assche
2012-11-12 22:51 ` Or Gerlitz
[not found] ` <CAJZOPZLHg84M3RUV00itGSGUZsigW0yw=TLOe6K63mUXH5v1pQ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2012-11-13 8:34 ` Bart Van Assche
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=508A872D.1010600@acm.org \
--to=bvanassche-hinycgiudog@public.gmane.org \
--cc=dillowda-1Heg1YXhbW8@public.gmane.org \
--cc=linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org \
--cc=roland-BHEL68pLQRGGvPXPguhicg@public.gmane.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).