From: Bart Van Assche <bvanassche-HInyCGIudOg@public.gmane.org>
To: "linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org"
<linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org>
Cc: David Dillow <dillowda-1Heg1YXhbW8@public.gmane.org>,
Roland Dreier <roland-BHEL68pLQRGGvPXPguhicg@public.gmane.org>
Subject: [PATCH 09/19] ib_srp: Make srp_disconnect_target() wait for IB completions
Date: Fri, 26 Oct 2012 14:50:53 +0200 [thread overview]
Message-ID: <508A872D.1010600@acm.org> (raw)
In-Reply-To: <508A85BB.1000505-HInyCGIudOg@public.gmane.org>
Modify srp_disconnect_target() such that it waits until it is
sure that no new IB completions will be received anymore.
Cc: David Dillow <dillowda-1Heg1YXhbW8@public.gmane.org>
Cc: Roland Dreier <roland-BHEL68pLQRGGvPXPguhicg@public.gmane.org>
Signed-off-by: Bart Van Assche <bvanassche-HInyCGIudOg@public.gmane.org>
---
drivers/infiniband/ulp/srp/ib_srp.c | 104 ++++++++++++++++++++++++++++++-----
drivers/infiniband/ulp/srp/ib_srp.h | 6 ++
2 files changed, 95 insertions(+), 15 deletions(-)
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 8ae2070..58d19a2 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -40,7 +40,7 @@
#include <linux/parser.h>
#include <linux/random.h>
#include <linux/jiffies.h>
-
+#include <linux/delay.h>
#include <linux/atomic.h>
#include <scsi/scsi.h>
@@ -229,14 +229,16 @@ static int srp_create_target_ib(struct srp_target_port *target)
return -ENOMEM;
target->recv_cq = ib_create_cq(target->srp_host->srp_dev->dev,
- srp_recv_completion, NULL, target, SRP_RQ_SIZE, 0);
+ srp_recv_completion, NULL, target,
+ SRP_RQ_SIZE + 1, 0);
if (IS_ERR(target->recv_cq)) {
ret = PTR_ERR(target->recv_cq);
goto err;
}
target->send_cq = ib_create_cq(target->srp_host->srp_dev->dev,
- srp_send_completion, NULL, target, SRP_SQ_SIZE, 0);
+ srp_send_completion, NULL, target,
+ SRP_SQ_SIZE + 1, 0);
if (IS_ERR(target->send_cq)) {
ret = PTR_ERR(target->send_cq);
goto err_recv_cq;
@@ -245,8 +247,8 @@ static int srp_create_target_ib(struct srp_target_port *target)
ib_req_notify_cq(target->recv_cq, IB_CQ_NEXT_COMP);
init_attr->event_handler = srp_qp_event;
- init_attr->cap.max_send_wr = SRP_SQ_SIZE;
- init_attr->cap.max_recv_wr = SRP_RQ_SIZE;
+ init_attr->cap.max_send_wr = SRP_SQ_SIZE + 1;
+ init_attr->cap.max_recv_wr = SRP_RQ_SIZE + 1;
init_attr->cap.max_recv_sge = 1;
init_attr->cap.max_send_sge = 1;
init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
@@ -460,11 +462,69 @@ static bool srp_change_conn_state(struct srp_target_port *target,
return changed;
}
+static void srp_wait_last_recv_wqe(struct srp_target_port *target)
+{
+ static struct ib_recv_wr wr = {
+ .wr_id = SRP_LAST_RECV,
+ };
+ struct ib_recv_wr *bad_wr;
+ int ret;
+
+ if (target->last_recv_wqe)
+ return;
+
+ ret = ib_post_recv(target->qp, &wr, &bad_wr);
+ if (ret < 0) {
+ shost_printk(KERN_ERR, target->scsi_host,
+ "ib_post_recv() failed (%d)\n", ret);
+ return;
+ }
+
+ ret = wait_event_timeout(target->qp_wq, target->last_recv_wqe,
+ target->rq_tmo_jiffies);
+ WARN(ret <= 0, "Timeout while waiting for last recv WQE (ret = %d)\n",
+ ret);
+}
+
+static void srp_wait_last_send_wqe(struct srp_target_port *target)
+{
+ static struct ib_send_wr wr = {
+ .wr_id = SRP_LAST_SEND,
+ };
+ struct ib_send_wr *bad_wr;
+ unsigned long deadline = jiffies + target->rq_tmo_jiffies;
+ int ret;
+
+ if (target->last_send_wqe)
+ return;
+
+ ret = ib_post_send(target->qp, &wr, &bad_wr);
+ if (ret < 0) {
+ shost_printk(KERN_ERR, target->scsi_host,
+ "ib_post_send() failed (%d)\n", ret);
+ return;
+ }
+
+ while (!target->last_send_wqe && time_before(jiffies, deadline)) {
+ srp_send_completion(target->send_cq, target);
+ msleep(20);
+ }
+
+ WARN_ON(!target->last_send_wqe);
+}
+
static void srp_disconnect_target(struct srp_target_port *target)
{
+ static struct ib_qp_attr qp_attr = {
+ .qp_state = IB_QPS_ERR
+ };
+ int ret;
+
if (srp_change_conn_state(target, false)) {
/* XXX should send SRP_I_LOGOUT request */
+ BUG_ON(!target->cm_id);
+
init_completion(&target->done);
if (ib_send_cm_dreq(target->cm_id, NULL, 0)) {
shost_printk(KERN_DEBUG, target->scsi_host,
@@ -473,6 +533,20 @@ static void srp_disconnect_target(struct srp_target_port *target)
wait_for_completion(&target->done);
}
}
+
+ if (target->cm_id) {
+ ib_destroy_cm_id(target->cm_id);
+ target->cm_id = NULL;
+ }
+
+ if (target->qp) {
+ ret = ib_modify_qp(target->qp, &qp_attr, IB_QP_STATE);
+ WARN(ret != 0, "ib_modify_qp() failed: %d\n", ret);
+
+ srp_wait_last_recv_wqe(target);
+
+ srp_wait_last_send_wqe(target);
+ }
}
static void srp_free_req_data(struct srp_target_port *target)
@@ -516,7 +590,6 @@ static void srp_remove_target(struct srp_target_port *target)
srp_remove_host(target->scsi_host);
scsi_remove_host(target->scsi_host);
srp_disconnect_target(target);
- ib_destroy_cm_id(target->cm_id);
srp_free_target_ib(target);
srp_free_req_data(target);
scsi_host_put(target->scsi_host);
@@ -544,6 +617,8 @@ static int srp_connect_target(struct srp_target_port *target)
WARN_ON(target->connected);
target->qp_in_error = false;
+ target->last_recv_wqe = false;
+ target->last_send_wqe = false;
ret = srp_lookup_path(target);
if (ret)
@@ -679,7 +754,6 @@ static int srp_reconnect_target(struct srp_target_port *target)
{
struct Scsi_Host *shost = target->scsi_host;
struct ib_qp_attr qp_attr;
- struct ib_wc wc;
int i, ret;
if (target->state != SRP_TARGET_LIVE)
@@ -705,11 +779,6 @@ static int srp_reconnect_target(struct srp_target_port *target)
if (ret)
goto err;
- while (ib_poll_cq(target->recv_cq, 1, &wc) > 0)
- ; /* nothing */
- while (ib_poll_cq(target->send_cq, 1, &wc) > 0)
- ; /* nothing */
-
for (i = 0; i < SRP_CMD_SQ_SIZE; ++i) {
struct srp_request *req = &target->req_ring[i];
if (req->scmnd)
@@ -1293,7 +1362,7 @@ static void srp_handle_qp_err(enum ib_wc_status wc_status,
enum ib_wc_opcode wc_opcode,
struct srp_target_port *target)
{
- if (target->connected)
+ if (target->connected && !target->qp_in_error)
shost_printk(KERN_ERR, target->scsi_host,
PFX "failed %s status %d\n",
wc_opcode & IB_WC_RECV ? "receive" : "send",
@@ -1311,8 +1380,11 @@ static void srp_recv_completion(struct ib_cq *cq, void *target_ptr)
if (likely(wc.status == IB_WC_SUCCESS)) {
srp_handle_recv(target, &wc);
} else {
+ if (wc.wr_id == SRP_LAST_RECV) {
+ target->last_recv_wqe = true;
+ wake_up(&target->qp_wq);
+ }
srp_handle_qp_err(wc.status, wc.opcode, target);
- break;
}
}
}
@@ -1328,8 +1400,9 @@ static void srp_send_completion(struct ib_cq *cq, void *target_ptr)
iu = (struct srp_iu *) (uintptr_t) wc.wr_id;
list_add(&iu->list, &target->free_tx);
} else {
+ if (wc.wr_id == SRP_LAST_SEND)
+ target->last_send_wqe = true;
srp_handle_qp_err(wc.status, wc.opcode, target);
- break;
}
}
}
@@ -2260,6 +2333,7 @@ static ssize_t srp_create_target(struct device *dev,
spin_lock_init(&target->lock);
INIT_LIST_HEAD(&target->free_tx);
INIT_LIST_HEAD(&target->free_reqs);
+ init_waitqueue_head(&target->qp_wq);
for (i = 0; i < SRP_CMD_SQ_SIZE; ++i) {
struct srp_request *req = &target->req_ring[i];
diff --git a/drivers/infiniband/ulp/srp/ib_srp.h b/drivers/infiniband/ulp/srp/ib_srp.h
index de2d0b3..1b11117 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.h
+++ b/drivers/infiniband/ulp/srp/ib_srp.h
@@ -76,6 +76,9 @@ enum {
SRP_MAP_ALLOW_FMR = 0,
SRP_MAP_NO_FMR = 1,
+
+ SRP_LAST_RECV = 0,
+ SRP_LAST_SEND = 0,
};
enum srp_target_state {
@@ -180,6 +183,9 @@ struct srp_target_port {
struct completion done;
int status;
bool qp_in_error;
+ bool last_recv_wqe;
+ bool last_send_wqe;
+ wait_queue_head_t qp_wq;
struct completion tsk_mgmt_done;
u8 tsk_mgmt_status;
--
1.7.10.4
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
next prev parent reply other threads:[~2012-10-26 12:50 UTC|newest]
Thread overview: 34+ messages / expand[flat|nested] mbox.gz Atom feed top
2012-10-26 12:44 [PATCH 00/19, v5] Make ib_srp better suited for H.A. purposes Bart Van Assche
2012-10-26 12:52 ` [PATCH 11/19] srp_transport: Fix attribute registration Bart Van Assche
2012-10-26 12:54 ` [PATCH 13/19] srp_transport: Document sysfs attributes Bart Van Assche
2012-10-26 12:55 ` [PATCH 14/19] ib_srp: Allow SRP disconnect through sysfs Bart Van Assche
[not found] ` <508A85BB.1000505-HInyCGIudOg@public.gmane.org>
2012-10-26 12:45 ` [PATCH 01/19] ib_srp: Enlarge block layer timeout Bart Van Assche
2012-10-26 12:46 ` [PATCH 02/19] ib_srp: Eliminate state SRP_TARGET_CONNECTING Bart Van Assche
2012-10-26 12:46 ` [PATCH 03/19] ib_srp: Introduce srp_handle_qp_err() Bart Van Assche
2012-10-26 12:47 ` [PATCH 04/19] ib_srp: Suppress superfluous error messages Bart Van Assche
2012-10-26 12:48 ` [PATCH 05/19] ib_srp: Avoid that SCSI error handling causes trouble Bart Van Assche
2012-10-26 12:49 ` [PATCH 06/19] ib_srp: Introduce the helper function srp_remove_target() Bart Van Assche
2012-10-26 12:49 ` [PATCH 07/19] ib_srp: Eliminate state SRP_TARGET_DEAD Bart Van Assche
2012-10-26 12:50 ` [PATCH 08/19] ib_srp: Keep processing commands during host removal Bart Van Assche
2012-10-26 12:50 ` Bart Van Assche [this message]
2012-10-26 12:51 ` [PATCH 10/19] ib_srp: Document sysfs attributes Bart Van Assche
2012-10-26 12:53 ` [PATCH 12/19] srp_transport: Simplify attribute initialization code Bart Van Assche
2012-10-26 12:55 ` [PATCH 15/19] ib_srp: Maintain a single connection per I_T nexus Bart Van Assche
2012-10-26 12:56 ` [PATCH 16/19] srp_transport: Add transport layer error handling Bart Van Assche
2012-10-26 12:57 ` [PATCH 17/19] ib_srp: Add dev_loss_tmo support Bart Van Assche
2012-10-26 12:58 ` [PATCH 18/19] ib_srp: Remove SCSI devices upon port down event Bart Van Assche
[not found] ` <508A88D8.2050905-HInyCGIudOg@public.gmane.org>
2012-11-12 22:40 ` Or Gerlitz
[not found] ` <CAJZOPZL8mKU2MsrPPACvWjiA59aGnWDj0HNTQQNhbDrMsE0+Tg-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2012-11-13 8:59 ` Bart Van Assche
[not found] ` <50A20C03.9040607-HInyCGIudOg@public.gmane.org>
2012-11-13 20:54 ` Or Gerlitz
[not found] ` <CAJZOPZ+PiDQ6GYLkDO4MaPTDxLr2XDMn8q3gTaX-COx04PSegg-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2012-11-13 21:20 ` Bart Van Assche
[not found] ` <50A2B989.8000600-HInyCGIudOg@public.gmane.org>
2012-11-13 21:23 ` Or Gerlitz
[not found] ` <CAJZOPZLSPz7f99tj2w-79sPbibrHP3WZY_ct0Cq07Q1so54kFQ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2012-11-13 21:35 ` Bart Van Assche
2012-10-26 12:58 ` [PATCH 19/19] scsi_transport_srp: Fail I/O faster Bart Van Assche
2012-11-12 22:36 ` [PATCH 00/19, v5] Make ib_srp better suited for H.A. purposes Or Gerlitz
[not found] ` <CAJZOPZJPQkJ-kkW3ro9sRJXQJg_Yz_tjoJ1Rwb=XEePO3j_iJw-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2012-11-13 8:41 ` Bart Van Assche
[not found] ` <50A207D5.6060207-HInyCGIudOg@public.gmane.org>
2012-11-13 21:04 ` Or Gerlitz
[not found] ` <CAJZOPZJXdLRH9NPCt0snGNP8LKODO+phtV7uts6Vj-gxEEjpsw-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2012-11-13 21:30 ` Bart Van Assche
[not found] ` <50A2BC01.40609-HInyCGIudOg@public.gmane.org>
2012-11-13 21:41 ` Or Gerlitz
[not found] ` <CAJZOPZLQ8B9UGvGdM5LvA6r+XDARO5BXGoMmtdSH6+8EMyMaXw-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2012-11-13 22:35 ` Bart Van Assche
2012-11-12 22:51 ` Or Gerlitz
[not found] ` <CAJZOPZLHg84M3RUV00itGSGUZsigW0yw=TLOe6K63mUXH5v1pQ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2012-11-13 8:34 ` Bart Van Assche
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=508A872D.1010600@acm.org \
--to=bvanassche-hinycgiudog@public.gmane.org \
--cc=dillowda-1Heg1YXhbW8@public.gmane.org \
--cc=linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org \
--cc=roland-BHEL68pLQRGGvPXPguhicg@public.gmane.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.