[PATCH 0/7] qla2xxx: Bug fixes for driver

[PATCH 0/7] qla2xxx: Bug fixes for driver

[Himanshu Madhani]

Hi Martin, 

This series addresses review comments for previously submitted series for
Target MQ and FC-NVMe support.

Patch 1, 2 addresses review comments by Johannes for FC-NVMe series 
(https://www.spinics.net/lists/linux-scsi/msg110077.html)
(https://www.spinics.net/lists/linux-scsi/msg109985.html)

Patch 3 adds completion wq for FC-NVMe error path.

Patch 4 adds ZIO support for FC-NVMe.

Patch 5 simplifies waiting for unregistration of local/remote FC-NVMe ports.

Patch 6 fixes handling of remote-port disconnect for FC-NVMe.

Patch 7 address review comments by Bart for Target Multi-queue. 
(https://www.spinics.net/lists/target-devel/msg15563.html)

Please apply these patches to scsi-fixes for inclusion in 4.13.0-rc2. 

Thanks,
Himanshu

Duane Grigsby (2):
  qla2xxx: Add command completion wq for error path
  qla2xxx: Added change to enable ZIO for FC-NVMe devices

Himanshu Madhani (4):
  qla2xxx: Cleanup NVMe code.
  qla2xxx: Move function prototype to correct header
  qla2xxx: Simpify unregistration of FC-NVMe local/remote ports
  qla2xxx: Fix remoteport disconnect for FC-NVMe

Quinn Tran (1):
  qla2xxx: Refactor usage of Active command arrays

 drivers/scsi/qla2xxx/qla_bsg.c    |  51 ++++++------
 drivers/scsi/qla2xxx/qla_dbg.c    |   2 +-
 drivers/scsi/qla2xxx/qla_def.h    |  45 +++++++----
 drivers/scsi/qla2xxx/qla_gbl.h    |  14 +---
 drivers/scsi/qla2xxx/qla_init.c   |   7 +-
 drivers/scsi/qla2xxx/qla_inline.h |  23 +++---
 drivers/scsi/qla2xxx/qla_iocb.c   |  16 ++--
 drivers/scsi/qla2xxx/qla_isr.c    |  74 ++++++++++++-----
 drivers/scsi/qla2xxx/qla_mbx.c    |  82 +++++++++++++++----
 drivers/scsi/qla2xxx/qla_mr.c     |  19 ++---
 drivers/scsi/qla2xxx/qla_nvme.c   | 162 +++++++++++++++++---------------------
 drivers/scsi/qla2xxx/qla_nvme.h   |  17 ++++
 drivers/scsi/qla2xxx/qla_nx.c     |   6 +-
 drivers/scsi/qla2xxx/qla_os.c     |  69 +++++++++++-----
 drivers/scsi/qla2xxx/qla_target.c |  18 +++--
 drivers/scsi/qla2xxx/qla_target.h |   5 --
 16 files changed, 371 insertions(+), 239 deletions(-)

-- 
2.12.0

[PATCH 1/7] qla2xxx: Cleanup NVMe code.

[Himanshu Madhani]

This patch cleaned up code to make it more readable

Cc: Johannes Thumshirn <jthumshirn@suse.de>
Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
---
 drivers/scsi/qla2xxx/qla_def.h  |  6 ++--
 drivers/scsi/qla2xxx/qla_nvme.c | 61 ++++++++++++++++++-----------------------
 drivers/scsi/qla2xxx/qla_os.c   |  6 ++--
 3 files changed, 33 insertions(+), 40 deletions(-)

diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index 0730b10b4280..1635e98867aa 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -470,7 +470,7 @@ typedef struct srb {
 	uint8_t cmd_type;
 	uint8_t pad[3];
 	atomic_t ref_count;
-	wait_queue_head_t nvme_ls_waitQ;
+	wait_queue_head_t nvme_ls_waitq;
 	struct fc_port *fcport;
 	struct scsi_qla_host *vha;
 	uint32_t handle;
@@ -2302,7 +2302,7 @@ typedef struct fc_port {
 
 	struct work_struct nvme_del_work;
 	atomic_t nvme_ref_count;
-	wait_queue_head_t nvme_waitQ;
+	wait_queue_head_t nvme_waitq;
 	uint32_t nvme_prli_service_param;
 #define NVME_PRLI_SP_CONF       BIT_7
 #define NVME_PRLI_SP_INITIATOR  BIT_5
@@ -4130,7 +4130,7 @@ typedef struct scsi_qla_host {
 
 	struct		nvme_fc_local_port *nvme_local_port;
 	atomic_t	nvme_ref_count;
-	wait_queue_head_t nvme_waitQ;
+	wait_queue_head_t nvme_waitq;
 	struct list_head nvme_rport_list;
 	atomic_t 	nvme_active_aen_cnt;
 	uint16_t	nvme_last_rptd_aen;
diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c
index f3710a75fe1f..3c58d1b71e6e 100644
--- a/drivers/scsi/qla2xxx/qla_nvme.c
+++ b/drivers/scsi/qla2xxx/qla_nvme.c
@@ -77,15 +77,15 @@ int qla_nvme_register_remote(scsi_qla_host_t *vha, fc_port_t *fcport)
 	fcport->nvme_remote_port->private = fcport;
 	fcport->nvme_flag |= NVME_FLAG_REGISTERED;
 	atomic_set(&fcport->nvme_ref_count, 1);
-	init_waitqueue_head(&fcport->nvme_waitQ);
+	init_waitqueue_head(&fcport->nvme_waitq);
 	rport->fcport = fcport;
 	list_add_tail(&rport->list, &vha->nvme_rport_list);
 	return 0;
 }
 
 /* Allocate a queue for NVMe traffic */
-static int qla_nvme_alloc_queue(struct nvme_fc_local_port *lport, unsigned int qidx,
-    u16 qsize, void **handle)
+static int qla_nvme_alloc_queue(struct nvme_fc_local_port *lport,
+    unsigned int qidx, u16 qsize, void **handle)
 {
 	struct scsi_qla_host *vha;
 	struct qla_hw_data *ha;
@@ -193,13 +193,11 @@ static void qla_nvme_ls_abort(struct nvme_fc_local_port *lport,
 	struct qla_hw_data *ha = fcport->vha->hw;
 
 	rval = ha->isp_ops->abort_command(sp);
-	if (rval != QLA_SUCCESS)
-		ql_log(ql_log_warn, fcport->vha, 0x2125,
-		    "%s: failed to abort LS command for SP:%p rval=%x\n",
-		    __func__, sp, rval);
 
 	ql_dbg(ql_dbg_io, fcport->vha, 0x212b,
-	    "%s: aborted sp:%p on fcport:%p\n", __func__, sp, fcport);
+	    "%s: %s LS command for sp=%p on fcport=%p rval=%x\n", __func__,
+	    (rval != QLA_SUCCESS) ? "Failed to abort" : "Aborted",
+	    sp, fcport, rval);
 }
 
 static void qla_nvme_ls_complete(struct work_struct *work)
@@ -214,7 +212,7 @@ static void qla_nvme_ls_complete(struct work_struct *work)
 static int qla_nvme_ls_req(struct nvme_fc_local_port *lport,
     struct nvme_fc_remote_port *rport, struct nvmefc_ls_req *fd)
 {
-	fc_port_t *fcport = (fc_port_t *)rport->private;
+	fc_port_t *fcport = rport->private;
 	struct srb_iocb   *nvme;
 	struct nvme_private *priv = fd->private;
 	struct scsi_qla_host *vha;
@@ -236,7 +234,7 @@ static int qla_nvme_ls_req(struct nvme_fc_local_port *lport,
 	sp->name = "nvme_ls";
 	sp->done = qla_nvme_sp_ls_done;
 	atomic_set(&sp->ref_count, 1);
-	init_waitqueue_head(&sp->nvme_ls_waitQ);
+	init_waitqueue_head(&sp->nvme_ls_waitq);
 	nvme = &sp->u.iocb_cmd;
 	priv->sp = sp;
 	priv->fd = fd;
@@ -258,7 +256,7 @@ static int qla_nvme_ls_req(struct nvme_fc_local_port *lport,
 		ql_log(ql_log_warn, vha, 0x700e,
 		    "qla2x00_start_sp failed = %d\n", rval);
 		atomic_dec(&sp->ref_count);
-		wake_up(&sp->nvme_ls_waitQ);
+		wake_up(&sp->nvme_ls_waitq);
 		return rval;
 	}
 
@@ -276,20 +274,18 @@ static void qla_nvme_fcp_abort(struct nvme_fc_local_port *lport,
 	struct qla_hw_data *ha = fcport->vha->hw;
 
 	rval = ha->isp_ops->abort_command(sp);
-	if (!rval)
-		ql_log(ql_log_warn, fcport->vha, 0x2127,
-		    "%s: failed to abort command for SP:%p rval=%x\n",
-		    __func__, sp, rval);
 
-	ql_dbg(ql_dbg_io, fcport->vha, 0x2126,
-	    "%s: aborted sp:%p on fcport:%p\n", __func__, sp, fcport);
+	ql_dbg(ql_dbg_io, fcport->vha, 0x2127,
+	    "%s: %s command for sp=%p on fcport=%p rval=%x\n", __func__,
+	    (rval != QLA_SUCCESS) ? "Failed to abort" : "Aborted",
+	    sp, fcport, rval);
 }
 
 static void qla_nvme_poll(struct nvme_fc_local_port *lport, void *hw_queue_handle)
 {
 	struct scsi_qla_host *vha = lport->private;
 	unsigned long flags;
-	struct qla_qpair *qpair = (struct qla_qpair *)hw_queue_handle;
+	struct qla_qpair *qpair = hw_queue_handle;
 
 	/* Acquire ring specific lock */
 	spin_lock_irqsave(&qpair->qp_lock, flags);
@@ -487,7 +483,7 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport,
 	struct scsi_qla_host *vha;
 	int rval = QLA_FUNCTION_FAILED;
 	srb_t *sp;
-	struct qla_qpair *qpair = (struct qla_qpair *)hw_queue_handle;
+	struct qla_qpair *qpair = hw_queue_handle;
 	struct nvme_private *priv;
 
 	if (!fd) {
@@ -496,7 +492,7 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport,
 	}
 
 	priv = fd->private;
-	fcport = (fc_port_t *)rport->private;
+	fcport = rport->private;
 	if (!fcport) {
 		ql_log(ql_log_warn, NULL, 0x210e, "No fcport ptr\n");
 		return rval;
@@ -512,7 +508,7 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport,
 		return -EIO;
 
 	atomic_set(&sp->ref_count, 1);
-	init_waitqueue_head(&sp->nvme_ls_waitQ);
+	init_waitqueue_head(&sp->nvme_ls_waitq);
 	priv->sp = sp;
 	sp->type = SRB_NVME_CMD;
 	sp->name = "nvme_cmd";
@@ -526,7 +522,7 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport,
 		ql_log(ql_log_warn, vha, 0x212d,
 		    "qla2x00_start_nvme_mq failed = %d\n", rval);
 		atomic_dec(&sp->ref_count);
-		wake_up(&sp->nvme_ls_waitQ);
+		wake_up(&sp->nvme_ls_waitq);
 		return -EIO;
 	}
 
@@ -538,7 +534,7 @@ static void qla_nvme_localport_delete(struct nvme_fc_local_port *lport)
 	struct scsi_qla_host *vha = lport->private;
 
 	atomic_dec(&vha->nvme_ref_count);
-	wake_up_all(&vha->nvme_waitQ);
+	wake_up_all(&vha->nvme_waitq);
 
 	ql_log(ql_log_info, vha, 0x210f,
 	    "localport delete of %p completed.\n", vha->nvme_local_port);
@@ -550,11 +546,11 @@ static void qla_nvme_remoteport_delete(struct nvme_fc_remote_port *rport)
 	fc_port_t *fcport;
 	struct nvme_rport *r_port, *trport;
 
-	fcport = (fc_port_t *)rport->private;
+	fcport = rport->private;
 	fcport->nvme_remote_port = NULL;
 	fcport->nvme_flag &= ~NVME_FLAG_REGISTERED;
 	atomic_dec(&fcport->nvme_ref_count);
-	wake_up_all(&fcport->nvme_waitQ);
+	wake_up_all(&fcport->nvme_waitq);
 
 	list_for_each_entry_safe(r_port, trport,
 	    &fcport->vha->nvme_rport_list, list) {
@@ -594,7 +590,7 @@ static int qla_nvme_wait_on_command(srb_t *sp)
 {
 	int ret = QLA_SUCCESS;
 
-	wait_event_timeout(sp->nvme_ls_waitQ, (atomic_read(&sp->ref_count) > 1),
+	wait_event_timeout(sp->nvme_ls_waitq, (atomic_read(&sp->ref_count) > 1),
 	    NVME_ABORT_POLLING_PERIOD*HZ);
 
 	if (atomic_read(&sp->ref_count) > 1)
@@ -607,7 +603,7 @@ static int qla_nvme_wait_on_rport_del(fc_port_t *fcport)
 {
 	int ret = QLA_SUCCESS;
 
-	wait_event_timeout(fcport->nvme_waitQ,
+	wait_event_timeout(fcport->nvme_waitq,
 	    atomic_read(&fcport->nvme_ref_count),
 	    NVME_ABORT_POLLING_PERIOD*HZ);
 
@@ -625,12 +621,9 @@ void qla_nvme_abort(struct qla_hw_data *ha, srb_t *sp)
 	int rval;
 
 	rval = ha->isp_ops->abort_command(sp);
-	if (!rval) {
-		if (!qla_nvme_wait_on_command(sp))
-			ql_log(ql_log_warn, NULL, 0x2112,
-			    "nvme_wait_on_command timed out waiting on sp=%p\n",
-			    sp);
-	}
+	if (!rval && !qla_nvme_wait_on_command(sp))
+		ql_log(ql_log_warn, NULL, 0x2112,
+		    "nvme_wait_on_comand timed out waiting on sp=%p\n", sp);
 }
 
 static void qla_nvme_abort_all(fc_port_t *fcport)
@@ -757,5 +750,5 @@ void qla_nvme_register_hba(scsi_qla_host_t *vha)
 	}
 	atomic_set(&vha->nvme_ref_count, 1);
 	vha->nvme_local_port->private = vha;
-	init_waitqueue_head(&vha->nvme_waitQ);
+	init_waitqueue_head(&vha->nvme_waitq);
 }
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index df57655779ed..635ce75c630b 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -710,7 +710,7 @@ qla2x00_sp_free_dma(void *ptr)
 	}
 
 end:
-	if ((sp->type != SRB_NVME_CMD) && (sp->type != SRB_NVME_LS)) {
+	if (sp->type != SRB_NVME_CMD && sp->type != SRB_NVME_LS) {
 		CMD_SP(cmd) = NULL;
 		qla2x00_rel_sp(sp);
 	}
@@ -1715,8 +1715,8 @@ qla2x00_abort_all_cmds(scsi_qla_host_t *vha, int res)
 			if (sp) {
 				req->outstanding_cmds[cnt] = NULL;
 				if (sp->cmd_type == TYPE_SRB) {
-					if ((sp->type == SRB_NVME_CMD) ||
-					    (sp->type == SRB_NVME_LS)) {
+					if (sp->type == SRB_NVME_CMD ||
+					    sp->type == SRB_NVME_LS) {
 						sp_get(sp);
 						spin_unlock_irqrestore(
 						    &ha->hardware_lock, flags);
-- 
2.12.0

[PATCH 2/7] qla2xxx: Move function prototype to correct header

[Himanshu Madhani]

Cc: Johannes Thumshirn <jthumshirn@suse.de>
Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
---
 drivers/scsi/qla2xxx/qla_gbl.h  | 11 -----------
 drivers/scsi/qla2xxx/qla_isr.c  |  4 ++--
 drivers/scsi/qla2xxx/qla_nvme.c |  9 ++++-----
 drivers/scsi/qla2xxx/qla_nvme.h | 17 +++++++++++++++++
 4 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
index cadb6e3baacc..659cdf592678 100644
--- a/drivers/scsi/qla2xxx/qla_gbl.h
+++ b/drivers/scsi/qla2xxx/qla_gbl.h
@@ -10,17 +10,6 @@
 #include <linux/interrupt.h>
 
 /*
- * Global functions prototype in qla_nvme.c source file.
- */
-extern void qla_nvme_register_hba(scsi_qla_host_t *);
-extern int  qla_nvme_register_remote(scsi_qla_host_t *, fc_port_t *);
-extern void qla_nvme_delete(scsi_qla_host_t *);
-extern void qla_nvme_abort(struct qla_hw_data *, srb_t *sp);
-extern void qla24xx_nvme_ls4_iocb(scsi_qla_host_t *, struct pt_ls4_request *,
-    struct req_que *);
-extern void qla24xx_async_gffid_sp_done(void *, int);
-
-/*
  * Global Function Prototypes in qla_init.c source file.
  */
 extern int qla2x00_initialize_adapter(scsi_qla_host_t *);
diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
index 7b3b702ef622..9127eee67478 100644
--- a/drivers/scsi/qla2xxx/qla_isr.c
+++ b/drivers/scsi/qla2xxx/qla_isr.c
@@ -2827,8 +2827,8 @@ qla24xx_abort_iocb_entry(scsi_qla_host_t *vha, struct req_que *req,
 	sp->done(sp, 0);
 }
 
-void qla24xx_nvme_ls4_iocb(scsi_qla_host_t *vha, struct pt_ls4_request *pkt,
-    struct req_que *req)
+void qla24xx_nvme_ls4_iocb(struct scsi_qla_host *vha,
+    struct pt_ls4_request *pkt, struct req_que *req)
 {
 	srb_t *sp;
 	const char func[] = "LS4_IOCB";
diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c
index 3c58d1b71e6e..11494f2f90b5 100644
--- a/drivers/scsi/qla2xxx/qla_nvme.c
+++ b/drivers/scsi/qla2xxx/qla_nvme.c
@@ -5,7 +5,6 @@
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
 #include "qla_nvme.h"
-#include "qla_def.h"
 #include <linux/scatterlist.h>
 #include <linux/delay.h>
 #include <linux/nvme.h>
@@ -15,7 +14,7 @@ static struct nvme_fc_port_template qla_nvme_fc_transport;
 
 static void qla_nvme_unregister_remote_port(struct work_struct *);
 
-int qla_nvme_register_remote(scsi_qla_host_t *vha, fc_port_t *fcport)
+int qla_nvme_register_remote(struct scsi_qla_host *vha, struct fc_port *fcport)
 {
 	struct nvme_rport *rport;
 	int ret;
@@ -616,7 +615,7 @@ static int qla_nvme_wait_on_rport_del(fc_port_t *fcport)
 	return ret;
 }
 
-void qla_nvme_abort(struct qla_hw_data *ha, srb_t *sp)
+void qla_nvme_abort(struct qla_hw_data *ha, struct srb *sp)
 {
 	int rval;
 
@@ -679,7 +678,7 @@ static void qla_nvme_unregister_remote_port(struct work_struct *work)
 	}
 }
 
-void qla_nvme_delete(scsi_qla_host_t *vha)
+void qla_nvme_delete(struct scsi_qla_host *vha)
 {
 	struct nvme_rport *rport, *trport;
 	fc_port_t *fcport;
@@ -711,7 +710,7 @@ void qla_nvme_delete(scsi_qla_host_t *vha)
 	}
 }
 
-void qla_nvme_register_hba(scsi_qla_host_t *vha)
+void qla_nvme_register_hba(struct scsi_qla_host *vha)
 {
 	struct nvme_fc_port_template *tmpl;
 	struct qla_hw_data *ha;
diff --git a/drivers/scsi/qla2xxx/qla_nvme.h b/drivers/scsi/qla2xxx/qla_nvme.h
index dfe56f207b28..7f05fa1c77db 100644
--- a/drivers/scsi/qla2xxx/qla_nvme.h
+++ b/drivers/scsi/qla2xxx/qla_nvme.h
@@ -12,12 +12,18 @@
 #include <uapi/scsi/fc/fc_els.h>
 #include <linux/nvme-fc-driver.h>
 
+#include "qla_def.h"
+
 #define NVME_ATIO_CMD_OFF 32
 #define NVME_FIRST_PACKET_CMDLEN (64 - NVME_ATIO_CMD_OFF)
 #define Q2T_NVME_NUM_TAGS 2048
 #define QLA_MAX_FC_SEGMENTS 64
 
+struct scsi_qla_host;
+struct qla_hw_data;
+struct req_que;
 struct srb;
+
 struct nvme_private {
 	struct srb	*sp;
 	struct nvmefc_ls_req *fd;
@@ -129,4 +135,15 @@ struct pt_ls4_rx_unsol {
 	uint32_t desc_len;
 	uint32_t payload[3];
 };
+
+/*
+ * Global functions prototype in qla_nvme.c source file.
+ */
+void qla_nvme_register_hba(struct scsi_qla_host *);
+int  qla_nvme_register_remote(struct scsi_qla_host *, struct fc_port *);
+void qla_nvme_delete(struct scsi_qla_host *);
+void qla_nvme_abort(struct qla_hw_data *, struct srb *sp);
+void qla24xx_nvme_ls4_iocb(struct scsi_qla_host *, struct pt_ls4_request *,
+    struct req_que *);
+void qla24xx_async_gffid_sp_done(void *, int);
 #endif
-- 
2.12.0

[PATCH 3/7] qla2xxx: Add command completion wq for error path

[Himanshu Madhani]

From: Duane Grigsby <duane.grigsby@cavium.com>

When NVMe commands encounter error NVMe FC transport needs to
teardown the connection. This patch adds worker thread to process
these IO errors.

Signed-off-by: Duane Grigsby <duane.grigsby@cavium.com>
Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
---
 drivers/scsi/qla2xxx/qla_def.h  |  2 ++
 drivers/scsi/qla2xxx/qla_nvme.c | 20 +++++++++++++++++++-
 drivers/scsi/qla2xxx/qla_os.c   | 10 +++++++++-
 3 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index 1635e98867aa..799d25564ed6 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -430,6 +430,7 @@ struct srb_iocb {
 		} nvme;
 	} u;
 
+	struct work_struct rq_work;
 	struct timer_list timer;
 	void (*timeout)(void *);
 };
@@ -4132,6 +4133,7 @@ typedef struct scsi_qla_host {
 	atomic_t	nvme_ref_count;
 	wait_queue_head_t nvme_waitq;
 	struct list_head nvme_rport_list;
+	struct workqueue_struct *nvme_io_wq;
 	atomic_t 	nvme_active_aen_cnt;
 	uint16_t	nvme_last_rptd_aen;
 
diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c
index 11494f2f90b5..7543f533edfb 100644
--- a/drivers/scsi/qla2xxx/qla_nvme.c
+++ b/drivers/scsi/qla2xxx/qla_nvme.c
@@ -156,6 +156,17 @@ static void qla_nvme_sp_ls_done(void *ptr, int res)
 	qla2x00_rel_sp(sp);
 }
 
+static void qla_nvme_io_work(struct work_struct *work)
+{
+	srb_t *sp;
+	struct srb_iocb *nvme = container_of(work, struct srb_iocb, rq_work);
+	struct nvmefc_fcp_req *fd = nvme->u.nvme.desc;
+	sp = container_of(nvme, srb_t, u.iocb_cmd);
+
+	fd->done(fd);
+	qla2xxx_rel_qpair_sp(sp->qpair, sp);
+}
+
 static void qla_nvme_sp_done(void *ptr, int res)
 {
 	srb_t *sp = ptr;
@@ -177,7 +188,13 @@ static void qla_nvme_sp_done(void *ptr, int res)
 		fd->status = 0;
 
 	fd->rcv_rsplen = nvme->u.nvme.rsp_pyld_len;
-	fd->done(fd);
+	if (res == QLA_FUNCTION_FAILED) {
+		INIT_WORK(&nvme->rq_work, qla_nvme_io_work);
+		queue_work(sp->fcport->vha->nvme_io_wq, &nvme->rq_work);
+		return;
+	} else {
+		fd->done(fd);
+	}
 rel:
 	qla2xxx_rel_qpair_sp(sp->qpair, sp);
 }
@@ -514,6 +531,7 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport,
 	sp->done = qla_nvme_sp_done;
 	sp->qpair = qpair;
 	nvme = &sp->u.iocb_cmd;
+	INIT_WORK(&nvme->rq_work, qla_nvme_io_work);
 	nvme->u.nvme.desc = fd;
 
 	rval = qla2x00_start_nvme_mq(sp);
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index 635ce75c630b..3329512b4b35 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -2751,7 +2751,6 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	spin_lock_init(&ha->tgt.sess_lock);
 	spin_lock_init(&ha->tgt.atio_lock);
 
-
 	/* Clear our data area */
 	ha->bars = bars;
 	ha->mem_only = mem_only;
@@ -3286,6 +3285,13 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	base_vha->flags.init_done = 1;
 	base_vha->flags.online = 1;
 	ha->prev_minidump_failed = 0;
+	atomic_set(&base_vha->nvme_active_aen_cnt, 0);
+	base_vha->nvme_io_wq = alloc_workqueue("qlnvme-io-wq", 0, 0);
+	if (!base_vha->nvme_io_wq) {
+		ql_log(ql_log_fatal, base_vha, 0x000b,
+		    "Unable to allocate workqueue for nvme_io_wq\n");
+		goto disable_device;
+	}
 
 	ql_dbg(ql_dbg_init, base_vha, 0x00f2,
 	    "Init done and hba is online.\n");
@@ -3559,6 +3565,8 @@ qla2x00_remove_one(struct pci_dev *pdev)
 	set_bit(UNLOADING, &base_vha->dpc_flags);
 
 	qla_nvme_delete(base_vha);
+	if (base_vha->nvme_io_wq)
+		destroy_workqueue(base_vha->nvme_io_wq);
 
 	dma_free_coherent(&ha->pdev->dev,
 		base_vha->gnl.size, base_vha->gnl.l, base_vha->gnl.ldma);
-- 
2.12.0

[PATCH 4/7] qla2xxx: Added change to enable ZIO for FC-NVMe devices

[Himanshu Madhani]

From: Duane Grigsby <duane.grigsby@cavium.com>

Add support to the driver to set the exchange threshold value for
the number of outstanding AENs.

Signed-off-by: Duane Grigsby <duane.grigsby@cavium.com>
Signed-off-by: Darren Trapp <darren.trapp@cavium.com>
Signed-off-by: Anil Gurumurthy <anil.gurumurthy@cavium.com>
Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
---
 drivers/scsi/qla2xxx/qla_dbg.c  |  2 +-
 drivers/scsi/qla2xxx/qla_def.h  |  5 +++
 drivers/scsi/qla2xxx/qla_gbl.h  |  3 ++
 drivers/scsi/qla2xxx/qla_isr.c  |  2 +-
 drivers/scsi/qla2xxx/qla_mbx.c  | 76 ++++++++++++++++++++++++++++++++++-------
 drivers/scsi/qla2xxx/qla_nvme.c | 14 ++++++--
 drivers/scsi/qla2xxx/qla_os.c   | 26 +++++++++++---
 7 files changed, 105 insertions(+), 23 deletions(-)

diff --git a/drivers/scsi/qla2xxx/qla_dbg.c b/drivers/scsi/qla2xxx/qla_dbg.c
index 26751d34bcf2..7b74973d5788 100644
--- a/drivers/scsi/qla2xxx/qla_dbg.c
+++ b/drivers/scsi/qla2xxx/qla_dbg.c
@@ -14,7 +14,7 @@
  * | Module Init and Probe        |       0x0193       | 0x0146         |
  * |                              |                    | 0x015b-0x0160	|
  * |                              |                    | 0x016e		|
- * | Mailbox commands             |       0x1199       | 0x1193		|
+ * | Mailbox commands             |       0x1205       | 0x11a2-0x11ff	|
  * | Device Discovery             |       0x2134       | 0x210e-0x2116  |
  * |				  | 		       | 0x211a         |
  * |                              |                    | 0x211c-0x2128  |
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index 799d25564ed6..015908f99e76 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -978,6 +978,7 @@ struct mbx_cmd_32 {
 #define MBC_ABORT_TARGET		0x17	/* Abort target (ID). */
 #define MBC_RESET			0x18	/* Reset. */
 #define MBC_GET_ADAPTER_LOOP_ID		0x20	/* Get loop id of ISP2200. */
+#define MBC_GET_SET_ZIO_THRESHOLD	0x21	/* Get/SET ZIO THRESHOLD. */
 #define MBC_GET_RETRY_COUNT		0x22	/* Get f/w retry cnt/delay. */
 #define MBC_DISABLE_VI			0x24	/* Disable VI operation. */
 #define MBC_ENABLE_VI			0x25	/* Enable VI operation. */
@@ -4018,6 +4019,9 @@ struct qla_hw_data {
 
 	struct qlt_hw_data tgt;
 	int	allow_cna_fw_dump;
+
+	atomic_t        nvme_active_aen_cnt;
+	uint16_t        nvme_last_rptd_aen;             /* Last recorded aen count */
 };
 
 /*
@@ -4090,6 +4094,7 @@ typedef struct scsi_qla_host {
 #define FX00_CRITEMP_RECOVERY	25
 #define FX00_HOST_INFO_RESEND	26
 #define QPAIR_ONLINE_CHECK_NEEDED	27
+#define SET_ZIO_THRESHOLD_NEEDED	28
 
 	unsigned long	pci_flags;
 #define PFLG_DISCONNECTED	0	/* PCI device removed */
diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
index 659cdf592678..67864d4492cd 100644
--- a/drivers/scsi/qla2xxx/qla_gbl.h
+++ b/drivers/scsi/qla2xxx/qla_gbl.h
@@ -484,6 +484,9 @@ int qla24xx_gidlist_wait(struct scsi_qla_host *, void *, dma_addr_t,
 int __qla24xx_parse_gpdb(struct scsi_qla_host *, fc_port_t *,
 	struct port_database_24xx *);
 
+extern int qla27xx_get_zio_threshold(scsi_qla_host_t *, uint16_t *);
+extern int qla27xx_set_zio_threshold(scsi_qla_host_t *, uint16_t );
+
 /*
  * Global Function Prototypes in qla_isr.c source file.
  */
diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
index 9127eee67478..317fe6026856 100644
--- a/drivers/scsi/qla2xxx/qla_isr.c
+++ b/drivers/scsi/qla2xxx/qla_isr.c
@@ -1823,7 +1823,7 @@ qla24xx_nvme_iocb_entry(scsi_qla_host_t *vha, struct req_que *req, void *tsk)
 	nvme = &sp->u.iocb_cmd;
 
 	if (unlikely(nvme->u.nvme.aen_op))
-		atomic_dec(&sp->vha->nvme_active_aen_cnt);
+		atomic_dec(&sp->vha->hw->nvme_active_aen_cnt);
 
 	/*
 	 * State flags: Bit 6 and 0.
diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c
index 7c6d1a404011..b271c3f78c85 100644
--- a/drivers/scsi/qla2xxx/qla_mbx.c
+++ b/drivers/scsi/qla2xxx/qla_mbx.c
@@ -947,20 +947,12 @@ qla2x00_get_fw_version(scsi_qla_host_t *vha)
 			    "%s: Firmware supports Exchange Offload 0x%x\n",
 			    __func__, ha->fw_attributes_h);
 
-		/* bit 26 of fw_attributes */
-		if ((ha->fw_attributes_h & 0x400) && ql2xnvmeenable) {
-			struct init_cb_24xx *icb;
-
-			icb = (struct init_cb_24xx *)ha->init_cb;
-			/*
-			 * fw supports nvme and driver load
-			 * parameter requested nvme
-			 */
+		/*
+		 * fw supports nvme and driver load parameter requested nvme
+		 * bit 26 of fw_attributes indicates NVMe support
+		 */
+		if ((ha->fw_attributes_h & 0x400) && ql2xnvmeenable)
 			vha->flags.nvme_enabled = 1;
-			icb->firmware_options_2 &= cpu_to_le32(~0xf);
-			ha->zio_mode = 0;
-			ha->zio_timer = 0;
-		}
 
 	}
 
@@ -6085,3 +6077,61 @@ int qla24xx_gidlist_wait(struct scsi_qla_host *vha,
 done:
 	return rval;
 }
+
+int qla27xx_set_zio_threshold(scsi_qla_host_t *vha, uint16_t value)
+{
+	int rval;
+	mbx_cmd_t	mc;
+	mbx_cmd_t	*mcp = &mc;
+
+	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1200,
+	    "Entered %s\n", __func__);
+
+	memset(mcp->mb, 0 , sizeof(mcp->mb));
+	mcp->mb[0] = MBC_GET_SET_ZIO_THRESHOLD;
+	mcp->mb[1] = cpu_to_le16(1);
+	mcp->mb[2] = cpu_to_le16(value);
+	mcp->out_mb = MBX_2|MBX_1|MBX_0;
+	mcp->in_mb = MBX_2|MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+
+	rval = qla2x00_mailbox_command(vha, mcp);
+	if (rval != QLA_SUCCESS) {
+		ql_dbg(ql_dbg_mbx, vha, 0x1201, "Failed=%x.\n", rval);
+	} else {
+		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1202,
+		    "Done %s.\n", __func__);
+	}
+
+	return rval;
+}
+
+int qla27xx_get_zio_threshold(scsi_qla_host_t *vha, uint16_t *value)
+{
+	int rval;
+	mbx_cmd_t	mc;
+	mbx_cmd_t	*mcp = &mc;
+
+	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1203,
+	    "Entered %s\n", __func__);
+
+	memset(mcp->mb, 0 , sizeof(mcp->mb));
+	mcp->mb[0] = MBC_GET_SET_ZIO_THRESHOLD;
+	mcp->mb[1] = cpu_to_le16(0);
+	mcp->out_mb = MBX_1|MBX_0;
+	mcp->in_mb = MBX_2|MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+
+	rval = qla2x00_mailbox_command(vha, mcp);
+	if (rval != QLA_SUCCESS) {
+		ql_dbg(ql_dbg_mbx, vha, 0x1204, "Failed=%x.\n", rval);
+	} else {
+		*value = mc.mb[2];
+		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1205,
+		    "Done %s.\n", __func__);
+	}
+
+	return rval;
+}
diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c
index 7543f533edfb..4cb5bd20065a 100644
--- a/drivers/scsi/qla2xxx/qla_nvme.c
+++ b/drivers/scsi/qla2xxx/qla_nvme.c
@@ -322,6 +322,7 @@ static int qla2x00_start_nvme_mq(srb_t *sp)
 	uint16_t	avail_dsds;
 	uint32_t	*cur_dsd;
 	struct req_que *req = NULL;
+	struct rsp_que *rsp = NULL;
 	struct scsi_qla_host *vha = sp->fcport->vha;
 	struct qla_hw_data *ha = vha->hw;
 	struct qla_qpair *qpair = sp->qpair;
@@ -330,13 +331,15 @@ static int qla2x00_start_nvme_mq(srb_t *sp)
 	struct nvmefc_fcp_req *fd = nvme->u.nvme.desc;
 	uint32_t        rval = QLA_SUCCESS;
 
-	/* Setup qpair pointers */
-	req = qpair->req;
 	tot_dsds = fd->sg_cnt;
 
 	/* Acquire qpair specific lock */
 	spin_lock_irqsave(&qpair->qp_lock, flags);
 
+	/* Setup qpair pointers */
+	req = qpair->req;
+	rsp = qpair->rsp;
+
 	/* Check for room in outstanding command list. */
 	handle = req->current_outstanding_cmd;
 	for (index = 1; index < req->num_outstanding_cmds; index++) {
@@ -371,7 +374,7 @@ static int qla2x00_start_nvme_mq(srb_t *sp)
 		struct nvme_fc_cmd_iu *cmd = fd->cmdaddr;
 		if (cmd->sqe.common.opcode == nvme_admin_async_event) {
 			nvme->u.nvme.aen_op = 1;
-			atomic_inc(&vha->nvme_active_aen_cnt);
+			atomic_inc(&vha->hw->nvme_active_aen_cnt);
 		}
 	}
 
@@ -484,6 +487,11 @@ static int qla2x00_start_nvme_mq(srb_t *sp)
 	/* Set chip new ring index. */
 	WRT_REG_DWORD(req->req_q_in, req->ring_index);
 
+	/* Manage unprocessed RIO/ZIO commands in response queue. */
+	if (vha->flags.process_response_queue &&
+	    rsp->ring_ptr->signature != RESPONSE_PROCESSED)
+		qla24xx_process_response_queue(vha, rsp);
+
 queuing_error:
 	spin_unlock_irqrestore(&qpair->qp_lock, flags);
 	return rval;
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index 3329512b4b35..256d3a34d4ee 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -2751,6 +2751,8 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	spin_lock_init(&ha->tgt.sess_lock);
 	spin_lock_init(&ha->tgt.atio_lock);
 
+	atomic_set(&ha->nvme_active_aen_cnt, 0);
+
 	/* Clear our data area */
 	ha->bars = bars;
 	ha->mem_only = mem_only;
@@ -5836,6 +5838,17 @@ qla2x00_do_dpc(void *data)
 			mutex_unlock(&ha->mq_lock);
 		}
 
+		if (test_and_clear_bit(SET_ZIO_THRESHOLD_NEEDED, &base_vha->dpc_flags)) {
+			ql_log(ql_log_info, base_vha, 0xffffff,
+				"nvme: SET ZIO Activity exchange threshold to %d.\n",
+						ha->nvme_last_rptd_aen);
+			if (qla27xx_set_zio_threshold(base_vha, ha->nvme_last_rptd_aen)) {
+				ql_log(ql_log_info, base_vha, 0xffffff,
+					"nvme: Unable to SET ZIO Activity exchange threshold to %d.\n",
+						ha->nvme_last_rptd_aen);
+			}
+		}
+
 		if (!IS_QLAFX00(ha))
 			qla2x00_do_dpc_all_vps(base_vha);
 
@@ -6033,12 +6046,15 @@ qla2x00_timer(scsi_qla_host_t *vha)
 	 * FC-NVME
 	 * see if the active AEN count has changed from what was last reported.
 	 */
-	if (atomic_read(&vha->nvme_active_aen_cnt) != vha->nvme_last_rptd_aen) {
-		vha->nvme_last_rptd_aen =
-		    atomic_read(&vha->nvme_active_aen_cnt);
+	if (!vha->vp_idx &&
+		atomic_read(&ha->nvme_active_aen_cnt) != ha->nvme_last_rptd_aen &&
+		ha->zio_mode == QLA_ZIO_MODE_6) {
 		ql_log(ql_log_info, vha, 0x3002,
-		    "reporting new aen count of %d to the fw\n",
-		    vha->nvme_last_rptd_aen);
+			"nvme: Sched: Set ZIO exchange threshold to %d.\n",
+			ha->nvme_last_rptd_aen);
+		ha->nvme_last_rptd_aen = atomic_read(&ha->nvme_active_aen_cnt);
+		set_bit(SET_ZIO_THRESHOLD_NEEDED, &vha->dpc_flags);
+		start_dpc++;
 	}
 
 	/* Schedule the DPC routine if needed */
-- 
2.12.0

[PATCH 5/7] qla2xxx: Simpify unregistration of FC-NVMe local/remote ports

[Himanshu Madhani]

Simplified waiting for unregister local/remote FC-NVMe ports
to complete cleanup.

Signed-off-by: Duane Grigsby <duane.grigsby@cavium.com>
Signed-off-by: Darren Trapp <darren.trapp@cavium.com>
Signed-off-by: Anil Gurumurthy <anil.gurumurthy@cavium.com>
Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
---
 drivers/scsi/qla2xxx/qla_def.h  |  5 ++--
 drivers/scsi/qla2xxx/qla_nvme.c | 59 +++++++----------------------------------
 2 files changed, 12 insertions(+), 52 deletions(-)

diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index 015908f99e76..caee4a2b4002 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -2304,7 +2304,7 @@ typedef struct fc_port {
 
 	struct work_struct nvme_del_work;
 	atomic_t nvme_ref_count;
-	wait_queue_head_t nvme_waitq;
+	struct completion nvme_del_done;
 	uint32_t nvme_prli_service_param;
 #define NVME_PRLI_SP_CONF       BIT_7
 #define NVME_PRLI_SP_INITIATOR  BIT_5
@@ -4135,8 +4135,7 @@ typedef struct scsi_qla_host {
 	uint8_t		fabric_node_name[WWN_SIZE];
 
 	struct		nvme_fc_local_port *nvme_local_port;
-	atomic_t	nvme_ref_count;
-	wait_queue_head_t nvme_waitq;
+	struct completion nvme_del_done;
 	struct list_head nvme_rport_list;
 	struct workqueue_struct *nvme_io_wq;
 	atomic_t 	nvme_active_aen_cnt;
diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c
index 4cb5bd20065a..ccafcdb228e8 100644
--- a/drivers/scsi/qla2xxx/qla_nvme.c
+++ b/drivers/scsi/qla2xxx/qla_nvme.c
@@ -75,8 +75,6 @@ int qla_nvme_register_remote(struct scsi_qla_host *vha, struct fc_port *fcport)
 
 	fcport->nvme_remote_port->private = fcport;
 	fcport->nvme_flag |= NVME_FLAG_REGISTERED;
-	atomic_set(&fcport->nvme_ref_count, 1);
-	init_waitqueue_head(&fcport->nvme_waitq);
 	rport->fcport = fcport;
 	list_add_tail(&rport->list, &vha->nvme_rport_list);
 	return 0;
@@ -250,7 +248,6 @@ static int qla_nvme_ls_req(struct nvme_fc_local_port *lport,
 	sp->name = "nvme_ls";
 	sp->done = qla_nvme_sp_ls_done;
 	atomic_set(&sp->ref_count, 1);
-	init_waitqueue_head(&sp->nvme_ls_waitq);
 	nvme = &sp->u.iocb_cmd;
 	priv->sp = sp;
 	priv->fd = fd;
@@ -558,12 +555,10 @@ static void qla_nvme_localport_delete(struct nvme_fc_local_port *lport)
 {
 	struct scsi_qla_host *vha = lport->private;
 
-	atomic_dec(&vha->nvme_ref_count);
-	wake_up_all(&vha->nvme_waitq);
-
 	ql_log(ql_log_info, vha, 0x210f,
 	    "localport delete of %p completed.\n", vha->nvme_local_port);
 	vha->nvme_local_port = NULL;
+	complete(&vha->nvme_del_done);
 }
 
 static void qla_nvme_remoteport_delete(struct nvme_fc_remote_port *rport)
@@ -574,8 +569,6 @@ static void qla_nvme_remoteport_delete(struct nvme_fc_remote_port *rport)
 	fcport = rport->private;
 	fcport->nvme_remote_port = NULL;
 	fcport->nvme_flag &= ~NVME_FLAG_REGISTERED;
-	atomic_dec(&fcport->nvme_ref_count);
-	wake_up_all(&fcport->nvme_waitq);
 
 	list_for_each_entry_safe(r_port, trport,
 	    &fcport->vha->nvme_rport_list, list) {
@@ -585,6 +578,7 @@ static void qla_nvme_remoteport_delete(struct nvme_fc_remote_port *rport)
 		}
 	}
 	kfree(r_port);
+	complete(&fcport->nvme_del_done);
 
 	ql_log(ql_log_info, fcport->vha, 0x2110,
 	    "remoteport_delete of %p completed.\n", fcport);
@@ -627,12 +621,11 @@ static int qla_nvme_wait_on_command(srb_t *sp)
 static int qla_nvme_wait_on_rport_del(fc_port_t *fcport)
 {
 	int ret = QLA_SUCCESS;
+	int timeout;
 
-	wait_event_timeout(fcport->nvme_waitq,
-	    atomic_read(&fcport->nvme_ref_count),
-	    NVME_ABORT_POLLING_PERIOD*HZ);
-
-	if (atomic_read(&fcport->nvme_ref_count)) {
+	timeout = wait_for_completion_timeout(&fcport->nvme_del_done,
+	    msecs_to_jiffies(2000));
+	if (!timeout) {
 		ret = QLA_FUNCTION_FAILED;
 		ql_log(ql_log_info, fcport->vha, 0x2111,
 		    "timed out waiting for fcport=%p to delete\n", fcport);
@@ -651,39 +644,6 @@ void qla_nvme_abort(struct qla_hw_data *ha, struct srb *sp)
 		    "nvme_wait_on_comand timed out waiting on sp=%p\n", sp);
 }
 
-static void qla_nvme_abort_all(fc_port_t *fcport)
-{
-	int que, cnt;
-	unsigned long flags;
-	srb_t *sp;
-	struct qla_hw_data *ha = fcport->vha->hw;
-	struct req_que *req;
-
-	spin_lock_irqsave(&ha->hardware_lock, flags);
-	for (que = 0; que < ha->max_req_queues; que++) {
-		req = ha->req_q_map[que];
-		if (!req)
-			continue;
-		if (!req->outstanding_cmds)
-			continue;
-		for (cnt = 1; cnt < req->num_outstanding_cmds; cnt++) {
-			sp = req->outstanding_cmds[cnt];
-			if ((sp) && ((sp->type == SRB_NVME_CMD) ||
-			    (sp->type == SRB_NVME_LS)) &&
-				(sp->fcport == fcport)) {
-				atomic_inc(&sp->ref_count);
-				spin_unlock_irqrestore(&ha->hardware_lock,
-				    flags);
-				qla_nvme_abort(ha, sp);
-				spin_lock_irqsave(&ha->hardware_lock, flags);
-				req->outstanding_cmds[cnt] = NULL;
-				sp->done(sp, 1);
-			}
-		}
-	}
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
-}
-
 static void qla_nvme_unregister_remote_port(struct work_struct *work)
 {
 	struct fc_port *fcport = container_of(work, struct fc_port,
@@ -719,12 +679,13 @@ void qla_nvme_delete(struct scsi_qla_host *vha)
 		ql_log(ql_log_info, fcport->vha, 0x2114, "%s: fcport=%p\n",
 		    __func__, fcport);
 
+		init_completion(&fcport->nvme_del_done);
 		nvme_fc_unregister_remoteport(fcport->nvme_remote_port);
 		qla_nvme_wait_on_rport_del(fcport);
-		qla_nvme_abort_all(fcport);
 	}
 
 	if (vha->nvme_local_port) {
+		init_completion(&vha->nvme_del_done);
 		nv_ret = nvme_fc_unregister_localport(vha->nvme_local_port);
 		if (nv_ret == 0)
 			ql_log(ql_log_info, vha, 0x2116,
@@ -733,6 +694,8 @@ void qla_nvme_delete(struct scsi_qla_host *vha)
 		else
 			ql_log(ql_log_info, vha, 0x2115,
 			    "Unregister of localport failed\n");
+		wait_for_completion_timeout(&vha->nvme_del_done,
+		    msecs_to_jiffies(5000));
 	}
 }
 
@@ -773,7 +736,5 @@ void qla_nvme_register_hba(struct scsi_qla_host *vha)
 		    "register_localport failed: ret=%x\n", ret);
 		return;
 	}
-	atomic_set(&vha->nvme_ref_count, 1);
 	vha->nvme_local_port->private = vha;
-	init_waitqueue_head(&vha->nvme_waitq);
 }
-- 
2.12.0

[PATCH 6/7] qla2xxx: Fix remoteport disconnect for FC-NVMe

[Himanshu Madhani]

Signed-off-by: Duane Grigsby <duane.grigsby@cavium.com>
Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
---
 drivers/scsi/qla2xxx/qla_isr.c  | 37 +++++++++++++++++++++++++++++++------
 drivers/scsi/qla2xxx/qla_nvme.c | 11 ++++++++---
 2 files changed, 39 insertions(+), 9 deletions(-)

diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
index 317fe6026856..c14fab35fc36 100644
--- a/drivers/scsi/qla2xxx/qla_isr.c
+++ b/drivers/scsi/qla2xxx/qla_isr.c
@@ -1856,17 +1856,42 @@ qla24xx_nvme_iocb_entry(scsi_qla_host_t *vha, struct req_que *req, void *tsk)
 	fd->transferred_length = fd->payload_length -
 	    le32_to_cpu(sts->residual_len);
 
+	/*
+	 * If transport error then Failure (HBA rejects request)
+	 * otherwise transport will handle.
+	 */
 	if (sts->entry_status) {
 		ql_log(ql_log_warn, fcport->vha, 0x5038,
 		    "NVME-%s error - hdl=%x entry-status(%x).\n",
 		    sp->name, sp->handle, sts->entry_status);
 		ret = QLA_FUNCTION_FAILED;
-	} else if (sts->comp_status != cpu_to_le16(CS_COMPLETE)) {
-		ql_log(ql_log_warn, fcport->vha, 0x5039,
-		    "NVME-%s error - hdl=%x completion status(%x) resid=%x  ox_id=%x\n",
-		    sp->name, sp->handle, sts->comp_status,
-		    le32_to_cpu(sts->residual_len), sts->ox_id);
-		ret = QLA_FUNCTION_FAILED;
+	} else  {
+		switch (le16_to_cpu(sts->comp_status)) {
+			case CS_COMPLETE:
+				ret = 0;
+			break;
+
+			case CS_ABORTED:
+			case CS_RESET:
+			case CS_PORT_UNAVAILABLE:
+			case CS_PORT_LOGGED_OUT:
+			case CS_PORT_BUSY:
+				ql_log(ql_log_warn, fcport->vha, 0x5060,
+				"NVME-%s ERR Handling - hdl=%x completion status(%x) resid=%x  ox_id=%x\n",
+				sp->name, sp->handle, sts->comp_status,
+				le32_to_cpu(sts->residual_len), sts->ox_id);
+				fd->transferred_length = fd->payload_length;
+				ret = QLA_ABORTED;
+			break;
+
+			default:
+				ql_log(ql_log_warn, fcport->vha, 0x5060,
+				"NVME-%s error - hdl=%x completion status(%x) resid=%x  ox_id=%x\n",
+				sp->name, sp->handle, sts->comp_status,
+				le32_to_cpu(sts->residual_len), sts->ox_id);
+				ret = QLA_FUNCTION_FAILED;
+				break;
+		}
 	}
 	sp->done(sp, ret);
 }
diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c
index ccafcdb228e8..da32a06f17a0 100644
--- a/drivers/scsi/qla2xxx/qla_nvme.c
+++ b/drivers/scsi/qla2xxx/qla_nvme.c
@@ -180,13 +180,13 @@ static void qla_nvme_sp_done(void *ptr, int res)
 	if (!(sp->fcport->nvme_flag & NVME_FLAG_REGISTERED))
 		goto rel;
 
-	if (unlikely(nvme->u.nvme.comp_status || res))
-		fd->status = -EINVAL;
+	if (unlikely(res == QLA_FUNCTION_FAILED))
+		fd->status = NVME_SC_FC_TRANSPORT_ERROR;
 	else
 		fd->status = 0;
 
 	fd->rcv_rsplen = nvme->u.nvme.rsp_pyld_len;
-	if (res == QLA_FUNCTION_FAILED) {
+	if (res) {
 		INIT_WORK(&nvme->rq_work, qla_nvme_io_work);
 		queue_work(sp->fcport->vha->nvme_io_wq, &nvme->rq_work);
 		return;
@@ -653,13 +653,18 @@ static void qla_nvme_unregister_remote_port(struct work_struct *work)
 	if (!IS_ENABLED(CONFIG_NVME_FC))
 		return;
 
+	ql_log(ql_log_warn, NULL, 0x2112,
+	    "%s: unregister remoteport on %p\n",__func__, fcport);
+
 	list_for_each_entry_safe(rport, trport,
 	    &fcport->vha->nvme_rport_list, list) {
 		if (rport->fcport == fcport) {
 			ql_log(ql_log_info, fcport->vha, 0x2113,
 			    "%s: fcport=%p\n", __func__, fcport);
+			init_completion(&fcport->nvme_del_done);
 			nvme_fc_unregister_remoteport(
 			    fcport->nvme_remote_port);
+			qla_nvme_wait_on_rport_del(fcport);
 		}
 	}
 }
-- 
2.12.0

[PATCH 7/7] qla2xxx: Refactor usage of Active command arrays

[Himanshu Madhani]

From: Quinn Tran <quinn.tran@cavium.com>

Cc: Bart Van Assche <Bart.VanAssche@wdc.com>
Signed-off-by: Quinn Tran <quinn.tran@cavium.com>
Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
---
 drivers/scsi/qla2xxx/qla_bsg.c    | 51 ++++++++++++++++++++-------------------
 drivers/scsi/qla2xxx/qla_def.h    | 31 +++++++++++++++---------
 drivers/scsi/qla2xxx/qla_init.c   |  7 +++---
 drivers/scsi/qla2xxx/qla_inline.h | 23 +++++++++++-------
 drivers/scsi/qla2xxx/qla_iocb.c   | 16 ++++++------
 drivers/scsi/qla2xxx/qla_isr.c    | 31 +++++++++++++++---------
 drivers/scsi/qla2xxx/qla_mbx.c    |  6 +++--
 drivers/scsi/qla2xxx/qla_mr.c     | 19 ++++++++-------
 drivers/scsi/qla2xxx/qla_nvme.c   |  2 +-
 drivers/scsi/qla2xxx/qla_nx.c     |  6 +++--
 drivers/scsi/qla2xxx/qla_os.c     | 29 +++++++++++++---------
 drivers/scsi/qla2xxx/qla_target.c | 18 ++++++++------
 drivers/scsi/qla2xxx/qla_target.h |  5 ----
 13 files changed, 139 insertions(+), 105 deletions(-)

diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c
index 2ea0ef93f5cb..356cea400d19 100644
--- a/drivers/scsi/qla2xxx/qla_bsg.c
+++ b/drivers/scsi/qla2xxx/qla_bsg.c
@@ -1920,7 +1920,7 @@ qla24xx_process_bidir_cmd(struct bsg_job *bsg_job)
 	return rval;
 
 done_free_srb:
-	mempool_free(sp, ha->srb_mempool);
+	mempool_free(SRB_TO_U(sp), ha->srb_mempool);
 done_unmap_sg:
 	dma_unmap_sg(&ha->pdev->dev,
 	    bsg_job->reply_payload.sg_list,
@@ -2043,7 +2043,7 @@ qlafx00_mgmt_cmd(struct bsg_job *bsg_job)
 	if (rval != QLA_SUCCESS) {
 		ql_log(ql_log_warn, vha, 0x70cd,
 		    "qla2x00_start_sp failed=%d.\n", rval);
-		mempool_free(sp, ha->srb_mempool);
+		mempool_free(SRB_TO_U(sp), ha->srb_mempool);
 		rval = -EIO;
 		goto done_free_fcport;
 	}
@@ -2533,6 +2533,7 @@ qla24xx_bsg_timeout(struct bsg_job *bsg_job)
 	int cnt, que;
 	unsigned long flags;
 	struct req_que *req;
+	struct unify_cmd *u;
 
 	/* find the bsg job from the active list of commands */
 	spin_lock_irqsave(&ha->hardware_lock, flags);
@@ -2542,30 +2543,30 @@ qla24xx_bsg_timeout(struct bsg_job *bsg_job)
 			continue;
 
 		for (cnt = 1; cnt < req->num_outstanding_cmds; cnt++) {
-			sp = req->outstanding_cmds[cnt];
-			if (sp) {
-				if (((sp->type == SRB_CT_CMD) ||
-					(sp->type == SRB_ELS_CMD_HST) ||
-					(sp->type == SRB_FXIOCB_BCMD))
-					&& (sp->u.bsg_job == bsg_job)) {
-					req->outstanding_cmds[cnt] = NULL;
-					spin_unlock_irqrestore(&ha->hardware_lock, flags);
-					if (ha->isp_ops->abort_command(sp)) {
-						ql_log(ql_log_warn, vha, 0x7089,
-						    "mbx abort_command "
-						    "failed.\n");
-						scsi_req(bsg_job->req)->result =
-						bsg_reply->result = -EIO;
-					} else {
-						ql_dbg(ql_dbg_user, vha, 0x708a,
-						    "mbx abort_command "
-						    "success.\n");
-						scsi_req(bsg_job->req)->result =
-						bsg_reply->result = 0;
-					}
-					spin_lock_irqsave(&ha->hardware_lock, flags);
-					goto done;
+			u = req->outstanding_cmds[cnt];
+			if (!u || u->cmd_type != TYPE_SRB)
+				continue;
+
+			sp = &u->srb;
+			if (((sp->type == SRB_CT_CMD) ||
+			    (sp->type == SRB_ELS_CMD_HST) ||
+			    (sp->type == SRB_FXIOCB_BCMD))
+				&& (sp->u.bsg_job == bsg_job)) {
+				req->outstanding_cmds[cnt] = NULL;
+				spin_unlock_irqrestore(&ha->hardware_lock, flags);
+				if (ha->isp_ops->abort_command(sp)) {
+					ql_log(ql_log_warn, vha, 0x7089,
+					    "mbx abort_command failed.\n");
+					scsi_req(bsg_job->req)->result =
+					    bsg_reply->result = -EIO;
+				} else {
+					ql_dbg(ql_dbg_user, vha, 0x708a,
+					    "mbx abort_command success.\n");
+					scsi_req(bsg_job->req)->result =
+					    bsg_reply->result = 0;
 				}
+				spin_lock_irqsave(&ha->hardware_lock, flags);
+				goto done;
 			}
 		}
 	}
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index caee4a2b4002..a17042649eab 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -291,6 +291,7 @@ struct name_list_extended {
 
 struct req_que;
 struct qla_tgt_sess;
+struct unify_cmd;
 
 /*
  * SCSI Request Block
@@ -458,18 +459,7 @@ struct srb_iocb {
 #define SRB_NVME_LS	20
 #define SRB_PRLI_CMD	21
 
-enum {
-	TYPE_SRB,
-	TYPE_TGT_CMD,
-};
-
 typedef struct srb {
-	/*
-	 * Do not move cmd_type field, it needs to
-	 * line up with qla_tgt_cmd->cmd_type
-	 */
-	uint8_t cmd_type;
-	uint8_t pad[3];
 	atomic_t ref_count;
 	wait_queue_head_t nvme_ls_waitq;
 	struct fc_port *fcport;
@@ -3286,7 +3276,7 @@ struct req_que {
 	uint16_t  qos;
 	uint16_t  vp_idx;
 	struct rsp_que *rsp;
-	srb_t **outstanding_cmds;
+	struct unify_cmd **outstanding_cmds;
 	uint32_t current_outstanding_cmd;
 	uint16_t num_outstanding_cmds;
 	int max_q_depth;
@@ -4383,6 +4373,23 @@ enum nexus_wait_type {
 	(IS_QLA27XX(_ha) || IS_QLA83XX(_ha)))
 
 #include "qla_target.h"
+
+typedef enum {
+	TYPE_SRB,
+	TYPE_TGT_CMD,
+} cmd_type_t;
+
+struct unify_cmd {
+	cmd_type_t  cmd_type;
+	union {
+		struct srb srb;
+		struct qla_tgt_cmd tcmd;
+	};
+};
+
+#define SRB_TO_U(_p) container_of(_p, struct unify_cmd, srb)
+#define TCMD_TO_U(_p) container_of(_p, struct unify_cmd, tcmd)
+
 #include "qla_gbl.h"
 #include "qla_dbg.h"
 #include "qla_inline.h"
diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c
index 072ad1aa5505..8c6dd13982be 100644
--- a/drivers/scsi/qla2xxx/qla_init.c
+++ b/drivers/scsi/qla2xxx/qla_init.c
@@ -1404,10 +1404,11 @@ qla24xx_async_abort_command(srb_t *sp)
 	struct scsi_qla_host *vha = fcport->vha;
 	struct qla_hw_data *ha = vha->hw;
 	struct req_que *req = vha->req;
+	struct unify_cmd *u = SRB_TO_U(sp);
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	for (handle = 1; handle < req->num_outstanding_cmds; handle++) {
-		if (req->outstanding_cmds[handle] == sp)
+		if (req->outstanding_cmds[handle] == u)
 			break;
 	}
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
@@ -2799,7 +2800,7 @@ qla2x00_alloc_outstanding_cmds(struct qla_hw_data *ha, struct req_que *req)
 			req->num_outstanding_cmds = ha->cur_fw_iocb_count;
 	}
 
-	req->outstanding_cmds = kzalloc(sizeof(srb_t *) *
+	req->outstanding_cmds = kzalloc(sizeof(struct unify_cmd *) *
 	    req->num_outstanding_cmds, GFP_KERNEL);
 
 	if (!req->outstanding_cmds) {
@@ -2808,7 +2809,7 @@ qla2x00_alloc_outstanding_cmds(struct qla_hw_data *ha, struct req_que *req)
 		 * initialization.
 		 */
 		req->num_outstanding_cmds = MIN_OUTSTANDING_COMMANDS;
-		req->outstanding_cmds = kzalloc(sizeof(srb_t *) *
+		req->outstanding_cmds = kzalloc(sizeof(struct unify_cmd *) *
 		    req->num_outstanding_cmds, GFP_KERNEL);
 
 		if (!req->outstanding_cmds) {
diff --git a/drivers/scsi/qla2xxx/qla_inline.h b/drivers/scsi/qla2xxx/qla_inline.h
index 9a2c86eacf44..a2c29334a5bd 100644
--- a/drivers/scsi/qla2xxx/qla_inline.h
+++ b/drivers/scsi/qla2xxx/qla_inline.h
@@ -206,6 +206,7 @@ qla2x00_reset_active(scsi_qla_host_t *vha)
 static inline srb_t *
 qla2xxx_get_qpair_sp(struct qla_qpair *qpair, fc_port_t *fcport, gfp_t flag)
 {
+	struct unify_cmd *u;
 	srb_t *sp = NULL;
 	uint8_t bail;
 
@@ -213,11 +214,13 @@ qla2xxx_get_qpair_sp(struct qla_qpair *qpair, fc_port_t *fcport, gfp_t flag)
 	if (unlikely(bail))
 		return NULL;
 
-	sp = mempool_alloc(qpair->srb_mempool, flag);
-	if (!sp)
+	u = mempool_alloc(qpair->srb_mempool, flag);
+	if (!u)
 		goto done;
 
-	memset(sp, 0, sizeof(*sp));
+	memset(u, 0, sizeof(*u));
+	u->cmd_type = TYPE_SRB;
+	sp = &u->srb;
 	sp->fcport = fcport;
 	sp->iocbs = 1;
 	sp->vha = qpair->vha;
@@ -230,7 +233,7 @@ qla2xxx_get_qpair_sp(struct qla_qpair *qpair, fc_port_t *fcport, gfp_t flag)
 static inline void
 qla2xxx_rel_qpair_sp(struct qla_qpair *qpair, srb_t *sp)
 {
-	mempool_free(sp, qpair->srb_mempool);
+	mempool_free(SRB_TO_U(sp), qpair->srb_mempool);
 	QLA_QPAIR_MARK_NOT_BUSY(qpair);
 }
 
@@ -238,19 +241,21 @@ static inline srb_t *
 qla2x00_get_sp(scsi_qla_host_t *vha, fc_port_t *fcport, gfp_t flag)
 {
 	srb_t *sp = NULL;
+	struct unify_cmd *u;
 	uint8_t bail;
 
 	QLA_VHA_MARK_BUSY(vha, bail);
 	if (unlikely(bail))
 		return NULL;
 
-	sp = mempool_alloc(vha->hw->srb_mempool, flag);
-	if (!sp)
+	u = mempool_alloc(vha->hw->srb_mempool, flag);
+	if (!u)
 		goto done;
 
-	memset(sp, 0, sizeof(*sp));
+	memset(u, 0, sizeof(*u));
+	u->cmd_type = TYPE_SRB;
+	sp = &u->srb;
 	sp->fcport = fcport;
-	sp->cmd_type = TYPE_SRB;
 	sp->iocbs = 1;
 	sp->vha = vha;
 done:
@@ -263,7 +268,7 @@ static inline void
 qla2x00_rel_sp(srb_t *sp)
 {
 	QLA_VHA_MARK_NOT_BUSY(sp->vha);
-	mempool_free(sp, sp->vha->hw->srb_mempool);
+	mempool_free(SRB_TO_U(sp), sp->vha->hw->srb_mempool);
 }
 
 static inline void
diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c
index a36c485fae50..838b5e338701 100644
--- a/drivers/scsi/qla2xxx/qla_iocb.c
+++ b/drivers/scsi/qla2xxx/qla_iocb.c
@@ -384,7 +384,7 @@ qla2x00_start_scsi(srb_t *sp)
 
 	/* Build command packet */
 	req->current_outstanding_cmd = handle;
-	req->outstanding_cmds[handle] = sp;
+	req->outstanding_cmds[handle] = SRB_TO_U(sp);
 	sp->handle = handle;
 	cmd->host_scribble = (unsigned char *)(unsigned long)handle;
 	req->cnt -= req_cnt;
@@ -1474,7 +1474,7 @@ qla24xx_start_scsi(srb_t *sp)
 
 	/* Build command packet. */
 	req->current_outstanding_cmd = handle;
-	req->outstanding_cmds[handle] = sp;
+	req->outstanding_cmds[handle] = SRB_TO_U(sp);
 	sp->handle = handle;
 	cmd->host_scribble = (unsigned char *)(unsigned long)handle;
 	req->cnt -= req_cnt;
@@ -1677,7 +1677,7 @@ qla24xx_dif_start_scsi(srb_t *sp)
 
 	/* Build header part of command packet (excluding the OPCODE). */
 	req->current_outstanding_cmd = handle;
-	req->outstanding_cmds[handle] = sp;
+	req->outstanding_cmds[handle] = SRB_TO_U(sp);
 	sp->handle = handle;
 	cmd->host_scribble = (unsigned char *)(unsigned long)handle;
 	req->cnt -= req_cnt;
@@ -1827,7 +1827,7 @@ qla2xxx_start_scsi_mq(srb_t *sp)
 
 	/* Build command packet. */
 	req->current_outstanding_cmd = handle;
-	req->outstanding_cmds[handle] = sp;
+	req->outstanding_cmds[handle] = SRB_TO_U(sp);
 	sp->handle = handle;
 	cmd->host_scribble = (unsigned char *)(unsigned long)handle;
 	req->cnt -= req_cnt;
@@ -2044,7 +2044,7 @@ qla2xxx_dif_start_scsi_mq(srb_t *sp)
 
 	/* Build header part of command packet (excluding the OPCODE). */
 	req->current_outstanding_cmd = handle;
-	req->outstanding_cmds[handle] = sp;
+	req->outstanding_cmds[handle] = SRB_TO_U(sp);
 	sp->handle = handle;
 	cmd->host_scribble = (unsigned char *)(unsigned long)handle;
 	req->cnt -= req_cnt;
@@ -2148,7 +2148,7 @@ __qla2x00_alloc_iocbs(struct qla_qpair *qpair, srb_t *sp)
 
 	/* Prep command array. */
 	req->current_outstanding_cmd = handle;
-	req->outstanding_cmds[handle] = sp;
+	req->outstanding_cmds[handle] = SRB_TO_U(sp);
 	sp->handle = handle;
 
 	/* Adjust entry-counts as needed. */
@@ -3033,7 +3033,7 @@ qla82xx_start_scsi(srb_t *sp)
 	}
 	/* Build command packet. */
 	req->current_outstanding_cmd = handle;
-	req->outstanding_cmds[handle] = sp;
+	req->outstanding_cmds[handle] = SRB_TO_U(sp);
 	sp->handle = handle;
 	cmd->host_scribble = (unsigned char *)(unsigned long)handle;
 	req->cnt -= req_cnt;
@@ -3459,7 +3459,7 @@ qla2x00_start_bidir(srb_t *sp, struct scsi_qla_host *vha, uint32_t tot_dsds)
 	cmd_pkt->entry_status = (uint8_t) rsp->id;
 	/* Build command packet. */
 	req->current_outstanding_cmd = handle;
-	req->outstanding_cmds[handle] = sp;
+	req->outstanding_cmds[handle] = SRB_TO_U(sp);
 	sp->handle = handle;
 	req->cnt -= req_cnt;
 
diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
index c14fab35fc36..fba8bfdd1627 100644
--- a/drivers/scsi/qla2xxx/qla_isr.c
+++ b/drivers/scsi/qla2xxx/qla_isr.c
@@ -1251,6 +1251,7 @@ qla2x00_process_completed_request(struct scsi_qla_host *vha,
 {
 	srb_t *sp;
 	struct qla_hw_data *ha = vha->hw;
+	struct unify_cmd *u;
 
 	/* Validate handle. */
 	if (index >= req->num_outstanding_cmds) {
@@ -1264,8 +1265,9 @@ qla2x00_process_completed_request(struct scsi_qla_host *vha,
 		return;
 	}
 
-	sp = req->outstanding_cmds[index];
-	if (sp) {
+	u = req->outstanding_cmds[index];
+	if (u) {
+		sp = &u->srb;
 		/* Free outstanding command slot. */
 		req->outstanding_cmds[index] = NULL;
 
@@ -1289,6 +1291,7 @@ qla2x00_get_sp_from_handle(scsi_qla_host_t *vha, const char *func,
 	sts_entry_t *pkt = iocb;
 	srb_t *sp = NULL;
 	uint16_t index;
+	struct unify_cmd *u;
 
 	index = LSW(pkt->handle);
 	if (index >= req->num_outstanding_cmds) {
@@ -1301,12 +1304,14 @@ qla2x00_get_sp_from_handle(scsi_qla_host_t *vha, const char *func,
 			set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
 		goto done;
 	}
-	sp = req->outstanding_cmds[index];
-	if (!sp) {
+	u = req->outstanding_cmds[index];
+	if (!u) {
 		ql_log(ql_log_warn, vha, 0x5032,
 		    "Invalid completion handle (%x) -- timed-out.\n", index);
-		return sp;
+		return NULL;
 	}
+	sp = &u->srb;
+
 	if (sp->handle != index) {
 		ql_log(ql_log_warn, vha, 0x5033,
 		    "SRB handle (%x) mismatch %x.\n", sp->handle, index);
@@ -2165,6 +2170,7 @@ qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt,
 	struct fc_bsg_reply *bsg_reply;
 	sts_entry_t *sts;
 	struct sts_entry_24xx *sts24;
+	struct unify_cmd *u;
 	sts = (sts_entry_t *) pkt;
 	sts24 = (struct sts_entry_24xx *) pkt;
 
@@ -2176,8 +2182,8 @@ qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt,
 		return;
 	}
 
-	sp = req->outstanding_cmds[index];
-	if (!sp) {
+	u = req->outstanding_cmds[index];
+	if (!u) {
 		ql_log(ql_log_warn, vha, 0x70b0,
 		    "Req:%d: Invalid ISP SCSI completion handle(0x%x)\n",
 		    req->id, index);
@@ -2188,6 +2194,7 @@ qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt,
 
 	/* Free outstanding command slot. */
 	req->outstanding_cmds[index] = NULL;
+	sp = &u->srb;
 	bsg_job = sp->u.bsg_job;
 	bsg_request = bsg_job->request;
 	bsg_reply = bsg_job->reply;
@@ -2330,6 +2337,7 @@ qla2x00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt)
 	uint16_t state_flags = 0;
 	uint16_t retry_delay = 0;
 	uint8_t no_logout = 0;
+	struct unify_cmd *u;
 
 	sts = (sts_entry_t *) pkt;
 	sts24 = (struct sts_entry_24xx *) pkt;
@@ -2356,8 +2364,8 @@ qla2x00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt)
 
 	/* Validate handle. */
 	if (handle < req->num_outstanding_cmds) {
-		sp = req->outstanding_cmds[handle];
-		if (!sp) {
+		u = req->outstanding_cmds[handle];
+		if (!u) {
 			ql_dbg(ql_dbg_io, vha, 0x3075,
 			    "%s(%ld): Already returned command for status handle (0x%x).\n",
 			    __func__, vha->host_no, sts->handle);
@@ -2378,14 +2386,15 @@ qla2x00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt)
 		return;
 	}
 
-	if (sp->cmd_type != TYPE_SRB) {
+	if (u->cmd_type != TYPE_SRB) {
 		req->outstanding_cmds[handle] = NULL;
 		ql_dbg(ql_dbg_io, vha, 0x3015,
 		    "Unknown sp->cmd_type %x %p).\n",
-		    sp->cmd_type, sp);
+		    u->cmd_type, u);
 		return;
 	}
 
+	sp = &u->srb;
 	/* NVME completion. */
 	if (sp->type == SRB_NVME_CMD) {
 		qla24xx_nvme_iocb_entry(vha, req, pkt);
diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c
index b271c3f78c85..582896c37af7 100644
--- a/drivers/scsi/qla2xxx/qla_mbx.c
+++ b/drivers/scsi/qla2xxx/qla_mbx.c
@@ -1306,7 +1306,8 @@ qla2x00_abort_command(srb_t *sp)
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	for (handle = 1; handle < req->num_outstanding_cmds; handle++) {
-		if (req->outstanding_cmds[handle] == sp)
+		struct unify_cmd *u = SRB_TO_U(sp);
+		if (req->outstanding_cmds[handle] == u)
 			break;
 	}
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
@@ -2923,7 +2924,8 @@ qla24xx_abort_command(srb_t *sp)
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	for (handle = 1; handle < req->num_outstanding_cmds; handle++) {
-		if (req->outstanding_cmds[handle] == sp)
+		struct unify_cmd *u = SRB_TO_U(sp);
+		if (req->outstanding_cmds[handle] == u)
 			break;
 	}
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
diff --git a/drivers/scsi/qla2xxx/qla_mr.c b/drivers/scsi/qla2xxx/qla_mr.c
index 10b742d27e16..fe5b990a88ad 100644
--- a/drivers/scsi/qla2xxx/qla_mr.c
+++ b/drivers/scsi/qla2xxx/qla_mr.c
@@ -2292,7 +2292,7 @@ qlafx00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt)
 	struct req_que *req;
 	int logit = 1;
 	int res = 0;
-
+	struct unify_cmd *u;
 	sts = (struct sts_entry_fx00 *) pkt;
 
 	comp_status = sts->comp_status;
@@ -2305,11 +2305,11 @@ qlafx00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt)
 
 	/* Validate handle. */
 	if (handle < req->num_outstanding_cmds)
-		sp = req->outstanding_cmds[handle];
+		u = req->outstanding_cmds[handle];
 	else
-		sp = NULL;
+		u = NULL;
 
-	if (sp == NULL) {
+	if (u == NULL) {
 		ql_dbg(ql_dbg_io, vha, 0x3034,
 		    "Invalid status handle (0x%x).\n", handle);
 
@@ -2317,6 +2317,7 @@ qlafx00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt)
 		qla2xxx_wake_dpc(vha);
 		return;
 	}
+	sp = &u->srb;
 
 	if (sp->type == SRB_TM_CMD) {
 		req->outstanding_cmds[handle] = NULL;
@@ -2626,7 +2627,7 @@ static void
 qlafx00_multistatus_entry(struct scsi_qla_host *vha,
 	struct rsp_que *rsp, void *pkt)
 {
-	srb_t		*sp;
+	struct unify_cmd *u;
 	struct multi_sts_entry_fx00 *stsmfx;
 	struct qla_hw_data *ha = vha->hw;
 	uint32_t handle, hindex, handle_count, i;
@@ -2656,11 +2657,11 @@ qlafx00_multistatus_entry(struct scsi_qla_host *vha,
 
 		/* Validate handle. */
 		if (handle < req->num_outstanding_cmds)
-			sp = req->outstanding_cmds[handle];
+			u = req->outstanding_cmds[handle];
 		else
-			sp = NULL;
+			u = NULL;
 
-		if (sp == NULL) {
+		if (u == NULL) {
 			ql_dbg(ql_dbg_io, vha, 0x3044,
 			    "Invalid status handle (0x%x).\n", handle);
 			set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
@@ -3134,7 +3135,7 @@ qlafx00_start_scsi(srb_t *sp)
 
 	/* Build command packet. */
 	req->current_outstanding_cmd = handle;
-	req->outstanding_cmds[handle] = sp;
+	req->outstanding_cmds[handle] = SRB_TO_U(sp);
 	sp->handle = handle;
 	cmd->host_scribble = (unsigned char *)(unsigned long)handle;
 	req->cnt -= req_cnt;
diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c
index da32a06f17a0..faa1b4a40f42 100644
--- a/drivers/scsi/qla2xxx/qla_nvme.c
+++ b/drivers/scsi/qla2xxx/qla_nvme.c
@@ -377,7 +377,7 @@ static int qla2x00_start_nvme_mq(srb_t *sp)
 
 	/* Build command packet. */
 	req->current_outstanding_cmd = handle;
-	req->outstanding_cmds[handle] = sp;
+	req->outstanding_cmds[handle] = SRB_TO_U(sp);
 	sp->handle = handle;
 	req->cnt -= req_cnt;
 
diff --git a/drivers/scsi/qla2xxx/qla_nx.c b/drivers/scsi/qla2xxx/qla_nx.c
index a77c33987703..8b86c2b3c286 100644
--- a/drivers/scsi/qla2xxx/qla_nx.c
+++ b/drivers/scsi/qla2xxx/qla_nx.c
@@ -3668,6 +3668,7 @@ qla82xx_chip_reset_cleanup(scsi_qla_host_t *vha)
 		int cnt, que;
 		srb_t *sp;
 		struct req_que *req;
+		struct unify_cmd *u;
 
 		spin_lock_irqsave(&ha->hardware_lock, flags);
 		for (que = 0; que < ha->max_req_queues; que++) {
@@ -3675,8 +3676,9 @@ qla82xx_chip_reset_cleanup(scsi_qla_host_t *vha)
 			if (!req)
 				continue;
 			for (cnt = 1; cnt < req->num_outstanding_cmds; cnt++) {
-				sp = req->outstanding_cmds[cnt];
-				if (sp) {
+				u = req->outstanding_cmds[cnt];
+				if (u) {
+					sp = &u->srb;
 					if ((!sp->u.scmd.ctx ||
 					    (sp->flags &
 						SRB_FCP_CMND_DMA_VALID)) &&
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index 256d3a34d4ee..45482c9c73b0 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -1335,6 +1335,7 @@ qla2x00_eh_wait_for_pending_commands(scsi_qla_host_t *vha, unsigned int t,
 	struct req_que *req;
 	srb_t *sp;
 	struct scsi_cmnd *cmd;
+	struct unify_cmd *u;
 
 	status = QLA_SUCCESS;
 
@@ -1342,9 +1343,11 @@ qla2x00_eh_wait_for_pending_commands(scsi_qla_host_t *vha, unsigned int t,
 	req = vha->req;
 	for (cnt = 1; status == QLA_SUCCESS &&
 		cnt < req->num_outstanding_cmds; cnt++) {
-		sp = req->outstanding_cmds[cnt];
-		if (!sp)
+		u = req->outstanding_cmds[cnt];
+		if (!u)
 			continue;
+
+		sp = &u->srb;
 		if (sp->type != SRB_SCSI_CMD)
 			continue;
 		if (vha->vp_idx != sp->vha->vp_idx)
@@ -1697,6 +1700,7 @@ qla2x00_abort_all_cmds(scsi_qla_host_t *vha, int res)
 	int que, cnt, status;
 	unsigned long flags;
 	srb_t *sp;
+	struct unify_cmd *u;
 	struct qla_hw_data *ha = vha->hw;
 	struct req_que *req;
 	struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
@@ -1711,10 +1715,11 @@ qla2x00_abort_all_cmds(scsi_qla_host_t *vha, int res)
 		if (!req->outstanding_cmds)
 			continue;
 		for (cnt = 1; cnt < req->num_outstanding_cmds; cnt++) {
-			sp = req->outstanding_cmds[cnt];
-			if (sp) {
+			u = req->outstanding_cmds[cnt];
+			if (u) {
 				req->outstanding_cmds[cnt] = NULL;
-				if (sp->cmd_type == TYPE_SRB) {
+				if (u->cmd_type == TYPE_SRB) {
+					sp = &u->srb;
 					if (sp->type == SRB_NVME_CMD ||
 					    sp->type == SRB_NVME_LS) {
 						sp_get(sp);
@@ -1768,7 +1773,7 @@ qla2x00_abort_all_cmds(scsi_qla_host_t *vha, int res)
 							    vha->dpc_flags);
 						continue;
 					}
-					cmd = (struct qla_tgt_cmd *)sp;
+					cmd = &u->tcmd;
 					qlt_abort_cmd_on_host_reset(cmd->vha,
 					    cmd);
 				}
@@ -5926,6 +5931,7 @@ qla2x00_timer(scsi_qla_host_t *vha)
 	uint16_t        w;
 	struct qla_hw_data *ha = vha->hw;
 	struct req_que *req;
+	struct unify_cmd *u;
 
 	if (ha->flags.eeh_busy) {
 		ql_dbg(ql_dbg_timer, vha, 0x6000,
@@ -5986,11 +5992,12 @@ qla2x00_timer(scsi_qla_host_t *vha)
 				    index++) {
 					fc_port_t *sfcp;
 
-					sp = req->outstanding_cmds[index];
-					if (!sp)
+					u = req->outstanding_cmds[index];
+					if (!u)
 						continue;
-					if (sp->cmd_type != TYPE_SRB)
+					if (u->cmd_type != TYPE_SRB)
 						continue;
+					sp = &u->srb;
 					if (sp->type != SRB_SCSI_CMD)
 						continue;
 					sfcp = sp->fcport;
@@ -6571,8 +6578,8 @@ qla2x00_module_init(void)
 	int ret = 0;
 
 	/* Allocate cache for SRBs. */
-	srb_cachep = kmem_cache_create("qla2xxx_srbs", sizeof(srb_t), 0,
-	    SLAB_HWCACHE_ALIGN, NULL);
+	srb_cachep = kmem_cache_create("qla2xxx_srbs", sizeof(struct unify_cmd),
+	    0, SLAB_HWCACHE_ALIGN, NULL);
 	if (srb_cachep == NULL) {
 		ql_log(ql_log_fatal, NULL, 0x0001,
 		    "Unable to allocate SRB cache...Failing load!.\n");
diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c
index e101cd3043b9..0aa0088f2988 100644
--- a/drivers/scsi/qla2xxx/qla_target.c
+++ b/drivers/scsi/qla2xxx/qla_target.c
@@ -2379,7 +2379,7 @@ static int qlt_24xx_build_ctio_pkt(struct qla_qpair *qpair,
 		 */
 		return -EAGAIN;
 	} else
-		qpair->req->outstanding_cmds[h] = (srb_t *)prm->cmd;
+		qpair->req->outstanding_cmds[h] = TCMD_TO_U(prm->cmd);
 
 	pkt->handle = MAKE_HANDLE(qpair->req->id, h);
 	pkt->handle |= CTIO_COMPLETION_HANDLE_MARK;
@@ -2908,7 +2908,7 @@ qlt_build_ctio_crc2_pkt(struct qla_qpair *qpair, struct qla_tgt_prm *prm)
 		 */
 		return -EAGAIN;
 	} else
-		qpair->req->outstanding_cmds[h] = (srb_t *)prm->cmd;
+		qpair->req->outstanding_cmds[h] = TCMD_TO_U(prm->cmd);
 
 	pkt->handle  = MAKE_HANDLE(qpair->req->id, h);
 	pkt->handle |= CTIO_COMPLETION_HANDLE_MARK;
@@ -3709,6 +3709,7 @@ static struct qla_tgt_cmd *qlt_ctio_to_cmd(struct scsi_qla_host *vha,
 	struct req_que *req;
 	int qid = GET_QID(handle);
 	uint32_t h = handle & ~QLA_TGT_HANDLE_MASK;
+	struct unify_cmd *u;
 
 	if (unlikely(h == QLA_TGT_SKIP_HANDLE))
 		return NULL;
@@ -3734,13 +3735,14 @@ static struct qla_tgt_cmd *qlt_ctio_to_cmd(struct scsi_qla_host *vha,
 			return NULL;
 		}
 
-		cmd = (struct qla_tgt_cmd *)req->outstanding_cmds[h];
-		if (unlikely(cmd == NULL)) {
+		u = req->outstanding_cmds[h];
+		if (unlikely(u == NULL)) {
 			ql_dbg(ql_dbg_async, vha, 0xe053,
 			    "qla_target(%d): Suspicious: unable to find the command with handle %x req->id %d rsp->id %d\n",
 				vha->vp_idx, handle, req->id, rsp->id);
 			return NULL;
 		}
+		cmd = &u->tcmd;
 		req->outstanding_cmds[h] = NULL;
 	} else if (ctio != NULL) {
 		/* We can't get loop ID from CTIO7 */
@@ -4161,14 +4163,16 @@ static struct qla_tgt_cmd *qlt_get_tag(scsi_qla_host_t *vha,
 	struct se_session *se_sess = sess->se_sess;
 	struct qla_tgt_cmd *cmd;
 	int tag;
+	struct unify_cmd *u;
 
 	tag = percpu_ida_alloc(&se_sess->sess_tag_pool, TASK_RUNNING);
 	if (tag < 0)
 		return NULL;
 
-	cmd = &((struct qla_tgt_cmd *)se_sess->sess_cmd_map)[tag];
-	memset(cmd, 0, sizeof(struct qla_tgt_cmd));
-	cmd->cmd_type = TYPE_TGT_CMD;
+	u = &((struct unify_cmd *)se_sess->sess_cmd_map)[tag];
+	memset(u, 0, sizeof(*u));
+	u->cmd_type = TYPE_TGT_CMD;
+	cmd = &u->tcmd;
 	memcpy(&cmd->atio, atio, sizeof(*atio));
 	cmd->state = QLA_TGT_STATE_NEW;
 	cmd->tgt = vha->vha_tgt.qla_tgt;
diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h
index 7fe02d036bdf..b2032a4efbc2 100644
--- a/drivers/scsi/qla2xxx/qla_target.h
+++ b/drivers/scsi/qla2xxx/qla_target.h
@@ -872,11 +872,6 @@ enum trace_flags {
 };
 
 struct qla_tgt_cmd {
-	/*
-	 * Do not move cmd_type field. it needs to line up with srb->cmd_type
-	 */
-	uint8_t cmd_type;
-	uint8_t pad[7];
 	struct se_cmd se_cmd;
 	struct fc_port *sess;
 	struct qla_qpair *qpair;
-- 
2.12.0

Re: [PATCH 1/7] qla2xxx: Cleanup NVMe code.

[Bart Van Assche]

On Wed, 2017-07-19 at 11:51 -0700, Himanshu Madhani wrote:
> This patch cleaned up code to make it more readable

Hello Himanshu,

"cleaned up code" is a very vague description. Please be more specific and
please also mention in the patch description what functionality has been
changed (if any).

Bart.

Re: [PATCH 3/7] qla2xxx: Add command completion wq for error path

[Bart Van Assche]

On Wed, 2017-07-19 at 11:51 -0700, Himanshu Madhani wrote:
> From: Duane Grigsby <duane.grigsby@cavium.com>

Hello Himanshu and Duane,

Too many drivers create workqueues for all kinds of purposes. Why is it
necessary to execute qla_nvme_io_work() on the context of a new workqueue?
If any of the existing system workqueues can be used please modify the patch
accordingly. Otherwise please explain why it is necessary to introduce a new
workqueue.

Bart.

Re: [PATCH 1/7] qla2xxx: Cleanup NVMe code.

[Madhani, Himanshu]

Hi Bart, 

> On Jul 19, 2017, at 1:44 PM, Bart Van Assche <Bart.VanAssche@wdc.com> wrote:
> 
> On Wed, 2017-07-19 at 11:51 -0700, Himanshu Madhani wrote:
>> This patch cleaned up code to make it more readable
> 
> Hello Himanshu,
> 
> "cleaned up code" is a very vague description. Please be more specific and
> please also mention in the patch description what functionality has been
> changed (if any).
> 
> Bart.

There is no functionality change in this patch, it addresses comments to make code readable. 

for example, 

Changing waitQ -> wait.
Combined multiple debug statements into single 
Drop extra parenthesis in if() statements

I’ll add this into commit message for more clarity.

Thanks,
- Himanshu

Re: [PATCH 1/7] qla2xxx: Cleanup NVMe code.

[Johannes Thumshirn]

On Wed, Jul 19, 2017 at 08:44:13PM +0000, Bart Van Assche wrote:
> "cleaned up code" is a very vague description. Please be more specific and
> please also mention in the patch description what functionality has been
> changed (if any).

Hi Bart,

These are just some style changes I requested when reviewing the last
patchset. But I agree at least a "no functional changes introduced" would be
very beneficial.

Thanks,
	Johannes
-- 
Johannes Thumshirn                                          Storage
jthumshirn@suse.de                                +49 911 74053 689
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 Nürnberg
GF: Felix Imendörffer, Jane Smithard, Graham Norton
HRB 21284 (AG Nürnberg)
Key fingerprint = EC38 9CAB C2C4 F25D 8600 D0D0 0393 969D 2D76 0850

Re: [PATCH 1/7] qla2xxx: Cleanup NVMe code.

[Johannes Thumshirn]

With an updated changelog,
Reviewed-by: Johannes Thumshirn <jthumshirn@suse.de>

-- 
Johannes Thumshirn                                          Storage
jthumshirn@suse.de                                +49 911 74053 689
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 Nürnberg
GF: Felix Imendörffer, Jane Smithard, Graham Norton
HRB 21284 (AG Nürnberg)
Key fingerprint = EC38 9CAB C2C4 F25D 8600 D0D0 0393 969D 2D76 0850

Re: [PATCH 2/7] qla2xxx: Move function prototype to correct header

[Johannes Thumshirn]

Thanks Himanshu,
Reviewed-by: Johannes Thumshirn <jthumshirn@suse.de>
-- 
Johannes Thumshirn                                          Storage
jthumshirn@suse.de                                +49 911 74053 689
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 Nürnberg
GF: Felix Imendörffer, Jane Smithard, Graham Norton
HRB 21284 (AG Nürnberg)
Key fingerprint = EC38 9CAB C2C4 F25D 8600 D0D0 0393 969D 2D76 0850

Re: [PATCH 4/7] qla2xxx: Added change to enable ZIO for FC-NVMe devices

[Johannes Thumshirn]

On Wed, Jul 19, 2017 at 11:51:48AM -0700, Himanshu Madhani wrote:
> +	mcp->out_mb = MBX_2|MBX_1|MBX_0;
> +	mcp->in_mb = MBX_2|MBX_0;

Please add spaces around the '|'. Checkpatch should've warned you about that.

> +	mcp->tov = MBX_TOV_SECONDS;
> +	mcp->flags = 0;
> +
> +	rval = qla2x00_mailbox_command(vha, mcp);
> +	if (rval != QLA_SUCCESS) {
> +		ql_dbg(ql_dbg_mbx, vha, 0x1201, "Failed=%x.\n", rval);
> +	} else {
> +		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1202,
> +		    "Done %s.\n", __func__);
> +	}

In patch 1 you change all the 
if (rval != QLA_SUCCESS) {
	qla_dbg();
} else {
	qla_dbg();
}
into a single qla_dbg. Now you re-introduce the pattern here.


> +int qla27xx_get_zio_threshold(scsi_qla_host_t *vha, uint16_t *value)

See above comment

Thanks,	
	Johannes
-- 
Johannes Thumshirn                                          Storage
jthumshirn@suse.de                                +49 911 74053 689
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 Nürnberg
GF: Felix Imendörffer, Jane Smithard, Graham Norton
HRB 21284 (AG Nürnberg)
Key fingerprint = EC38 9CAB C2C4 F25D 8600 D0D0 0393 969D 2D76 0850

Re: [PATCH 5/7] qla2xxx: Simpify unregistration of FC-NVMe local/remote ports

[Johannes Thumshirn]

On Wed, Jul 19, 2017 at 11:51:49AM -0700, Himanshu Madhani wrote:
> -static void qla_nvme_abort_all(fc_port_t *fcport)
> -{
> -	int que, cnt;
> -	unsigned long flags;
> -	srb_t *sp;
> -	struct qla_hw_data *ha = fcport->vha->hw;
> -	struct req_que *req;
> -
> -	spin_lock_irqsave(&ha->hardware_lock, flags);
> -	for (que = 0; que < ha->max_req_queues; que++) {
> -		req = ha->req_q_map[que];
> -		if (!req)
> -			continue;
> -		if (!req->outstanding_cmds)
> -			continue;
> -		for (cnt = 1; cnt < req->num_outstanding_cmds; cnt++) {
> -			sp = req->outstanding_cmds[cnt];
> -			if ((sp) && ((sp->type == SRB_NVME_CMD) ||
> -			    (sp->type == SRB_NVME_LS)) &&
> -				(sp->fcport == fcport)) {
> -				atomic_inc(&sp->ref_count);
> -				spin_unlock_irqrestore(&ha->hardware_lock,
> -				    flags);
> -				qla_nvme_abort(ha, sp);
> -				spin_lock_irqsave(&ha->hardware_lock, flags);
> -				req->outstanding_cmds[cnt] = NULL;
> -				sp->done(sp, 1);
> -			}
> -		}
> -	}
> -	spin_unlock_irqrestore(&ha->hardware_lock, flags);
> -}
> -
>  static void qla_nvme_unregister_remote_port(struct work_struct *work)
>  {
>  	struct fc_port *fcport = container_of(work, struct fc_port,
> @@ -719,12 +679,13 @@ void qla_nvme_delete(struct scsi_qla_host *vha)
>  		ql_log(ql_log_info, fcport->vha, 0x2114, "%s: fcport=%p\n",
>  		    __func__, fcport);
>  
> +		init_completion(&fcport->nvme_del_done);
>  		nvme_fc_unregister_remoteport(fcport->nvme_remote_port);
>  		qla_nvme_wait_on_rport_del(fcport);
> -		qla_nvme_abort_all(fcport);

What changed the need to abort all outstanding commands?

-- 
Johannes Thumshirn                                          Storage
jthumshirn@suse.de                                +49 911 74053 689
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 Nürnberg
GF: Felix Imendörffer, Jane Smithard, Graham Norton
HRB 21284 (AG Nürnberg)
Key fingerprint = EC38 9CAB C2C4 F25D 8600 D0D0 0393 969D 2D76 0850

Re: [PATCH 6/7] qla2xxx: Fix remoteport disconnect for FC-NVMe

[Johannes Thumshirn]

Looks good,
Reviewed-by: Johannes Thumshirn <jthumshirn@suse.de>
-- 
Johannes Thumshirn                                          Storage
jthumshirn@suse.de                                +49 911 74053 689
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 Nürnberg
GF: Felix Imendörffer, Jane Smithard, Graham Norton
HRB 21284 (AG Nürnberg)
Key fingerprint = EC38 9CAB C2C4 F25D 8600 D0D0 0393 969D 2D76 0850

Re: [PATCH 4/7] qla2xxx: Added change to enable ZIO for FC-NVMe devices

[Madhani, Himanshu]

> On Jul 20, 2017, at 12:00 AM, Johannes Thumshirn <jthumshirn@suse.de> wrote:
> 
> On Wed, Jul 19, 2017 at 11:51:48AM -0700, Himanshu Madhani wrote:
>> +	mcp->out_mb = MBX_2|MBX_1|MBX_0;
>> +	mcp->in_mb = MBX_2|MBX_0;
> 
> Please add spaces around the '|'. Checkpatch should've warned you about that.
> 

Sure. 

>> +	mcp->tov = MBX_TOV_SECONDS;
>> +	mcp->flags = 0;
>> +
>> +	rval = qla2x00_mailbox_command(vha, mcp);
>> +	if (rval != QLA_SUCCESS) {
>> +		ql_dbg(ql_dbg_mbx, vha, 0x1201, "Failed=%x.\n", rval);
>> +	} else {
>> +		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1202,
>> +		    "Done %s.\n", __func__);
>> +	}
> 
> In patch 1 you change all the 
> if (rval != QLA_SUCCESS) {
> 	qla_dbg();
> } else {
> 	qla_dbg();
> }
> into a single qla_dbg. Now you re-introduce the pattern here.
> 

Good catch.. somehow this missed my eye. I’ll fix in v2 of this series. 

> 
>> +int qla27xx_get_zio_threshold(scsi_qla_host_t *vha, uint16_t *value)
> 
> See above comment
> 
> Thanks,	
> 	Johannes
> -- 
> Johannes Thumshirn                                          Storage
> jthumshirn@suse.de                                +49 911 74053 689
> SUSE LINUX GmbH, Maxfeldstr. 5, 90409 Nürnberg
> GF: Felix Imendörffer, Jane Smithard, Graham Norton
> HRB 21284 (AG Nürnberg)
> Key fingerprint = EC38 9CAB C2C4 F25D 8600 D0D0 0393 969D 2D76 0850

Thanks,
- Himanshu

Re: [PATCH 3/7] qla2xxx: Add command completion wq for error path

[Madhani, Himanshu]

Hi Bart, 


> On Jul 19, 2017, at 1:46 PM, Bart Van Assche <bart.vanassche@wdc.com> wrote:
> 
> On Wed, 2017-07-19 at 11:51 -0700, Himanshu Madhani wrote:
>> From: Duane Grigsby <duane.grigsby@cavium.com>
> 
> Hello Himanshu and Duane,
> 
> Too many drivers create workqueues for all kinds of purposes. Why is it
> necessary to execute qla_nvme_io_work() on the context of a new workqueue?
> If any of the existing system workqueues can be used please modify the patch
> accordingly. Otherwise please explain why it is necessary to introduce a new
> workqueue.
> 
> Bart.

Good point. We went with this approach so that we could release qpair lock on a
particular queue when we get error. however, we’ll rework this patch and submit
it again to use one of already existing work queue.

Thanks,
- Himanshu

Re: [PATCH 5/7] qla2xxx: Simpify unregistration of FC-NVMe local/remote ports

[Madhani, Himanshu]

Hi Johannes, 

> On Jul 20, 2017, at 12:03 AM, Johannes Thumshirn <jthumshirn@suse.de> wrote:
> 
> On Wed, Jul 19, 2017 at 11:51:49AM -0700, Himanshu Madhani wrote:
>> -static void qla_nvme_abort_all(fc_port_t *fcport)
>> -{
>> -	int que, cnt;
>> -	unsigned long flags;
>> -	srb_t *sp;
>> -	struct qla_hw_data *ha = fcport->vha->hw;
>> -	struct req_que *req;
>> -
>> -	spin_lock_irqsave(&ha->hardware_lock, flags);
>> -	for (que = 0; que < ha->max_req_queues; que++) {
>> -		req = ha->req_q_map[que];
>> -		if (!req)
>> -			continue;
>> -		if (!req->outstanding_cmds)
>> -			continue;
>> -		for (cnt = 1; cnt < req->num_outstanding_cmds; cnt++) {
>> -			sp = req->outstanding_cmds[cnt];
>> -			if ((sp) && ((sp->type == SRB_NVME_CMD) ||
>> -			    (sp->type == SRB_NVME_LS)) &&
>> -				(sp->fcport == fcport)) {
>> -				atomic_inc(&sp->ref_count);
>> -				spin_unlock_irqrestore(&ha->hardware_lock,
>> -				    flags);
>> -				qla_nvme_abort(ha, sp);
>> -				spin_lock_irqsave(&ha->hardware_lock, flags);
>> -				req->outstanding_cmds[cnt] = NULL;
>> -				sp->done(sp, 1);
>> -			}
>> -		}
>> -	}
>> -	spin_unlock_irqrestore(&ha->hardware_lock, flags);
>> -}
>> -
>> static void qla_nvme_unregister_remote_port(struct work_struct *work)
>> {
>> 	struct fc_port *fcport = container_of(work, struct fc_port,
>> @@ -719,12 +679,13 @@ void qla_nvme_delete(struct scsi_qla_host *vha)
>> 		ql_log(ql_log_info, fcport->vha, 0x2114, "%s: fcport=%p\n",
>> 		    __func__, fcport);
>> 
>> +		init_completion(&fcport->nvme_del_done);
>> 		nvme_fc_unregister_remoteport(fcport->nvme_remote_port);
>> 		qla_nvme_wait_on_rport_del(fcport);
>> -		qla_nvme_abort_all(fcport);
> 
> What changed the need to abort all outstanding commands?
> 

FC NVMe transport now handles _abort_ so we do not need to call qla_nvme_abort_all()

> -- 
> Johannes Thumshirn                                          Storage
> jthumshirn@suse.de                                +49 911 74053 689
> SUSE LINUX GmbH, Maxfeldstr. 5, 90409 Nürnberg
> GF: Felix Imendörffer, Jane Smithard, Graham Norton
> HRB 21284 (AG Nürnberg)
> Key fingerprint = EC38 9CAB C2C4 F25D 8600 D0D0 0393 969D 2D76 0850

Thanks,
- Himanshu