[PATCH 1/9] scsi: Add helper code so transport classes/driver can control queueing

[PATCH 1/9] scsi: Add helper code so transport classes/driver can control queueing

[michaelc]

From: Mike Christie <michaelc@cs.wisc.edu>

SCSI-ml manages the queueing limits for the device and host, but
does not do so at the target level. However something something similar
can come in userful when a driver is transitioning a transport object to
the the blocked state, becuase at that time we do not want to queue
io and we do not want the queuecommand to be called again.

The patch adds code similar to the exisiting SCSI_ML_*BUSY handlers.
You can now return SCSI_MLQUEUE_TARGET_BUSY when we hit
a transport level queueing issue like the hw cannot allocate some
resource at the iscsi session/connection level, or the target has temporarily
closed or shrunk the queueing window, or if we are transitioning
to the blocked state.

For bnx2i we will also need to be able to limit queueing at this level.
bnx2i will hook into libiscsi, but will allocate a scsi host per
netdevice/hba, so unlike pure software iscsi/iser which is allocating
a host per session, it cannot set the scsi_host->can_queue and return
SCSI_MLQUEUE_HOST_BUSY to reflect queueing limits on the transport.

The iscsi class/driver can also set a scsi_target->can_queue value which
reflects the max commands the driver/class can support. For iscsi this
reflects the number of commands we can support for each session due to
session/connection hw limits, driver limits, and to also reflect the
session/targets's queueing window.

Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
---
 drivers/scsi/scsi.c        |   11 +++++--
 drivers/scsi/scsi_lib.c    |   72 +++++++++++++++++++++++++++++++++++++++-----
 drivers/scsi/scsi_scan.c   |    1 +
 include/scsi/scsi.h        |    1 +
 include/scsi/scsi_device.h |   10 ++++++
 5 files changed, 84 insertions(+), 11 deletions(-)

diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c
index fecba05..a26f976 100644
--- a/drivers/scsi/scsi.c
+++ b/drivers/scsi/scsi.c
@@ -628,9 +628,12 @@ int scsi_dispatch_cmd(struct scsi_cmnd *cmd)
 	if (rtn) {
 		if (scsi_delete_timer(cmd)) {
 			atomic_inc(&cmd->device->iodone_cnt);
-			scsi_queue_insert(cmd,
-					  (rtn == SCSI_MLQUEUE_DEVICE_BUSY) ?
-					  rtn : SCSI_MLQUEUE_HOST_BUSY);
+
+			if (rtn != SCSI_MLQUEUE_DEVICE_BUSY &&
+			    rtn != SCSI_MLQUEUE_TARGET_BUSY)
+				rtn = SCSI_MLQUEUE_HOST_BUSY;
+
+			scsi_queue_insert(cmd, rtn);
 		}
 		SCSI_LOG_MLQUEUE(3,
 		    printk("queuecommand : request rejected\n"));
@@ -729,6 +732,7 @@ static struct scsi_driver *scsi_cmd_to_driver(struct scsi_cmnd *cmd)
 void scsi_finish_command(struct scsi_cmnd *cmd)
 {
 	struct scsi_device *sdev = cmd->device;
+	struct scsi_target *starget = scsi_target(sdev);
 	struct Scsi_Host *shost = sdev->host;
 	struct scsi_driver *drv;
 	unsigned int good_bytes;
@@ -744,6 +748,7 @@ void scsi_finish_command(struct scsi_cmnd *cmd)
 	 * XXX(hch): What about locking?
          */
         shost->host_blocked = 0;
+	starget->target_blocked = 0;
         sdev->device_blocked = 0;
 
 	/*
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index ba21d97..92b4f0e 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -114,6 +114,7 @@ int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
 {
 	struct Scsi_Host *host = cmd->device->host;
 	struct scsi_device *device = cmd->device;
+	struct scsi_target *starget = scsi_target(device);
 	struct request_queue *q = device->request_queue;
 	unsigned long flags;
 
@@ -133,10 +134,17 @@ int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
 	 * if a command is requeued with no other commands outstanding
 	 * either for the device or for the host.
 	 */
-	if (reason == SCSI_MLQUEUE_HOST_BUSY)
+	switch (reason) {
+	case SCSI_MLQUEUE_HOST_BUSY:
 		host->host_blocked = host->max_host_blocked;
-	else if (reason == SCSI_MLQUEUE_DEVICE_BUSY)
+		break;
+	case SCSI_MLQUEUE_DEVICE_BUSY:
 		device->device_blocked = device->max_device_blocked;
+		break;
+	case SCSI_MLQUEUE_TARGET_BUSY:
+		starget->target_blocked = starget->max_target_blocked;
+		break;
+	}
 
 	/*
 	 * Decrement the counters, since these commands are no longer
@@ -451,10 +459,12 @@ static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
 void scsi_device_unbusy(struct scsi_device *sdev)
 {
 	struct Scsi_Host *shost = sdev->host;
+	struct scsi_target *starget = scsi_target(sdev);
 	unsigned long flags;
 
 	spin_lock_irqsave(shost->host_lock, flags);
 	shost->host_busy--;
+	starget->target_busy--;
 	if (unlikely(scsi_host_in_recovery(shost) &&
 		     (shost->host_failed || shost->host_eh_scheduled)))
 		scsi_eh_wakeup(shost);
@@ -1295,6 +1305,51 @@ static inline int scsi_dev_queue_ready(struct request_queue *q,
 	return 1;
 }
 
+
+/*
+ * scsi_target_queue_ready: checks if there we can send commands to target
+ * @sdev: scsi device on starget to check.
+ *
+ * Called with the host lock held.
+ */
+static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
+					   struct scsi_device *sdev)
+{
+	struct scsi_target *starget = scsi_target(sdev);
+
+	if (starget->single_lun) {
+		if (starget->starget_sdev_user &&
+		    starget->starget_sdev_user != sdev)
+			return 0;
+		starget->starget_sdev_user = sdev;
+	}
+
+	if (starget->target_busy == 0 && starget->target_blocked) {
+		/*
+		 * unblock after target_blocked iterates to zero
+		 */
+		if (--starget->target_blocked == 0) {
+			SCSI_LOG_MLQUEUE(3,
+				   starget_printk(KERN_INFO, starget,
+				   "unblocking target at zero depth\n"));
+		} else {
+			blk_plug_device(sdev->request_queue);
+			return 0;
+		}
+	}
+
+	if ((starget->can_queue > 0 &&
+	     starget->target_busy >= starget->can_queue) ||
+	     starget->target_blocked) {
+		if (list_empty(&sdev->starved_entry))
+			list_add_tail(&sdev->starved_entry,
+				      &shost->starved_list);
+		return 0;
+	}
+
+	return 1;
+}
+
 /*
  * scsi_host_queue_ready: if we can send requests to shost, return 1 else
  * return 0. We must end up running the queue again whenever 0 is
@@ -1342,6 +1397,7 @@ static void scsi_kill_request(struct request *req, struct request_queue *q)
 {
 	struct scsi_cmnd *cmd = req->special;
 	struct scsi_device *sdev = cmd->device;
+	struct scsi_target *starget = scsi_target(sdev);
 	struct Scsi_Host *shost = sdev->host;
 
 	blkdev_dequeue_request(req);
@@ -1365,6 +1421,7 @@ static void scsi_kill_request(struct request *req, struct request_queue *q)
 	spin_unlock(sdev->request_queue->queue_lock);
 	spin_lock(shost->host_lock);
 	shost->host_busy++;
+	starget->target_busy++;
 	spin_unlock(shost->host_lock);
 	spin_lock(sdev->request_queue->queue_lock);
 
@@ -1478,14 +1535,13 @@ static void scsi_request_fn(struct request_queue *q)
 		}
 		spin_lock(shost->host_lock);
 
+		if (!scsi_target_queue_ready(shost, sdev))
+			goto not_ready;
+
 		if (!scsi_host_queue_ready(q, shost, sdev))
 			goto not_ready;
-		if (scsi_target(sdev)->single_lun) {
-			if (scsi_target(sdev)->starget_sdev_user &&
-			    scsi_target(sdev)->starget_sdev_user != sdev)
-				goto not_ready;
-			scsi_target(sdev)->starget_sdev_user = sdev;
-		}
+
+		scsi_target(sdev)->target_busy++;
 		shost->host_busy++;
 
 		/*
diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c
index 1dc165a..c9f7b74 100644
--- a/drivers/scsi/scsi_scan.c
+++ b/drivers/scsi/scsi_scan.c
@@ -396,6 +396,7 @@ static struct scsi_target *scsi_alloc_target(struct device *parent,
 		shost->host_no, channel, id);
 	starget->id = id;
 	starget->channel = channel;
+	starget->can_queue = 0;
 	INIT_LIST_HEAD(&starget->siblings);
 	INIT_LIST_HEAD(&starget->devices);
 	starget->state = STARGET_RUNNING;
diff --git a/include/scsi/scsi.h b/include/scsi/scsi.h
index 1f74bcd..2160b36 100644
--- a/include/scsi/scsi.h
+++ b/include/scsi/scsi.h
@@ -381,6 +381,7 @@ struct scsi_lun {
 #define SCSI_MLQUEUE_HOST_BUSY   0x1055
 #define SCSI_MLQUEUE_DEVICE_BUSY 0x1056
 #define SCSI_MLQUEUE_EH_RETRY    0x1057
+#define SCSI_MLQUEUE_TARGET_BUSY 0x1058
 
 /*
  *  Use these to separate status msg and our bytes
diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h
index ab7acbe..f8e4184 100644
--- a/include/scsi/scsi_device.h
+++ b/include/scsi/scsi_device.h
@@ -205,6 +205,16 @@ struct scsi_target {
 						 * for the device at a time. */
 	unsigned int		pdt_1f_for_no_lun;	/* PDT = 0x1f */
 						/* means no lun present */
+	/* commands actually active on LLD. protected by host lock. */
+	unsigned int		target_busy;
+	/*
+	 * LLDs should set this in the slave_alloc host template callout.
+	 * If set to zero then there is not limit.
+	 */
+	unsigned int		can_queue;
+	unsigned int		target_blocked;
+	unsigned int		max_target_blocked;
+#define SCSI_DEFAULT_TARGET_BLOCKED	3
 
 	char			scsi_level;
 	struct execute_work	ew;
-- 
1.5.4.1

[PATCH 2/9] iscsi: user per target can_queue and return SCSI_MLQUEUE_TARGET_BUSY

[michaelc]

From: Mike Christie <michaelc@cs.wisc.edu>

This hooks iscsi_tcp and libiscsi into the target->can_queue
code and it has libiscsi use SCSI_MLQUEUE_TARGET_BUSY.

Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
---
 drivers/infiniband/ulp/iser/iscsi_iser.c |    1 +
 drivers/scsi/iscsi_tcp.c                 |    1 +
 drivers/scsi/libiscsi.c                  |   31 +++++++++++++++++++----------
 include/scsi/libiscsi.h                  |    1 +
 4 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c
index be1b9fb..c818707 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.c
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
@@ -552,6 +552,7 @@ static struct scsi_host_template iscsi_iser_sht = {
 	.sg_tablesize           = ISCSI_ISER_SG_TABLESIZE,
 	.max_sectors		= 1024,
 	.cmd_per_lun            = ISCSI_MAX_CMD_PER_LUN,
+	.slave_alloc		= iscsi_slave_alloc,
 	.eh_abort_handler       = iscsi_eh_abort,
 	.eh_device_reset_handler= iscsi_eh_device_reset,
 	.eh_host_reset_handler	= iscsi_eh_host_reset,
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index 8a17867..c63e0e8 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -1936,6 +1936,7 @@ static struct scsi_host_template iscsi_sht = {
 	.eh_device_reset_handler= iscsi_eh_device_reset,
 	.eh_host_reset_handler	= iscsi_eh_host_reset,
 	.use_clustering         = DISABLE_CLUSTERING,
+	.slave_alloc		= iscsi_slave_alloc,
 	.slave_configure        = iscsi_tcp_slave_configure,
 	.proc_name		= "iscsi_tcp",
 	.this_id		= -1,
diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
index bdd7de7..156ec1a 100644
--- a/drivers/scsi/libiscsi.c
+++ b/drivers/scsi/libiscsi.c
@@ -1021,13 +1021,7 @@ int iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
 		goto fault;
 	}
 
-	/*
-	 * ISCSI_STATE_FAILED is a temp. state. The recovery
-	 * code will decide what is best to do with command queued
-	 * during this time
-	 */
-	if (session->state != ISCSI_STATE_LOGGED_IN &&
-	    session->state != ISCSI_STATE_FAILED) {
+	if (session->state != ISCSI_STATE_LOGGED_IN) {
 		/*
 		 * to handle the race between when we set the recovery state
 		 * and block the session we requeue here (commands could
@@ -1035,10 +1029,10 @@ int iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
 		 * up because the block code is not locked)
 		 */
 		switch (session->state) {
+		case ISCSI_STATE_FAILED:
 		case ISCSI_STATE_IN_RECOVERY:
 			reason = FAILURE_SESSION_IN_RECOVERY;
-			sc->result = DID_IMM_RETRY << 16;
-			break;
+			goto reject;
 		case ISCSI_STATE_LOGGING_OUT:
 			reason = FAILURE_SESSION_LOGGING_OUT;
 			sc->result = DID_IMM_RETRY << 16;
@@ -1097,7 +1091,7 @@ reject:
 	spin_unlock(&session->lock);
 	debug_scsi("cmd 0x%x rejected (%d)\n", sc->cmnd[0], reason);
 	spin_lock(host->host_lock);
-	return SCSI_MLQUEUE_HOST_BUSY;
+	return SCSI_MLQUEUE_TARGET_BUSY;
 
 fault:
 	spin_unlock(&session->lock);
@@ -1118,6 +1112,21 @@ int iscsi_change_queue_depth(struct scsi_device *sdev, int depth)
 }
 EXPORT_SYMBOL_GPL(iscsi_change_queue_depth);
 
+int iscsi_slave_alloc(struct scsi_device *sdev)
+{
+	struct scsi_target *starget = scsi_target(sdev);
+	struct iscsi_cls_session *cls_session = starget_to_session(starget);
+	struct iscsi_session *session;
+
+	if (!cls_session || iscsi_session_chkready(cls_session))
+		return -ENXIO;
+
+	session = class_to_transport_session(cls_session);
+	starget->can_queue = session->cmds_max;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(iscsi_slave_alloc);
+
 void iscsi_session_recovery_timedout(struct iscsi_cls_session *cls_session)
 {
 	struct iscsi_session *session = class_to_transport_session(cls_session);
@@ -1724,7 +1733,7 @@ iscsi_session_setup(struct iscsi_transport *iscsit,
 		return NULL;
 
 	/* the iscsi layer takes one task for reserve */
-	shost->can_queue = cmds_max - 1;
+	shost->can_queue = cmds_max;
 	shost->cmd_per_lun = qdepth;
 	shost->max_id = 1;
 	shost->max_channel = 0;
diff --git a/include/scsi/libiscsi.h b/include/scsi/libiscsi.h
index 7b90b63..3c2bdc4 100644
--- a/include/scsi/libiscsi.h
+++ b/include/scsi/libiscsi.h
@@ -308,6 +308,7 @@ struct iscsi_session {
 /*
  * scsi host template
  */
+extern int iscsi_slave_alloc(struct scsi_device *sdev);
 extern int iscsi_change_queue_depth(struct scsi_device *sdev, int depth);
 extern int iscsi_eh_abort(struct scsi_cmnd *sc);
 extern int iscsi_eh_host_reset(struct scsi_cmnd *sc);
-- 
1.5.4.1

[PATCH 3/9] iscsi class: add iscsi queueing helper

[michaelc]

From: Mike Christie <michaelc@cs.wisc.edu>

When we transition to blocked return SCSI_MLQUEUE_TARGET_BUSY
instead of DID_IMM_RETRY, because we do not want the command
to come back around.

Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
---
 drivers/scsi/scsi_transport_iscsi.c |   35 +++++++++++++++++++++++++++++++++++
 include/scsi/scsi_transport_iscsi.h |    3 +++
 2 files changed, 38 insertions(+), 0 deletions(-)

diff --git a/drivers/scsi/scsi_transport_iscsi.c b/drivers/scsi/scsi_transport_iscsi.c
index dfb026b..534a149 100644
--- a/drivers/scsi/scsi_transport_iscsi.c
+++ b/drivers/scsi/scsi_transport_iscsi.c
@@ -24,6 +24,7 @@
 #include <linux/mutex.h>
 #include <net/tcp.h>
 #include <scsi/scsi.h>
+#include <scsi/scsi_cmnd.h>
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_device.h>
 #include <scsi/scsi_transport.h>
@@ -245,6 +246,40 @@ static const char *iscsi_session_state_name(int state)
 	return name;
 }
 
+/**
+ * iscsi_session_queue_ready - check if a session queue can accpet IO
+ * @session: iscsi session to check
+ * @cmd: scsi command to process
+ *
+ * Returns 0 if IO can be queued or if there the cmd should be
+ * completed by a call to cmd->scsi_done. Caller must check cmd->result.
+ * Returns SCSI_MLQUEUE value if IO cannot be queued.
+ */
+int iscsi_session_queue_ready(struct iscsi_cls_session *session,
+			      struct scsi_cmnd *cmd)
+{
+	unsigned long flags;
+	int err = 0;
+
+	cmd->result = 0;
+	spin_lock_irqsave(&session->lock, flags);
+	switch (session->state) {
+	case ISCSI_SESSION_LOGGED_IN:
+		break;
+	case ISCSI_SESSION_FAILED:
+		err = SCSI_MLQUEUE_TARGET_BUSY;
+		break;
+	case ISCSI_SESSION_FREE:
+		/* fall through */
+	default:
+		cmd->result = DID_NO_CONNECT << 16;
+		break;
+	}
+	spin_unlock_irqrestore(&session->lock, flags);
+	return err;
+}
+EXPORT_SYMBOL_GPL(iscsi_session_queue_ready);
+
 int iscsi_session_chkready(struct iscsi_cls_session *session)
 {
 	unsigned long flags;
diff --git a/include/scsi/scsi_transport_iscsi.h b/include/scsi/scsi_transport_iscsi.h
index dbc96ef..5d3ffb3 100644
--- a/include/scsi/scsi_transport_iscsi.h
+++ b/include/scsi/scsi_transport_iscsi.h
@@ -31,6 +31,7 @@
 struct scsi_transport_template;
 struct iscsi_transport;
 struct Scsi_Host;
+struct scsi_cmd;
 struct iscsi_cls_conn;
 struct iscsi_conn;
 struct iscsi_cmd_task;
@@ -218,6 +219,8 @@ struct iscsi_host {
 #define iscsi_cls_conn_printk(prefix, _cls_conn, fmt, a...) \
 	dev_printk(prefix, &(_cls_conn)->dev, fmt, ##a)
 
+extern int iscsi_session_queue_ready(struct iscsi_cls_session *session,
+				     struct scsi_cmnd *cmd);
 extern int iscsi_session_chkready(struct iscsi_cls_session *session);
 extern struct iscsi_cls_session *iscsi_alloc_session(struct Scsi_Host *shost,
 					struct iscsi_transport *transport);
-- 
1.5.4.1

[PATCH 4/9] qla4xxx: use SCSI_ML_TARGET_BUSY and iscsi helper

[michaelc]

From: Mike Christie <michaelc@cs.wisc.edu>

When transitioning to the blocked state use SCSI_ML_TARGET_BUSY
instead of returning with DID_IMM_RETRY, because DID_IMM_RETRY
causes a requeue and then possibly (if under recursion limits)
a queuecommand call.

Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
---
 drivers/scsi/qla4xxx/ql4_os.c |   37 ++++++++++++++++++++-----------------
 1 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c
index c3c59d7..40ad624 100644
--- a/drivers/scsi/qla4xxx/ql4_os.c
+++ b/drivers/scsi/qla4xxx/ql4_os.c
@@ -341,9 +341,7 @@ void qla4xxx_mark_device_missing(struct scsi_qla_host *ha,
 }
 
 static struct srb* qla4xxx_get_new_srb(struct scsi_qla_host *ha,
-				       struct ddb_entry *ddb_entry,
-				       struct scsi_cmnd *cmd,
-				       void (*done)(struct scsi_cmnd *))
+				       struct ddb_entry *ddb_entry)
 {
 	struct srb *srb;
 
@@ -354,11 +352,7 @@ static struct srb* qla4xxx_get_new_srb(struct scsi_qla_host *ha,
 	atomic_set(&srb->ref_count, 1);
 	srb->ha = ha;
 	srb->ddb = ddb_entry;
-	srb->cmd = cmd;
 	srb->flags = 0;
-	cmd->SCp.ptr = (void *)srb;
-	cmd->scsi_done = done;
-
 	return srb;
 }
 
@@ -407,33 +401,41 @@ static int qla4xxx_queuecommand(struct scsi_cmnd *cmd,
 	struct srb *srb;
 	int rval;
 
+	cmd->scsi_done = done;
+	cmd->result = 0;
+
 	if (!sess) {
 		cmd->result = DID_IMM_RETRY << 16;
 		goto qc_fail_command;
 	}
 
-	rval = iscsi_session_chkready(sess);
-	if (rval) {
-		cmd->result = rval;
+	rval = iscsi_session_queue_ready(sess, cmd);
+	if (rval)
+		goto qc_busy;
+	if (cmd->result)
 		goto qc_fail_command;
-	}
 
 	if (atomic_read(&ddb_entry->state) != DDB_STATE_ONLINE) {
 		if (atomic_read(&ddb_entry->state) == DDB_STATE_DEAD) {
 			cmd->result = DID_NO_CONNECT << 16;
 			goto qc_fail_command;
 		}
-		goto qc_host_busy;
+		rval = SCSI_MLQUEUE_TARGET_BUSY;
+		goto qc_busy;
 	}
 
-	if (test_bit(DPC_RESET_HA_INTR, &ha->dpc_flags))
-		goto qc_host_busy;
+	if (test_bit(DPC_RESET_HA_INTR, &ha->dpc_flags)) {
+		rval = SCSI_MLQUEUE_HOST_BUSY;
+		goto qc_busy;
+	}
 
 	spin_unlock_irq(ha->host->host_lock);
 
-	srb = qla4xxx_get_new_srb(ha, ddb_entry, cmd, done);
+	srb = qla4xxx_get_new_srb(ha, ddb_entry);
 	if (!srb)
 		goto qc_host_busy_lock;
+	cmd->SCp.ptr = (void *)srb;
+	srb->cmd = cmd;
 
 	rval = qla4xxx_send_command_to_isp(ha, srb);
 	if (rval != QLA_SUCCESS)
@@ -447,10 +449,11 @@ qc_host_busy_free_sp:
 	mempool_free(srb, ha->srb_mempool);
 
 qc_host_busy_lock:
+	rval = SCSI_MLQUEUE_HOST_BUSY;
 	spin_lock_irq(ha->host->host_lock);
 
-qc_host_busy:
-	return SCSI_MLQUEUE_HOST_BUSY;
+qc_busy:
+	return rval;
 
 qc_fail_command:
 	done(cmd);
-- 
1.5.4.1

[PATCH 5/9] libiscsi: use iscsi queueing helper.

[michaelc]

From: Mike Christie <michaelc@cs.wisc.edu>

Convert libiscsi to use new helper.

Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
---
 drivers/scsi/libiscsi.c |   18 +++++++++++-------
 1 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
index 156ec1a..1ee1cc9 100644
--- a/drivers/scsi/libiscsi.c
+++ b/drivers/scsi/libiscsi.c
@@ -1000,7 +1000,7 @@ enum {
 int iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
 {
 	struct Scsi_Host *host;
-	int reason = 0;
+	int reason = 0, err = 0;
 	struct iscsi_session *session;
 	struct iscsi_conn *conn;
 	struct iscsi_cmd_task *ctask = NULL;
@@ -1014,12 +1014,13 @@ int iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
 
 	session = iscsi_hostdata(host->hostdata);
 	spin_lock(&session->lock);
-
-	reason = iscsi_session_chkready(session_to_cls(session));
-	if (reason) {
-		sc->result = reason;
-		goto fault;
+	err = iscsi_session_queue_ready(session_to_cls(session), sc);
+	if (err) {
+		reason = FAILURE_SESSION_NOT_READY;
+		goto reject;
 	}
+	if (sc->result)
+		goto fault;
 
 	if (session->state != ISCSI_STATE_LOGGED_IN) {
 		/*
@@ -1032,6 +1033,7 @@ int iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
 		case ISCSI_STATE_FAILED:
 		case ISCSI_STATE_IN_RECOVERY:
 			reason = FAILURE_SESSION_IN_RECOVERY;
+			err = SCSI_MLQUEUE_TARGET_BUSY;
 			goto reject;
 		case ISCSI_STATE_LOGGING_OUT:
 			reason = FAILURE_SESSION_LOGGING_OUT;
@@ -1061,12 +1063,14 @@ int iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
 
 	if (iscsi_check_cmdsn_window_closed(conn)) {
 		reason = FAILURE_WINDOW_CLOSED;
+		err = SCSI_MLQUEUE_TARGET_BUSY;
 		goto reject;
 	}
 
 	if (!__kfifo_get(session->cmdpool.queue, (void*)&ctask,
 			 sizeof(void*))) {
 		reason = FAILURE_OOM;
+		err = SCSI_MLQUEUE_TARGET_BUSY;
 		goto reject;
 	}
 	session->queued_cmdsn++;
@@ -1091,7 +1095,7 @@ reject:
 	spin_unlock(&session->lock);
 	debug_scsi("cmd 0x%x rejected (%d)\n", sc->cmnd[0], reason);
 	spin_lock(host->host_lock);
-	return SCSI_MLQUEUE_TARGET_BUSY;
+	return err;
 
 fault:
 	spin_unlock(&session->lock);
-- 
1.5.4.1

[PATCH 6/9] fc class: add fc queueing helper

[michaelc]

From: Mike Christie <michaelc@cs.wisc.edu>

Add a queueing helper which will return SCSI_MLQUEUE_TARGET_BUSY
if we are transitioning because there is no need to call into
the driver anymore at that point.

Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
---
 include/scsi/scsi_transport_fc.h |   33 +++++++++++++++++++++++++++++++++
 1 files changed, 33 insertions(+), 0 deletions(-)

diff --git a/include/scsi/scsi_transport_fc.h b/include/scsi/scsi_transport_fc.h
index 4769efd..62a6a74 100644
--- a/include/scsi/scsi_transport_fc.h
+++ b/include/scsi/scsi_transport_fc.h
@@ -29,6 +29,7 @@
 
 #include <linux/sched.h>
 #include <scsi/scsi.h>
+#include <scsi/scsi_cmnd.h>
 #include <scsi/scsi_netlink.h>
 
 struct scsi_transport_template;
@@ -640,6 +641,38 @@ struct fc_function_template {
 	unsigned long	disable_target_scan:1;
 };
 
+/**
+ * fc_remote_port_queue_ready - called to validate the remote port state
+ * @rport: remote port to be checked
+ *
+ * Returns 0 if IO can be queued or if there the cmd should be
+ * completed by a call to cmd->scsi_done. Caller must check cmd->result.
+ * Returns SCSI_MLQUEUE value if IO cannot be queued temporarily.
+ **/
+static inline int
+fc_remote_port_queue_ready(struct fc_rport *rport, struct scsi_cmnd *cmd)
+{
+	int result = 0;
+
+	cmd->result = 0;
+	switch (rport->port_state) {
+	case FC_PORTSTATE_ONLINE:
+		if (rport->roles & FC_PORT_ROLE_FCP_TARGET)
+			result = 0;
+		else if (rport->flags & FC_RPORT_DEVLOSS_PENDING)
+			result = SCSI_MLQUEUE_TARGET_BUSY;
+		else
+			cmd->result = DID_NO_CONNECT << 16;
+		break;
+	case FC_PORTSTATE_BLOCKED:
+		result = SCSI_MLQUEUE_TARGET_BUSY;
+		break;
+	default:
+		cmd->result = DID_NO_CONNECT << 16;
+		break;
+	}
+	return result;
+}
 
 /**
  * fc_remote_port_chkready - called to validate the remote port state
-- 
1.5.4.1

[PATCH 7/9] qla2xxx: convert to use fc helper and SCSI_MLQUEUE_TARGET_BUSY

[michaelc]

From: Mike Christie <michaelc@cs.wisc.edu>

When transitioning to the blocked state use SCSI_ML_TARGET_BUSY
instead of returning with DID_IMM_RETRY, because DID_IMM_RETRY
causes a requeue and then possibly (if under recursion limits)
a queuecommand call.

Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
---
 drivers/scsi/qla2xxx/qla_os.c |   32 ++++++++++++++++++--------------
 1 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index 3c1b433..36f1d9e 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -379,11 +379,11 @@ qla2x00_queuecommand(struct scsi_cmnd *cmd, void (*done)(struct scsi_cmnd *))
 		goto qc_fail_command;
 	}
 
-	rval = fc_remote_port_chkready(rport);
-	if (rval) {
-		cmd->result = rval;
+	rval = fc_remote_port_queue_ready(rport, cmd);
+	if (rval)
+		goto qc_busy;
+	if (cmd->result)
 		goto qc_fail_command;
-	}
 
 	/* Close window on fcport/rport state-transitioning. */
 	if (!*(fc_port_t **)rport->dd_data) {
@@ -397,7 +397,8 @@ qla2x00_queuecommand(struct scsi_cmnd *cmd, void (*done)(struct scsi_cmnd *))
 			cmd->result = DID_NO_CONNECT << 16;
 			goto qc_fail_command;
 		}
-		goto qc_host_busy;
+		rval = SCSI_MLQUEUE_HOST_BUSY;
+		goto qc_busy;
 	}
 
 	spin_unlock_irq(ha->host->host_lock);
@@ -419,10 +420,11 @@ qc_host_busy_free_sp:
 	mempool_free(sp, ha->srb_mempool);
 
 qc_host_busy_lock:
+	rval = SCSI_MLQUEUE_HOST_BUSY;
 	spin_lock_irq(ha->host->host_lock);
 
-qc_host_busy:
-	return SCSI_MLQUEUE_HOST_BUSY;
+qc_busy:
+	return rval;
 
 qc_fail_command:
 	done(cmd);
@@ -446,11 +448,11 @@ qla24xx_queuecommand(struct scsi_cmnd *cmd, void (*done)(struct scsi_cmnd *))
 		goto qc24_fail_command;
 	}
 
-	rval = fc_remote_port_chkready(rport);
-	if (rval) {
-		cmd->result = rval;
+	rval = fc_remote_port_queue_ready(rport, cmd);
+	if (rval)
+		goto qc24_busy;
+	if (cmd->result)
 		goto qc24_fail_command;
-	}
 
 	/* Close window on fcport/rport state-transitioning. */
 	if (!*(fc_port_t **)rport->dd_data) {
@@ -464,7 +466,8 @@ qla24xx_queuecommand(struct scsi_cmnd *cmd, void (*done)(struct scsi_cmnd *))
 			cmd->result = DID_NO_CONNECT << 16;
 			goto qc24_fail_command;
 		}
-		goto qc24_host_busy;
+		rval = SCSI_MLQUEUE_HOST_BUSY;
+		goto qc24_busy;
 	}
 
 	spin_unlock_irq(ha->host->host_lock);
@@ -486,10 +489,11 @@ qc24_host_busy_free_sp:
 	mempool_free(sp, pha->srb_mempool);
 
 qc24_host_busy_lock:
+	rval = SCSI_MLQUEUE_HOST_BUSY;
 	spin_lock_irq(ha->host->host_lock);
 
-qc24_host_busy:
-	return SCSI_MLQUEUE_HOST_BUSY;
+qc24_busy:
+	return rval;
 
 qc24_fail_command:
 	done(cmd);
-- 
1.5.4.1

[PATCH 8/9] lpfc: convert to use fc helper and SCSI_MLQUEUE_TARGET_BUSY

[michaelc]

From: Mike Christie <michaelc@cs.wisc.edu>

When transitioning to the blocked state use SCSI_ML_TARGET_BUSY
instead of returning with DID_IMM_RETRY, because DID_IMM_RETRY
causes a requeue and then possibly (if under recursion limits)
a queuecommand call.

Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
---
 drivers/scsi/lpfc/lpfc_scsi.c |   17 +++++++++--------
 1 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
index 70255c1..a7810f4 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -949,12 +949,11 @@ lpfc_queuecommand(struct scsi_cmnd *cmnd, void (*done) (struct scsi_cmnd *))
 	struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
 	int err;
 
-	err = fc_remote_port_chkready(rport);
-	if (err) {
-		cmnd->result = err;
+	err = fc_remote_port_queue_ready(rport, cmnd);
+	if (err)
+		goto out_busy;
+	if (cmnd->result)
 		goto out_fail_command;
-	}
-
 	/*
 	 * Catch race where our node has transitioned, but the
 	 * transport is still transitioning.
@@ -970,7 +969,8 @@ lpfc_queuecommand(struct scsi_cmnd *cmnd, void (*done) (struct scsi_cmnd *))
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
 				 "0707 driver's buffer pool is empty, "
 				 "IO busied\n");
-		goto out_host_busy;
+		err = SCSI_MLQUEUE_HOST_BUSY;
+		goto out_busy;
 	}
 
 	/*
@@ -1003,10 +1003,11 @@ lpfc_queuecommand(struct scsi_cmnd *cmnd, void (*done) (struct scsi_cmnd *))
 	return 0;
 
  out_host_busy_free_buf:
+	err = SCSI_MLQUEUE_HOST_BUSY;
 	lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd);
 	lpfc_release_scsi_buf(phba, lpfc_cmd);
- out_host_busy:
-	return SCSI_MLQUEUE_HOST_BUSY;
+ out_busy:
+	return err;
 
  out_fail_command:
 	done(cmnd);
-- 
1.5.4.1

[PATCH 9/9] mptfc: convert mptfc to use fc queueing helper.

[michaelc]

From: Mike Christie <michaelc@cs.wisc.edu>

Convert mptfc to use fc queueing helper.

Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
---
 drivers/message/fusion/mptfc.c |    7 ++++---
 1 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/drivers/message/fusion/mptfc.c b/drivers/message/fusion/mptfc.c
index 3cdd4e9..1189bbe 100644
--- a/drivers/message/fusion/mptfc.c
+++ b/drivers/message/fusion/mptfc.c
@@ -656,9 +656,10 @@ mptfc_qcmd(struct scsi_cmnd *SCpnt, void (*done)(struct scsi_cmnd *))
 		return 0;
 	}
 
-	err = fc_remote_port_chkready(rport);
-	if (unlikely(err)) {
-		SCpnt->result = err;
+	err = fc_remote_port_queue_ready(rport, SCpnt);
+	if (err)
+		return err;
+	if (SCpnt->result) {
 		done(SCpnt);
 		return 0;
 	}
-- 
1.5.4.1

Re: [PATCH 1/9] scsi: Add helper code so transport classes/driver can control queueing

[Mike Christie]

Oh yeah, these patches were made over scsi-rc-fixes and can be applied 
over the target resets ones (the target reset ones can be applied over 
these too).

michaelc@cs.wisc.edu wrote:
> From: Mike Christie <michaelc@cs.wisc.edu>
> 
> SCSI-ml manages the queueing limits for the device and host, but
> does not do so at the target level. However something something similar
> can come in userful when a driver is transitioning a transport object to
> the the blocked state, becuase at that time we do not want to queue
> io and we do not want the queuecommand to be called again.
> 
> The patch adds code similar to the exisiting SCSI_ML_*BUSY handlers.
> You can now return SCSI_MLQUEUE_TARGET_BUSY when we hit
> a transport level queueing issue like the hw cannot allocate some
> resource at the iscsi session/connection level, or the target has temporarily
> closed or shrunk the queueing window, or if we are transitioning
> to the blocked state.
> 
> For bnx2i we will also need to be able to limit queueing at this level.
> bnx2i will hook into libiscsi, but will allocate a scsi host per
> netdevice/hba, so unlike pure software iscsi/iser which is allocating
> a host per session, it cannot set the scsi_host->can_queue and return
> SCSI_MLQUEUE_HOST_BUSY to reflect queueing limits on the transport.
> 
> The iscsi class/driver can also set a scsi_target->can_queue value which
> reflects the max commands the driver/class can support. For iscsi this
> reflects the number of commands we can support for each session due to
> session/connection hw limits, driver limits, and to also reflect the
> session/targets's queueing window.
> 

Re: [PATCH 4/9] qla4xxx: use SCSI_ML_TARGET_BUSY and iscsi helper

[David C Somayajulu]

On Sun, 2008-03-02 at 03:03 -0600, michaelc@cs.wisc.edu wrote:
> From: Mike Christie <michaelc@cs.wisc.edu>
> 
> When transitioning to the blocked state use SCSI_ML_TARGET_BUSY
> instead of returning with DID_IMM_RETRY, because DID_IMM_RETRY
> causes a requeue and then possibly (if under recursion limits)
> a queuecommand call.
> 
> Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
> ---
>  drivers/scsi/qla4xxx/ql4_os.c |   37 ++++++++++++++++++++-----------------
>  1 files changed, 20 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c
> index c3c59d7..40ad624 100644
> --- a/drivers/scsi/qla4xxx/ql4_os.c
> +++ b/drivers/scsi/qla4xxx/ql4_os.c
> @@ -341,9 +341,7 @@ void qla4xxx_mark_device_missing(struct scsi_qla_host *ha,
>  }
>  
>  static struct srb* qla4xxx_get_new_srb(struct scsi_qla_host *ha,
> -				       struct ddb_entry *ddb_entry,
> -				       struct scsi_cmnd *cmd,
> -				       void (*done)(struct scsi_cmnd *))
> +				       struct ddb_entry *ddb_entry)
>  {
>  	struct srb *srb;
>  
> @@ -354,11 +352,7 @@ static struct srb* qla4xxx_get_new_srb(struct scsi_qla_host *ha,
>  	atomic_set(&srb->ref_count, 1);
>  	srb->ha = ha;
>  	srb->ddb = ddb_entry;
> -	srb->cmd = cmd;
>  	srb->flags = 0;
> -	cmd->SCp.ptr = (void *)srb;
> -	cmd->scsi_done = done;
> -
>  	return srb;
>  }
>  
> @@ -407,33 +401,41 @@ static int qla4xxx_queuecommand(struct scsi_cmnd *cmd,
>  	struct srb *srb;
>  	int rval;
>  
> +	cmd->scsi_done = done;
> +	cmd->result = 0;
> +
>  	if (!sess) {
>  		cmd->result = DID_IMM_RETRY << 16;
>  		goto qc_fail_command;
>  	}
>  
> -	rval = iscsi_session_chkready(sess);
> -	if (rval) {
> -		cmd->result = rval;
> +	rval = iscsi_session_queue_ready(sess, cmd);
> +	if (rval)
> +		goto qc_busy;
> +	if (cmd->result)
>  		goto qc_fail_command;
> -	}
>  
>  	if (atomic_read(&ddb_entry->state) != DDB_STATE_ONLINE) {
>  		if (atomic_read(&ddb_entry->state) == DDB_STATE_DEAD) {
>  			cmd->result = DID_NO_CONNECT << 16;
>  			goto qc_fail_command;
>  		}
> -		goto qc_host_busy;
> +		rval = SCSI_MLQUEUE_TARGET_BUSY;
> +		goto qc_busy;
>  	}
>  
> -	if (test_bit(DPC_RESET_HA_INTR, &ha->dpc_flags))
> -		goto qc_host_busy;
> +	if (test_bit(DPC_RESET_HA_INTR, &ha->dpc_flags)) {
> +		rval = SCSI_MLQUEUE_HOST_BUSY;
> +		goto qc_busy;
> +	}
>  
>  	spin_unlock_irq(ha->host->host_lock);
>  
> -	srb = qla4xxx_get_new_srb(ha, ddb_entry, cmd, done);
> +	srb = qla4xxx_get_new_srb(ha, ddb_entry);
>  	if (!srb)On Sun, 2008-03-02 at 03:03 -0600, michaelc@cs.wisc.edu wrote:
> From: Mike Christie <michaelc@cs.wisc.edu>
> 
> When transitioning to the blocked state use SCSI_ML_TARGET_BUSY
> instead of returning with DID_IMM_RETRY, because DID_IMM_RETRY
> causes a requeue and then possibly (if under recursion limits)
> a queuecommand call.
> 
> Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
> ---
>  drivers/scsi/qla4xxx/ql4_os.c |   37 ++++++++++++++++++++-----------------
>  1 files changed, 20 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c
> index c3c59d7..40ad624 100644
> --- a/drivers/scsi/qla4xxx/ql4_os.c
> +++ b/drivers/scsi/qla4xxx/ql4_os.c
> @@ -341,9 +341,7 @@ void qla4xxx_mark_device_missing(struct scsi_qla_host *ha,
>  }
>  
>  static struct srb* qla4xxx_get_new_srb(struct scsi_qla_host *ha,
> -				       struct ddb_entry *ddb_entry,
> -				       struct scsi_cmnd *cmd,
> -				       void (*done)(struct scsi_cmnd *))
> +				       struct ddb_entry *ddb_entry)
>  {
>  	struct srb *srb;
>  
> @@ -354,11 +352,7 @@ static struct srb* qla4xxx_get_new_srb(struct scsi_qla_host *ha,
>  	atomic_set(&srb->ref_count, 1);
>  	srb->ha = ha;
>  	srb->ddb = ddb_entry;
> -	srb->cmd = cmd;On Sun, 2008-03-02 at 03:03 -0600, michaelc@cs.wisc.edu wrote:
> From: Mike Christie <michaelc@cs.wisc.edu>
> 
> When transitioning to the blocked state use SCSI_ML_TARGET_BUSY
> instead of returning with DID_IMM_RETRY, because DID_IMM_RETRY
> causes a requeue and then possibly (if under recursion limits)
> a queuecommand call.
> 
> Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
> ---
>  drivers/scsi/qla4xxx/ql4_os.c |   37 ++++++++++++++++++++-----------------
>  1 files changed, 20 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c
> index c3c59d7..40ad624 100644
> --- a/drivers/scsi/qla4xxx/ql4_os.c
> +++ b/drivers/scsi/qla4xxx/ql4_os.c
> @@ -341,9 +341,7 @@ void qla4xxx_mark_device_missing(struct scsi_qla_host *ha,
>  }
>  
>  static struct srb* qla4xxx_get_new_srb(struct scsi_qla_host *ha,
> -				       struct ddb_entry *ddb_entry,
> -				       struct scsi_cmnd *cmd,
> -				       void (*done)(struct scsi_cmnd *))
> +				       struct ddb_entry *ddb_entry)
>  {
>  	struct srb *srb;
>  
> @@ -354,11 +352,7 @@ static struct srb* qla4xxx_get_new_srb(struct scsi_qla_host *ha,
>  	atomic_set(&srb->ref_count, 1);
>  	srb->ha = ha;
>  	srb->ddb = ddb_entry;
> -	srb->cmd = cmd;
>  	srb->flags = 0;
> -	cmd->SCp.ptr = (void *)srb;
> -	cmd->scsi_done = done;
> -
>  	return srb;
>  }
>  
> @@ -407,33 +401,41 @@ static int qla4xxx_queuecommand(struct scsi_cmnd *cmd,
>  	struct srb *srb;
>  	int rval;
>  
> +	cmd->scsi_done = done;
> +	cmd->result = 0;
> +
>  	if (!sess) {
>  		cmd->result = DID_IMM_RETRY << 16;
>  		goto qc_fail_command;
>  	}
>  
> -	rval = iscsi_session_chkready(sess);
> -	if (rval) {
> -		cmd->result = rval;
> +	rval = iscsi_session_queue_ready(sess, cmd);
> +	if (rval)
> +		goto qc_busy;
> +	if (cmd->result)
>  		goto qc_fail_command;
> -	}
>  
>  	if (atomic_read(&ddb_entry->state) != DDB_STATE_ONLINE) {
>  		if (atomic_read(&ddb_entry->state) == DDB_STATE_DEAD) {
>  			cmd->result = DID_NO_CONNECT << 16;
>  			goto qc_fail_command;
>  		}
> -		goto qc_host_busy;
> +		rval = SCSI_MLQUEUE_TARGET_BUSY;
> +		goto qc_busy;
>  	}
>  
> -	if (test_bit(DPC_RESET_HA_INTR, &ha->dpc_flags))
> -		goto qc_host_busy;
> +	if (test_bit(DPC_RESET_HA_INTR, &ha->dpc_flags)) {
> +		rval = SCSI_MLQUEUE_HOST_BUSY;
> +		goto qc_busy;
> +	}
>  
>  	spin_unlock_irq(ha->host->host_lock);
>  
> -	srb = qla4xxx_get_new_srb(ha, ddb_entry, cmd, done);
> +	srb = qla4xxx_get_new_srb(ha, ddb_entry);
>  	if (!srb)
>  		goto qc_host_busy_lock;
> +	cmd->SCp.ptr = (void *)srb;
> +	srb->cmd = cmd;
>  
>  	rval = qla4xxx_send_command_to_isp(ha, srb);
>  	if (rval != QLA_SUCCESS)
> @@ -447,10 +449,11 @@ qc_host_busy_free_sp:
>  	mempool_free(srb, ha->srb_mempool);
>  
>  qc_host_busy_lock:
> +	rval = SCSI_MLQUEUE_HOST_BUSY;
>  	spin_lock_irq(ha->host->host_lock);
>  
> -qc_host_busy:
> -	return SCSI_MLQUEUE_HOST_BUSY;
> +qc_busy:
> +	return rval;
>  
>  qc_fail_command:
>  	done(cmd);
>  	srb->flags = 0;
> -	cmd->SCp.ptr = (void *)srb;
> -	cmd->scsi_done = done;
> -On Sun, 2008-03-02 at 03:03 -0600, michaelc@cs.wisc.edu wrote:
> From: Mike Christie <michaelc@cs.wisc.edu>
> 
> When transitioning to the blocked state use SCSI_ML_TARGET_BUSY
> instead of returning with DID_IMM_RETRY, because DID_IMM_RETRY
> causes a requeue and then possibly (if under recursion limits)
> a queuecommand call.
> 
> Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
> ---
>  drivers/scsi/qla4xxx/ql4_os.c |   37 ++++++++++++++++++++-----------------
>  1 files changed, 20 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c
> index c3c59d7..40ad624 100644
> --- a/drivers/scsi/qla4xxx/ql4_os.c
> +++ b/drivers/scsi/qla4xxx/ql4_os.c
> @@ -341,9 +341,7 @@ void qla4xxx_mark_device_missing(struct scsi_qla_host *ha,
>  }
>  
>  static struct srb* qla4xxx_get_new_srb(struct scsi_qla_host *ha,
> -				       struct ddb_entry *ddb_entry,
> -				       struct scsi_cmnd *cmd,
> -				       void (*done)(struct scsi_cmnd *))
> +				       struct ddb_entry *ddb_entry)
>  {
>  	struct srb *srb;
>  
> @@ -354,11 +352,7 @@ static struct srb* qla4xxx_get_new_srb(struct scsi_qla_host *ha,
>  	atomic_set(&srb->ref_count, 1);
>  	srb->ha = ha;
>  	srb->ddb = ddb_entry;
> -	srb->cmd = cmd;
>  	srb->flags = 0;
> -	cmd->SCp.ptr = (void *)srb;
> -	cmd->scsi_done = done;
> -
>  	return srb;
>  }
>  
> @@ -407,33 +401,41 @@ static int qla4xxx_queuecommand(struct scsi_cmnd *cmd,
>  	struct srb *srb;
>  	int rval;
>  
> +	cmd->scsi_done = done;
> +	cmd->result = 0;
> +
>  	if (!sess) {
>  		cmd->result = DID_IMM_RETRY << 16;
>  		goto qc_fail_command;
>  	}
>  
> -	rval = iscsi_session_chkready(sess);
> -	if (rval) {
> -		cmd->result = rval;
> +	rval = iscsi_session_queue_ready(sess, cmd);
> +	if (rval)
> +		goto qc_busy;
> +	if (cmd->result)
>  		goto qc_fail_command;
> -	}
>  
>  	if (atomic_read(&ddb_entry->state) != DDB_STATE_ONLINE) {
>  		if (atomic_read(&ddb_entry->state) == DDB_STATE_DEAD) {
>  			cmd->result = DID_NO_CONNECT << 16;
>  			goto qc_fail_command;
>  		}
> -		goto qc_host_busy;
> +		rval = SCSI_MLQUEUE_TARGET_BUSY;
> +		goto qc_busy;
>  	}
>  
> -	if (test_bit(DPC_RESET_HA_INTR, &ha->dpc_flags))
> -		goto qc_host_busy;
> +	if (test_bit(DPC_RESET_HA_INTR, &ha->dpc_flags)) {
> +		rval = SCSI_MLQUEUE_HOST_BUSY;
> +		goto qc_busy;
> +	}
>  
>  	spin_unlock_irq(ha->host->host_lock);
>  
> -	srb = qla4xxx_get_new_srb(ha, ddb_entry, cmd, done);On Sun, 2008-03-02 at 03:03 -0600, michaelc@cs.wisc.edu wrote:
> From: Mike Christie <michaelc@cs.wisc.edu>
> 
> When transitioning to the blocked state use SCSI_ML_TARGET_BUSY
> instead of returning with DID_IMM_RETRY, because DID_IMM_RETRY
> causes a requeue and then possibly (if under recursion limits)
> a queuecommand call.
> 
> Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
> ---
>  drivers/scsi/qla4xxx/ql4_os.c |   37 ++++++++++++++++++++-----------------
>  1 files changed, 20 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c
> index c3c59d7..40ad624 100644
> --- a/drivers/scsi/qla4xxx/ql4_os.c
> +++ b/drivers/scsi/qla4xxx/ql4_os.c
> @@ -341,9 +341,7 @@ void qla4xxx_mark_device_missing(struct scsi_qla_host *ha,
>  }
>  
>  static struct srb* qla4xxx_get_new_srb(struct scsi_qla_host *ha,
> -				       struct ddb_entry *ddb_entry,
> -				       struct scsi_cmnd *cmd,
> -				       void (*done)(struct scsi_cmnd *))
> +				       struct ddb_entry *ddb_entry)
>  {
>  	struct srb *srb;
>  
> @@ -354,11 +352,7 @@ static struct srb* qla4xxx_get_new_srb(struct scsi_qla_host *ha,
>  	atomic_set(&srb->ref_count, 1);
>  	srb->ha = ha;
>  	srb->ddb = ddb_entry;
> -	srb->cmd = cmd;
>  	srb->flags = 0;
> -	cmd->SCp.ptr = (void *)srb;
> -	cmd->scsi_done = done;
> -
>  	return srb;
>  }
>  
> @@ -407,33 +401,41 @@ static int qla4xxx_queuecommand(struct scsi_cmnd *cmd,
>  	struct srb *srb;
>  	int rval;
>  
> +	cmd->scsi_done = done;
> +	cmd->result = 0;
> +
>  	if (!sess) {
>  		cmd->result = DID_IMM_RETRY << 16;
>  		goto qc_fail_command;
>  	}
>  
> -	rval = iscsi_session_chkready(sess);
> -	if (rval) {
> -		cmd->result = rval;
> +	rval = iscsi_session_queue_ready(sess, cmd);
> +	if (rval)
> +		goto qc_busy;
> +	if (cmd->result)
>  		goto qc_fail_command;
> -	}
>  
>  	if (atomic_read(&ddb_entry->state) != DDB_STATE_ONLINE) {
>  		if (atomic_read(&ddb_entry->state) == DDB_STATE_DEAD) {
>  			cmd->result = DID_NO_CONNECT << 16;
>  			goto qc_fail_command;
>  		}
> -		goto qc_host_busy;
> +		rval = SCSI_MLQUEUE_TARGET_BUSY;
> +		goto qc_busy;
>  	}
>  
> -	if (test_bit(DPC_RESET_HA_INTR, &ha->dpc_flags))
> -		goto qc_host_busy;
> +	if (test_bit(DPC_RESET_HA_INTR, &ha->dpc_flags)) {
> +		rval = SCSI_MLQUEUE_HOST_BUSY;
> +		goto qc_busy;
> +	}
>  
>  	spin_unlock_irq(ha->host->host_lock);
>  
> -	srb = qla4xxx_get_new_srb(ha, ddb_entry, cmd, done);
> +	srb = qla4xxx_get_new_srb(ha, ddb_entry);
>  	if (!srb)
>  		goto qc_host_busy_lock;
> +	cmd->SCp.ptr = (void *)srb;
> +	srb->cmd = cmd;
>  
>  	rval = qla4xxx_send_command_to_isp(ha, srb);
>  	if (rval != QLA_SUCCESS)
> @@ -447,10 +449,11 @@ qc_host_busy_free_sp:
>  	mempool_free(srb, ha->srb_mempool);
>  
>  qc_host_busy_lock:
> +	rval = SCSI_MLQUEUE_HOST_BUSY;
>  	spin_lock_irq(ha->host->host_lock);
>  
> -qc_host_busy:
> -	return SCSI_MLQUEUE_HOST_BUSY;
> +qc_busy:
> +	return rval;
>  
>  qc_fail_command:
>  	done(cmd);
> +	srb = qla4xxx_get_new_srb(ha, ddb_entry);
>  	if (!srb)
>  		goto qc_host_busy_lock;
> +	cmd->SCp.ptr = (void *)srb;
> +	srb->cmd = cmd;
>  
>  	rval = qla4xxx_send_command_to_isp(ha, srb);
>  	if (rval != QLA_SUCCESS)
> @@ -447,10 +449,11 @@ qc_host_busy_free_sp:
>  	mempool_free(srb, ha->srb_mempool);
>  
>  qc_host_busy_lock:
> +	rval = SCSI_MLQUEUE_HOST_BUSY;
>  	spin_lock_irq(ha->host->host_lock);
>  
> -qc_host_busy:
> -	return SCSI_MLQUEUE_HOST_BUSY;
> +qc_busy:
> +	return rval;
>  
>  qc_fail_command:
>  	done(cmd);
>  	return srb;
>  }
>  
> @@ -407,33 +401,41 @@ static int qla4xxx_queuecommand(struct scsi_cmnd *cmd,
>  	struct srb *srb;
>  	int rval;
>  
> +	cmd->scsi_done = done;
> +	cmd->result = 0;
> +
>  	if (!sess) {
>  		cmd->result = DID_IMM_RETRY << 16;
>  		goto qc_fail_command;
>  	}
>  
> -	rval = iscsi_session_chkready(sess);
> -	if (rval) {
> -		cmd->result = rval;
> +	rval = iscsi_session_queue_ready(sess, cmd);
> +	if (rval)
> +		goto qc_busy;
> +	if (cmd->result)
>  		goto qc_fail_command;
> -	}
>  
>  	if (atomic_read(&ddb_entry->state) != DDB_STATE_ONLINE) {
>  		if (atomic_read(&ddb_entry->state) == DDB_STATE_DEAD) {
>  			cmd->result = DID_NO_CONNECT << 16;
>  			goto qc_fail_command;
>  		}
> -		goto qc_host_busy;
> +		rval = SCSI_MLQUEUE_TARGET_BUSY;
> +		goto qc_busy;
>  	}
>  
> -	if (test_bit(DPC_RESET_HA_INTR, &ha->dpc_flags))
> -		goto qc_host_busy;
> +	if (test_bit(DPC_RESET_HA_INTR, &ha->dpc_flags)) {
> +		rval = SCSI_MLQUEUE_HOST_BUSY;
> +		goto qc_busy;
> +	}
>  
>  	spin_unlock_irq(ha->host->host_lock);
>  
> -	srb = qla4xxx_get_new_srb(ha, ddb_entry, cmd, done);
> +	srb = qla4xxx_get_new_srb(ha, ddb_entry);
>  	if (!srb)
>  		goto qc_host_busy_lock;
> +	cmd->SCp.ptr = (void *)srb;
> +	srb->cmd = cmd;
>  
>  	rval = qla4xxx_send_command_to_isp(ha, srb);
>  	if (rval != QLA_SUCCESS)
> @@ -447,10 +449,11 @@ qc_host_busy_free_sp:
>  	mempool_free(srb, ha->srb_mempool);
>  
>  qc_host_busy_lock:
> +	rval = SCSI_MLQUEUE_HOST_BUSY;
>  	spin_lock_irq(ha->host->host_lock);
>  
> -qc_host_busy:
> -	return SCSI_MLQUEUE_HOST_BUSY;
> +qc_busy:
> +	return rval;
>  
>  qc_fail_command:
>  	done(cmd);
>  		goto qc_host_busy_lock;
> +	cmd->SCp.ptr = (void *)srb;
> +	srb->cmd = cmd;
>  
>  	rval = qla4xxx_send_command_to_isp(ha, srb);
>  	if (rval != QLA_SUCCESS)
> @@ -447,10 +449,11 @@ qc_host_busy_free_sp:
>  	mempool_free(srb, ha->srb_mempool);
>  
>  qc_host_busy_lock:
> +	rval = SCSI_MLQUEUE_HOST_BUSY;
>  	spin_lock_irq(ha->host->host_lock);
>  
> -qc_host_busy:
> -	return SCSI_MLQUEUE_HOST_BUSY;
> +qc_busy:
> +	return rval;
>  
>  qc_fail_command:
>  	done(cmd);
Signed-off-by: David C Somayajulu <david.somayajulu@qlogic.com>

Re: [PATCH 6/9] fc class: add fc queueing helper

[Michael Reed]

michaelc@cs.wisc.edu wrote:
> From: Mike Christie <michaelc@cs.wisc.edu>
> 
> Add a queueing helper which will return SCSI_MLQUEUE_TARGET_BUSY
> if we are transitioning because there is no need to call into
> the driver anymore at that point.
> 
> Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
> ---
>  include/scsi/scsi_transport_fc.h |   33 +++++++++++++++++++++++++++++++++
>  1 files changed, 33 insertions(+), 0 deletions(-)
> 
> diff --git a/include/scsi/scsi_transport_fc.h b/include/scsi/scsi_transport_fc.h
> index 4769efd..62a6a74 100644
> --- a/include/scsi/scsi_transport_fc.h
> +++ b/include/scsi/scsi_transport_fc.h
> @@ -29,6 +29,7 @@
>  
>  #include <linux/sched.h>
>  #include <scsi/scsi.h>
> +#include <scsi/scsi_cmnd.h>
>  #include <scsi/scsi_netlink.h>
>  
>  struct scsi_transport_template;
> @@ -640,6 +641,38 @@ struct fc_function_template {
>  	unsigned long	disable_target_scan:1;
>  };
>  
> +/**
> + * fc_remote_port_queue_ready - called to validate the remote port state
> + * @rport: remote port to be checked
> + *
> + * Returns 0 if IO can be queued or if there the cmd should be

Remove "there" from the comment.

> + * completed by a call to cmd->scsi_done. Caller must check cmd->result.
> + * Returns SCSI_MLQUEUE value if IO cannot be queued temporarily.

How 'bout: "Returns a SCSI_MLQUEUE value if the IO cannot be queued due
to an rport state transition." ?

> + **/
> +static inline int
> +fc_remote_port_queue_ready(struct fc_rport *rport, struct scsi_cmnd *cmd)
> +{
> +	int result = 0;
> +
> +	cmd->result = 0;
> +	switch (rport->port_state) {
> +	case FC_PORTSTATE_ONLINE:
> +		if (rport->roles & FC_PORT_ROLE_FCP_TARGET)
> +			result = 0;
> +		else if (rport->flags & FC_RPORT_DEVLOSS_PENDING)
> +			result = SCSI_MLQUEUE_TARGET_BUSY;
> +		else
> +			cmd->result = DID_NO_CONNECT << 16;
> +		break;
> +	case FC_PORTSTATE_BLOCKED:
> +		result = SCSI_MLQUEUE_TARGET_BUSY;
> +		break;
> +	default:
> +		cmd->result = DID_NO_CONNECT << 16;
> +		break;
> +	}
> +	return result;
> +}
>  
>  /**
>   * fc_remote_port_chkready - called to validate the remote port state

Re: [PATCH 6/9] fc class: add fc queueing helper

[Mike Christie]

Michael Reed wrote:
> 
> michaelc@cs.wisc.edu wrote:
>> From: Mike Christie <michaelc@cs.wisc.edu>
>>
>> Add a queueing helper which will return SCSI_MLQUEUE_TARGET_BUSY
>> if we are transitioning because there is no need to call into
>> the driver anymore at that point.
>>
>> Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
>> ---
>>  include/scsi/scsi_transport_fc.h |   33 +++++++++++++++++++++++++++++++++
>>  1 files changed, 33 insertions(+), 0 deletions(-)
>>
>> diff --git a/include/scsi/scsi_transport_fc.h b/include/scsi/scsi_transport_fc.h
>> index 4769efd..62a6a74 100644
>> --- a/include/scsi/scsi_transport_fc.h
>> +++ b/include/scsi/scsi_transport_fc.h
>> @@ -29,6 +29,7 @@
>>  
>>  #include <linux/sched.h>
>>  #include <scsi/scsi.h>
>> +#include <scsi/scsi_cmnd.h>
>>  #include <scsi/scsi_netlink.h>
>>  
>>  struct scsi_transport_template;
>> @@ -640,6 +641,38 @@ struct fc_function_template {
>>  	unsigned long	disable_target_scan:1;
>>  };
>>  
>> +/**
>> + * fc_remote_port_queue_ready - called to validate the remote port state
>> + * @rport: remote port to be checked
>> + *
>> + * Returns 0 if IO can be queued or if there the cmd should be
> 
> Remove "there" from the comment.
> 
>> + * completed by a call to cmd->scsi_done. Caller must check cmd->result.
>> + * Returns SCSI_MLQUEUE value if IO cannot be queued temporarily.
> 
> How 'bout: "Returns a SCSI_MLQUEUE value if the IO cannot be queued due
> to an rport state transition." ?
> 

A lot better :) I will fix that and the iscsi comment too. Thanks.

Re: [PATCH 9/9] mptfc: convert mptfc to use fc queueing helper.

[Michael Reed]

michaelc@cs.wisc.edu wrote:
> From: Mike Christie <michaelc@cs.wisc.edu>
> 
> Convert mptfc to use fc queueing helper.

I've been trying to exercise the fusion FC change.
I can't seem to force the code into returning either
TARGET_BUSY or DID_NO_CONNECT.  Any suggestions on
how I might get a command down to the driver while
the rport is blocked or in DEVLOSS_PENDING?

I've been running 32 outstanding commands to 16 targets,
disabling/enabling the various FC switch ports, AND
I disabled the usage of the scsi_host_lock in
fc_remote_port_delete() and I unlock/relock it in the
mptfc_qmnd() function.

Thanks,
 Mike

> 
> Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
> ---
>  drivers/message/fusion/mptfc.c |    7 ++++---
>  1 files changed, 4 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/message/fusion/mptfc.c b/drivers/message/fusion/mptfc.c
> index 3cdd4e9..1189bbe 100644
> --- a/drivers/message/fusion/mptfc.c
> +++ b/drivers/message/fusion/mptfc.c
> @@ -656,9 +656,10 @@ mptfc_qcmd(struct scsi_cmnd *SCpnt, void (*done)(struct scsi_cmnd *))
>  		return 0;
>  	}
>  
> -	err = fc_remote_port_chkready(rport);
> -	if (unlikely(err)) {
> -		SCpnt->result = err;
> +	err = fc_remote_port_queue_ready(rport, SCpnt);
> +	if (err)
> +		return err;
> +	if (SCpnt->result) {
>  		done(SCpnt);
>  		return 0;
>  	}

Re: [PATCH 9/9] mptfc: convert mptfc to use fc queueing helper.

[Mike Christie]

Michael Reed wrote:
> 
> michaelc@cs.wisc.edu wrote:
>> From: Mike Christie <michaelc@cs.wisc.edu>
>>
>> Convert mptfc to use fc queueing helper.
> 
> I've been trying to exercise the fusion FC change.
> I can't seem to force the code into returning either
> TARGET_BUSY or DID_NO_CONNECT.  Any suggestions on

Did you mean DID_IMM_RETRY in the original code?

> how I might get a command down to the driver while
> the rport is blocked or in DEVLOSS_PENDING?
> 
> I've been running 32 outstanding commands to 16 targets,
> disabling/enabling the various FC switch ports, AND
> I disabled the usage of the scsi_host_lock in
> fc_remote_port_delete() and I unlock/relock it in the
> mptfc_qmnd() function.
> 

For FC it is a small window where the rport is being blocked but the 
scsi_device is not yet in the blocked state. Instead of lots of targets 
you want lots of devices on just one target.

To test it with just one device you can just add a sleep in 
fc_remote_port_delete after the port_state and flags are set and the 
host lock is dropped and before the call to scsi_target_block.

        rport->port_state = FC_PORTSTATE_BLOCKED;

         rport->flags |= FC_RPORT_DEVLOSS_PENDING;

         spin_unlock_irqrestore(shost->host_lock, flags);

         if (rport->roles & FC_PORT_ROLE_FCP_INITIATOR &&
             shost->active_mode & MODE_TARGET)
                 fc_tgt_it_nexus_destroy(shost, (unsigned long)rport);


ssleep(1);

         scsi_target_block(&rport->dev);

Re: [PATCH 9/9] mptfc: convert mptfc to use fc queueing helper.

[Mike Christie]

Mike Christie wrote:
> Michael Reed wrote:
>>
>> michaelc@cs.wisc.edu wrote:
>>> From: Mike Christie <michaelc@cs.wisc.edu>
>>>
>>> Convert mptfc to use fc queueing helper.
>>
>> I've been trying to exercise the fusion FC change.
>> I can't seem to force the code into returning either
>> TARGET_BUSY or DID_NO_CONNECT.  Any suggestions on
> 
> Did you mean DID_IMM_RETRY in the original code?
> 
>> how I might get a command down to the driver while
>> the rport is blocked or in DEVLOSS_PENDING?
>>
>> I've been running 32 outstanding commands to 16 targets,
>> disabling/enabling the various FC switch ports, AND
>> I disabled the usage of the scsi_host_lock in
>> fc_remote_port_delete() and I unlock/relock it in the
>> mptfc_qmnd() function.
>>
> 
> For FC it is a small window where the rport is being blocked but the 
> scsi_device is not yet in the blocked state. Instead of lots of targets 
> you want lots of devices on just one target.
> 
> To test it with just one device you can just add a sleep in 
> fc_remote_port_delete after the port_state and flags are set and the 
> host lock is dropped and before the call to scsi_target_block.
> 
>        rport->port_state = FC_PORTSTATE_BLOCKED;
> 
>         rport->flags |= FC_RPORT_DEVLOSS_PENDING;
> 
>         spin_unlock_irqrestore(shost->host_lock, flags);
> 
>         if (rport->roles & FC_PORT_ROLE_FCP_INITIATOR &&
>             shost->active_mode & MODE_TARGET)
>                 fc_tgt_it_nexus_destroy(shost, (unsigned long)rport);
> 
> 
> ssleep(1);
> 
>         scsi_target_block(&rport->dev);

I think we could actually avoid the transition checks if we just held 
the host lock around the code we set the port state and when we call 
scsi_target_block. scsi_target_block would just need to be modified so 
that it did not grab the host lock (the driver model code does not use a 
mutex anymore).

On the scsi_target_unblock side we would grab the host lock and set the 
rport state and unblock the devices at the same time. Again we would 
need to not take the host lock when looping over the devices and we 
could just plug the queue instead of running the request_fn (because 
performance is not an issue in this code path instead of using 
blk_start_queue we could call somehing that cleared the stop flag and 
always plugged the queue).

Re: [PATCH 1/9] scsi: Add helper code so transport classes/driver can control queueing

[Mike Christie]

James,

Ignore this patchset for now. I will resend later.

I was trying to reuse the starved list, but scsi_run_queue's check and 
break when the blk_run_queue call put the device back on the starved 
list will cause problems when there are multiple targets and one is 
ready to go but the one before it is not.



michaelc@cs.wisc.edu wrote:
> From: Mike Christie <michaelc@cs.wisc.edu>
> 
> SCSI-ml manages the queueing limits for the device and host, but
> does not do so at the target level. However something something similar
> can come in userful when a driver is transitioning a transport object to
> the the blocked state, becuase at that time we do not want to queue
> io and we do not want the queuecommand to be called again.
> 
> The patch adds code similar to the exisiting SCSI_ML_*BUSY handlers.
> You can now return SCSI_MLQUEUE_TARGET_BUSY when we hit
> a transport level queueing issue like the hw cannot allocate some
> resource at the iscsi session/connection level, or the target has temporarily
> closed or shrunk the queueing window, or if we are transitioning
> to the blocked state.
> 
> For bnx2i we will also need to be able to limit queueing at this level.
> bnx2i will hook into libiscsi, but will allocate a scsi host per
> netdevice/hba, so unlike pure software iscsi/iser which is allocating
> a host per session, it cannot set the scsi_host->can_queue and return
> SCSI_MLQUEUE_HOST_BUSY to reflect queueing limits on the transport.
> 
> The iscsi class/driver can also set a scsi_target->can_queue value which
> reflects the max commands the driver/class can support. For iscsi this
> reflects the number of commands we can support for each session due to
> session/connection hw limits, driver limits, and to also reflect the
> session/targets's queueing window.
> 
> Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
> ---
>  drivers/scsi/scsi.c        |   11 +++++--
>  drivers/scsi/scsi_lib.c    |   72 +++++++++++++++++++++++++++++++++++++++-----
>  drivers/scsi/scsi_scan.c   |    1 +
>  include/scsi/scsi.h        |    1 +
>  include/scsi/scsi_device.h |   10 ++++++
>  5 files changed, 84 insertions(+), 11 deletions(-)
> 
> diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c
> index fecba05..a26f976 100644
> --- a/drivers/scsi/scsi.c
> +++ b/drivers/scsi/scsi.c
> @@ -628,9 +628,12 @@ int scsi_dispatch_cmd(struct scsi_cmnd *cmd)
>  	if (rtn) {
>  		if (scsi_delete_timer(cmd)) {
>  			atomic_inc(&cmd->device->iodone_cnt);
> -			scsi_queue_insert(cmd,
> -					  (rtn == SCSI_MLQUEUE_DEVICE_BUSY) ?
> -					  rtn : SCSI_MLQUEUE_HOST_BUSY);
> +
> +			if (rtn != SCSI_MLQUEUE_DEVICE_BUSY &&
> +			    rtn != SCSI_MLQUEUE_TARGET_BUSY)
> +				rtn = SCSI_MLQUEUE_HOST_BUSY;
> +
> +			scsi_queue_insert(cmd, rtn);
>  		}
>  		SCSI_LOG_MLQUEUE(3,
>  		    printk("queuecommand : request rejected\n"));
> @@ -729,6 +732,7 @@ static struct scsi_driver *scsi_cmd_to_driver(struct scsi_cmnd *cmd)
>  void scsi_finish_command(struct scsi_cmnd *cmd)
>  {
>  	struct scsi_device *sdev = cmd->device;
> +	struct scsi_target *starget = scsi_target(sdev);
>  	struct Scsi_Host *shost = sdev->host;
>  	struct scsi_driver *drv;
>  	unsigned int good_bytes;
> @@ -744,6 +748,7 @@ void scsi_finish_command(struct scsi_cmnd *cmd)
>  	 * XXX(hch): What about locking?
>           */
>          shost->host_blocked = 0;
> +	starget->target_blocked = 0;
>          sdev->device_blocked = 0;
>  
>  	/*
> diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
> index ba21d97..92b4f0e 100644
> --- a/drivers/scsi/scsi_lib.c
> +++ b/drivers/scsi/scsi_lib.c
> @@ -114,6 +114,7 @@ int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
>  {
>  	struct Scsi_Host *host = cmd->device->host;
>  	struct scsi_device *device = cmd->device;
> +	struct scsi_target *starget = scsi_target(device);
>  	struct request_queue *q = device->request_queue;
>  	unsigned long flags;
>  
> @@ -133,10 +134,17 @@ int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
>  	 * if a command is requeued with no other commands outstanding
>  	 * either for the device or for the host.
>  	 */
> -	if (reason == SCSI_MLQUEUE_HOST_BUSY)
> +	switch (reason) {
> +	case SCSI_MLQUEUE_HOST_BUSY:
>  		host->host_blocked = host->max_host_blocked;
> -	else if (reason == SCSI_MLQUEUE_DEVICE_BUSY)
> +		break;
> +	case SCSI_MLQUEUE_DEVICE_BUSY:
>  		device->device_blocked = device->max_device_blocked;
> +		break;
> +	case SCSI_MLQUEUE_TARGET_BUSY:
> +		starget->target_blocked = starget->max_target_blocked;
> +		break;
> +	}
>  
>  	/*
>  	 * Decrement the counters, since these commands are no longer
> @@ -451,10 +459,12 @@ static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
>  void scsi_device_unbusy(struct scsi_device *sdev)
>  {
>  	struct Scsi_Host *shost = sdev->host;
> +	struct scsi_target *starget = scsi_target(sdev);
>  	unsigned long flags;
>  
>  	spin_lock_irqsave(shost->host_lock, flags);
>  	shost->host_busy--;
> +	starget->target_busy--;
>  	if (unlikely(scsi_host_in_recovery(shost) &&
>  		     (shost->host_failed || shost->host_eh_scheduled)))
>  		scsi_eh_wakeup(shost);
> @@ -1295,6 +1305,51 @@ static inline int scsi_dev_queue_ready(struct request_queue *q,
>  	return 1;
>  }
>  
> +
> +/*
> + * scsi_target_queue_ready: checks if there we can send commands to target
> + * @sdev: scsi device on starget to check.
> + *
> + * Called with the host lock held.
> + */
> +static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
> +					   struct scsi_device *sdev)
> +{
> +	struct scsi_target *starget = scsi_target(sdev);
> +
> +	if (starget->single_lun) {
> +		if (starget->starget_sdev_user &&
> +		    starget->starget_sdev_user != sdev)
> +			return 0;
> +		starget->starget_sdev_user = sdev;
> +	}
> +
> +	if (starget->target_busy == 0 && starget->target_blocked) {
> +		/*
> +		 * unblock after target_blocked iterates to zero
> +		 */
> +		if (--starget->target_blocked == 0) {
> +			SCSI_LOG_MLQUEUE(3,
> +				   starget_printk(KERN_INFO, starget,
> +				   "unblocking target at zero depth\n"));
> +		} else {
> +			blk_plug_device(sdev->request_queue);
> +			return 0;
> +		}
> +	}
> +
> +	if ((starget->can_queue > 0 &&
> +	     starget->target_busy >= starget->can_queue) ||
> +	     starget->target_blocked) {
> +		if (list_empty(&sdev->starved_entry))
> +			list_add_tail(&sdev->starved_entry,
> +				      &shost->starved_list);
> +		return 0;
> +	}
> +
> +	return 1;
> +}
> +
>  /*
>   * scsi_host_queue_ready: if we can send requests to shost, return 1 else
>   * return 0. We must end up running the queue again whenever 0 is
> @@ -1342,6 +1397,7 @@ static void scsi_kill_request(struct request *req, struct request_queue *q)
>  {
>  	struct scsi_cmnd *cmd = req->special;
>  	struct scsi_device *sdev = cmd->device;
> +	struct scsi_target *starget = scsi_target(sdev);
>  	struct Scsi_Host *shost = sdev->host;
>  
>  	blkdev_dequeue_request(req);
> @@ -1365,6 +1421,7 @@ static void scsi_kill_request(struct request *req, struct request_queue *q)
>  	spin_unlock(sdev->request_queue->queue_lock);
>  	spin_lock(shost->host_lock);
>  	shost->host_busy++;
> +	starget->target_busy++;
>  	spin_unlock(shost->host_lock);
>  	spin_lock(sdev->request_queue->queue_lock);
>  
> @@ -1478,14 +1535,13 @@ static void scsi_request_fn(struct request_queue *q)
>  		}
>  		spin_lock(shost->host_lock);
>  
> +		if (!scsi_target_queue_ready(shost, sdev))
> +			goto not_ready;
> +
>  		if (!scsi_host_queue_ready(q, shost, sdev))
>  			goto not_ready;
> -		if (scsi_target(sdev)->single_lun) {
> -			if (scsi_target(sdev)->starget_sdev_user &&
> -			    scsi_target(sdev)->starget_sdev_user != sdev)
> -				goto not_ready;
> -			scsi_target(sdev)->starget_sdev_user = sdev;
> -		}
> +
> +		scsi_target(sdev)->target_busy++;
>  		shost->host_busy++;
>  
>  		/*
> diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c
> index 1dc165a..c9f7b74 100644
> --- a/drivers/scsi/scsi_scan.c
> +++ b/drivers/scsi/scsi_scan.c
> @@ -396,6 +396,7 @@ static struct scsi_target *scsi_alloc_target(struct device *parent,
>  		shost->host_no, channel, id);
>  	starget->id = id;
>  	starget->channel = channel;
> +	starget->can_queue = 0;
>  	INIT_LIST_HEAD(&starget->siblings);
>  	INIT_LIST_HEAD(&starget->devices);
>  	starget->state = STARGET_RUNNING;
> diff --git a/include/scsi/scsi.h b/include/scsi/scsi.h
> index 1f74bcd..2160b36 100644
> --- a/include/scsi/scsi.h
> +++ b/include/scsi/scsi.h
> @@ -381,6 +381,7 @@ struct scsi_lun {
>  #define SCSI_MLQUEUE_HOST_BUSY   0x1055
>  #define SCSI_MLQUEUE_DEVICE_BUSY 0x1056
>  #define SCSI_MLQUEUE_EH_RETRY    0x1057
> +#define SCSI_MLQUEUE_TARGET_BUSY 0x1058
>  
>  /*
>   *  Use these to separate status msg and our bytes
> diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h
> index ab7acbe..f8e4184 100644
> --- a/include/scsi/scsi_device.h
> +++ b/include/scsi/scsi_device.h
> @@ -205,6 +205,16 @@ struct scsi_target {
>  						 * for the device at a time. */
>  	unsigned int		pdt_1f_for_no_lun;	/* PDT = 0x1f */
>  						/* means no lun present */
> +	/* commands actually active on LLD. protected by host lock. */
> +	unsigned int		target_busy;
> +	/*
> +	 * LLDs should set this in the slave_alloc host template callout.
> +	 * If set to zero then there is not limit.
> +	 */
> +	unsigned int		can_queue;
> +	unsigned int		target_blocked;
> +	unsigned int		max_target_blocked;
> +#define SCSI_DEFAULT_TARGET_BLOCKED	3
>  
>  	char			scsi_level;
>  	struct execute_work	ew;