[PATCH] delay transition requeues for 2 seconds - alua

[PATCH] delay transition requeues for 2 seconds - alua

[Rob Evers]

From: Rob Evers <revers@redhat.com>

When alua targets are transitioning, the scsi midlayer retry mechanism
continuously retries the scsi commands that are returning with not ready
transitioning status.  The target is not capable of handling the
commands for time on the order of several seconds during these transistions.

This patch delays the device queue for 2 seconds, which is in the same
order of aas transition time.

Also, handle all other cases where ADD_TO_MLQUEUE_DELAY could be returned
instead of ADD_TO_MLQUEUE as if ADD_TO_MLQUEUE were being returned.

Problem found by array partner testing

change MLQUEUE_DEV_DLY_RTY to MLQUEUE_DELAYED_RETRY

Signed-off-by: Rob Evers <revers@redhat.com>
---
 drivers/scsi/device_handler/scsi_dh_alua.c |    7 ++++---
 drivers/scsi/scsi.c                        |    3 +++
 drivers/scsi/scsi_error.c                  |    1 +
 drivers/scsi/scsi_lib.c                    |    9 ++++++++-
 include/scsi/scsi.h                        |   12 +++++++-----
 5 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c
index 4ef0212..33b8df7 100644
--- a/drivers/scsi/device_handler/scsi_dh_alua.c
+++ b/drivers/scsi/device_handler/scsi_dh_alua.c
@@ -233,7 +233,7 @@ static void stpg_endio(struct request *req, int error)
 			goto done;
 		}
 		err = alua_check_sense(h->sdev, &sense_hdr);
-		if (err == ADD_TO_MLQUEUE) {
+		if (err == ADD_TO_MLQUEUE || err == ADD_TO_MLQUEUE_DELAY) {
 			err = SCSI_DH_RETRY;
 			goto done;
 		}
@@ -443,7 +443,7 @@ static int alua_check_sense(struct scsi_device *sdev,
 			/*
 			 * LUN Not Accessible - ALUA state transition
 			 */
-			return ADD_TO_MLQUEUE;
+			return ADD_TO_MLQUEUE_DELAY;
 		if (sense_hdr->asc == 0x04 && sense_hdr->ascq == 0x0b)
 			/*
 			 * LUN Not Accessible -- Target port in standby state
@@ -521,7 +521,8 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
 			return SCSI_DH_IO;
 
 		err = alua_check_sense(sdev, &sense_hdr);
-		if (err == ADD_TO_MLQUEUE && time_before(jiffies, expiry))
+		if ((err == ADD_TO_MLQUEUE || err == ADD_TO_MLQUEUE_DELAY) &&
+		    time_before(jiffies, expiry))
 			goto retry;
 		sdev_printk(KERN_INFO, sdev,
 			    "%s: rtpg sense code %02x/%02x/%02x\n",
diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c
index 2aeb2e9..ced0085 100644
--- a/drivers/scsi/scsi.c
+++ b/drivers/scsi/scsi.c
@@ -600,6 +600,9 @@ void scsi_log_completion(struct scsi_cmnd *cmd, int disposition)
 			case ADD_TO_MLQUEUE:
 				printk("MLQUEUE\n");
 				break;
+			case ADD_TO_MLQUEUE_DELAY:
+				printk("MLQUEUE_DELAY\n");
+				break;
 			case FAILED:
 				printk("FAILED\n");
 				break;
diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
index dc6131e..775f822 100644
--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -812,6 +812,7 @@ static int scsi_send_eh_cmnd(struct scsi_cmnd *scmd, unsigned char *cmnd,
 		case TARGET_ERROR:
 			break;
 		case ADD_TO_MLQUEUE:
+		case ADD_TO_MLQUEUE_DELAY:
 			rtn = NEEDS_RETRY;
 			break;
 		default:
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 06bc265..02a7421 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -139,6 +139,7 @@ static int __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
 		break;
 	case SCSI_MLQUEUE_DEVICE_BUSY:
 	case SCSI_MLQUEUE_EH_RETRY:
+	case SCSI_MLQUEUE_DELAYED_RETRY:
 		device->device_blocked = device->max_device_blocked;
 		break;
 	case SCSI_MLQUEUE_TARGET_BUSY:
@@ -161,7 +162,10 @@ static int __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
 	blk_requeue_request(q, cmd->request);
 	spin_unlock_irqrestore(q->queue_lock, flags);
 
-	kblockd_schedule_work(q, &device->requeue_work);
+	if (reason == SCSI_MLQUEUE_DELAYED_RETRY)
+		blk_delay_queue(q, 2000);
+	else
+		kblockd_schedule_work(q, &device->requeue_work);
 
 	return 0;
 }
@@ -1465,6 +1469,9 @@ static void scsi_softirq_done(struct request *rq)
 		case ADD_TO_MLQUEUE:
 			scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
 			break;
+		case ADD_TO_MLQUEUE_DELAY:
+			scsi_queue_insert(cmd, SCSI_MLQUEUE_DELAYED_RETRY);
+			break;
 		default:
 			if (!scsi_eh_scmd_add(cmd, 0))
 				scsi_finish_command(cmd);
diff --git a/include/scsi/scsi.h b/include/scsi/scsi.h
index 8001ae4..699d24c 100644
--- a/include/scsi/scsi.h
+++ b/include/scsi/scsi.h
@@ -466,17 +466,19 @@ static inline int scsi_is_wlun(unsigned int lun)
 #define SOFT_ERROR      0x2005
 #define ADD_TO_MLQUEUE  0x2006
 #define TIMEOUT_ERROR   0x2007
-#define SCSI_RETURN_NOT_HANDLED   0x2008
+#define SCSI_RETURN_NOT_HANDLED	0x2008
 #define FAST_IO_FAIL	0x2009
 #define TARGET_ERROR    0x200A
+#define ADD_TO_MLQUEUE_DELAY	0x200B
 
 /*
  * Midlevel queue return values.
  */
-#define SCSI_MLQUEUE_HOST_BUSY   0x1055
-#define SCSI_MLQUEUE_DEVICE_BUSY 0x1056
-#define SCSI_MLQUEUE_EH_RETRY    0x1057
-#define SCSI_MLQUEUE_TARGET_BUSY 0x1058
+#define SCSI_MLQUEUE_HOST_BUSY     0x1055
+#define SCSI_MLQUEUE_DEVICE_BUSY   0x1056
+#define SCSI_MLQUEUE_EH_RETRY      0x1057
+#define SCSI_MLQUEUE_TARGET_BUSY   0x1058
+#define SCSI_MLQUEUE_DELAYED_RETRY 0x1059
 
 /*
  *  Use these to separate status msg and our bytes
-- 
1.7.4.2

Re: [PATCH] delay transition requeues for 2 seconds - alua

[Rob Evers]

On 01/03/2012 02:20 PM, Rob Evers wrote:
> From: Rob Evers<revers@redhat.com>
>
> When alua targets are transitioning, the scsi midlayer retry mechanism
> continuously retries the scsi commands that are returning with not ready
> transitioning status.  The target is not capable of handling the
> commands for time on the order of several seconds during these transistions.
>
> This patch delays the device queue for 2 seconds, which is in the same
> order of aas transition time.
>
> Also, handle all other cases where ADD_TO_MLQUEUE_DELAY could be returned
> instead of ADD_TO_MLQUEUE as if ADD_TO_MLQUEUE were being returned.
>
> Problem found by array partner testing
>
> change MLQUEUE_DEV_DLY_RTY to MLQUEUE_DELAYED_RETRY

Hi James,

I noticed this didn't get included yet.

If you are planning to include this, please hold off.
I misunderstood the testing status initially and we
need to collect more information.  It may take some
time.

Rob

Re: [PATCH] delay transition requeues for 2 seconds - alua

[Rob Evers]

On 01/07/2012 10:06 AM, Rob Evers wrote:
> On 01/03/2012 02:20 PM, Rob Evers wrote:
>> From: Rob Evers<revers@redhat.com>
>>
>> When alua targets are transitioning, the scsi midlayer retry mechanism
>> continuously retries the scsi commands that are returning with not ready
>> transitioning status.  The target is not capable of handling the
>> commands for time on the order of several seconds during these 
>> transistions.
>>
>> This patch delays the device queue for 2 seconds, which is in the same
>> order of aas transition time.
>>
>> Also, handle all other cases where ADD_TO_MLQUEUE_DELAY could be 
>> returned
>> instead of ADD_TO_MLQUEUE as if ADD_TO_MLQUEUE were being returned.
>>
>> Problem found by array partner testing
>>
>> change MLQUEUE_DEV_DLY_RTY to MLQUEUE_DELAYED_RETRY
>
> Hi James,
>
> I noticed this didn't get included yet.
>
> If you are planning to include this, please hold off.
> I misunderstood the testing status initially and we
> need to collect more information.  It may take some
> time.
>
> Rob

Our partner provided some updated results on testing
of this and the results are mixed.

Read/write traffic is reduced to the array during
IO transitions as was expected.

However, IO stalls during alua array transitions
have been seen to get longer in some cases.

Based on this, I withdraw the request to include
this patch.

Rob

Re: [PATCH] delay transition requeues for 2 seconds - alua

[Hannes Reinecke]

On 01/03/2012 08:20 PM, Rob Evers wrote:
> From: Rob Evers <revers@redhat.com>
> 
> When alua targets are transitioning, the scsi midlayer retry mechanism
> continuously retries the scsi commands that are returning with not ready
> transitioning status.  The target is not capable of handling the
> commands for time on the order of several seconds during these transistions.
> 
> This patch delays the device queue for 2 seconds, which is in the same
> order of aas transition time.
> 
> Also, handle all other cases where ADD_TO_MLQUEUE_DELAY could be returned
> instead of ADD_TO_MLQUEUE as if ADD_TO_MLQUEUE were being returned.
> 
> Problem found by array partner testing
> 
> change MLQUEUE_DEV_DLY_RTY to MLQUEUE_DELAYED_RETRY
> 
I have been working on a different solution, whic
> Signed-off-by: Rob Evers <revers@redhat.com>
> ---
>  drivers/scsi/device_handler/scsi_dh_alua.c |    7 ++++---
>  drivers/scsi/scsi.c                        |    3 +++
>  drivers/scsi/scsi_error.c                  |    1 +
>  drivers/scsi/scsi_lib.c                    |    9 ++++++++-
>  include/scsi/scsi.h                        |   12 +++++++-----
>  5 files changed, 23 insertions(+), 9 deletions(-)
> 
> diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c
> index 4ef0212..33b8df7 100644
> --- a/drivers/scsi/device_handler/scsi_dh_alua.c
> +++ b/drivers/scsi/device_handler/scsi_dh_alua.c
> @@ -233,7 +233,7 @@ static void stpg_endio(struct request *req, int error)
>  			goto done;
>  		}
>  		err = alua_check_sense(h->sdev, &sense_hdr);
> -		if (err == ADD_TO_MLQUEUE) {
> +		if (err == ADD_TO_MLQUEUE || err == ADD_TO_MLQUEUE_DELAY) {
>  			err = SCSI_DH_RETRY;
>  			goto done;
>  		}
> @@ -443,7 +443,7 @@ static int alua_check_sense(struct scsi_device *sdev,
>  			/*
>  			 * LUN Not Accessible - ALUA state transition
>  			 */
> -			return ADD_TO_MLQUEUE;
> +			return ADD_TO_MLQUEUE_DELAY;
>  		if (sense_hdr->asc == 0x04 && sense_hdr->ascq == 0x0b)
>  			/*
>  			 * LUN Not Accessible -- Target port in standby state
> @@ -521,7 +521,8 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
>  			return SCSI_DH_IO;
>  
>  		err = alua_check_sense(sdev, &sense_hdr);
> -		if (err == ADD_TO_MLQUEUE && time_before(jiffies, expiry))
> +		if ((err == ADD_TO_MLQUEUE || err == ADD_TO_MLQUEUE_DELAY) &&
> +		    time_before(jiffies, expiry))
>  			goto retry;
>  		sdev_printk(KERN_INFO, sdev,
>  			    "%s: rtpg sense code %02x/%02x/%02x\n",
Actually, this doesn't help if the RTPG command returns with the
mentioned error; then you'll just continue flooding the array with
RTPG commands. You'll need to delay the RTPG commands, too.

I'll be sending an updated patch.

Cheers,

Hannes
-- 
Dr. Hannes Reinecke		      zSeries & Storage
hare@suse.de			      +49 911 74053 688
SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg
GF: J. Hawn, J. Guild, F. Imendörffer, HRB 16746 (AG Nürnberg)
--
To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] delay transition requeues for 2 seconds - alua

[Rob Evers]

On 01/12/2012 04:43 AM, Hannes Reinecke wrote:
> On 01/03/2012 08:20 PM, Rob Evers wrote:
>> From: Rob Evers<revers@redhat.com>
>>
>> When alua targets are transitioning, the scsi midlayer retry mechanism
>> continuously retries the scsi commands that are returning with not ready
>> transitioning status.  The target is not capable of handling the
>> commands for time on the order of several seconds during these transistions.
>>
>> This patch delays the device queue for 2 seconds, which is in the same
>> order of aas transition time.
>>
>> Also, handle all other cases where ADD_TO_MLQUEUE_DELAY could be returned
>> instead of ADD_TO_MLQUEUE as if ADD_TO_MLQUEUE were being returned.
>>
>> Problem found by array partner testing
>>
>> change MLQUEUE_DEV_DLY_RTY to MLQUEUE_DELAYED_RETRY
>>
> I have been working on a different solution, whic
>> Signed-off-by: Rob Evers<revers@redhat.com>
>> ---
>>   drivers/scsi/device_handler/scsi_dh_alua.c |    7 ++++---
>>   drivers/scsi/scsi.c                        |    3 +++
>>   drivers/scsi/scsi_error.c                  |    1 +
>>   drivers/scsi/scsi_lib.c                    |    9 ++++++++-
>>   include/scsi/scsi.h                        |   12 +++++++-----
>>   5 files changed, 23 insertions(+), 9 deletions(-)
>>
>> diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c
>> index 4ef0212..33b8df7 100644
>> --- a/drivers/scsi/device_handler/scsi_dh_alua.c
>> +++ b/drivers/scsi/device_handler/scsi_dh_alua.c
>> @@ -233,7 +233,7 @@ static void stpg_endio(struct request *req, int error)
>>   			goto done;
>>   		}
>>   		err = alua_check_sense(h->sdev,&sense_hdr);
>> -		if (err == ADD_TO_MLQUEUE) {
>> +		if (err == ADD_TO_MLQUEUE || err == ADD_TO_MLQUEUE_DELAY) {
>>   			err = SCSI_DH_RETRY;
>>   			goto done;
>>   		}
>> @@ -443,7 +443,7 @@ static int alua_check_sense(struct scsi_device *sdev,
>>   			/*
>>   			 * LUN Not Accessible - ALUA state transition
>>   			 */
>> -			return ADD_TO_MLQUEUE;
>> +			return ADD_TO_MLQUEUE_DELAY;
>>   		if (sense_hdr->asc == 0x04&&  sense_hdr->ascq == 0x0b)
>>   			/*
>>   			 * LUN Not Accessible -- Target port in standby state
>> @@ -521,7 +521,8 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
>>   			return SCSI_DH_IO;
>>
>>   		err = alua_check_sense(sdev,&sense_hdr);
>> -		if (err == ADD_TO_MLQUEUE&&  time_before(jiffies, expiry))
>> +		if ((err == ADD_TO_MLQUEUE || err == ADD_TO_MLQUEUE_DELAY)&&
>> +		    time_before(jiffies, expiry))
>>   			goto retry;
>>   		sdev_printk(KERN_INFO, sdev,
>>   			    "%s: rtpg sense code %02x/%02x/%02x\n",
> Actually, this doesn't help if the RTPG command returns with the
> mentioned error; then you'll just continue flooding the array with
> RTPG commands. You'll need to delay the RTPG commands, too.

I thought that the rtpg command would get requeued into the
device queue that is being delayed anyway.

Isn't that true?

Rob

Re: [PATCH] delay transition requeues for 2 seconds - alua

[Hannes Reinecke]

On 01/12/2012 06:03 PM, Rob Evers wrote:
> On 01/12/2012 04:43 AM, Hannes Reinecke wrote:
>> On 01/03/2012 08:20 PM, Rob Evers wrote:
>>> From: Rob Evers<revers@redhat.com>
>>>
>>> When alua targets are transitioning, the scsi midlayer retry mechanism
>>> continuously retries the scsi commands that are returning with not ready
>>> transitioning status. The target is not capable of handling the
>>> commands for time on the order of several seconds during these
>>> transistions.
>>>
>>> This patch delays the device queue for 2 seconds, which is in the same
>>> order of aas transition time.
>>>
>>> Also, handle all other cases where ADD_TO_MLQUEUE_DELAY could be
>>> returned
>>> instead of ADD_TO_MLQUEUE as if ADD_TO_MLQUEUE were being returned.
>>>
>>> Problem found by array partner testing
>>>
>>> change MLQUEUE_DEV_DLY_RTY to MLQUEUE_DELAYED_RETRY
>>>
>> I have been working on a different solution, whic
>>> Signed-off-by: Rob Evers<revers@redhat.com>
>>> ---
>>> drivers/scsi/device_handler/scsi_dh_alua.c | 7 ++++---
>>> drivers/scsi/scsi.c | 3 +++
>>> drivers/scsi/scsi_error.c | 1 +
>>> drivers/scsi/scsi_lib.c | 9 ++++++++-
>>> include/scsi/scsi.h | 12 +++++++-----
>>> 5 files changed, 23 insertions(+), 9 deletions(-)
>>>
>>> diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c
>>> b/drivers/scsi/device_handler/scsi_dh_alua.c
>>> index 4ef0212..33b8df7 100644
>>> --- a/drivers/scsi/device_handler/scsi_dh_alua.c
>>> +++ b/drivers/scsi/device_handler/scsi_dh_alua.c
>>> @@ -233,7 +233,7 @@ static void stpg_endio(struct request *req, int
>>> error)
>>> goto done;
>>> }
>>> err = alua_check_sense(h->sdev,&sense_hdr);
>>> - if (err == ADD_TO_MLQUEUE) {
>>> + if (err == ADD_TO_MLQUEUE || err == ADD_TO_MLQUEUE_DELAY) {
>>> err = SCSI_DH_RETRY;
>>> goto done;
>>> }
>>> @@ -443,7 +443,7 @@ static int alua_check_sense(struct scsi_device
>>> *sdev,
>>> /*
>>> * LUN Not Accessible - ALUA state transition
>>> */
>>> - return ADD_TO_MLQUEUE;
>>> + return ADD_TO_MLQUEUE_DELAY;
>>> if (sense_hdr->asc == 0x04&& sense_hdr->ascq == 0x0b)
>>> /*
>>> * LUN Not Accessible -- Target port in standby state
>>> @@ -521,7 +521,8 @@ static int alua_rtpg(struct scsi_device *sdev,
>>> struct alua_dh_data *h)
>>> return SCSI_DH_IO;
>>>
>>> err = alua_check_sense(sdev,&sense_hdr);
>>> - if (err == ADD_TO_MLQUEUE&& time_before(jiffies, expiry))
>>> + if ((err == ADD_TO_MLQUEUE || err == ADD_TO_MLQUEUE_DELAY)&&
>>> + time_before(jiffies, expiry))
>>> goto retry;
>>> sdev_printk(KERN_INFO, sdev,
>>> "%s: rtpg sense code %02x/%02x/%02x\n",
>> Actually, this doesn't help if the RTPG command returns with the
>> mentioned error; then you'll just continue flooding the array with
>> RTPG commands. You'll need to delay the RTPG commands, too.
>
> I thought that the rtpg command would get requeued into the
> device queue that is being delayed anyway.
>
> Isn't that true?
>
Nope.

rtpg is being send via the SG_IO path, for which the error is returned 
directly without being retried.

So we'll need to fix it up in the alua handler.

Cheers,

Hannes
-- 
Dr. Hannes Reinecke		      zSeries & Storage
hare@suse.de			      +49 911 74053 688
SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg
GF: Markus Rex, HRB 16746 (AG Nürnberg)

--
To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] delay transition requeues for 2 seconds - alua

[Mike Christie]

On 01/12/2012 03:23 PM, Hannes Reinecke wrote:
> On 01/12/2012 06:03 PM, Rob Evers wrote:
>> On 01/12/2012 04:43 AM, Hannes Reinecke wrote:
>>> On 01/03/2012 08:20 PM, Rob Evers wrote:
>>>> From: Rob Evers<revers@redhat.com>
>>>>
>>>> When alua targets are transitioning, the scsi midlayer retry mechanism
>>>> continuously retries the scsi commands that are returning with not
>>>> ready
>>>> transitioning status. The target is not capable of handling the
>>>> commands for time on the order of several seconds during these
>>>> transistions.
>>>>
>>>> This patch delays the device queue for 2 seconds, which is in the same
>>>> order of aas transition time.
>>>>
>>>> Also, handle all other cases where ADD_TO_MLQUEUE_DELAY could be
>>>> returned
>>>> instead of ADD_TO_MLQUEUE as if ADD_TO_MLQUEUE were being returned.
>>>>
>>>> Problem found by array partner testing
>>>>
>>>> change MLQUEUE_DEV_DLY_RTY to MLQUEUE_DELAYED_RETRY
>>>>
>>> I have been working on a different solution, whic
>>>> Signed-off-by: Rob Evers<revers@redhat.com>
>>>> ---
>>>> drivers/scsi/device_handler/scsi_dh_alua.c | 7 ++++---
>>>> drivers/scsi/scsi.c | 3 +++
>>>> drivers/scsi/scsi_error.c | 1 +
>>>> drivers/scsi/scsi_lib.c | 9 ++++++++-
>>>> include/scsi/scsi.h | 12 +++++++-----
>>>> 5 files changed, 23 insertions(+), 9 deletions(-)
>>>>
>>>> diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c
>>>> b/drivers/scsi/device_handler/scsi_dh_alua.c
>>>> index 4ef0212..33b8df7 100644
>>>> --- a/drivers/scsi/device_handler/scsi_dh_alua.c
>>>> +++ b/drivers/scsi/device_handler/scsi_dh_alua.c
>>>> @@ -233,7 +233,7 @@ static void stpg_endio(struct request *req, int
>>>> error)
>>>> goto done;
>>>> }
>>>> err = alua_check_sense(h->sdev,&sense_hdr);
>>>> - if (err == ADD_TO_MLQUEUE) {
>>>> + if (err == ADD_TO_MLQUEUE || err == ADD_TO_MLQUEUE_DELAY) {
>>>> err = SCSI_DH_RETRY;
>>>> goto done;
>>>> }
>>>> @@ -443,7 +443,7 @@ static int alua_check_sense(struct scsi_device
>>>> *sdev,
>>>> /*
>>>> * LUN Not Accessible - ALUA state transition
>>>> */
>>>> - return ADD_TO_MLQUEUE;
>>>> + return ADD_TO_MLQUEUE_DELAY;
>>>> if (sense_hdr->asc == 0x04&& sense_hdr->ascq == 0x0b)
>>>> /*
>>>> * LUN Not Accessible -- Target port in standby state
>>>> @@ -521,7 +521,8 @@ static int alua_rtpg(struct scsi_device *sdev,
>>>> struct alua_dh_data *h)
>>>> return SCSI_DH_IO;
>>>>
>>>> err = alua_check_sense(sdev,&sense_hdr);
>>>> - if (err == ADD_TO_MLQUEUE&& time_before(jiffies, expiry))
>>>> + if ((err == ADD_TO_MLQUEUE || err == ADD_TO_MLQUEUE_DELAY)&&
>>>> + time_before(jiffies, expiry))
>>>> goto retry;
>>>> sdev_printk(KERN_INFO, sdev,
>>>> "%s: rtpg sense code %02x/%02x/%02x\n",
>>> Actually, this doesn't help if the RTPG command returns with the
>>> mentioned error; then you'll just continue flooding the array with
>>> RTPG commands. You'll need to delay the RTPG commands, too.
>>
>> I thought that the rtpg command would get requeued into the
>> device queue that is being delayed anyway.
>>
>> Isn't that true?
>>
> Nope.
> 
> rtpg is being send via the SG_IO path, for which the error is returned
> directly without being retried.
> 

It should get retried by the scsi_decide_disposition/scsi_softirq_done
code. It should be going from:

scsi_softirq_done->scsi_decide_disposition->scsi_check_sense->scsi_dh->check_sense->alua_check_sense

alua_check_sense will return ADD_TO_MLQUEUE_DELAY then scsi_check_sense
will pass that up and scsi_decide_disposition will return that right
away. And then in scsi_softirq_done we will just requeue in the code the
patch added:

+		case ADD_TO_MLQUEUE_DELAY:
+			scsi_queue_insert(cmd, SCSI_MLQUEUE_DELAYED_RETRY);
+			break;

Re: [PATCH] delay transition requeues for 2 seconds - alua

[Mike Christie]

On 01/12/2012 01:54 PM, Mike Christie wrote:
> alua_check_sense will return ADD_TO_MLQUEUE_DELAY then scsi_check_sense
> will pass that up and scsi_decide_disposition will return that right
> away.

I mean it is one of those weird ones where we do not do the goto
maybe_retry in scsi_decide_disposition, so we do not see the fast fail
bit set. This happens for ADD_TO_MLQUEUE_DELAY and ADD_TO_MLQUEUE.

Re: [PATCH] delay transition requeues for 2 seconds - alua

[Hannes Reinecke]

On 01/12/2012 08:59 PM, Mike Christie wrote:
> On 01/12/2012 01:54 PM, Mike Christie wrote:
>> alua_check_sense will return ADD_TO_MLQUEUE_DELAY then scsi_check_sense
>> will pass that up and scsi_decide_disposition will return that right
>> away.
>
> I mean it is one of those weird ones where we do not do the goto
> maybe_retry in scsi_decide_disposition, so we do not see the fast fail
> bit set. This happens for ADD_TO_MLQUEUE_DELAY and ADD_TO_MLQUEUE.
>
Hmm. Not sure here.
With the above reasoning SG_IO would be retried, too.
Which it most definitely isn't.

I'll be digging deeper here tomorrow.

Cheers,

Hannes
-- 
Dr. Hannes Reinecke		      zSeries & Storage
hare@suse.de			      +49 911 74053 688
SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg
GF: Markus Rex, HRB 16746 (AG Nürnberg)

--
To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] delay transition requeues for 2 seconds - alua

[Rob Evers]

On 01/12/2012 05:13 PM, Hannes Reinecke wrote:
> On 01/12/2012 08:59 PM, Mike Christie wrote:
>> On 01/12/2012 01:54 PM, Mike Christie wrote:
>>> alua_check_sense will return ADD_TO_MLQUEUE_DELAY then scsi_check_sense
>>> will pass that up and scsi_decide_disposition will return that right
>>> away.
>>
>> I mean it is one of those weird ones where we do not do the goto
>> maybe_retry in scsi_decide_disposition, so we do not see the fast fail
>> bit set. This happens for ADD_TO_MLQUEUE_DELAY and ADD_TO_MLQUEUE.
>>
> Hmm. Not sure here.
> With the above reasoning SG_IO would be retried, too.
> Which it most definitely isn't.
>
> I'll be digging deeper here tomorrow.
>
> Cheers,
>
> Hannes

Hannes,

I ran some tests today to verify what you said about rtpg not ending
up executing the ADD_TO_MLQUEUE_DELAY path via scsi_softirq_done.

So yes, looks like another delay is required in alua_rtpg.

Rob

Re: [PATCH] delay transition requeues for 2 seconds - alua

[Rob Evers]

On 01/13/2012 12:51 PM, Rob Evers wrote:
> On 01/12/2012 05:13 PM, Hannes Reinecke wrote:
>> On 01/12/2012 08:59 PM, Mike Christie wrote:
>>> On 01/12/2012 01:54 PM, Mike Christie wrote:
>>>> alua_check_sense will return ADD_TO_MLQUEUE_DELAY then 
>>>> scsi_check_sense
>>>> will pass that up and scsi_decide_disposition will return that right
>>>> away.
>>>
>>> I mean it is one of those weird ones where we do not do the goto
>>> maybe_retry in scsi_decide_disposition, so we do not see the fast fail
>>> bit set. This happens for ADD_TO_MLQUEUE_DELAY and ADD_TO_MLQUEUE.
>>>
>> Hmm. Not sure here.
>> With the above reasoning SG_IO would be retried, too.
>> Which it most definitely isn't.
>>
>> I'll be digging deeper here tomorrow.
>>
>> Cheers,
>>
>> Hannes
>
> Hannes,
>
> I ran some tests today to verify what you said about rtpg not ending
> up executing the ADD_TO_MLQUEUE_DELAY path via scsi_softirq_done.
>
> So yes, looks like another delay is required in alua_rtpg.

It turns out that the rtpg activity on an array is limited during these
transitions and scales with the number of paths connected to the array.

What I have seen for rtpg sense codes after the first alua_rtpg retry is:

06/29/00
06/2a/06

and 1-2 retries in alua_rtpg for the first retry condition.  This is for
every path to an array.

This activity follows shortly after an array controller reboot.
The rtpg retries all occur within a second of each other.

The rtpg sense codes never match the modifications to alua_check_sense
where a ADD_TO_MLQUEUE_DELAY condition gets triggered (2/4/a).  This is
confirmed by our array vendor partner.

The 1st rtpg retries in alua_rtpg don't get delayed by the sdevice queue 
being
delayed, at least they are never seperated by 2 seconds as would 
indicate that.

I could use an explanation of why the rtpgs retries don't get delayed, 
if someone knows and would be so kind.

The alua_check_sense 2/4/a triggers the ADD_TO_MLQUEUE_DELAY condition
and this does cause the sdevice queue to delay, and this repeats every 2 
seconds
as expected during the transitions.

Hannes,

Can you revisit this?

Thanks, Rob

[PATCH] delay transition requeues for 2 seconds - alua

[Rob Evers]

When alua targets are transitioning, the scsi midlayer retry mechanism
continuously retries the scsi commands that are returning with not ready
transitioning status.  The target is not capable of handling the
commands for time on the order of several seconds during these transistions.

This patch delays the device queue for 2 seconds, which is in the same
order of aas transition time.

Also, handle all other cases where ADD_TO_MLQUEUE_DELAY could be returned
instead of ADD_TO_MLQUEUE as if ADD_TO_MLQUEUE were being returned.

Problem found by array partner testing

Signed-off-by: Rob Evers <revers@redhat.com>
---
 drivers/scsi/device_handler/scsi_dh_alua.c |    7 ++++---
 drivers/scsi/scsi.c                        |    3 +++
 drivers/scsi/scsi_error.c                  |    1 +
 drivers/scsi/scsi_lib.c                    |    9 ++++++++-
 include/scsi/scsi.h                        |    4 +++-
 5 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c
index 4ef0212..33b8df7 100644
--- a/drivers/scsi/device_handler/scsi_dh_alua.c
+++ b/drivers/scsi/device_handler/scsi_dh_alua.c
@@ -233,7 +233,7 @@ static void stpg_endio(struct request *req, int error)
 			goto done;
 		}
 		err = alua_check_sense(h->sdev, &sense_hdr);
-		if (err == ADD_TO_MLQUEUE) {
+		if (err == ADD_TO_MLQUEUE || err == ADD_TO_MLQUEUE_DELAY) {
 			err = SCSI_DH_RETRY;
 			goto done;
 		}
@@ -443,7 +443,7 @@ static int alua_check_sense(struct scsi_device *sdev,
 			/*
 			 * LUN Not Accessible - ALUA state transition
 			 */
-			return ADD_TO_MLQUEUE;
+			return ADD_TO_MLQUEUE_DELAY;
 		if (sense_hdr->asc == 0x04 && sense_hdr->ascq == 0x0b)
 			/*
 			 * LUN Not Accessible -- Target port in standby state
@@ -521,7 +521,8 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
 			return SCSI_DH_IO;
 
 		err = alua_check_sense(sdev, &sense_hdr);
-		if (err == ADD_TO_MLQUEUE && time_before(jiffies, expiry))
+		if ((err == ADD_TO_MLQUEUE || err == ADD_TO_MLQUEUE_DELAY) &&
+		    time_before(jiffies, expiry))
 			goto retry;
 		sdev_printk(KERN_INFO, sdev,
 			    "%s: rtpg sense code %02x/%02x/%02x\n",
diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c
index 2aeb2e9..ced0085 100644
--- a/drivers/scsi/scsi.c
+++ b/drivers/scsi/scsi.c
@@ -600,6 +600,9 @@ void scsi_log_completion(struct scsi_cmnd *cmd, int disposition)
 			case ADD_TO_MLQUEUE:
 				printk("MLQUEUE\n");
 				break;
+			case ADD_TO_MLQUEUE_DELAY:
+				printk("MLQUEUE_DELAY\n");
+				break;
 			case FAILED:
 				printk("FAILED\n");
 				break;
diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
index dc6131e..775f822 100644
--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -812,6 +812,7 @@ static int scsi_send_eh_cmnd(struct scsi_cmnd *scmd, unsigned char *cmnd,
 		case TARGET_ERROR:
 			break;
 		case ADD_TO_MLQUEUE:
+		case ADD_TO_MLQUEUE_DELAY:
 			rtn = NEEDS_RETRY;
 			break;
 		default:
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 06bc265..fb35a76 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -139,6 +139,7 @@ static int __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
 		break;
 	case SCSI_MLQUEUE_DEVICE_BUSY:
 	case SCSI_MLQUEUE_EH_RETRY:
+	case SCSI_MLQUEUE_DEV_DLY_RTY:
 		device->device_blocked = device->max_device_blocked;
 		break;
 	case SCSI_MLQUEUE_TARGET_BUSY:
@@ -161,7 +162,10 @@ static int __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
 	blk_requeue_request(q, cmd->request);
 	spin_unlock_irqrestore(q->queue_lock, flags);
 
-	kblockd_schedule_work(q, &device->requeue_work);
+	if (reason == SCSI_MLQUEUE_DEV_DLY_RTY)
+		blk_delay_queue(q, 2000);
+	else
+		kblockd_schedule_work(q, &device->requeue_work);
 
 	return 0;
 }
@@ -1465,6 +1469,9 @@ static void scsi_softirq_done(struct request *rq)
 		case ADD_TO_MLQUEUE:
 			scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
 			break;
+		case ADD_TO_MLQUEUE_DELAY:
+			scsi_queue_insert(cmd, SCSI_MLQUEUE_DEV_DLY_RTY);
+			break;
 		default:
 			if (!scsi_eh_scmd_add(cmd, 0))
 				scsi_finish_command(cmd);
diff --git a/include/scsi/scsi.h b/include/scsi/scsi.h
index 8001ae4..df0e353 100644
--- a/include/scsi/scsi.h
+++ b/include/scsi/scsi.h
@@ -466,9 +466,10 @@ static inline int scsi_is_wlun(unsigned int lun)
 #define SOFT_ERROR      0x2005
 #define ADD_TO_MLQUEUE  0x2006
 #define TIMEOUT_ERROR   0x2007
-#define SCSI_RETURN_NOT_HANDLED   0x2008
+#define SCSI_RETURN_NOT_HANDLED	0x2008
 #define FAST_IO_FAIL	0x2009
 #define TARGET_ERROR    0x200A
+#define ADD_TO_MLQUEUE_DELAY	0x200B
 
 /*
  * Midlevel queue return values.
@@ -477,6 +478,7 @@ static inline int scsi_is_wlun(unsigned int lun)
 #define SCSI_MLQUEUE_DEVICE_BUSY 0x1056
 #define SCSI_MLQUEUE_EH_RETRY    0x1057
 #define SCSI_MLQUEUE_TARGET_BUSY 0x1058
+#define SCSI_MLQUEUE_DEV_DLY_RTY 0x1059
 
 /*
  *  Use these to separate status msg and our bytes
-- 
1.7.4.2

Re: [PATCH] delay transition requeues for 2 seconds - alua

[Hannes Reinecke]

On 12/13/2011 11:40 PM, Rob Evers wrote:
> When alua targets are transitioning, the scsi midlayer retry mechanism
> continuously retries the scsi commands that are returning with not ready
> transitioning status.  The target is not capable of handling the
> commands for time on the order of several seconds during these transistions.
> 
> This patch delays the device queue for 2 seconds, which is in the same
> order of aas transition time.
> 
> Also, handle all other cases where ADD_TO_MLQUEUE_DELAY could be returned
> instead of ADD_TO_MLQUEUE as if ADD_TO_MLQUEUE were being returned.
> 
> Problem found by array partner testing
> 
> Signed-off-by: Rob Evers <revers@redhat.com>
> ---
>  drivers/scsi/device_handler/scsi_dh_alua.c |    7 ++++---
>  drivers/scsi/scsi.c                        |    3 +++
>  drivers/scsi/scsi_error.c                  |    1 +
>  drivers/scsi/scsi_lib.c                    |    9 ++++++++-
>  include/scsi/scsi.h                        |    4 +++-
>  5 files changed, 19 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c
> index 4ef0212..33b8df7 100644
> --- a/drivers/scsi/device_handler/scsi_dh_alua.c
> +++ b/drivers/scsi/device_handler/scsi_dh_alua.c
> @@ -233,7 +233,7 @@ static void stpg_endio(struct request *req, int error)
>  			goto done;
>  		}
>  		err = alua_check_sense(h->sdev, &sense_hdr);
> -		if (err == ADD_TO_MLQUEUE) {
> +		if (err == ADD_TO_MLQUEUE || err == ADD_TO_MLQUEUE_DELAY) {
>  			err = SCSI_DH_RETRY;
>  			goto done;
>  		}
> @@ -443,7 +443,7 @@ static int alua_check_sense(struct scsi_device *sdev,
>  			/*
>  			 * LUN Not Accessible - ALUA state transition
>  			 */
> -			return ADD_TO_MLQUEUE;
> +			return ADD_TO_MLQUEUE_DELAY;
>  		if (sense_hdr->asc == 0x04 && sense_hdr->ascq == 0x0b)
>  			/*
>  			 * LUN Not Accessible -- Target port in standby state
> @@ -521,7 +521,8 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
>  			return SCSI_DH_IO;
>  
>  		err = alua_check_sense(sdev, &sense_hdr);
> -		if (err == ADD_TO_MLQUEUE && time_before(jiffies, expiry))
> +		if ((err == ADD_TO_MLQUEUE || err == ADD_TO_MLQUEUE_DELAY) &&
> +		    time_before(jiffies, expiry))
>  			goto retry;
>  		sdev_printk(KERN_INFO, sdev,
>  			    "%s: rtpg sense code %02x/%02x/%02x\n",
> diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c
> index 2aeb2e9..ced0085 100644
> --- a/drivers/scsi/scsi.c
> +++ b/drivers/scsi/scsi.c
> @@ -600,6 +600,9 @@ void scsi_log_completion(struct scsi_cmnd *cmd, int disposition)
>  			case ADD_TO_MLQUEUE:
>  				printk("MLQUEUE\n");
>  				break;
> +			case ADD_TO_MLQUEUE_DELAY:
> +				printk("MLQUEUE_DELAY\n");
> +				break;
>  			case FAILED:
>  				printk("FAILED\n");
>  				break;
> diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
> index dc6131e..775f822 100644
> --- a/drivers/scsi/scsi_error.c
> +++ b/drivers/scsi/scsi_error.c
> @@ -812,6 +812,7 @@ static int scsi_send_eh_cmnd(struct scsi_cmnd *scmd, unsigned char *cmnd,
>  		case TARGET_ERROR:
>  			break;
>  		case ADD_TO_MLQUEUE:
> +		case ADD_TO_MLQUEUE_DELAY:
>  			rtn = NEEDS_RETRY;
>  			break;
>  		default:
> diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
> index 06bc265..fb35a76 100644
> --- a/drivers/scsi/scsi_lib.c
> +++ b/drivers/scsi/scsi_lib.c
> @@ -139,6 +139,7 @@ static int __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
>  		break;
>  	case SCSI_MLQUEUE_DEVICE_BUSY:
>  	case SCSI_MLQUEUE_EH_RETRY:
> +	case SCSI_MLQUEUE_DEV_DLY_RTY:
>  		device->device_blocked = device->max_device_blocked;
>  		break;
>  	case SCSI_MLQUEUE_TARGET_BUSY:
> @@ -161,7 +162,10 @@ static int __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
>  	blk_requeue_request(q, cmd->request);
>  	spin_unlock_irqrestore(q->queue_lock, flags);
>  
> -	kblockd_schedule_work(q, &device->requeue_work);
> +	if (reason == SCSI_MLQUEUE_DEV_DLY_RTY)
> +		blk_delay_queue(q, 2000);
> +	else
> +		kblockd_schedule_work(q, &device->requeue_work);
>  
>  	return 0;
>  }
> @@ -1465,6 +1469,9 @@ static void scsi_softirq_done(struct request *rq)
>  		case ADD_TO_MLQUEUE:
>  			scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
>  			break;
> +		case ADD_TO_MLQUEUE_DELAY:
> +			scsi_queue_insert(cmd, SCSI_MLQUEUE_DEV_DLY_RTY);
> +			break;
>  		default:
>  			if (!scsi_eh_scmd_add(cmd, 0))
>  				scsi_finish_command(cmd);
> diff --git a/include/scsi/scsi.h b/include/scsi/scsi.h
> index 8001ae4..df0e353 100644
> --- a/include/scsi/scsi.h
> +++ b/include/scsi/scsi.h
> @@ -466,9 +466,10 @@ static inline int scsi_is_wlun(unsigned int lun)
>  #define SOFT_ERROR      0x2005
>  #define ADD_TO_MLQUEUE  0x2006
>  #define TIMEOUT_ERROR   0x2007
> -#define SCSI_RETURN_NOT_HANDLED   0x2008
> +#define SCSI_RETURN_NOT_HANDLED	0x2008
>  #define FAST_IO_FAIL	0x2009
>  #define TARGET_ERROR    0x200A
> +#define ADD_TO_MLQUEUE_DELAY	0x200B
>  
>  /*
>   * Midlevel queue return values.
> @@ -477,6 +478,7 @@ static inline int scsi_is_wlun(unsigned int lun)
>  #define SCSI_MLQUEUE_DEVICE_BUSY 0x1056
>  #define SCSI_MLQUEUE_EH_RETRY    0x1057
>  #define SCSI_MLQUEUE_TARGET_BUSY 0x1058
> +#define SCSI_MLQUEUE_DEV_DLY_RTY 0x1059
>  
Can't we just name it 'MLQUEUE_DELAYED_RETRY'?
'MLQUEUE_DEV_DLY_RTY' is a bit unwieldy ...

Other than that: Looks good.

Cheers,

Hannes
-- 
Dr. Hannes Reinecke		      zSeries & Storage
hare@suse.de			      +49 911 74053 688
SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg
GF: J. Hawn, J. Guild, F. Imendörffer, HRB 16746 (AG Nürnberg)
--
To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html