[PATCH 1/3] alua: implement 'implied transition timeout'

[PATCH 1/3] alua: implement 'implied transition timeout'

[Rob Evers]

From: Rob Evers <revers@redhat.com>

During alua transitions, an array can return transitioning
status in response to rtpg requests.  These requests get
retried for a maximum of 60 seconds by default before timing
out.  Sometimes this timeout isn't sufficient to allow the
array to complete the transition.  T10-spc4 addresses this
under 'Report Target Port Groups' command.

This update retrieves the timeout value from the storage
array if available and retries the transitioning rtpgs
for up to the 'implied transitioning timeout' value

Signed-off-by: Rob Evers <revers@redhat.com>
---
 drivers/scsi/device_handler/scsi_dh_alua.c |   41 ++++++++++++++++++++++++---
 include/scsi/scsi.h                        |    2 +
 2 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c
index 04c5cea..6a7efd3 100644
--- a/drivers/scsi/device_handler/scsi_dh_alua.c
+++ b/drivers/scsi/device_handler/scsi_dh_alua.c
@@ -46,13 +46,16 @@
 #define TPGS_SUPPORT_OFFLINE		0x40
 #define TPGS_SUPPORT_TRANSITION		0x80
 
+#define RTPG_FMT_MASK			0x70
+#define RTPG_FMT_EXT_HDR		0x10
+
 #define TPGS_MODE_UNINITIALIZED		 -1
 #define TPGS_MODE_NONE			0x0
 #define TPGS_MODE_IMPLICIT		0x1
 #define TPGS_MODE_EXPLICIT		0x2
 
 #define ALUA_INQUIRY_SIZE		36
-#define ALUA_FAILOVER_TIMEOUT		(60 * HZ)
+#define ALUA_FAILOVER_TIMEOUT		60
 #define ALUA_FAILOVER_RETRIES		5
 
 struct alua_dh_data {
@@ -63,6 +66,7 @@ struct alua_dh_data {
 	unsigned char		inq[ALUA_INQUIRY_SIZE];
 	unsigned char		*buff;
 	int			bufflen;
+	unsigned char		transition_tmo;
 	unsigned char		sense[SCSI_SENSE_BUFFERSIZE];
 	int			senselen;
 	struct scsi_device	*sdev;
@@ -123,7 +127,7 @@ static struct request *get_alua_req(struct scsi_device *sdev,
 	rq->cmd_flags |= REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
 			 REQ_FAILFAST_DRIVER;
 	rq->retries = ALUA_FAILOVER_RETRIES;
-	rq->timeout = ALUA_FAILOVER_TIMEOUT;
+	rq->timeout = ALUA_FAILOVER_TIMEOUT * HZ;
 
 	return rq;
 }
@@ -180,7 +184,7 @@ static unsigned submit_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
 
 	/* Prepare the command. */
 	rq->cmd[0] = MAINTENANCE_IN;
-	rq->cmd[1] = MI_REPORT_TARGET_PGS;
+	rq->cmd[1] = MI_REPORT_TARGET_PGS | MI_EXT_HDR_PARAM_FMT;
 	rq->cmd[6] = (h->bufflen >> 24) & 0xff;
 	rq->cmd[7] = (h->bufflen >> 16) & 0xff;
 	rq->cmd[8] = (h->bufflen >>  8) & 0xff;
@@ -514,8 +518,14 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
 	unsigned char *ucp;
 	unsigned err;
 	unsigned long expiry, interval = 1000;
+	unsigned int tpg_desc_tbl_off;
+	unsigned char orig_transition_tmo;
+
+	if (!h->transition_tmo)
+		expiry = round_jiffies_up(jiffies + ALUA_FAILOVER_TIMEOUT * HZ);
+	else
+		expiry = round_jiffies_up(jiffies + h->transition_tmo * HZ);
 
-	expiry = round_jiffies_up(jiffies + ALUA_FAILOVER_TIMEOUT);
  retry:
 	err = submit_rtpg(sdev, h);
 
@@ -551,7 +561,28 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
 		goto retry;
 	}
 
-	for (k = 4, ucp = h->buff + 4; k < len; k += off, ucp += off) {
+	orig_transition_tmo = h->transition_tmo;
+	if ((h->buff[4] & RTPG_FMT_MASK) == RTPG_FMT_EXT_HDR && h->buff[5] != 0)
+		h->transition_tmo = h->buff[5];
+	else
+		h->transition_tmo = ALUA_FAILOVER_TIMEOUT;
+
+	if (orig_transition_tmo != h->transition_tmo) {
+		sdev_printk(KERN_INFO, sdev,
+			    "%s: transition timeout set to %d seconds\n",
+			    ALUA_DH_NAME, h->transition_tmo);
+		expiry = jiffies + h->transition_tmo * HZ;
+	}
+
+	if ((h->buff[4] & RTPG_FMT_MASK) == RTPG_FMT_EXT_HDR)
+		tpg_desc_tbl_off = 8;
+	else
+		tpg_desc_tbl_off = 4;
+
+	for (k = tpg_desc_tbl_off, ucp = h->buff + tpg_desc_tbl_off;
+	     k < len;
+	     k += off, ucp += off) {
+
 		if (h->group_id == (ucp[2] << 8) + ucp[3]) {
 			h->state = ucp[0] & 0x0f;
 			valid_states = ucp[1];
diff --git a/include/scsi/scsi.h b/include/scsi/scsi.h
index f34a5a8..c6f0974 100644
--- a/include/scsi/scsi.h
+++ b/include/scsi/scsi.h
@@ -161,6 +161,8 @@ struct scsi_cmnd;
 #define MI_REPORT_PRIORITY   0x0e
 #define MI_REPORT_TIMESTAMP  0x0f
 #define MI_MANAGEMENT_PROTOCOL_IN 0x10
+/* value for MI_REPORT_TARGET_PGS ext header */
+#define MI_EXT_HDR_PARAM_FMT  0x20
 /* values for maintenance out */
 #define MO_SET_IDENTIFYING_INFORMATION 0x06
 #define MO_SET_TARGET_PGS     0x0a
-- 
1.7.7.2

[PATCH 2/3] alua: retry alua rtpg extended header for illegal request response

[Rob Evers]

From: Rob Evers <revers@redhat.com>

Some storage arrays are known to return 'illegal request'
when an rtpg extended header request is made.  T10 says the
array should ignore the bit, and return the non-extended
rtpg as the array doesn't support the request.  Working
around this by retrying the rtpg request without the extended
header bit set when the extended rtpg request results in
illegal request.

Signed-off-by: Rob Evers <revers@redhat.com>
---
 drivers/scsi/device_handler/scsi_dh_alua.c |   25 ++++++++++++++++++++++---
 1 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c
index 6a7efd3..ca414ae 100644
--- a/drivers/scsi/device_handler/scsi_dh_alua.c
+++ b/drivers/scsi/device_handler/scsi_dh_alua.c
@@ -173,7 +173,8 @@ done:
  * submit_rtpg - Issue a REPORT TARGET GROUP STATES command
  * @sdev: sdev the command should be sent to
  */
-static unsigned submit_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
+static unsigned submit_rtpg(struct scsi_device *sdev, struct alua_dh_data *h,
+			    bool rtpg_ext_hdr_req)
 {
 	struct request *rq;
 	int err = SCSI_DH_RES_TEMP_UNAVAIL;
@@ -184,7 +185,10 @@ static unsigned submit_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
 
 	/* Prepare the command. */
 	rq->cmd[0] = MAINTENANCE_IN;
-	rq->cmd[1] = MI_REPORT_TARGET_PGS | MI_EXT_HDR_PARAM_FMT;
+	if (rtpg_ext_hdr_req)
+		rq->cmd[1] = MI_REPORT_TARGET_PGS | MI_EXT_HDR_PARAM_FMT;
+	else
+		rq->cmd[1] = MI_REPORT_TARGET_PGS;
 	rq->cmd[6] = (h->bufflen >> 24) & 0xff;
 	rq->cmd[7] = (h->bufflen >> 16) & 0xff;
 	rq->cmd[8] = (h->bufflen >>  8) & 0xff;
@@ -517,6 +521,7 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
 	int len, k, off, valid_states = 0;
 	unsigned char *ucp;
 	unsigned err;
+	bool rtpg_ext_hdr_req = 1;
 	unsigned long expiry, interval = 1000;
 	unsigned int tpg_desc_tbl_off;
 	unsigned char orig_transition_tmo;
@@ -527,7 +532,7 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
 		expiry = round_jiffies_up(jiffies + h->transition_tmo * HZ);
 
  retry:
-	err = submit_rtpg(sdev, h);
+	err = submit_rtpg(sdev, h, rtpg_ext_hdr_req);
 
 	if (err == SCSI_DH_IO && h->senselen > 0) {
 		err = scsi_normalize_sense(h->sense, SCSI_SENSE_BUFFERSIZE,
@@ -535,6 +540,20 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
 		if (!err)
 			return SCSI_DH_IO;
 
+		/*
+		 * submit_rtpb() has failed on existing arrays
+		 * when requesting extended header info, and
+		 * the array doesn't support extended headers,
+		 * even though it shouldn't according to T10.
+		 * The retry without the ALUA_RTPG_EXT_HDR_REQ
+		 * handles this.
+		 */
+		if (sense_hdr.sense_key == ILLEGAL_REQUEST &&
+		    sense_hdr.asc == 0x24 && sense_hdr.ascq == 0) {
+			rtpg_ext_hdr_req = 0;
+			goto retry;
+		}
+
 		err = alua_check_sense(sdev, &sense_hdr);
 		if (err == ADD_TO_MLQUEUE && time_before(jiffies, expiry))
 			goto retry;
-- 
1.7.7.2

[PATCH 3/3] alua: backoff alua rtpg retry linearly vs. geometrically

[Rob Evers]

From: Rob Evers <revers@redhat.com>

Currently the backoff algorithm for when to retry alua rtpg
requests progresses geometrically as so:

2, 4, 8, 16, 32, 64... seconds.

This progression can lead to un-needed delay in retrying
alua rtpg requests when the rtpgs are delayed.  A less
aggressive backoff algorithm that is additive would not
lead to such large jumps when delays start getting long, but
would backoff linearly:

2, 4, 6, 8, 10... seconds.

Signed-off-by: Rob Evers <revers@redhat.com>
---
 drivers/scsi/device_handler/scsi_dh_alua.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c
index ca414ae..e0749e9 100644
--- a/drivers/scsi/device_handler/scsi_dh_alua.c
+++ b/drivers/scsi/device_handler/scsi_dh_alua.c
@@ -522,7 +522,7 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
 	unsigned char *ucp;
 	unsigned err;
 	bool rtpg_ext_hdr_req = 1;
-	unsigned long expiry, interval = 1000;
+	unsigned long expiry, interval = 0;
 	unsigned int tpg_desc_tbl_off;
 	unsigned char orig_transition_tmo;
 
@@ -624,7 +624,7 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
 	case TPGS_STATE_TRANSITIONING:
 		if (time_before(jiffies, expiry)) {
 			/* State transition, retry */
-			interval *= 2;
+			interval += 2000;
 			msleep(interval);
 			goto retry;
 		}
-- 
1.7.7.2

RE: [PATCH 2/3] alua: retry alua rtpg extended header for illegal request response

[Moger, Babu]

Rob, Minor comments below.

> -----Original Message-----
> From: linux-scsi-owner@vger.kernel.org [mailto:linux-scsi-
> owner@vger.kernel.org] On Behalf Of Rob Evers
> Sent: Wednesday, May 16, 2012 9:15 AM
> To: linux-scsi@vger.kernel.org
> Subject: [PATCH 2/3] alua: retry alua rtpg extended header for illegal request
> response
> 
> From: Rob Evers <revers@redhat.com>
> 
> Some storage arrays are known to return 'illegal request'
> when an rtpg extended header request is made.  T10 says the
> array should ignore the bit, and return the non-extended
> rtpg as the array doesn't support the request.  Working
> around this by retrying the rtpg request without the extended
> header bit set when the extended rtpg request results in
> illegal request.
> 
> Signed-off-by: Rob Evers <revers@redhat.com>
> ---
>  drivers/scsi/device_handler/scsi_dh_alua.c |   25
> ++++++++++++++++++++++---
>  1 files changed, 22 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c
> b/drivers/scsi/device_handler/scsi_dh_alua.c
> index 6a7efd3..ca414ae 100644
> --- a/drivers/scsi/device_handler/scsi_dh_alua.c
> +++ b/drivers/scsi/device_handler/scsi_dh_alua.c
> @@ -173,7 +173,8 @@ done:
>   * submit_rtpg - Issue a REPORT TARGET GROUP STATES command
>   * @sdev: sdev the command should be sent to
>   */
> -static unsigned submit_rtpg(struct scsi_device *sdev, struct alua_dh_data
> *h)
> +static unsigned submit_rtpg(struct scsi_device *sdev, struct alua_dh_data
> *h,
> +			    bool rtpg_ext_hdr_req)
>  {
>  	struct request *rq;
>  	int err = SCSI_DH_RES_TEMP_UNAVAIL;
> @@ -184,7 +185,10 @@ static unsigned submit_rtpg(struct scsi_device
> *sdev, struct alua_dh_data *h)
> 
>  	/* Prepare the command. */
>  	rq->cmd[0] = MAINTENANCE_IN;
> -	rq->cmd[1] = MI_REPORT_TARGET_PGS |
> MI_EXT_HDR_PARAM_FMT;
> +	if (rtpg_ext_hdr_req)
> +		rq->cmd[1] = MI_REPORT_TARGET_PGS |
> MI_EXT_HDR_PARAM_FMT;
> +	else
> +		rq->cmd[1] = MI_REPORT_TARGET_PGS;
>  	rq->cmd[6] = (h->bufflen >> 24) & 0xff;
>  	rq->cmd[7] = (h->bufflen >> 16) & 0xff;
>  	rq->cmd[8] = (h->bufflen >>  8) & 0xff;
> @@ -517,6 +521,7 @@ static int alua_rtpg(struct scsi_device *sdev, struct
> alua_dh_data *h)
>  	int len, k, off, valid_states = 0;
>  	unsigned char *ucp;
>  	unsigned err;
> +	bool rtpg_ext_hdr_req = 1;
>  	unsigned long expiry, interval = 1000;
>  	unsigned int tpg_desc_tbl_off;
>  	unsigned char orig_transition_tmo;
> @@ -527,7 +532,7 @@ static int alua_rtpg(struct scsi_device *sdev, struct
> alua_dh_data *h)
>  		expiry = round_jiffies_up(jiffies + h->transition_tmo * HZ);
> 
>   retry:
> -	err = submit_rtpg(sdev, h);
> +	err = submit_rtpg(sdev, h, rtpg_ext_hdr_req);
> 
>  	if (err == SCSI_DH_IO && h->senselen > 0) {
>  		err = scsi_normalize_sense(h->sense,
> SCSI_SENSE_BUFFERSIZE,
> @@ -535,6 +540,20 @@ static int alua_rtpg(struct scsi_device *sdev, struct
> alua_dh_data *h)
>  		if (!err)
>  			return SCSI_DH_IO;
> 
> +		/*
> +		 * submit_rtpb() has failed on existing arrays

There is a typo.  Should be submit_rtpg.
 
> +		 * when requesting extended header info, and
> +		 * the array doesn't support extended headers,
> +		 * even though it shouldn't according to T10.
> +		 * The retry without the ALUA_RTPG_EXT_HDR_REQ
> +		 * handles this.
> +		 */
> +		if (sense_hdr.sense_key == ILLEGAL_REQUEST &&
> +		    sense_hdr.asc == 0x24 && sense_hdr.ascq == 0) {
> +			rtpg_ext_hdr_req = 0;
> +			goto retry;

We should probably have an exit criteria here.  What if target misbehaves
(with ILLEGAL_REQUEST) even with rtpg_ext_hdr_req = 0.


> +		}
> +
>  		err = alua_check_sense(sdev, &sense_hdr);
>  		if (err == ADD_TO_MLQUEUE && time_before(jiffies,
> expiry))
>  			goto retry;
> --
> 1.7.7.2
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 2/3] alua: retry alua rtpg extended header for illegal request response

[Rob Evers]

On 05/17/2012 05:02 PM, Moger, Babu wrote:
> Rob, Minor comments below.

Thanks for looking.

>> -----Original Message-----
>> From: linux-scsi-owner@vger.kernel.org [mailto:linux-scsi-
>> owner@vger.kernel.org] On Behalf Of Rob Evers
>> Sent: Wednesday, May 16, 2012 9:15 AM
>> To: linux-scsi@vger.kernel.org
>> Subject: [PATCH 2/3] alua: retry alua rtpg extended header for illegal request
>> response
>>
>> From: Rob Evers<revers@redhat.com>
>>
>> Some storage arrays are known to return 'illegal request'
>> when an rtpg extended header request is made.  T10 says the
>> array should ignore the bit, and return the non-extended
>> rtpg as the array doesn't support the request.  Working
>> around this by retrying the rtpg request without the extended
>> header bit set when the extended rtpg request results in
>> illegal request.
>>
>> Signed-off-by: Rob Evers<revers@redhat.com>
>> ---
>>   drivers/scsi/device_handler/scsi_dh_alua.c |   25
>> ++++++++++++++++++++++---
>>   1 files changed, 22 insertions(+), 3 deletions(-)
>>
>> diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c
>> b/drivers/scsi/device_handler/scsi_dh_alua.c
>> index 6a7efd3..ca414ae 100644
>> --- a/drivers/scsi/device_handler/scsi_dh_alua.c
>> +++ b/drivers/scsi/device_handler/scsi_dh_alua.c
>> @@ -173,7 +173,8 @@ done:
>>    * submit_rtpg - Issue a REPORT TARGET GROUP STATES command
>>    * @sdev: sdev the command should be sent to
>>    */
>> -static unsigned submit_rtpg(struct scsi_device *sdev, struct alua_dh_data
>> *h)
>> +static unsigned submit_rtpg(struct scsi_device *sdev, struct alua_dh_data
>> *h,
>> +			    bool rtpg_ext_hdr_req)
>>   {
>>   	struct request *rq;
>>   	int err = SCSI_DH_RES_TEMP_UNAVAIL;
>> @@ -184,7 +185,10 @@ static unsigned submit_rtpg(struct scsi_device
>> *sdev, struct alua_dh_data *h)
>>
>>   	/* Prepare the command. */
>>   	rq->cmd[0] = MAINTENANCE_IN;
>> -	rq->cmd[1] = MI_REPORT_TARGET_PGS |
>> MI_EXT_HDR_PARAM_FMT;
>> +	if (rtpg_ext_hdr_req)
>> +		rq->cmd[1] = MI_REPORT_TARGET_PGS |
>> MI_EXT_HDR_PARAM_FMT;
>> +	else
>> +		rq->cmd[1] = MI_REPORT_TARGET_PGS;
>>   	rq->cmd[6] = (h->bufflen>>  24)&  0xff;
>>   	rq->cmd[7] = (h->bufflen>>  16)&  0xff;
>>   	rq->cmd[8] = (h->bufflen>>   8)&  0xff;
>> @@ -517,6 +521,7 @@ static int alua_rtpg(struct scsi_device *sdev, struct
>> alua_dh_data *h)
>>   	int len, k, off, valid_states = 0;
>>   	unsigned char *ucp;
>>   	unsigned err;
>> +	bool rtpg_ext_hdr_req = 1;
>>   	unsigned long expiry, interval = 1000;
>>   	unsigned int tpg_desc_tbl_off;
>>   	unsigned char orig_transition_tmo;
>> @@ -527,7 +532,7 @@ static int alua_rtpg(struct scsi_device *sdev, struct
>> alua_dh_data *h)
>>   		expiry = round_jiffies_up(jiffies + h->transition_tmo * HZ);
>>
>>    retry:
>> -	err = submit_rtpg(sdev, h);
>> +	err = submit_rtpg(sdev, h, rtpg_ext_hdr_req);
>>
>>   	if (err == SCSI_DH_IO&&  h->senselen>  0) {
>>   		err = scsi_normalize_sense(h->sense,
>> SCSI_SENSE_BUFFERSIZE,
>> @@ -535,6 +540,20 @@ static int alua_rtpg(struct scsi_device *sdev, struct
>> alua_dh_data *h)
>>   		if (!err)
>>   			return SCSI_DH_IO;
>>
>> +		/*
>> +		 * submit_rtpb() has failed on existing arrays
> There is a typo.  Should be submit_rtpg.

yes.

>
>> +		 * when requesting extended header info, and
>> +		 * the array doesn't support extended headers,
>> +		 * even though it shouldn't according to T10.
>> +		 * The retry without the ALUA_RTPG_EXT_HDR_REQ

typo here too, ALUA_RTPG_EXT_HDR_REQ should be rtpg_ext_hdr_req

>> +		 * handles this.
>> +		 */
>> +		if (sense_hdr.sense_key == ILLEGAL_REQUEST&&
>> +		    sense_hdr.asc == 0x24&&  sense_hdr.ascq == 0) {
>> +			rtpg_ext_hdr_req = 0;
>> +			goto retry;
> We should probably have an exit criteria here.  What if target misbehaves
> (with ILLEGAL_REQUEST) even with rtpg_ext_hdr_req = 0.

Do you have a more detailed suggestion on the
exit criteria?

What I can say is that this works for the only case I know of where
an array doesn't work as it is supposed to according to T10 w.r.t.
extended rtpg format request.

>
>> +		}
>> +
>>   		err = alua_check_sense(sdev,&sense_hdr);
>>   		if (err == ADD_TO_MLQUEUE&&  time_before(jiffies,
>> expiry))
>>   			goto retry;
>> --
>> 1.7.7.2
>>
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 2/3] alua: retry alua rtpg extended header for illegal request response

[Rob Evers]

On 05/17/2012 10:00 PM, Rob Evers wrote:
> On 05/17/2012 05:02 PM, Moger, Babu wrote:
>
>>> +         * handles this.
>>> +         */
>>> +        if (sense_hdr.sense_key == ILLEGAL_REQUEST&&
>>> +            sense_hdr.asc == 0x24&&  sense_hdr.ascq == 0) {
>>> +            rtpg_ext_hdr_req = 0;
>>> +            goto retry;
>> We should probably have an exit criteria here.  What if target 
>> misbehaves
>> (with ILLEGAL_REQUEST) even with rtpg_ext_hdr_req = 0.
>
> Do you have a more detailed suggestion on the
> exit criteria?
>

Yes I got your point.  I'll repost.