[PATCH] nvme: enable retries for authentication commands

[PATCH] nvme: enable retries for authentication commands

[hare]

From: Hannes Reinecke <hare@suse.de>

Authentication commands are normal NVMe commands, and as such
can return a status where NVME_SC_DNR is not set, indicating
that the command should be retried.
Rather than checking NVME_SC_DNR manually for each command completion
this patch adds a flag 'retry' to __nvme_submit_sync_cmd(), causing
the REQ_FAILFAST_DRIVER option to be cleared and the command to
be retried via the normal mechanism.

Reported-by: Martin George <marting@netapp.com>
Signed-off-by: Hannes Reinecke <hare@suse.de>
---
 drivers/nvme/host/auth.c    |  2 +-
 drivers/nvme/host/core.c    | 10 ++++++----
 drivers/nvme/host/fabrics.c | 10 +++++-----
 drivers/nvme/host/nvme.h    |  2 +-
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/drivers/nvme/host/auth.c b/drivers/nvme/host/auth.c
index 35a460cfb4bf..c3ab7c86065e 100644
--- a/drivers/nvme/host/auth.c
+++ b/drivers/nvme/host/auth.c
@@ -82,7 +82,7 @@ static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid,
 
 	ret = __nvme_submit_sync_cmd(q, &cmd, NULL, data, data_len,
 				     qid == 0 ? NVME_QID_ANY : qid,
-				     0, flags);
+				     0, flags, true);
 	if (ret > 0)
 		dev_warn(ctrl->device,
 			"qid %d auth_send failed with status %d\n", qid, ret);
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 376efb310eb2..dade4fcb88d2 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -1049,7 +1049,7 @@ EXPORT_SYMBOL_NS_GPL(nvme_execute_rq, NVME_TARGET_PASSTHRU);
  */
 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 		union nvme_result *result, void *buffer, unsigned bufflen,
-		int qid, int at_head, blk_mq_req_flags_t flags)
+		int qid, int at_head, blk_mq_req_flags_t flags, bool retry)
 {
 	struct request *req;
 	int ret;
@@ -1063,6 +1063,8 @@ int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 	nvme_init_request(req, cmd);
+	if (retry)
+		req->cmd_flags &= ~REQ_FAILFAST_DRIVER;
 
 	if (buffer && bufflen) {
 		ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
@@ -1083,7 +1085,7 @@ int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 		void *buffer, unsigned bufflen)
 {
 	return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen,
-			NVME_QID_ANY, 0, 0);
+			NVME_QID_ANY, 0, 0, false);
 }
 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
 
@@ -1547,7 +1549,7 @@ static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
 	c.features.dword11 = cpu_to_le32(dword11);
 
 	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
-			buffer, buflen, NVME_QID_ANY, 0, 0);
+			buffer, buflen, NVME_QID_ANY, 0, 0, false);
 	if (ret >= 0 && result)
 		*result = le32_to_cpu(res.u32);
 	return ret;
@@ -2149,7 +2151,7 @@ static int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t l
 	cmd.common.cdw11 = cpu_to_le32(len);
 
 	return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
-			NVME_QID_ANY, 1, 0);
+			NVME_QID_ANY, 1, 0, false);
 }
 
 static void nvme_configure_opal(struct nvme_ctrl *ctrl, bool was_suspended)
diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c
index 8140d8166c3e..66799d59fcd1 100644
--- a/drivers/nvme/host/fabrics.c
+++ b/drivers/nvme/host/fabrics.c
@@ -180,7 +180,7 @@ int nvmf_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
 	cmd.prop_get.offset = cpu_to_le32(off);
 
 	ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, &res, NULL, 0,
-			NVME_QID_ANY, 0, 0);
+			NVME_QID_ANY, 0, 0, false);
 
 	if (ret >= 0)
 		*val = le64_to_cpu(res.u64);
@@ -226,7 +226,7 @@ int nvmf_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
 	cmd.prop_get.offset = cpu_to_le32(off);
 
 	ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, &res, NULL, 0,
-			NVME_QID_ANY, 0, 0);
+			NVME_QID_ANY, 0, 0, false);
 
 	if (ret >= 0)
 		*val = le64_to_cpu(res.u64);
@@ -271,7 +271,7 @@ int nvmf_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
 	cmd.prop_set.value = cpu_to_le64(val);
 
 	ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, NULL, NULL, 0,
-			NVME_QID_ANY, 0, 0);
+			NVME_QID_ANY, 0, 0, false);
 	if (unlikely(ret))
 		dev_err(ctrl->device,
 			"Property Set error: %d, offset %#x\n",
@@ -451,7 +451,7 @@ int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl)
 
 	ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, &res,
 			data, sizeof(*data), NVME_QID_ANY, 1,
-			BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
+			BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT, false);
 	if (ret) {
 		nvmf_log_connect_error(ctrl, ret, le32_to_cpu(res.u32),
 				       &cmd, data);
@@ -527,7 +527,7 @@ int nvmf_connect_io_queue(struct nvme_ctrl *ctrl, u16 qid)
 
 	ret = __nvme_submit_sync_cmd(ctrl->connect_q, &cmd, &res,
 			data, sizeof(*data), qid, 1,
-			BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
+			BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT, false);
 	if (ret) {
 		nvmf_log_connect_error(ctrl, ret, le32_to_cpu(res.u32),
 				       &cmd, data);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 396e6e72b6a3..743ef474d282 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -829,7 +829,7 @@ int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 		union nvme_result *result, void *buffer, unsigned bufflen,
 		int qid, int at_head,
-		blk_mq_req_flags_t flags);
+		blk_mq_req_flags_t flags, bool retry);
 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
 		      unsigned int dword11, void *buffer, size_t buflen,
 		      u32 *result);
-- 
2.35.3

Re: [PATCH] nvme: enable retries for authentication commands

[Jens Axboe]

On 1/25/24 6:09 AM, hare@kernel.org wrote:
> From: Hannes Reinecke <hare@suse.de>
> 
> Authentication commands are normal NVMe commands, and as such
> can return a status where NVME_SC_DNR is not set, indicating
> that the command should be retried.
> Rather than checking NVME_SC_DNR manually for each command completion
> this patch adds a flag 'retry' to __nvme_submit_sync_cmd(), causing
> the REQ_FAILFAST_DRIVER option to be cleared and the command to
> be retried via the normal mechanism.

Can we please do this without adding Yet Another parameter to that
function?

-- 
Jens Axboe

Re: [PATCH] nvme: enable retries for authentication commands

[Christoph Hellwig]

On Thu, Jan 25, 2024 at 08:42:10AM -0700, Jens Axboe wrote:
> On 1/25/24 6:09 AM, hare@kernel.org wrote:
> > From: Hannes Reinecke <hare@suse.de>
> > 
> > Authentication commands are normal NVMe commands, and as such
> > can return a status where NVME_SC_DNR is not set, indicating
> > that the command should be retried.
> > Rather than checking NVME_SC_DNR manually for each command completion
> > this patch adds a flag 'retry' to __nvme_submit_sync_cmd(), causing
> > the REQ_FAILFAST_DRIVER option to be cleared and the command to
> > be retried via the normal mechanism.
> 
> Can we please do this without adding Yet Another parameter to that
> function?

Heh, I wrote about the same reply a few minutes ago.  I'll try to think
what we can do instead.

Re: [PATCH] nvme: enable retries for authentication commands

[Hannes Reinecke]

On 1/25/24 16:48, Christoph Hellwig wrote:
> On Thu, Jan 25, 2024 at 08:42:10AM -0700, Jens Axboe wrote:
>> On 1/25/24 6:09 AM, hare@kernel.org wrote:
>>> From: Hannes Reinecke <hare@suse.de>
>>>
>>> Authentication commands are normal NVMe commands, and as such
>>> can return a status where NVME_SC_DNR is not set, indicating
>>> that the command should be retried.
>>> Rather than checking NVME_SC_DNR manually for each command completion
>>> this patch adds a flag 'retry' to __nvme_submit_sync_cmd(), causing
>>> the REQ_FAILFAST_DRIVER option to be cleared and the command to
>>> be retried via the normal mechanism.
>>
>> Can we please do this without adding Yet Another parameter to that
>> function?
> 
> Heh, I wrote about the same reply a few minutes ago.  I'll try to think
> what we can do instead.

We could do something like this:

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 8ebdfd623e0f..200326dc33c1 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -1057,6 +1057,8 @@ int __nvme_submit_sync_cmd(struct request_queue 
*q, struct nvme_command *cmd,
         if (IS_ERR(req))
                 return PTR_ERR(req);
         nvme_init_request(req, cmd);
+       if (flags & BLK_MQ_REQ_RETRY)
+               req->cmd_flags &= ~REQ_FAILFAST_DRIVER;

         if (buffer && bufflen) {
                 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 1ab3081c82ed..7983e1abb622 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -719,6 +719,8 @@ enum {
         BLK_MQ_REQ_RESERVED     = (__force blk_mq_req_flags_t)(1 << 1),
         /* set RQF_PM */
         BLK_MQ_REQ_PM           = (__force blk_mq_req_flags_t)(1 << 2),
+       /* enable command retries */
+       BLK_MQ_REQ_RETRY        = (__force blk_mq_req_flags_t)(1 << 3),
  };

  struct request *blk_mq_alloc_request(struct request_queue *q, 
blk_opf_t opf,

but that's arguably a misuse of block layer flags ...

Cheers,

Hannes
-- 
Dr. Hannes Reinecke                Kernel Storage Architect
hare@suse.de                              +49 911 74053 688
SUSE Software Solutions GmbH, Maxfeldstr. 5, 90409 Nürnberg
HRB 36809 (AG Nürnberg), GF: Ivo Totev, Andrew McDonald,
Werner Knoblich

Re: [PATCH] nvme: enable retries for authentication commands

[Jens Axboe]

On 1/25/24 9:20 AM, Hannes Reinecke wrote:
> On 1/25/24 16:48, Christoph Hellwig wrote:
>> On Thu, Jan 25, 2024 at 08:42:10AM -0700, Jens Axboe wrote:
>>> On 1/25/24 6:09 AM, hare@kernel.org wrote:
>>>> From: Hannes Reinecke <hare@suse.de>
>>>>
>>>> Authentication commands are normal NVMe commands, and as such
>>>> can return a status where NVME_SC_DNR is not set, indicating
>>>> that the command should be retried.
>>>> Rather than checking NVME_SC_DNR manually for each command completion
>>>> this patch adds a flag 'retry' to __nvme_submit_sync_cmd(), causing
>>>> the REQ_FAILFAST_DRIVER option to be cleared and the command to
>>>> be retried via the normal mechanism.
>>>
>>> Can we please do this without adding Yet Another parameter to that
>>> function?
>>
>> Heh, I wrote about the same reply a few minutes ago.  I'll try to think
>> what we can do instead.
> 
> We could do something like this:

That won't really do much, and yeah wouldn't be the way even if
complete. How about just not setting the flag in the first place?
nvme_init_request() defaults to doing it, so either pass in the init
flags to that, or just clear it in there and have the callers decide if
they want to set it.

-- 
Jens Axboe

Re: [PATCH] nvme: enable retries for authentication commands

[Hannes Reinecke]

On 1/25/24 17:50, Jens Axboe wrote:
> On 1/25/24 9:20 AM, Hannes Reinecke wrote:
>> On 1/25/24 16:48, Christoph Hellwig wrote:
>>> On Thu, Jan 25, 2024 at 08:42:10AM -0700, Jens Axboe wrote:
>>>> On 1/25/24 6:09 AM, hare@kernel.org wrote:
>>>>> From: Hannes Reinecke <hare@suse.de>
>>>>>
>>>>> Authentication commands are normal NVMe commands, and as such
>>>>> can return a status where NVME_SC_DNR is not set, indicating
>>>>> that the command should be retried.
>>>>> Rather than checking NVME_SC_DNR manually for each command completion
>>>>> this patch adds a flag 'retry' to __nvme_submit_sync_cmd(), causing
>>>>> the REQ_FAILFAST_DRIVER option to be cleared and the command to
>>>>> be retried via the normal mechanism.
>>>>
>>>> Can we please do this without adding Yet Another parameter to that
>>>> function?
>>>
>>> Heh, I wrote about the same reply a few minutes ago.  I'll try to think
>>> what we can do instead.
>>
>> We could do something like this:
> 
> That won't really do much, and yeah wouldn't be the way even if
> complete. How about just not setting the flag in the first place?
> nvme_init_request() defaults to doing it, so either pass in the init
> flags to that, or just clear it in there and have the callers decide if
> they want to set it.
> 
But that's the thing; the request is never visible to the caller.
So moving things out of the nvme_init_request() doesn't really help.

And open-coding __nvme_submit_sync_cmd() also doesn't look very
appealing.

Cheers,

Hannes
-- 
Dr. Hannes Reinecke                Kernel Storage Architect
hare@suse.de                              +49 911 74053 688
SUSE Software Solutions GmbH, Maxfeldstr. 5, 90409 Nürnberg
HRB 36809 (AG Nürnberg), GF: Ivo Totev, Andrew McDonald,
Werner Knoblich

Re: [PATCH] nvme: enable retries for authentication commands

[Jens Axboe]

On 1/25/24 10:09 AM, Hannes Reinecke wrote:
> On 1/25/24 17:50, Jens Axboe wrote:
>> On 1/25/24 9:20 AM, Hannes Reinecke wrote:
>>> On 1/25/24 16:48, Christoph Hellwig wrote:
>>>> On Thu, Jan 25, 2024 at 08:42:10AM -0700, Jens Axboe wrote:
>>>>> On 1/25/24 6:09 AM, hare@kernel.org wrote:
>>>>>> From: Hannes Reinecke <hare@suse.de>
>>>>>>
>>>>>> Authentication commands are normal NVMe commands, and as such
>>>>>> can return a status where NVME_SC_DNR is not set, indicating
>>>>>> that the command should be retried.
>>>>>> Rather than checking NVME_SC_DNR manually for each command completion
>>>>>> this patch adds a flag 'retry' to __nvme_submit_sync_cmd(), causing
>>>>>> the REQ_FAILFAST_DRIVER option to be cleared and the command to
>>>>>> be retried via the normal mechanism.
>>>>>
>>>>> Can we please do this without adding Yet Another parameter to that
>>>>> function?
>>>>
>>>> Heh, I wrote about the same reply a few minutes ago.  I'll try to think
>>>> what we can do instead.
>>>
>>> We could do something like this:
>>
>> That won't really do much, and yeah wouldn't be the way even if
>> complete. How about just not setting the flag in the first place?
>> nvme_init_request() defaults to doing it, so either pass in the init
>> flags to that, or just clear it in there and have the callers decide if
>> they want to set it.
>>
> But that's the thing; the request is never visible to the caller.
> So moving things out of the nvme_init_request() doesn't really help.
> 
> And open-coding __nvme_submit_sync_cmd() also doesn't look very
> appealing.

__nvme_submit_sync_cmd() already takes another bool (at_head) and blk-mq
flags. You could have it take some nvme specific flags instead, and fold
those two parameters into a single flags. Once that is done, adding the
retry flag on top would be trivial.

And as a side benefit, it gets RID of a bool flag rather than add
another random one.

-- 
Jens Axboe