[PATCH 3/6] nvme: add per request retry timer

[Sagi Grimberg <sagi@grimberg.me>]

The existing command retry mechanism adds a request to retry
to a retry list and modifies a request-queue (controller) wide
dealyed queue timer.

The issue is that the existing requests in this queue may wait
for longer periods of time as more requests are completed with
a retry crd level.

Instead, add a per-request timer that will allow different
requests retry in a way that is independent of other requests.

Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
---
 drivers/nvme/host/apple.c  |  1 +
 drivers/nvme/host/core.c   | 24 ++++++++++++++++++++++--
 drivers/nvme/host/fc.c     |  1 +
 drivers/nvme/host/nvme.h   |  2 ++
 drivers/nvme/host/pci.c    |  1 +
 drivers/nvme/host/rdma.c   |  1 +
 drivers/nvme/host/tcp.c    |  1 +
 drivers/nvme/target/loop.c |  1 +
 8 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c
index be3b91b43ea5..c584b9551f7d 100644
--- a/drivers/nvme/host/apple.c
+++ b/drivers/nvme/host/apple.c
@@ -829,6 +829,7 @@ static int apple_nvme_init_request(struct blk_mq_tag_set *set,
 	iod->q = q;
 	nreq->ctrl = &anv->ctrl;
 	nreq->cmd = &iod->cmd;
+	nvme_req_retry_timer_init(nreq);
 
 	return 0;
 }
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index a49a86f96dd6..11c24f89f4fd 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -328,8 +328,11 @@ static void nvme_retry_req(struct request *req)
 	if (ns)
 		atomic_long_inc(&ns->retries);
 
-	blk_mq_requeue_request(req, false);
-	blk_mq_delay_kick_requeue_list(req->q, delay);
+	if (delay)
+		mod_timer(&nvme_req(req)->retry_timer,
+			jiffies + msecs_to_jiffies(delay));
+	else
+		blk_mq_requeue_request(req, true);
 }
 
 static void nvme_log_error(struct request *req)
@@ -538,6 +541,7 @@ bool nvme_cancel_request(struct request *req, void *data)
 	if (blk_mq_rq_state(req) != MQ_RQ_IN_FLIGHT)
 		return true;
 
+	timer_delete_sync(&nvme_req(req)->retry_timer);
 	nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
 	nvme_req(req)->flags |= NVME_REQ_CANCELLED;
 	blk_mq_complete_request(req);
@@ -720,6 +724,7 @@ EXPORT_SYMBOL_NS_GPL(nvme_put_ns, "NVME_TARGET_PASSTHRU");
 
 static inline void nvme_clear_nvme_request(struct request *req)
 {
+	WARN_ON_ONCE(timer_pending(&nvme_req(req)->retry_timer));
 	nvme_req(req)->status = 0;
 	nvme_req(req)->retries = 0;
 	nvme_req(req)->flags = 0;
@@ -1074,6 +1079,21 @@ static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
 	return 0;
 }
 
+static void nvme_retry_timer_fn(struct timer_list *t)
+{
+	struct nvme_request *nr = timer_container_of(nr, t, retry_timer);
+
+	if (unlikely(nr->flags & NVME_REQ_CANCELLED))
+		return;
+	blk_mq_requeue_request(blk_mq_rq_from_pdu(nr), true);
+}
+
+void nvme_req_retry_timer_init(struct nvme_request *req)
+{
+	timer_setup(&req->retry_timer, nvme_retry_timer_fn, 0);
+}
+EXPORT_SYMBOL_GPL(nvme_req_retry_timer_init);
+
 void nvme_cleanup_cmd(struct request *req)
 {
 	if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
index 04363b9c4489..4117cf0f7ac2 100644
--- a/drivers/nvme/host/fc.c
+++ b/drivers/nvme/host/fc.c
@@ -2124,6 +2124,7 @@ nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq,
 	op->op.fcp_req.private = &op->priv[0];
 	nvme_req(rq)->ctrl = &ctrl->ctrl;
 	nvme_req(rq)->cmd = &op->op.cmd_iu.sqe;
+	nvme_req_retry_timer_init(nvme_req(rq));
 	return res;
 }
 
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 6d8f6c541a61..944462e912f6 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -249,6 +249,7 @@ struct nvme_request {
 	unsigned long		start_time;
 #endif
 	struct nvme_ctrl	*ctrl;
+	struct timer_list	retry_timer;
 };
 
 /*
@@ -931,6 +932,7 @@ static inline unsigned long nvme_crd_msecs(struct nvme_request *req)
 	return req->ctrl->crdt[crd - 1] * 100;
 }
 
+void nvme_req_retry_timer_init(struct nvme_request *req);
 #define NVME_QID_ANY -1
 void nvme_init_request(struct request *req, struct nvme_command *cmd);
 void nvme_cleanup_cmd(struct request *req);
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 8438c904ec49..aaaa72ce7cbf 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -671,6 +671,7 @@ static int nvme_pci_init_request(struct blk_mq_tag_set *set,
 
 	nvme_req(req)->ctrl = set->driver_data;
 	nvme_req(req)->cmd = &iod->cmd;
+	nvme_req_retry_timer_init(nvme_req(req));
 	return 0;
 }
 
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 6909e3542794..434999acc6ea 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -312,6 +312,7 @@ static int nvme_rdma_init_request(struct blk_mq_tag_set *set,
 
 	req->queue = queue;
 	nvme_req(rq)->cmd = req->sqe.data;
+	nvme_req_retry_timer_init(nvme_req(rq));
 
 	return 0;
 }
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index ba5c7b3e2a7c..3503e3ad4d6b 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -562,6 +562,7 @@ static int nvme_tcp_init_request(struct blk_mq_tag_set *set,
 	nvme_req(rq)->cmd = &pdu->cmd;
 	init_llist_node(&req->lentry);
 	INIT_LIST_HEAD(&req->entry);
+	nvme_req_retry_timer_init(nvme_req(rq));
 
 	return 0;
 }
diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c
index fcb1f8186fdd..86c21631345d 100644
--- a/drivers/nvme/target/loop.c
+++ b/drivers/nvme/target/loop.c
@@ -209,6 +209,7 @@ static int nvme_loop_init_request(struct blk_mq_tag_set *set,
 
 	nvme_req(req)->ctrl = &ctrl->ctrl;
 	nvme_req(req)->cmd = &iod->cmd;
+	nvme_req_retry_timer_init(nvme_req(req));
 	return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req),
 			(set == &ctrl->tag_set) ? hctx_idx + 1 : 0);
 }
-- 
2.43.0