[PATCH v2 0/2] nvmet: passthru cleanup and fixup I/O hotpath

[PATCH v2 0/2] nvmet: passthru cleanup and fixup I/O hotpath

[Nilay Shroff]

Hi,

This series addresses a race in the passthru I/O hotpath where
concurrently disabling the passthru controller while I/Os are in flight
can result in a use-after-free bug.

We were able to reproduce this bug with NVMe/TCP configured and by
injecting an additional delay into the target-side I/O processing code.
If the passthru controller is disabled while an I/O is still in flight,
it results in the following kernel crash:

BUG: Kernel NULL pointer dereference on read at 0x00000030
[...]
CPU: 9 UID: 0 PID: 4143 Comm: kworker/9:5H Kdump: loaded Not tainted 7.3.0-rc3+ #14 PREEMPT 
Hardware name: IBM,9080-HEX Power11 (architected) 0x820200 0xf000007 of:IBM,FW1110.00 (NH1110_031) hv:phyp pSeries
Workqueue: nvmet_tcp_wq nvmet_tcp_io_work [nvmet_tcp]
[...]
NIP [c0080000156d8fe0] nvmet_passthru_execute_cmd+0x58/0x45c [nvmet]
LR [c0080000156d8fc4] nvmet_passthru_execute_cmd+0x3c/0x45c [nvmet]
Call Trace:
 nvmet_passthru_execute_cmd+0x3c/0x45c [nvmet] (unreliable)
 nvmet_tcp_done_recv_pdu+0x2d0/0x718 [nvmet_tcp]
 nvmet_tcp_try_recv_pdu+0x29c/0x348 [nvmet_tcp]
 nvmet_tcp_io_work+0xe8/0x838 [nvmet_tcp]
 process_one_work+0x224/0x5ec
 worker_thread+0x1f8/0x3e8
 kthread+0x178/0x1ac
 start_kernel_thread+0x14/0x18

There are two patches in this series. The first patch groups all
passthru-related fields into a separate struct nvmet_passthru, which
makes the code easier to maintain and reason about. The second patch
fixes the use-after-free bug described above.

As usual, code review comments and feedback are most welcome!

Thanks!

Changes from v1:
  - In patch 1/2, dynamically allocate struct nvmet_passthru when the
    passthru controller is configured, reducing memory overhead for
    subsystems that don't use passthru. (hch)

Nilay Shroff (2):
  nvmet: introduce struct nvmet_passthru
  nvmet: fix use-after-free in passthru I/O hotpath

 drivers/nvme/target/configfs.c |  90 ++++++++++++++++++++++++----
 drivers/nvme/target/core.c     |  15 ++++-
 drivers/nvme/target/nvmet.h    |  66 +++++++++++++++++++--
 drivers/nvme/target/passthru.c | 103 ++++++++++++++++++++++++++-------
 4 files changed, 232 insertions(+), 42 deletions(-)

-- 
2.53.0

[PATCH v2 1/2] nvmet: introduce struct nvmet_passthru

[Nilay Shroff]

Currently, passthru parameters are grouped under struct nvmet_subsys.
Since passthru can be configured through configfs and all passthru
parameters are exposed under a separate configfs subdirectory, group
these parameters under a new struct nvmet_passthru.

Grouping passthru-specific parameters in a separate structure makes
the code easier to maintain and reason about.

This also allows the passthru structure to be allocated on demand,
avoiding the memory overhead for subsystems that do not use passthru.
The structure is allocated when passthru configuration is needed and
its configfs attributes are accessed under subsys->lock. The passthru
configfs group is still created when the subsystem is allocated.

There are no functional changes intended.

Signed-off-by: Nilay Shroff <nilay@linux.ibm.com>
---
 drivers/nvme/target/configfs.c | 90 +++++++++++++++++++++++++++++-----
 drivers/nvme/target/core.c     | 12 ++++-
 drivers/nvme/target/nvmet.h    | 34 ++++++++++---
 drivers/nvme/target/passthru.c | 52 ++++++++++++--------
 4 files changed, 148 insertions(+), 40 deletions(-)

diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c
index b03b5d1c2dc3..6bac66c48bff 100644
--- a/drivers/nvme/target/configfs.c
+++ b/drivers/nvme/target/configfs.c
@@ -905,21 +905,34 @@ static ssize_t nvmet_passthru_device_path_show(struct config_item *item,
 		char *page)
 {
 	struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
+	ssize_t ret;
 
-	return snprintf(page, PAGE_SIZE, "%s\n", subsys->passthru_ctrl_path);
+	mutex_lock(&subsys->lock);
+	ret = snprintf(page, PAGE_SIZE, "%s\n",
+		       subsys->passthru && subsys->passthru->ctrl_path ?
+		       subsys->passthru->ctrl_path : "");
+	mutex_unlock(&subsys->lock);
+	return ret;
 }
 
 static ssize_t nvmet_passthru_device_path_store(struct config_item *item,
 		const char *page, size_t count)
 {
 	struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
+	struct nvmet_passthru *passthru;
 	size_t len;
 	int ret;
 
 	mutex_lock(&subsys->lock);
 
+	passthru = nvmet_subsys_passthru(subsys);
+	if (!passthru) {
+		ret = -ENOMEM;
+		goto out_unlock;
+	}
+
 	ret = -EBUSY;
-	if (subsys->passthru_ctrl)
+	if (passthru->ctrl)
 		goto out_unlock;
 
 	ret = -EINVAL;
@@ -927,10 +940,10 @@ static ssize_t nvmet_passthru_device_path_store(struct config_item *item,
 	if (!len)
 		goto out_unlock;
 
-	kfree(subsys->passthru_ctrl_path);
+	kfree(passthru->ctrl_path);
 	ret = -ENOMEM;
-	subsys->passthru_ctrl_path = kstrndup(page, len, GFP_KERNEL);
-	if (!subsys->passthru_ctrl_path)
+	passthru->ctrl_path = kstrndup(page, len, GFP_KERNEL);
+	if (!passthru->ctrl_path)
 		goto out_unlock;
 
 	mutex_unlock(&subsys->lock);
@@ -946,8 +959,13 @@ static ssize_t nvmet_passthru_enable_show(struct config_item *item,
 		char *page)
 {
 	struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
+	ssize_t ret;
 
-	return sprintf(page, "%d\n", subsys->passthru_ctrl ? 1 : 0);
+	mutex_lock(&subsys->lock);
+	ret = sprintf(page, "%d\n",
+		subsys->passthru && subsys->passthru->ctrl ? 1 : 0);
+	mutex_unlock(&subsys->lock);
+	return ret;
 }
 
 static ssize_t nvmet_passthru_enable_store(struct config_item *item,
@@ -972,18 +990,34 @@ CONFIGFS_ATTR(nvmet_passthru_, enable);
 static ssize_t nvmet_passthru_admin_timeout_show(struct config_item *item,
 		char *page)
 {
-	return sprintf(page, "%u\n", to_subsys(item->ci_parent)->admin_timeout);
+	struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
+	ssize_t ret;
+
+	mutex_lock(&subsys->lock);
+	ret = sprintf(page, "%u\n", subsys->passthru ?
+			subsys->passthru->admin_timeout : 0);
+	mutex_unlock(&subsys->lock);
+	return ret;
 }
 
 static ssize_t nvmet_passthru_admin_timeout_store(struct config_item *item,
 		const char *page, size_t count)
 {
 	struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
+	struct nvmet_passthru *passthru;
 	unsigned int timeout;
 
 	if (kstrtouint(page, 0, &timeout))
 		return -EINVAL;
-	subsys->admin_timeout = timeout;
+
+	mutex_lock(&subsys->lock);
+	passthru = nvmet_subsys_passthru(subsys);
+	if (!passthru) {
+		mutex_unlock(&subsys->lock);
+		return -ENOMEM;
+	}
+	passthru->admin_timeout = timeout;
+	mutex_unlock(&subsys->lock);
 	return count;
 }
 CONFIGFS_ATTR(nvmet_passthru_, admin_timeout);
@@ -991,18 +1025,34 @@ CONFIGFS_ATTR(nvmet_passthru_, admin_timeout);
 static ssize_t nvmet_passthru_io_timeout_show(struct config_item *item,
 		char *page)
 {
-	return sprintf(page, "%u\n", to_subsys(item->ci_parent)->io_timeout);
+	struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
+	ssize_t ret;
+
+	mutex_lock(&subsys->lock);
+	ret = sprintf(page, "%u\n",
+		subsys->passthru ? subsys->passthru->io_timeout : 0);
+	mutex_unlock(&subsys->lock);
+	return ret;
 }
 
 static ssize_t nvmet_passthru_io_timeout_store(struct config_item *item,
 		const char *page, size_t count)
 {
 	struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
+	struct nvmet_passthru *passthru;
 	unsigned int timeout;
 
 	if (kstrtouint(page, 0, &timeout))
 		return -EINVAL;
-	subsys->io_timeout = timeout;
+
+	mutex_lock(&subsys->lock);
+	passthru = nvmet_subsys_passthru(subsys);
+	if (!passthru) {
+		mutex_unlock(&subsys->lock);
+		return -ENOMEM;
+	}
+	passthru->io_timeout = timeout;
+	mutex_unlock(&subsys->lock);
 	return count;
 }
 CONFIGFS_ATTR(nvmet_passthru_, io_timeout);
@@ -1010,18 +1060,34 @@ CONFIGFS_ATTR(nvmet_passthru_, io_timeout);
 static ssize_t nvmet_passthru_clear_ids_show(struct config_item *item,
 		char *page)
 {
-	return sprintf(page, "%u\n", to_subsys(item->ci_parent)->clear_ids);
+	struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
+	ssize_t ret;
+
+	mutex_lock(&subsys->lock);
+	ret = sprintf(page, "%u\n",
+		subsys->passthru ? subsys->passthru->clear_ids : 0);
+	mutex_unlock(&subsys->lock);
+	return ret;
 }
 
 static ssize_t nvmet_passthru_clear_ids_store(struct config_item *item,
 		const char *page, size_t count)
 {
 	struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
+	struct nvmet_passthru *passthru;
 	unsigned int clear_ids;
 
 	if (kstrtouint(page, 0, &clear_ids))
 		return -EINVAL;
-	subsys->clear_ids = clear_ids;
+
+	mutex_lock(&subsys->lock);
+	passthru = nvmet_subsys_passthru(subsys);
+	if (!passthru) {
+		mutex_unlock(&subsys->lock);
+		return -ENOMEM;
+	}
+	passthru->clear_ids = clear_ids;
+	mutex_unlock(&subsys->lock);
 	return count;
 }
 CONFIGFS_ATTR(nvmet_passthru_, clear_ids);
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index 8eea0a504308..b09681cb4a1f 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -1645,8 +1645,16 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args)
 
 #ifdef CONFIG_NVME_TARGET_PASSTHRU
 	/* By default, set loop targets to clear IDS by default */
-	if (ctrl->port->disc_addr.trtype == NVMF_TRTYPE_LOOP)
-		subsys->clear_ids = 1;
+	if (ctrl->port->disc_addr.trtype == NVMF_TRTYPE_LOOP) {
+		mutex_lock(&subsys->lock);
+		subsys->passthru = nvmet_subsys_passthru(subsys);
+		if (!subsys->passthru) {
+			mutex_unlock(&subsys->lock);
+			goto out_free_ctrl;
+		}
+		subsys->passthru->clear_ids = 1;
+		mutex_unlock(&subsys->lock);
+	}
 #endif
 
 	INIT_WORK(&ctrl->async_event_work, nvmet_async_event_work);
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 162e2fdd848e..eb0f965b1a7e 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -319,6 +319,14 @@ struct nvmet_ctrl {
 	struct nvmet_pr_log_mgr pr_log_mgr;
 };
 
+struct nvmet_passthru {
+	struct nvme_ctrl	*ctrl;
+	char			*ctrl_path;
+	unsigned int		admin_timeout;
+	unsigned int		io_timeout;
+	unsigned int		clear_ids;
+};
+
 struct nvmet_subsys {
 	enum nvme_subsys_type	type;
 
@@ -358,12 +366,8 @@ struct nvmet_subsys {
 	char			*firmware_rev;
 
 #ifdef CONFIG_NVME_TARGET_PASSTHRU
-	struct nvme_ctrl	*passthru_ctrl;
-	char			*passthru_ctrl_path;
+	struct nvmet_passthru	*passthru;
 	struct config_group	passthru_group;
-	unsigned int		admin_timeout;
-	unsigned int		io_timeout;
-	unsigned int		clear_ids;
 #endif /* CONFIG_NVME_TARGET_PASSTHRU */
 
 #ifdef CONFIG_BLK_DEV_ZONED
@@ -793,7 +797,20 @@ u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req);
 u16 nvmet_parse_passthru_io_cmd(struct nvmet_req *req);
 static inline bool nvmet_is_passthru_subsys(struct nvmet_subsys *subsys)
 {
-	return subsys->passthru_ctrl;
+	return subsys->passthru && subsys->passthru->ctrl;
+}
+
+static inline struct nvmet_passthru *nvmet_subsys_passthru(
+		struct nvmet_subsys *subsys)
+{
+	lockdep_assert_held(&subsys->lock);
+
+	if (!subsys->passthru) {
+		subsys->passthru = kzalloc_obj(*subsys->passthru);
+		if (!subsys->passthru)
+			return NULL;
+	}
+	return subsys->passthru;
 }
 #else /* CONFIG_NVME_TARGET_PASSTHRU */
 static inline void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys)
@@ -814,6 +831,11 @@ static inline bool nvmet_is_passthru_subsys(struct nvmet_subsys *subsys)
 {
 	return NULL;
 }
+static inline struct nvmet_passthru *nvmet_subsys_passthru(
+		struct nvmet_subsys *subsys)
+{
+	return NULL;
+}
 #endif /* CONFIG_NVME_TARGET_PASSTHRU */
 
 static inline bool nvmet_is_passthru_req(struct nvmet_req *req)
diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c
index fa6527c537e2..16cd3fdf98ec 100644
--- a/drivers/nvme/target/passthru.c
+++ b/drivers/nvme/target/passthru.c
@@ -26,7 +26,7 @@ void nvmet_passthrough_override_cap(struct nvmet_ctrl *ctrl)
 	 * Multiple command set support can only be declared if the underlying
 	 * controller actually supports it.
 	 */
-	if (!nvme_multi_css(ctrl->subsys->passthru_ctrl))
+	if (!nvme_multi_css(ctrl->subsys->passthru->ctrl))
 		ctrl->cap &= ~(1ULL << 43);
 }
 
@@ -39,7 +39,7 @@ static u16 nvmet_passthru_override_id_descs(struct nvmet_req *req)
 	void *data;
 	u8 csi;
 
-	if (!ctrl->subsys->clear_ids)
+	if (!ctrl->subsys->passthru->clear_ids)
 		return status;
 
 	data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
@@ -89,7 +89,7 @@ static u16 nvmet_passthru_override_id_descs(struct nvmet_req *req)
 static u16 nvmet_passthru_override_id_ctrl(struct nvmet_req *req)
 {
 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
-	struct nvme_ctrl *pctrl = ctrl->subsys->passthru_ctrl;
+	struct nvme_ctrl *pctrl = ctrl->subsys->passthru->ctrl;
 	u16 status = NVME_SC_SUCCESS;
 	struct nvme_id_ctrl *id;
 	unsigned int max_hw_sectors;
@@ -208,7 +208,7 @@ static u16 nvmet_passthru_override_id_ns(struct nvmet_req *req)
 	 */
 	id->mc = 0;
 
-	if (req->sq->ctrl->subsys->clear_ids) {
+	if (req->sq->ctrl->subsys->passthru->clear_ids) {
 		memset(id->nguid, 0, NVME_NIDT_NGUID_LEN);
 		memset(id->eui64, 0, NVME_NIDT_EUI64_LEN);
 	}
@@ -305,7 +305,8 @@ static int nvmet_passthru_map_sg(struct nvmet_req *req, struct request *rq)
 
 static void nvmet_passthru_execute_cmd(struct nvmet_req *req)
 {
-	struct nvme_ctrl *ctrl = nvmet_req_subsys(req)->passthru_ctrl;
+	struct nvmet_passthru *passthru = nvmet_req_subsys(req)->passthru;
+	struct nvme_ctrl *ctrl = passthru->ctrl;
 	struct request_queue *q = ctrl->admin_q;
 	struct nvme_ns *ns = NULL;
 	struct request *rq = NULL;
@@ -325,9 +326,9 @@ static void nvmet_passthru_execute_cmd(struct nvmet_req *req)
 		}
 
 		q = ns->queue;
-		timeout = nvmet_req_subsys(req)->io_timeout;
+		timeout = passthru->io_timeout;
 	} else {
-		timeout = nvmet_req_subsys(req)->admin_timeout;
+		timeout = passthru->admin_timeout;
 	}
 
 	rq = blk_mq_alloc_request(q, nvme_req_op(req->cmd), 0);
@@ -386,7 +387,7 @@ static void nvmet_passthru_execute_cmd(struct nvmet_req *req)
  */
 static void nvmet_passthru_set_host_behaviour(struct nvmet_req *req)
 {
-	struct nvme_ctrl *ctrl = nvmet_req_subsys(req)->passthru_ctrl;
+	struct nvme_ctrl *ctrl = nvmet_req_subsys(req)->passthru->ctrl;
 	struct nvme_feat_host_behavior *host;
 	u16 status = NVME_SC_INTERNAL;
 	int ret;
@@ -586,15 +587,17 @@ u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req)
 
 int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
 {
+	struct nvmet_passthru *passthru;
 	struct nvme_ctrl *ctrl;
 	struct file *file;
 	int ret = -EINVAL;
 	void *old;
 
 	mutex_lock(&subsys->lock);
-	if (!subsys->passthru_ctrl_path)
+	passthru = subsys->passthru;
+	if (!passthru || !passthru->ctrl_path)
 		goto out_unlock;
-	if (subsys->passthru_ctrl)
+	if (passthru->ctrl)
 		goto out_unlock;
 
 	if (subsys->nr_namespaces) {
@@ -602,7 +605,7 @@ int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
 		goto out_unlock;
 	}
 
-	file = filp_open(subsys->passthru_ctrl_path, O_RDWR, 0);
+	file = filp_open(passthru->ctrl_path, O_RDWR, 0);
 	if (IS_ERR(file)) {
 		ret = PTR_ERR(file);
 		goto out_unlock;
@@ -611,7 +614,7 @@ int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
 	ctrl = nvme_ctrl_from_file(file);
 	if (!ctrl) {
 		pr_err("failed to open nvme controller %s\n",
-		       subsys->passthru_ctrl_path);
+		       passthru->ctrl_path);
 
 		goto out_put_file;
 	}
@@ -626,7 +629,7 @@ int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
 	if (old)
 		goto out_put_file;
 
-	subsys->passthru_ctrl = ctrl;
+	passthru->ctrl = ctrl;
 	subsys->ver = ctrl->vs;
 
 	if (subsys->ver < NVME_VS(1, 2, 1)) {
@@ -636,7 +639,7 @@ int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
 		subsys->ver = NVME_VS(1, 2, 1);
 	}
 	nvme_get_ctrl(ctrl);
-	__module_get(subsys->passthru_ctrl->ops->module);
+	__module_get(passthru->ctrl->ops->module);
 	ret = 0;
 
 out_put_file:
@@ -648,12 +651,17 @@ int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
 
 static void __nvmet_passthru_ctrl_disable(struct nvmet_subsys *subsys)
 {
-	if (subsys->passthru_ctrl) {
-		xa_erase(&passthru_subsystems, subsys->passthru_ctrl->instance);
-		module_put(subsys->passthru_ctrl->ops->module);
-		nvme_put_ctrl(subsys->passthru_ctrl);
+	struct nvmet_passthru *passthru = subsys->passthru;
+
+	if (!passthru)
+		return;
+
+	if (passthru->ctrl) {
+		xa_erase(&passthru_subsystems, passthru->ctrl->instance);
+		module_put(passthru->ctrl->ops->module);
+		nvme_put_ctrl(passthru->ctrl);
 	}
-	subsys->passthru_ctrl = NULL;
+	passthru->ctrl = NULL;
 	subsys->ver = NVMET_DEFAULT_VS;
 }
 
@@ -668,6 +676,10 @@ void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys)
 {
 	mutex_lock(&subsys->lock);
 	__nvmet_passthru_ctrl_disable(subsys);
+	if (subsys->passthru) {
+		kfree(subsys->passthru->ctrl_path);
+		kfree(subsys->passthru);
+		subsys->passthru = NULL;
+	}
 	mutex_unlock(&subsys->lock);
-	kfree(subsys->passthru_ctrl_path);
 }
-- 
2.53.0

Re: [PATCH v2 1/2] nvmet: introduce struct nvmet_passthru

[Christoph Hellwig]

Looks good:

Reviewed-by: Christoph Hellwig <hch@lst.de>

[PATCH v2 2/2] nvmet: fix use-after-free in passthru I/O hotpath

[Nilay Shroff]

Concurrently disabling a passthru controller while passthru I/Os are
in flight can potentially result in a use-after-free. Introduce a
percpu refcount, an atomic flag, and an nvmet request flag to protect
the passthru controller lifetime.

When enabling the passthru controller, initialize the percpu refcount
and set the enabled flag. Each passthru I/O acquires a reference before
entering the passthru hotpath and sets an nvmet request flag to record
that the reference is held. The reference is released when the I/O
completes.

When disabling the passthru controller, clear the enabled flag, kill
the percpu refcount, and wait for all in-flight I/Os to release their
references before releasing the passthru controller.

Disable operations are serialized by subsys->lock. The enabled flag
ensures that only the first disable operation proceeds and subsequent
attempts observe the flag as cleared and return.

Once disabling starts, new I/Os either fail to be routed to the
passthru path or fail to acquire a live reference, and therefore cannot
dereference the passthru controller and thus avoid use-after-free.

Signed-off-by: Nilay Shroff <nilay@linux.ibm.com>
---
 drivers/nvme/target/core.c     |  3 ++
 drivers/nvme/target/nvmet.h    | 34 +++++++++++++++++-
 drivers/nvme/target/passthru.c | 65 +++++++++++++++++++++++++++++-----
 3 files changed, 92 insertions(+), 10 deletions(-)

diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index b09681cb4a1f..fa1420065e30 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -819,6 +819,8 @@ static void __nvmet_req_complete(struct nvmet_req *req, u16 status)
 		nvmet_pr_put_ns_pc_ref(pc_ref);
 	if (ns)
 		nvmet_put_namespace(ns);
+
+	nvmet_put_passthru_ref(req);
 }
 
 void nvmet_req_complete(struct nvmet_req *req, u16 status)
@@ -1195,6 +1197,7 @@ bool nvmet_req_init(struct nvmet_req *req, struct nvmet_sq *sq,
 	req->error_loc = NVMET_NO_ERROR_LOC;
 	req->error_slba = 0;
 	req->pc_ref = NULL;
+	req->p.ref_held = false;
 
 	/* no support for fused commands yet */
 	if (unlikely(flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND))) {
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index eb0f965b1a7e..04dc33004cb7 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -320,6 +320,11 @@ struct nvmet_ctrl {
 };
 
 struct nvmet_passthru {
+	struct percpu_ref	ref;
+	struct completion	disable_done;
+#define NVMET_PASSTHRU_ENABLED	0
+	unsigned long		flags;
+
 	struct nvme_ctrl	*ctrl;
 	char			*ctrl_path;
 	unsigned int		admin_timeout;
@@ -478,6 +483,7 @@ struct nvmet_req {
 			struct request		*rq;
 			struct work_struct      work;
 			bool			use_workqueue;
+			bool			ref_held;
 		} p;
 #ifdef CONFIG_BLK_DEV_ZONED
 		struct {
@@ -797,7 +803,8 @@ u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req);
 u16 nvmet_parse_passthru_io_cmd(struct nvmet_req *req);
 static inline bool nvmet_is_passthru_subsys(struct nvmet_subsys *subsys)
 {
-	return subsys->passthru && subsys->passthru->ctrl;
+	return subsys->passthru &&
+		test_bit(NVMET_PASSTHRU_ENABLED, &subsys->passthru->flags);
 }
 
 static inline struct nvmet_passthru *nvmet_subsys_passthru(
@@ -812,6 +819,24 @@ static inline struct nvmet_passthru *nvmet_subsys_passthru(
 	}
 	return subsys->passthru;
 }
+
+static inline bool nvmet_get_passthru_ref(struct nvmet_req *req)
+{
+	if (!percpu_ref_tryget_live(&nvmet_req_subsys(req)->passthru->ref))
+		return false;
+
+	req->p.ref_held = true;
+	return true;
+}
+
+static inline void nvmet_put_passthru_ref(struct nvmet_req *req)
+{
+	if (!req->p.ref_held)
+		return;
+
+	req->p.ref_held = false;
+	percpu_ref_put(&nvmet_req_subsys(req)->passthru->ref);
+}
 #else /* CONFIG_NVME_TARGET_PASSTHRU */
 static inline void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys)
 {
@@ -836,6 +861,13 @@ static inline struct nvmet_passthru *nvmet_subsys_passthru(
 {
 	return NULL;
 }
+static inline bool nvmet_get_passthru_ref(struct nvmet_req *req)
+{
+	return false;
+}
+static inline void nvmet_put_passthru_ref(struct nvmet_req *req)
+{
+}
 #endif /* CONFIG_NVME_TARGET_PASSTHRU */
 
 static inline bool nvmet_is_passthru_req(struct nvmet_req *req)
diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c
index 16cd3fdf98ec..6b67880c7bb2 100644
--- a/drivers/nvme/target/passthru.c
+++ b/drivers/nvme/target/passthru.c
@@ -305,9 +305,9 @@ static int nvmet_passthru_map_sg(struct nvmet_req *req, struct request *rq)
 
 static void nvmet_passthru_execute_cmd(struct nvmet_req *req)
 {
-	struct nvmet_passthru *passthru = nvmet_req_subsys(req)->passthru;
-	struct nvme_ctrl *ctrl = passthru->ctrl;
-	struct request_queue *q = ctrl->admin_q;
+	struct nvmet_passthru *passthru;
+	struct nvme_ctrl *ctrl;
+	struct request_queue *q;
 	struct nvme_ns *ns = NULL;
 	struct request *rq = NULL;
 	unsigned int timeout;
@@ -315,6 +315,15 @@ static void nvmet_passthru_execute_cmd(struct nvmet_req *req)
 	u16 status;
 	int ret;
 
+	if (!nvmet_get_passthru_ref(req)) {
+		status = NVME_SC_INTERNAL | NVME_STATUS_DNR;
+		goto out;
+	}
+
+	passthru = nvmet_req_subsys(req)->passthru;
+	ctrl = passthru->ctrl;
+	q = ctrl->admin_q;
+
 	if (likely(req->sq->qid != 0)) {
 		u32 nsid = le32_to_cpu(req->cmd->common.nsid);
 
@@ -387,11 +396,17 @@ static void nvmet_passthru_execute_cmd(struct nvmet_req *req)
  */
 static void nvmet_passthru_set_host_behaviour(struct nvmet_req *req)
 {
-	struct nvme_ctrl *ctrl = nvmet_req_subsys(req)->passthru->ctrl;
+	struct nvme_ctrl *ctrl;
 	struct nvme_feat_host_behavior *host;
 	u16 status = NVME_SC_INTERNAL;
 	int ret;
 
+	if (!nvmet_get_passthru_ref(req)) {
+		status |= NVME_STATUS_DNR;
+		goto out_complete_req;
+	}
+	ctrl = nvmet_req_subsys(req)->passthru->ctrl;
+
 	host = kzalloc(sizeof(*host) * 2, GFP_KERNEL);
 	if (!host)
 		goto out_complete_req;
@@ -585,6 +600,14 @@ u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req)
 	}
 }
 
+static void nvmet_release_passthru_ctrl(struct percpu_ref *ref)
+{
+	struct nvmet_passthru *passthru = container_of(ref,
+				struct nvmet_passthru, ref);
+
+	complete(&passthru->disable_done);
+}
+
 int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
 {
 	struct nvmet_passthru *passthru;
@@ -629,6 +652,13 @@ int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
 	if (old)
 		goto out_put_file;
 
+	ret = percpu_ref_init(&passthru->ref, nvmet_release_passthru_ctrl,
+			0, GFP_KERNEL);
+	if (ret) {
+		xa_erase(&passthru_subsystems, ctrl->instance);
+		goto out_put_file;
+	}
+	init_completion(&passthru->disable_done);
 	passthru->ctrl = ctrl;
 	subsys->ver = ctrl->vs;
 
@@ -640,6 +670,7 @@ int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
 	}
 	nvme_get_ctrl(ctrl);
 	__module_get(passthru->ctrl->ops->module);
+	set_bit(NVMET_PASSTHRU_ENABLED, &passthru->flags);
 	ret = 0;
 
 out_put_file:
@@ -653,14 +684,30 @@ static void __nvmet_passthru_ctrl_disable(struct nvmet_subsys *subsys)
 {
 	struct nvmet_passthru *passthru = subsys->passthru;
 
+	lockdep_assert_held(&subsys->lock);
+
 	if (!passthru)
 		return;
 
-	if (passthru->ctrl) {
-		xa_erase(&passthru_subsystems, passthru->ctrl->instance);
-		module_put(passthru->ctrl->ops->module);
-		nvme_put_ctrl(passthru->ctrl);
-	}
+	if (!test_and_clear_bit(NVMET_PASSTHRU_ENABLED, &passthru->flags))
+		return;
+
+	mutex_unlock(&subsys->lock);
+	/*
+	 * Now new I/Os should not enter passthru hotpath as we cleared the
+	 * enabled flag. Kill percpu reference and wait for in-flight I/Os
+	 * to drain.
+	 */
+	percpu_ref_kill(&passthru->ref);
+	wait_for_completion(&passthru->disable_done);
+	percpu_ref_exit(&passthru->ref);
+
+	mutex_lock(&subsys->lock);
+
+	xa_erase(&passthru_subsystems, passthru->ctrl->instance);
+	module_put(passthru->ctrl->ops->module);
+	nvme_put_ctrl(passthru->ctrl);
+
 	passthru->ctrl = NULL;
 	subsys->ver = NVMET_DEFAULT_VS;
 }
-- 
2.53.0

Re: [PATCH v2 2/2] nvmet: fix use-after-free in passthru I/O hotpath

[Christoph Hellwig]

Looks good:

Reviewed-by: Christoph Hellwig <hch@lst.de>

Re: [PATCH v2 2/2] nvmet: fix use-after-free in passthru I/O hotpath

[Shin'ichiro Kawasaki]

On Sep 29, 2026 / 16:56, Nilay Shroff wrote:
> Concurrently disabling a passthru controller while passthru I/Os are
> in flight can potentially result in a use-after-free. Introduce a
> percpu refcount, an atomic flag, and an nvmet request flag to protect
> the passthru controller lifetime.
> 
> When enabling the passthru controller, initialize the percpu refcount
> and set the enabled flag. Each passthru I/O acquires a reference before
> entering the passthru hotpath and sets an nvmet request flag to record
> that the reference is held. The reference is released when the I/O
> completes.
> 
> When disabling the passthru controller, clear the enabled flag, kill
> the percpu refcount, and wait for all in-flight I/Os to release their
> references before releasing the passthru controller.
> 
> Disable operations are serialized by subsys->lock. The enabled flag
> ensures that only the first disable operation proceeds and subsequent
> attempts observe the flag as cleared and return.
> 
> Once disabling starts, new I/Os either fail to be routed to the
> passthru path or fail to acquire a live reference, and therefore cannot
> dereference the passthru controller and thus avoid use-after-free.
> 
> Signed-off-by: Nilay Shroff <nilay@linux.ibm.com>

Hi, Nilay

I ran blktests with the kernel v7.4-rc1, and found the test case nvme/037 hangs
when it runs for rdma transport and siw driver [1]. I bisected and found this
patch as the commit 5e12d81874a1 is the trigger. If this han is a new issue, fix
actions will be appreciated.


[1] dmesg at nvme/037 hang

[ 3800.092349] [  T66191] run blktests nvme/037 at 2026-11-02 13:46:33.187037133
[ 3800.407876] [  T66934] nvmet_rdma: enabling port 0 (10.0.2.15:4420)
[ 3800.579559] [  T62940] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-0 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349.
[ 3800.601419] [  T66942] nvme nvme5: creating 4 I/O queues.
[ 3800.603382] [  T66942] nvme nvme5: mapped 4/0/0 default/read/poll queues.
[ 3800.728997] [  T66942] nvme nvme5: new ctrl: NQhN "blktests-subsystem-0", addr 10.0.2.15:4420, hostnqn: nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349
[ 3801.087205] [  T66966] nvme nvme5: Removing ctrl: NQN "blktests-subsystem-0"
[ 3801.531391] [  T66991] nvmet_rdma: enabling port 0 (10.0.2.15:4420)
[ 3801.688110] [  T62940] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-1 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349.
[ 3801.704598] [  T66999] nvme nvme5: creating 4 I/O queues.
[ 3801.706191] [  T66999] nvme nvme5: mapped 4/0/0 default/read/poll queues.
[ 3801.824182] [  T66999] nvme nvme5: new ctrl: NQN "blktests-subsystem-1", addr 10.0.2.15:4420, hostnqn: nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349
[ 3802.184483] [  T67023] nvme nvme5: Removing ctrl: NQN "blktests-subsystem-1"
[ 3802.641645] [  T67048] nvmet_rdma: enabling port 0 (10.0.2.15:4420)
[ 3802.799232] [  T62940] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-2 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349.
[ 3802.815664] [  T67056] nvme nvme5: creating 4 I/O queues.
[ 3802.816931] [  T67056] nvme nvme5: mapped 4/0/0 default/read/poll queues.
[ 3802.935988] [  T67056] nvme nvme5: new ctrl: NQN "blktests-subsystem-2", addr 10.0.2.15:4420, hostnqn: nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349
[ 3803.290667] [  T67080] nvme nvme5: Removing ctrl: NQN "blktests-subsystem-2"
[ 3803.749797] [  T67105] nvmet_rdma: enabling port 0 (10.0.2.15:4420)
[ 3803.909791] [  T62940] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-3 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349.
[ 3803.932485] [  T67113] nvme nvme5: creating 4 I/O queues.
[ 3803.933765] [  T67113] nvme nvme5: mapped 4/0/0 default/read/poll queues.
[ 3804.054973] [  T67113] nvme nvme5: new ctrl: NQN "blktests-subsystem-3", addr 10.0.2.15:4420, hostnqn: nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349
[ 3804.413732] [  T67137] nvme nvme5: Removing ctrl: NQN "blktests-subsystem-3"
[ 3804.857065] [  T67162] nvmet_rdma: enabling port 0 (10.0.2.15:4420)
[ 3805.010639] [  T62940] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-4 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349.
[ 3805.028058] [  T67170] nvme nvme5: creating 4 I/O queues.
[ 3805.030124] [  T67170] nvme nvme5: mapped 4/0/0 default/read/poll queues.
[ 3805.155236] [  T67170] nvme nvme5: new ctrl: NQN "blktests-subsystem-4", addr 10.0.2.15:4420, hostnqn: nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349
[ 3805.527884] [  T67194] nvme nvme5: Removing ctrl: NQN "blktests-subsystem-4"
[ 3805.971809] [  T67219] nvmet_rdma: enabling port 0 (10.0.2.15:4420)
[ 3806.137857] [  T62940] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-5 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349.
[ 3806.155758] [  T67227] nvme nvme5: creating 4 I/O queues.
[ 3806.157395] [  T67227] nvme nvme5: mapped 4/0/0 default/read/poll queues.
[ 3806.277467] [  T67227] nvme nvme5: new ctrl: NQN "blktests-subsystem-5", addr 10.0.2.15:4420, hostnqn: nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349
[ 3806.641474] [  T67251] nvme nvme5: Removing ctrl: NQN "blktests-subsystem-5"
[ 3807.084278] [  T67276] nvmet_rdma: enabling port 0 (10.0.2.15:4420)
[ 3807.278228] [  T62940] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-6 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349.
[ 3807.308262] [  T67284] nvme nvme5: creating 4 I/O queues.
[ 3807.317068] [  T67284] nvme nvme5: mapped 4/0/0 default/read/poll queues.
[ 3807.469771] [  T67284] nvme nvme5: new ctrl: NQN "blktests-subsystem-6", addr 10.0.2.15:4420, hostnqn: nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349
[ 3807.937039] [  T67308] nvme nvme5: Removing ctrl: NQN "blktests-subsystem-6"
[ 3809.420886] [  T67333] nvmet_rdma: enabling port 0 (10.0.2.15:4420)
[ 3809.592288] [  T62940] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-7 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349.
[ 3809.628746] [  T67341] nvme nvme5: creating 4 I/O queues.
[ 3809.630196] [  T67341] nvme nvme5: mapped 4/0/0 default/read/poll queues.
[ 3809.768300] [  T67341] nvme nvme5: new ctrl: NQN "blktests-subsystem-7", addr 10.0.2.15:4420, hostnqn: nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349
[ 3810.178568] [  T67365] nvme nvme5: Removing ctrl: NQN "blktests-subsystem-7"
[ 3811.140562] [  T67390] nvmet_rdma: enabling port 0 (10.0.2.15:4420)
[ 3811.332824] [  T62940] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-8 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349.
[ 3811.351258] [  T67398] nvme nvme5: creating 4 I/O queues.
[ 3811.352718] [  T67398] nvme nvme5: mapped 4/0/0 default/read/poll queues.
[ 3811.491120] [  T67398] nvme nvme5: new ctrl: NQN "blktests-subsystem-8", addr 10.0.2.15:4420, hostnqn: nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349
[ 3812.000917] [  T67422] nvme nvme5: Removing ctrl: NQN "blktests-subsystem-8"
[ 3812.828312] [  T67447] nvmet_rdma: enabling port 0 (10.0.2.15:4420)
[ 3812.998293] [  T62940] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-9 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349.
[ 3813.017919] [  T67455] nvme nvme5: creating 4 I/O queues.
[ 3813.020332] [  T67455] nvme nvme5: mapped 4/0/0 default/read/poll queues.
[ 3813.161933] [  T67455] nvme nvme5: new ctrl: NQN "blktests-subsystem-9", addr 10.0.2.15:4420, hostnqn: nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349
[ 3813.539457] [  T67479] nvme nvme5: Removing ctrl: NQN "blktests-subsystem-9"
[ 4056.926027] [     T48] INFO: task check:66191 blocked for more than 122 seconds.
[ 4056.931295] [     T48]       Not tainted 7.4.0-rc1 #487
[ 4056.931983] [     T48] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4056.932706] [     T48] task:check           state:D stack:0     pid:66191 tgid:66191 ppid:23692  task_flags:0x400140 flags:0x00080000
[ 4056.933524] [     T48] Call Trace:
[ 4056.934363] [     T48]  <TASK>
[ 4056.935110] [     T48]  __schedule+0x1d90/0x6ea0
[ 4056.935828] [     T48]  ? __pfx___schedule+0x10/0x10
[ 4056.936501] [     T48]  ? lock_acquire+0x1c1/0x340
[ 4056.937175] [     T48]  ? schedule+0x24a/0x2e0
[ 4056.937848] [     T48]  ? lock_release+0x1b5/0x340
[ 4056.939024] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4056.939719] [     T48]  schedule+0xe3/0x2e0
[ 4056.940371] [     T48]  schedule_timeout+0x1ac/0x290
[ 4056.940975] [     T48]  ? __pfx_schedule_timeout+0x10/0x10
[ 4056.941587] [     T48]  ? lockdep_hardirqs_on_prepare+0xd5/0x1b0
[ 4056.942166] [     T48]  ? trace_hardirqs_on+0x14/0x190
[ 4056.942796] [     T48]  wait_for_completion+0x17a/0x400
[ 4056.943440] [     T48]  ? __pfx_wait_for_completion+0x10/0x10
[ 4056.944065] [     T48]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 4056.944669] [     T48]  __nvmet_passthru_ctrl_disable+0xcf/0x320 [nvmet]
[ 4056.945333] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4056.945950] [     T48]  nvmet_passthru_ctrl_disable+0x24/0x40 [nvmet]
[ 4056.946530] [     T48]  ? __kasan_kmalloc+0x96/0xb0
[ 4056.947106] [     T48]  nvmet_passthru_enable_store+0xa0/0x100 [nvmet]
[ 4056.947713] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4056.948304] [     T48]  ? configfs_write_iter+0x240/0x4a0
[ 4056.948874] [     T48]  configfs_write_iter+0x2ad/0x4a0
[ 4056.949405] [     T48]  ? __pfx_configfs_write_iter+0x10/0x10
[ 4056.949949] [     T48]  vfs_write+0x4b3/0xf40
[ 4056.950465] [     T48]  ? __pfx_vfs_write+0x10/0x10
[ 4056.951003] [     T48]  ? __lock_acquire+0x54a/0x2490
[ 4056.951499] [     T48]  ? lock_acquire+0x1b1/0x340
[ 4056.952119] [     T48]  ksys_write+0x112/0x250
[ 4056.952640] [     T48]  ? __pfx_ksys_write+0x10/0x10
[ 4056.953138] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4056.953557] [     T48]  do_syscall_64+0xf2/0x690
[ 4056.954046] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4056.954486] [     T48]  ? do_syscall_64+0x2d/0x690
[ 4056.954952] [     T48]  ? trace_hardirqs_off_finish+0x139/0x180
[ 4056.955384] [     T48]  ? lockdep_hardirqs_off+0xaf/0xf0
[ 4056.955844] [     T48]  ? do_syscall_64+0x60/0x690
[ 4056.956254] [     T48]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 4056.956621] [     T48] RIP: 0033:0x7fb94423d54e
[ 4056.956995] [     T48] RSP: 002b:00007ffef5d375a0 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 4056.957416] [     T48] RAX: ffffffffffffffda RBX: 00007fb9443be580 RCX: 00007fb94423d54e
[ 4056.957852] [     T48] RDX: 0000000000000001 RSI: 000055c9e7267150 RDI: 0000000000000001
[ 4056.958265] [     T48] RBP: 00007ffef5d375b0 R08: 0000000000000000 R09: 0000000000000000
[ 4056.958585] [     T48] R10: 0000000000000000 R11: 0000000000000202 R12: 0000000000000001
[ 4056.958999] [     T48] R13: 0000000000000001 R14: 000055c9e7267150 R15: 0000000000000000
[ 4056.959369] [     T48]  </TASK>
[ 4056.959741] [     T48] 
                          Showing all locks held in the system:
[ 4056.960385] [     T48] locks held by pr/ttyS0/47: 2, last CPU#2:
[ 4056.960702] [     T48]  #0: ffffffff8b89ffd8 (console_srcu){....}-{0:0}, at: console_srcu_read_lock+0x45/0x60
[ 4056.961033] [     T48]  #1: ffffffffa2730278 (&port_lock_key){-.-.}-{3:3}, at: univ8250_console_device_lock+0x52/0xa0
[ 4056.961403] [     T48] locks held by khungtaskd/48: 1, last CPU#3:
[ 4056.961740] [     T48]  #0: ffffffff8b8b1e00 (rcu_read_lock){....}-{1:3}, at: debug_show_all_locks+0x39/0x180
[ 4056.962137] [     T48] locks held by kworker/3:11/65511: 4, on CPU#3:
[ 4056.962506] [     T48]  #0: ffff888100194148 ((wq_completion)events){+.+.}-{0:0}, at: process_one_work+0x1109/0x1760
[ 4056.962894] [     T48]  #1: ffff88814eeefd10 ((work_completion)(&helper->damage_work)){+.+.}-{0:0}, at: process_one_work+0x8c0/0x1760
[ 4056.963301] [     T48]  #2: ffff8881325dca70 (&helper->lock){+.+.}-{4:4}, at: drm_fb_helper_damage_work+0x175/0xac0 [drm_client_lib]
[ 4056.963690] [     T48]  #3: ffff8881333e0128 (&dev->master_mutex){+.+.}-{4:4}, at: drm_master_internal_acquire+0x1b/0x60 [drm]
[ 4056.964134] [     T48] locks held by check/66191: 3, on CPU#1:
[ 4056.964542] [     T48]  #0: ffff888142c0e478 (sb_writers#12){.+.+}-{0:0}, at: ksys_write+0x112/0x250
[ 4056.964950] [     T48]  #1: ffff888144737e80 (&buffer->mutex){+.+.}-{4:4}, at: configfs_write_iter+0x6d/0x4a0
[ 4056.965918] [     T48]  #2: ffff88813c09b070 (&p->frag_sem){++++}-{4:4}, at: configfs_write_iter+0x1db/0x4a0

[ 4056.966796] [     T48] =============================================

[ 4179.806492] [     T48] INFO: task check:66191 blocked for more than 245 seconds.
[ 4179.809371] [     T48]       Not tainted 7.4.0-rc1 #487
[ 4179.812094] [     T48] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4179.812633] [     T48] task:check           state:D stack:0     pid:66191 tgid:66191 ppid:23692  task_flags:0x400140 flags:0x00080000
[ 4179.813102] [     T48] Call Trace:
[ 4179.813539] [     T48]  <TASK>
[ 4179.813987] [     T48]  __schedule+0x1d90/0x6ea0
[ 4179.814452] [     T48]  ? __pfx___schedule+0x10/0x10
[ 4179.814877] [     T48]  ? lock_acquire+0x1c1/0x340
[ 4179.815318] [     T48]  ? schedule+0x24a/0x2e0
[ 4179.816338] [     T48]  ? lock_release+0x1b5/0x340
[ 4179.816784] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4179.817265] [     T48]  schedule+0xe3/0x2e0
[ 4179.817700] [     T48]  schedule_timeout+0x1ac/0x290
[ 4179.818169] [     T48]  ? __pfx_schedule_timeout+0x10/0x10
[ 4179.818623] [     T48]  ? lockdep_hardirqs_on_prepare+0xd5/0x1b0
[ 4179.819043] [     T48]  ? trace_hardirqs_on+0x14/0x190
[ 4179.819481] [     T48]  wait_for_completion+0x17a/0x400
[ 4179.819934] [     T48]  ? __pfx_wait_for_completion+0x10/0x10
[ 4179.820418] [     T48]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 4179.820861] [     T48]  __nvmet_passthru_ctrl_disable+0xcf/0x320 [nvmet]
[ 4179.821371] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4179.821853] [     T48]  nvmet_passthru_ctrl_disable+0x24/0x40 [nvmet]
[ 4179.822361] [     T48]  ? __kasan_kmalloc+0x96/0xb0
[ 4179.822829] [     T48]  nvmet_passthru_enable_store+0xa0/0x100 [nvmet]
[ 4179.823275] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4179.823734] [     T48]  ? configfs_write_iter+0x240/0x4a0
[ 4179.824162] [     T48]  configfs_write_iter+0x2ad/0x4a0
[ 4179.824612] [     T48]  ? __pfx_configfs_write_iter+0x10/0x10
[ 4179.825044] [     T48]  vfs_write+0x4b3/0xf40
[ 4179.825477] [     T48]  ? __pfx_vfs_write+0x10/0x10
[ 4179.825922] [     T48]  ? __lock_acquire+0x54a/0x2490
[ 4179.826351] [     T48]  ? lock_acquire+0x1b1/0x340
[ 4179.826790] [     T48]  ksys_write+0x112/0x250
[ 4179.827204] [     T48]  ? __pfx_ksys_write+0x10/0x10
[ 4179.827627] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4179.828025] [     T48]  do_syscall_64+0xf2/0x690
[ 4179.828414] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4179.828816] [     T48]  ? do_syscall_64+0x2d/0x690
[ 4179.829322] [     T48]  ? trace_hardirqs_off_finish+0x139/0x180
[ 4179.829741] [     T48]  ? lockdep_hardirqs_off+0xaf/0xf0
[ 4179.830125] [     T48]  ? do_syscall_64+0x60/0x690
[ 4179.830485] [     T48]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 4179.830856] [     T48] RIP: 0033:0x7fb94423d54e
[ 4179.831216] [     T48] RSP: 002b:00007ffef5d375a0 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 4179.831583] [     T48] RAX: ffffffffffffffda RBX: 00007fb9443be580 RCX: 00007fb94423d54e
[ 4179.831941] [     T48] RDX: 0000000000000001 RSI: 000055c9e7267150 RDI: 0000000000000001
[ 4179.832296] [     T48] RBP: 00007ffef5d375b0 R08: 0000000000000000 R09: 0000000000000000
[ 4179.832665] [     T48] R10: 0000000000000000 R11: 0000000000000202 R12: 0000000000000001
[ 4179.833014] [     T48] R13: 0000000000000001 R14: 000055c9e7267150 R15: 0000000000000000
[ 4179.833390] [     T48]  </TASK>
[ 4179.833749] [     T48] 
                          Showing all locks held in the system:
[ 4179.834379] [     T48] locks held by pr/ttyS0/47: 2, last CPU#2:
[ 4179.834713] [     T48]  #0: ffffffff8b89ffd8 (console_srcu){....}-{0:0}, at: console_srcu_read_lock+0x45/0x60
[ 4179.835055] [     T48]  #1: ffffffffa2730278 (&port_lock_key){-.-.}-{3:3}, at: univ8250_console_device_lock+0x52/0xa0
[ 4179.835400] [     T48] locks held by khungtaskd/48: 1, last CPU#3:
[ 4179.835756] [     T48]  #0: ffffffff8b8b1e00 (rcu_read_lock){....}-{1:3}, at: debug_show_all_locks+0x39/0x180
[ 4179.836149] [     T48] locks held by kworker/3:11/65511: 4, on CPU#3:
[ 4179.836481] [     T48]  #0: ffff888100194148 ((wq_completion)events){+.+.}-{0:0}, at: process_one_work+0x1109/0x1760
[ 4179.836874] [     T48]  #1: ffff88814eeefd10 ((work_completion)(&helper->damage_work)){+.+.}-{0:0}, at: process_one_work+0x8c0/0x1760
[ 4179.837276] [     T48]  #2: ffff8881325dca70 (&helper->lock){+.+.}-{4:4}, at: drm_fb_helper_damage_work+0x175/0xac0 [drm_client_lib]
[ 4179.837701] [     T48]  #3: ffff8881333e0128 (&dev->master_mutex){+.+.}-{4:4}, at: drm_master_internal_acquire+0x1b/0x60 [drm]
[ 4179.838196] [     T48] locks held by check/66191: 3, on CPU#1:
[ 4179.838590] [     T48]  #0: ffff888142c0e478 (sb_writers#12){.+.+}-{0:0}, at: ksys_write+0x112/0x250
[ 4179.839010] [     T48]  #1: ffff888144737e80 (&buffer->mutex){+.+.}-{4:4}, at: configfs_write_iter+0x6d/0x4a0
[ 4179.839429] [     T48]  #2: ffff88813c09b070 (&p->frag_sem){++++}-{4:4}, at: configfs_write_iter+0x1db/0x4a0

[ 4179.840265] [     T48] =============================================

[ 4302.686843] [     T48] INFO: task check:66191 blocked for more than 368 seconds.
[ 4302.689729] [     T48]       Not tainted 7.4.0-rc1 #487
[ 4302.692595] [     T48] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4302.695736] [     T48] task:check           state:D stack:0     pid:66191 tgid:66191 ppid:23692  task_flags:0x400140 flags:0x00080000
[ 4302.696226] [     T48] Call Trace:
[ 4302.696654] [     T48]  <TASK>
[ 4302.697069] [     T48]  __schedule+0x1d90/0x6ea0
[ 4302.697535] [     T48]  ? __pfx___schedule+0x10/0x10
[ 4302.697953] [     T48]  ? lock_acquire+0x1c1/0x340
[ 4302.698390] [     T48]  ? schedule+0x24a/0x2e0
[ 4302.698805] [     T48]  ? lock_release+0x1b5/0x340
[ 4302.699211] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4302.699676] [     T48]  schedule+0xe3/0x2e0
[ 4302.700092] [     T48]  schedule_timeout+0x1ac/0x290
[ 4302.700524] [     T48]  ? __pfx_schedule_timeout+0x10/0x10
[ 4302.700955] [     T48]  ? lockdep_hardirqs_on_prepare+0xd5/0x1b0
[ 4302.701394] [     T48]  ? trace_hardirqs_on+0x14/0x190
[ 4302.701815] [     T48]  wait_for_completion+0x17a/0x400
[ 4302.702239] [     T48]  ? __pfx_wait_for_completion+0x10/0x10
[ 4302.702677] [     T48]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 4302.703098] [     T48]  __nvmet_passthru_ctrl_disable+0xcf/0x320 [nvmet]
[ 4302.703557] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4302.704002] [     T48]  nvmet_passthru_ctrl_disable+0x24/0x40 [nvmet]
[ 4302.704463] [     T48]  ? __kasan_kmalloc+0x96/0xb0
[ 4302.704882] [     T48]  nvmet_passthru_enable_store+0xa0/0x100 [nvmet]
[ 4302.705349] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4302.705792] [     T48]  ? configfs_write_iter+0x240/0x4a0
[ 4302.706368] [     T48]  configfs_write_iter+0x2ad/0x4a0
[ 4302.706814] [     T48]  ? __pfx_configfs_write_iter+0x10/0x10
[ 4302.707242] [     T48]  vfs_write+0x4b3/0xf40
[ 4302.707672] [     T48]  ? __pfx_vfs_write+0x10/0x10
[ 4302.708103] [     T48]  ? __lock_acquire+0x54a/0x2490
[ 4302.708538] [     T48]  ? lock_acquire+0x1b1/0x340
[ 4302.708966] [     T48]  ksys_write+0x112/0x250
[ 4302.709386] [     T48]  ? __pfx_ksys_write+0x10/0x10
[ 4302.709781] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4302.710180] [     T48]  do_syscall_64+0xf2/0x690
[ 4302.710593] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4302.710979] [     T48]  ? do_syscall_64+0x2d/0x690
[ 4302.711369] [     T48]  ? trace_hardirqs_off_finish+0x139/0x180
[ 4302.711739] [     T48]  ? lockdep_hardirqs_off+0xaf/0xf0
[ 4302.712098] [     T48]  ? do_syscall_64+0x60/0x690
[ 4302.712470] [     T48]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 4302.712811] [     T48] RIP: 0033:0x7fb94423d54e
[ 4302.713160] [     T48] RSP: 002b:00007ffef5d375a0 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 4302.713523] [     T48] RAX: ffffffffffffffda RBX: 00007fb9443be580 RCX: 00007fb94423d54e
[ 4302.713861] [     T48] RDX: 0000000000000001 RSI: 000055c9e7267150 RDI: 0000000000000001
[ 4302.714208] [     T48] RBP: 00007ffef5d375b0 R08: 0000000000000000 R09: 0000000000000000
[ 4302.714561] [     T48] R10: 0000000000000000 R11: 0000000000000202 R12: 0000000000000001
[ 4302.714900] [     T48] R13: 0000000000000001 R14: 000055c9e7267150 R15: 0000000000000000
[ 4302.715244] [     T48]  </TASK>
[ 4302.715592] [     T48] 
                          Showing all locks held in the system:
[ 4302.716268] [     T48] locks held by pr/ttyS0/47: 2, last CPU#2:
[ 4302.716616] [     T48]  #0: ffffffff8b89ffd8 (console_srcu){....}-{0:0}, at: console_srcu_read_lock+0x45/0x60
[ 4302.716979] [     T48]  #1: ffffffffa2730278 (&port_lock_key){-.-.}-{3:3}, at: univ8250_console_device_lock+0x52/0xa0
[ 4302.717336] [     T48] locks held by khungtaskd/48: 1, last CPU#3:
[ 4302.717688] [     T48]  #0: ffffffff8b8b1e00 (rcu_read_lock){....}-{1:3}, at: debug_show_all_locks+0x39/0x180
[ 4302.718111] [     T48] locks held by kworker/3:11/65511: 4, on CPU#3:
[ 4302.718494] [     T48]  #0: ffff888100194148 ((wq_completion)events){+.+.}-{0:0}, at: process_one_work+0x1109/0x1760
[ 4302.718860] [     T48]  #1: ffff88814eeefd10 ((work_completion)(&helper->damage_work)){+.+.}-{0:0}, at: process_one_work+0x8c0/0x1760
[ 4302.719260] [     T48]  #2: ffff8881325dca70 (&helper->lock){+.+.}-{4:4}, at: drm_atomic_helper_dirtyfb+0xdf/0x770 [drm_kms_helper]
[ 4302.720166] [     T48]  #3: ffff88814eeefa58 (crtc_ww_class_mutex){+.+.}-{4:4}, at: drm_atomic_helper_dirtyfb+0xdf/0x770 [drm_kms_helper]
[ 4302.720646] [     T48] locks held by check/66191: 3, on CPU#1:
[ 4302.721038] [     T48]  #0: ffff888142c0e478 (sb_writers#12){.+.+}-{0:0}, at: ksys_write+0x112/0x250
[ 4302.721499] [     T48]  #1: ffff888144737e80 (&buffer->mutex){+.+.}-{4:4}, at: configfs_write_iter+0x6d/0x4a0
[ 4302.721908] [     T48]  #2: ffff88813c09b070 (&p->frag_sem){++++}-{4:4}, at: configfs_write_iter+0x1db/0x4a0

[ 4302.722791] [     T48] =============================================

[ 4425.567220] [     T48] INFO: task check:66191 blocked for more than 491 seconds.
[ 4425.570359] [     T48]       Not tainted 7.4.0-rc1 #487
[ 4425.573459] [     T48] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4425.576758] [     T48] task:check           state:D stack:0     pid:66191 tgid:66191 ppid:23692  task_flags:0x400140 flags:0x00080000
[ 4425.578392] [     T48] Call Trace:
[ 4425.578891] [     T48]  <TASK>
[ 4425.579409] [     T48]  __schedule+0x1d90/0x6ea0
[ 4425.579937] [     T48]  ? __pfx___schedule+0x10/0x10
[ 4425.580417] [     T48]  ? lock_acquire+0x1c1/0x340
[ 4425.580885] [     T48]  ? schedule+0x24a/0x2e0
[ 4425.581345] [     T48]  ? lock_release+0x1b5/0x340
[ 4425.581812] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4425.582313] [     T48]  schedule+0xe3/0x2e0
[ 4425.582785] [     T48]  schedule_timeout+0x1ac/0x290
[ 4425.583734] [     T48]  ? __pfx_schedule_timeout+0x10/0x10
[ 4425.584167] [     T48]  ? lockdep_hardirqs_on_prepare+0xd5/0x1b0
[ 4425.584604] [     T48]  ? trace_hardirqs_on+0x14/0x190
[ 4425.585073] [     T48]  wait_for_completion+0x17a/0x400
[ 4425.585518] [     T48]  ? __pfx_wait_for_completion+0x10/0x10
[ 4425.585968] [     T48]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 4425.586398] [     T48]  __nvmet_passthru_ctrl_disable+0xcf/0x320 [nvmet]
[ 4425.586836] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4425.587329] [     T48]  nvmet_passthru_ctrl_disable+0x24/0x40 [nvmet]
[ 4425.587801] [     T48]  ? __kasan_kmalloc+0x96/0xb0
[ 4425.588279] [     T48]  nvmet_passthru_enable_store+0xa0/0x100 [nvmet]
[ 4425.588743] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4425.589183] [     T48]  ? configfs_write_iter+0x240/0x4a0
[ 4425.589597] [     T48]  configfs_write_iter+0x2ad/0x4a0
[ 4425.590021] [     T48]  ? __pfx_configfs_write_iter+0x10/0x10
[ 4425.590432] [     T48]  vfs_write+0x4b3/0xf40
[ 4425.590850] [     T48]  ? __pfx_vfs_write+0x10/0x10
[ 4425.591261] [     T48]  ? __lock_acquire+0x54a/0x2490
[ 4425.591682] [     T48]  ? lock_acquire+0x1b1/0x340
[ 4425.592085] [     T48]  ksys_write+0x112/0x250
[ 4425.592483] [     T48]  ? __pfx_ksys_write+0x10/0x10
[ 4425.592889] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4425.593292] [     T48]  do_syscall_64+0xf2/0x690
[ 4425.593660] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4425.594065] [     T48]  ? do_syscall_64+0x2d/0x690
[ 4425.594454] [     T48]  ? trace_hardirqs_off_finish+0x139/0x180
[ 4425.594819] [     T48]  ? lockdep_hardirqs_off+0xaf/0xf0
[ 4425.595186] [     T48]  ? do_syscall_64+0x60/0x690
[ 4425.595541] [     T48]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 4425.595925] [     T48] RIP: 0033:0x7fb94423d54e
[ 4425.596289] [     T48] RSP: 002b:00007ffef5d375a0 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 4425.596657] [     T48] RAX: ffffffffffffffda RBX: 00007fb9443be580 RCX: 00007fb94423d54e
[ 4425.597015] [     T48] RDX: 0000000000000001 RSI: 000055c9e7267150 RDI: 0000000000000001
[ 4425.597419] [     T48] RBP: 00007ffef5d375b0 R08: 0000000000000000 R09: 0000000000000000
[ 4425.597803] [     T48] R10: 0000000000000000 R11: 0000000000000202 R12: 0000000000000001
[ 4425.598146] [     T48] R13: 0000000000000001 R14: 000055c9e7267150 R15: 0000000000000000
[ 4425.598537] [     T48]  </TASK>
[ 4425.598950] [     T48] 
                          Showing all locks held in the system:
[ 4425.599590] [     T48] locks held by pr/ttyS0/47: 2, last CPU#2:
[ 4425.599932] [     T48]  #0: ffffffff8b89ffd8 (console_srcu){....}-{0:0}, at: console_srcu_read_lock+0x45/0x60
[ 4425.600277] [     T48]  #1: ffffffffa2730278 (&port_lock_key){-.-.}-{3:3}, at: univ8250_console_device_lock+0x52/0xa0
[ 4425.600605] [     T48] locks held by khungtaskd/48: 1, last CPU#1:
[ 4425.600981] [     T48]  #0: ffffffff8b8b1e00 (rcu_read_lock){....}-{1:3}, at: debug_show_all_locks+0x39/0x180
[ 4425.601405] [     T48] locks held by check/66191: 3, on CPU#1:
[ 4425.601792] [     T48]  #0: ffff888142c0e478 (sb_writers#12){.+.+}-{0:0}, at: ksys_write+0x112/0x250
[ 4425.602160] [     T48]  #1: ffff888144737e80 (&buffer->mutex){+.+.}-{4:4}, at: configfs_write_iter+0x6d/0x4a0
[ 4425.602548] [     T48]  #2: ffff88813c09b070 (&p->frag_sem){++++}-{4:4}, at: configfs_write_iter+0x1db/0x4a0
[ 4425.602948] [     T48] locks held by kworker/3:0/67500: 4, on CPU#3:
[ 4425.603342] [     T48]  #0: ffff888100194148 ((wq_completion)events){+.+.}-{0:0}, at: process_one_work+0x1109/0x1760
[ 4425.603756] [     T48]  #1: ffff88812b2e7d10 ((work_completion)(&helper->damage_work)){+.+.}-{0:0}, at: process_one_work+0x8c0/0x1760
[ 4425.604169] [     T48]  #2: ffff8881325dca70 (&helper->lock){+.+.}-{4:4}, at: drm_fb_helper_damage_work+0x175/0xac0 [drm_client_lib]
[ 4425.604591] [     T48]  #3: ffff8881333e0128 (&dev->master_mutex){+.+.}-{4:4}, at: drm_master_internal_acquire+0x1b/0x60 [drm]

[ 4425.605522] [     T48] =============================================

[ 4548.447529] [     T48] INFO: task check:66191 blocked for more than 614 seconds.
[ 4548.450305] [     T48]       Not tainted 7.4.0-rc1 #487
[ 4548.452756] [     T48] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4548.453461] [     T48] task:check           state:D stack:0     pid:66191 tgid:66191 ppid:23692  task_flags:0x400140 flags:0x00080000
[ 4548.453922] [     T48] Call Trace:
[ 4548.454336] [     T48]  <TASK>
[ 4548.454739] [     T48]  __schedule+0x1d90/0x6ea0
[ 4548.455244] [     T48]  ? __pfx___schedule+0x10/0x10
[ 4548.455697] [     T48]  ? lock_acquire+0x1c1/0x340
[ 4548.456102] [     T48]  ? schedule+0x24a/0x2e0
[ 4548.456549] [     T48]  ? lock_release+0x1b5/0x340
[ 4548.457013] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4548.457478] [     T48]  schedule+0xe3/0x2e0
[ 4548.457900] [     T48]  schedule_timeout+0x1ac/0x290
[ 4548.458341] [     T48]  ? __pfx_schedule_timeout+0x10/0x10
[ 4548.458773] [     T48]  ? lockdep_hardirqs_on_prepare+0xd5/0x1b0
[ 4548.459211] [     T48]  ? trace_hardirqs_on+0x14/0x190
[ 4548.459639] [     T48]  wait_for_completion+0x17a/0x400
[ 4548.460103] [     T48]  ? __pfx_wait_for_completion+0x10/0x10
[ 4548.460531] [     T48]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 4548.460998] [     T48]  __nvmet_passthru_ctrl_disable+0xcf/0x320 [nvmet]
[ 4548.461471] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4548.461942] [     T48]  nvmet_passthru_ctrl_disable+0x24/0x40 [nvmet]
[ 4548.462383] [     T48]  ? __kasan_kmalloc+0x96/0xb0
[ 4548.462819] [     T48]  nvmet_passthru_enable_store+0xa0/0x100 [nvmet]
[ 4548.463313] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4548.463741] [     T48]  ? configfs_write_iter+0x240/0x4a0
[ 4548.464191] [     T48]  configfs_write_iter+0x2ad/0x4a0
[ 4548.464625] [     T48]  ? __pfx_configfs_write_iter+0x10/0x10
[ 4548.465107] [     T48]  vfs_write+0x4b3/0xf40
[ 4548.465531] [     T48]  ? __pfx_vfs_write+0x10/0x10
[ 4548.465965] [     T48]  ? __lock_acquire+0x54a/0x2490
[ 4548.466401] [     T48]  ? lock_acquire+0x1b1/0x340
[ 4548.466849] [     T48]  ksys_write+0x112/0x250
[ 4548.467295] [     T48]  ? __pfx_ksys_write+0x10/0x10
[ 4548.467687] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4548.468108] [     T48]  do_syscall_64+0xf2/0x690
[ 4548.468496] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4548.468893] [     T48]  ? do_syscall_64+0x2d/0x690
[ 4548.469260] [     T48]  ? trace_hardirqs_off_finish+0x139/0x180
[ 4548.469663] [     T48]  ? lockdep_hardirqs_off+0xaf/0xf0
[ 4548.470008] [     T48]  ? do_syscall_64+0x60/0x690
[ 4548.470386] [     T48]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 4548.470747] [     T48] RIP: 0033:0x7fb94423d54e
[ 4548.471149] [     T48] RSP: 002b:00007ffef5d375a0 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 4548.471506] [     T48] RAX: ffffffffffffffda RBX: 00007fb9443be580 RCX: 00007fb94423d54e
[ 4548.471869] [     T48] RDX: 0000000000000001 RSI: 000055c9e7267150 RDI: 0000000000000001
[ 4548.472223] [     T48] RBP: 00007ffef5d375b0 R08: 0000000000000000 R09: 0000000000000000
[ 4548.472553] [     T48] R10: 0000000000000000 R11: 0000000000000202 R12: 0000000000000001
[ 4548.472895] [     T48] R13: 0000000000000001 R14: 000055c9e7267150 R15: 0000000000000000
[ 4548.473260] [     T48]  </TASK>
[ 4548.473585] [     T48] 
                          Showing all locks held in the system:
[ 4548.474272] [     T48] locks held by pr/ttyS0/47: 2, last CPU#2:
[ 4548.474572] [     T48]  #0: ffffffff8b89ffd8 (console_srcu){....}-{0:0}, at: console_srcu_read_lock+0x45/0x60
[ 4548.474909] [     T48]  #1: ffffffffa2730278 (&port_lock_key){-.-.}-{3:3}, at: univ8250_console_device_lock+0x52/0xa0
[ 4548.475280] [     T48] locks held by khungtaskd/48: 1, last CPU#1:
[ 4548.475633] [     T48]  #0: ffffffff8b8b1e00 (rcu_read_lock){....}-{1:3}, at: debug_show_all_locks+0x39/0x180
[ 4548.476031] [     T48] locks held by check/66191: 3, on CPU#1:
[ 4548.476389] [     T48]  #0: ffff888142c0e478 (sb_writers#12){.+.+}-{0:0}, at: ksys_write+0x112/0x250
[ 4548.476787] [     T48]  #1: ffff888144737e80 (&buffer->mutex){+.+.}-{4:4}, at: configfs_write_iter+0x6d/0x4a0
[ 4548.477199] [     T48]  #2: ffff88813c09b070 (&p->frag_sem){++++}-{4:4}, at: configfs_write_iter+0x1db/0x4a0
[ 4548.477591] [     T48] locks held by kworker/2:1/67503: 4, last CPU#2:
[ 4548.477953] [     T48]  #0: ffff888100194148 ((wq_completion)events){+.+.}-{0:0}, at: process_one_work+0x1109/0x1760
[ 4548.478360] [     T48]  #1: ffff888149d8fd10 ((work_completion)(&helper->damage_work)){+.+.}-{0:0}, at: process_one_work+0x8c0/0x1760
[ 4548.478766] [     T48]  #2: ffff888149d8fa30 (crtc_ww_class_acquire){+.+.}-{0:0}, at: drm_atomic_helper_dirtyfb+0xdf/0x770 [drm_kms_helper]
[ 4548.479238] [     T48]  #3: ffff888149d8fa58 (crtc_ww_class_mutex){+.+.}-{4:4}, at: drm_atomic_helper_dirtyfb+0xdf/0x770 [drm_kms_helper]

[ 4548.480086] [     T48] =============================================

[ 4671.327902] [     T48] INFO: task check:66191 blocked for more than 737 seconds.
[ 4671.330876] [     T48]       Not tainted 7.4.0-rc1 #487
[ 4671.333073] [     T48] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4671.333513] [     T48] task:check           state:D stack:0     pid:66191 tgid:66191 ppid:23692  task_flags:0x400140 flags:0x00080000
[ 4671.333952] [     T48] Call Trace:
[ 4671.334436] [     T48]  <TASK>
[ 4671.334920] [     T48]  __schedule+0x1d90/0x6ea0
[ 4671.335389] [     T48]  ? __pfx___schedule+0x10/0x10
[ 4671.335804] [     T48]  ? lock_acquire+0x1c1/0x340
[ 4671.336222] [     T48]  ? schedule+0x24a/0x2e0
[ 4671.336651] [     T48]  ? lock_release+0x1b5/0x340
[ 4671.337072] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4671.338063] [     T48]  schedule+0xe3/0x2e0
[ 4671.338510] [     T48]  schedule_timeout+0x1ac/0x290
[ 4671.338932] [     T48]  ? __pfx_schedule_timeout+0x10/0x10
[ 4671.339383] [     T48]  ? lockdep_hardirqs_on_prepare+0xd5/0x1b0
[ 4671.339819] [     T48]  ? trace_hardirqs_on+0x14/0x190
[ 4671.340272] [     T48]  wait_for_completion+0x17a/0x400
[ 4671.340721] [     T48]  ? __pfx_wait_for_completion+0x10/0x10
[ 4671.341222] [     T48]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 4671.341678] [     T48]  __nvmet_passthru_ctrl_disable+0xcf/0x320 [nvmet]
[ 4671.342148] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4671.342725] [     T48]  nvmet_passthru_ctrl_disable+0x24/0x40 [nvmet]
[ 4671.343169] [     T48]  ? __kasan_kmalloc+0x96/0xb0
[ 4671.343583] [     T48]  nvmet_passthru_enable_store+0xa0/0x100 [nvmet]
[ 4671.344083] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4671.344581] [     T48]  ? configfs_write_iter+0x240/0x4a0
[ 4671.345036] [     T48]  configfs_write_iter+0x2ad/0x4a0
[ 4671.345463] [     T48]  ? __pfx_configfs_write_iter+0x10/0x10
[ 4671.345869] [     T48]  vfs_write+0x4b3/0xf40
[ 4671.346274] [     T48]  ? __pfx_vfs_write+0x10/0x10
[ 4671.346694] [     T48]  ? __lock_acquire+0x54a/0x2490
[ 4671.347117] [     T48]  ? lock_acquire+0x1b1/0x340
[ 4671.347536] [     T48]  ksys_write+0x112/0x250
[ 4671.347930] [     T48]  ? __pfx_ksys_write+0x10/0x10
[ 4671.348322] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4671.348720] [     T48]  do_syscall_64+0xf2/0x690
[ 4671.349100] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4671.349485] [     T48]  ? do_syscall_64+0x2d/0x690
[ 4671.349846] [     T48]  ? trace_hardirqs_off_finish+0x139/0x180
[ 4671.350209] [     T48]  ? lockdep_hardirqs_off+0xaf/0xf0
[ 4671.350578] [     T48]  ? do_syscall_64+0x60/0x690
[ 4671.350927] [     T48]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 4671.351420] [     T48] RIP: 0033:0x7fb94423d54e
[ 4671.351784] [     T48] RSP: 002b:00007ffef5d375a0 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 4671.352135] [     T48] RAX: ffffffffffffffda RBX: 00007fb9443be580 RCX: 00007fb94423d54e
[ 4671.352490] [     T48] RDX: 0000000000000001 RSI: 000055c9e7267150 RDI: 0000000000000001
[ 4671.352822] [     T48] RBP: 00007ffef5d375b0 R08: 0000000000000000 R09: 0000000000000000
[ 4671.353155] [     T48] R10: 0000000000000000 R11: 0000000000000202 R12: 0000000000000001
[ 4671.353501] [     T48] R13: 0000000000000001 R14: 000055c9e7267150 R15: 0000000000000000
[ 4671.353837] [     T48]  </TASK>
[ 4671.354177] [     T48] 
                          Showing all locks held in the system:
[ 4671.354803] [     T48] locks held by pr/ttyS0/47: 2, last CPU#0:
[ 4671.355107] [     T48]  #0: ffffffff8b89ffd8 (console_srcu){....}-{0:0}, at: console_srcu_read_lock+0x45/0x60
[ 4671.355450] [     T48]  #1: ffffffffa2730278 (&port_lock_key){-.-.}-{3:3}, at: univ8250_console_device_lock+0x52/0xa0
[ 4671.355787] [     T48] locks held by khungtaskd/48: 1, last CPU#1:
[ 4671.356110] [     T48]  #0: ffffffff8b8b1e00 (rcu_read_lock){....}-{1:3}, at: debug_show_all_locks+0x39/0x180
[ 4671.356500] [     T48] locks held by check/66191: 3, on CPU#1:
[ 4671.356833] [     T48]  #0: ffff888142c0e478 (sb_writers#12){.+.+}-{0:0}, at: ksys_write+0x112/0x250
[ 4671.357193] [     T48]  #1: ffff888144737e80 (&buffer->mutex){+.+.}-{4:4}, at: configfs_write_iter+0x6d/0x4a0
[ 4671.357568] [     T48]  #2: ffff88813c09b070 (&p->frag_sem){++++}-{4:4}, at: configfs_write_iter+0x1db/0x4a0
[ 4671.357931] [     T48] locks held by kworker/1:0/67499: 4, on CPU#1:
[ 4671.358285] [     T48]  #0: ffff888100194148 ((wq_completion)events){+.+.}-{0:0}, at: process_one_work+0x1109/0x1760
[ 4671.358678] [     T48]  #1: ffff888132427d10 ((work_completion)(&helper->damage_work)){+.+.}-{0:0}, at: process_one_work+0x8c0/0x1760
[ 4671.359068] [     T48]  #2: ffff8881325dca70 (&helper->lock){+.+.}-{4:4}, at: drm_fb_helper_damage_work+0x175/0xac0 [drm_client_lib]
[ 4671.359479] [     T48]  #3: ffff8881333e0128 (&dev->master_mutex){+.+.}-{4:4}, at: drm_master_internal_acquire+0x1b/0x60 [drm]

[ 4671.360318] [     T48] =============================================

[ 4794.208100] [     T48] INFO: task check:66191 blocked for more than 860 seconds.
[ 4794.211144] [     T48]       Not tainted 7.4.0-rc1 #487
[ 4794.215677] [     T48] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4794.224293] [     T48] task:check           state:D stack:0     pid:66191 tgid:66191 ppid:23692  task_flags:0x400140 flags:0x00080000
[ 4794.229298] [     T48] Call Trace:
[ 4794.234298] [     T48]  <TASK>
[ 4794.237617] [     T48]  __schedule+0x1d90/0x6ea0
[ 4794.239988] [     T48]  ? __pfx___schedule+0x10/0x10
[ 4794.242296] [     T48]  ? lock_acquire+0x1c1/0x340
[ 4794.244784] [     T48]  ? schedule+0x24a/0x2e0
[ 4794.245819] [     T48]  ? lock_release+0x1b5/0x340
[ 4794.246801] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4794.247868] [     T48]  schedule+0xe3/0x2e0
[ 4794.248685] [     T48]  schedule_timeout+0x1ac/0x290
[ 4794.249510] [     T48]  ? __pfx_schedule_timeout+0x10/0x10
[ 4794.250158] [     T48]  ? lockdep_hardirqs_on_prepare+0xd5/0x1b0
[ 4794.250623] [     T48]  ? trace_hardirqs_on+0x14/0x190
[ 4794.251035] [     T48]  wait_for_completion+0x17a/0x400
[ 4794.251440] [     T48]  ? __pfx_wait_for_completion+0x10/0x10
[ 4794.251893] [     T48]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 4794.252297] [     T48]  __nvmet_passthru_ctrl_disable+0xcf/0x320 [nvmet]
[ 4794.252766] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4794.253190] [     T48]  nvmet_passthru_ctrl_disable+0x24/0x40 [nvmet]
[ 4794.253640] [     T48]  ? __kasan_kmalloc+0x96/0xb0
[ 4794.254053] [     T48]  nvmet_passthru_enable_store+0xa0/0x100 [nvmet]
[ 4794.254476] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4794.254908] [     T48]  ? configfs_write_iter+0x240/0x4a0
[ 4794.255353] [     T48]  configfs_write_iter+0x2ad/0x4a0
[ 4794.255767] [     T48]  ? __pfx_configfs_write_iter+0x10/0x10
[ 4794.256195] [     T48]  vfs_write+0x4b3/0xf40
[ 4794.256592] [     T48]  ? __pfx_vfs_write+0x10/0x10
[ 4794.257031] [     T48]  ? __lock_acquire+0x54a/0x2490
[ 4794.257499] [     T48]  ? lock_acquire+0x1b1/0x340
[ 4794.257931] [     T48]  ksys_write+0x112/0x250
[ 4794.258364] [     T48]  ? __pfx_ksys_write+0x10/0x10
[ 4794.258795] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4794.259198] [     T48]  do_syscall_64+0xf2/0x690
[ 4794.259606] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4794.259980] [     T48]  ? do_syscall_64+0x2d/0x690
[ 4794.260367] [     T48]  ? trace_hardirqs_off_finish+0x139/0x180
[ 4794.260764] [     T48]  ? lockdep_hardirqs_off+0xaf/0xf0
[ 4794.261142] [     T48]  ? do_syscall_64+0x60/0x690
[ 4794.261487] [     T48]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 4794.261855] [     T48] RIP: 0033:0x7fb94423d54e
[ 4794.262230] [     T48] RSP: 002b:00007ffef5d375a0 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 4794.262573] [     T48] RAX: ffffffffffffffda RBX: 00007fb9443be580 RCX: 00007fb94423d54e
[ 4794.262918] [     T48] RDX: 0000000000000001 RSI: 000055c9e7267150 RDI: 0000000000000001
[ 4794.263315] [     T48] RBP: 00007ffef5d375b0 R08: 0000000000000000 R09: 0000000000000000
[ 4794.263639] [     T48] R10: 0000000000000000 R11: 0000000000000202 R12: 0000000000000001
[ 4794.263977] [     T48] R13: 0000000000000001 R14: 000055c9e7267150 R15: 0000000000000000
[ 4794.264341] [     T48]  </TASK>
[ 4794.264694] [     T48] 
                          Showing all locks held in the system:
[ 4794.265323] [     T48] locks held by pr/ttyS0/47: 2, last CPU#0:
[ 4794.265648] [     T48]  #0: ffffffff8b89ffd8 (console_srcu){....}-{0:0}, at: console_srcu_read_lock+0x45/0x60
[ 4794.266013] [     T48]  #1: ffffffffa2730278 (&port_lock_key){-.-.}-{3:3}, at: univ8250_console_device_lock+0x52/0xa0
[ 4794.266377] [     T48] locks held by khungtaskd/48: 1, last CPU#3:
[ 4794.266691] [     T48]  #0: ffffffff8b8b1e00 (rcu_read_lock){....}-{1:3}, at: debug_show_all_locks+0x39/0x180
[ 4794.267074] [     T48] locks held by check/66191: 3, on CPU#1:
[ 4794.267433] [     T48]  #0: ffff888142c0e478 (sb_writers#12){.+.+}-{0:0}, at: ksys_write+0x112/0x250
[ 4794.267836] [     T48]  #1: ffff888144737e80 (&buffer->mutex){+.+.}-{4:4}, at: configfs_write_iter+0x6d/0x4a0
[ 4794.268249] [     T48]  #2: ffff88813c09b070 (&p->frag_sem){++++}-{4:4}, at: configfs_write_iter+0x1db/0x4a0
[ 4794.268636] [     T48] locks held by kworker/1:0/67499: 5, last CPU#1:
[ 4794.268996] [     T48]  #0: ffff888100194148 ((wq_completion)events){+.+.}-{0:0}, at: process_one_work+0x1109/0x1760
[ 4794.269437] [     T48]  #1: ffff888132427d10 ((work_completion)(&helper->damage_work)){+.+.}-{0:0}, at: process_one_work+0x8c0/0x1760
[ 4794.269829] [     T48]  #2: ffff888132427a30 (crtc_ww_class_acquire){+.+.}-{0:0}, at: drm_atomic_helper_dirtyfb+0xdf/0x770 [drm_kms_helper]
[ 4794.270292] [     T48]  #3: ffff888132427a58 (crtc_ww_class_mutex){+.+.}-{4:4}, at: drm_atomic_helper_dirtyfb+0xdf/0x770 [drm_kms_helper]
[ 4794.270733] [     T48]  #4: ffff8883ac6c63e0 (&rq->__lock){-.-.}-{2:2}, at: raw_spin_rq_lock_nested+0x30/0x170

[ 4794.271606] [     T48] =============================================

[ 4917.088498] [     T48] INFO: task check:66191 blocked for more than 983 seconds.
[ 4917.091381] [     T48]       Not tainted 7.4.0-rc1 #487
[ 4917.094687] [     T48] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4917.095130] [     T48] task:check           state:D stack:0     pid:66191 tgid:66191 ppid:23692  task_flags:0x400140 flags:0x00080000
[ 4917.095578] [     T48] Call Trace:
[ 4917.096050] [     T48]  <TASK>
[ 4917.096474] [     T48]  __schedule+0x1d90/0x6ea0
[ 4917.096919] [     T48]  ? __pfx___schedule+0x10/0x10
[ 4917.097339] [     T48]  ? lock_acquire+0x1c1/0x340
[ 4917.097747] [     T48]  ? schedule+0x24a/0x2e0
[ 4917.098169] [     T48]  ? lock_release+0x1b5/0x340
[ 4917.098574] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4917.099014] [     T48]  schedule+0xe3/0x2e0
[ 4917.099436] [     T48]  schedule_timeout+0x1ac/0x290
[ 4917.099848] [     T48]  ? __pfx_schedule_timeout+0x10/0x10
[ 4917.100378] [     T48]  ? lockdep_hardirqs_on_prepare+0xd5/0x1b0
[ 4917.100841] [     T48]  ? trace_hardirqs_on+0x14/0x190
[ 4917.101259] [     T48]  wait_for_completion+0x17a/0x400
[ 4917.101698] [     T48]  ? __pfx_wait_for_completion+0x10/0x10
[ 4917.102123] [     T48]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 4917.102563] [     T48]  __nvmet_passthru_ctrl_disable+0xcf/0x320 [nvmet]
[ 4917.103023] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4917.103465] [     T48]  nvmet_passthru_ctrl_disable+0x24/0x40 [nvmet]
[ 4917.103922] [     T48]  ? __kasan_kmalloc+0x96/0xb0
[ 4917.104337] [     T48]  nvmet_passthru_enable_store+0xa0/0x100 [nvmet]
[ 4917.104794] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 4917.105350] [     T48]  ? configfs_write_iter+0x240/0x4a0
[ 4917.105815] [     T48]  configfs_write_iter+0x2ad/0x4a0
[ 4917.106244] [     T48]  ? __pfx_configfs_write_iter+0x10/0x10
[ 4917.106643] [     T48]  vfs_write+0x4b3/0xf40
[ 4917.107082] [     T48]  ? __pfx_vfs_write+0x10/0x10
[ 4917.107498] [     T48]  ? __lock_acquire+0x54a/0x2490
[ 4917.107935] [     T48]  ? lock_acquire+0x1b1/0x340
[ 4917.108345] [     T48]  ksys_write+0x112/0x250
[ 4917.108765] [     T48]  ? __pfx_ksys_write+0x10/0x10
[ 4917.109159] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4917.109533] [     T48]  do_syscall_64+0xf2/0x690
[ 4917.109934] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 4917.110315] [     T48]  ? do_syscall_64+0x2d/0x690
[ 4917.110692] [     T48]  ? trace_hardirqs_off_finish+0x139/0x180
[ 4917.111058] [     T48]  ? lockdep_hardirqs_off+0xaf/0xf0
[ 4917.111418] [     T48]  ? do_syscall_64+0x60/0x690
[ 4917.111795] [     T48]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 4917.112146] [     T48] RIP: 0033:0x7fb94423d54e
[ 4917.112506] [     T48] RSP: 002b:00007ffef5d375a0 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 4917.112876] [     T48] RAX: ffffffffffffffda RBX: 00007fb9443be580 RCX: 00007fb94423d54e
[ 4917.113218] [     T48] RDX: 0000000000000001 RSI: 000055c9e7267150 RDI: 0000000000000001
[ 4917.113541] [     T48] RBP: 00007ffef5d375b0 R08: 0000000000000000 R09: 0000000000000000
[ 4917.113899] [     T48] R10: 0000000000000000 R11: 0000000000000202 R12: 0000000000000001
[ 4917.114242] [     T48] R13: 0000000000000001 R14: 000055c9e7267150 R15: 0000000000000000
[ 4917.114570] [     T48]  </TASK>
[ 4917.114927] [     T48] 
                          Showing all locks held in the system:
[ 4917.115548] [     T48] locks held by pr/ttyS0/47: 2, last CPU#0:
[ 4917.115876] [     T48]  #0: ffffffff8b89ffd8 (console_srcu){....}-{0:0}, at: console_srcu_read_lock+0x45/0x60
[ 4917.116211] [     T48]  #1: ffffffffa2730278 (&port_lock_key){-.-.}-{3:3}, at: univ8250_console_device_lock+0x52/0xa0
[ 4917.116539] [     T48] locks held by khungtaskd/48: 1, last CPU#3:
[ 4917.116908] [     T48]  #0: ffffffff8b8b1e00 (rcu_read_lock){....}-{1:3}, at: debug_show_all_locks+0x39/0x180
[ 4917.117360] [     T48] locks held by check/66191: 3, on CPU#1:
[ 4917.117779] [     T48]  #0: ffff888142c0e478 (sb_writers#12){.+.+}-{0:0}, at: ksys_write+0x112/0x250
[ 4917.118219] [     T48]  #1: ffff888144737e80 (&buffer->mutex){+.+.}-{4:4}, at: configfs_write_iter+0x6d/0x4a0
[ 4917.118640] [     T48]  #2: ffff88813c09b070 (&p->frag_sem){++++}-{4:4}, at: configfs_write_iter+0x1db/0x4a0
[ 4917.119634] [     T48] locks held by kworker/3:1/67508: 4, on CPU#3:
[ 4917.120083] [     T48]  #0: ffff888100194148 ((wq_completion)events){+.+.}-{0:0}, at: process_one_work+0x1109/0x1760
[ 4917.120552] [     T48]  #1: ffff8881f7c6fd10 ((work_completion)(&helper->damage_work)){+.+.}-{0:0}, at: process_one_work+0x8c0/0x1760
[ 4917.121046] [     T48]  #2: ffff8881f7c6fa30 (crtc_ww_class_acquire){+.+.}-{0:0}, at: drm_atomic_helper_dirtyfb+0xdf/0x770 [drm_kms_helper]
[ 4917.121530] [     T48]  #3: ffff8881f7c6fa58 (crtc_ww_class_mutex){+.+.}-{4:4}, at: drm_atomic_helper_dirtyfb+0xdf/0x770 [drm_kms_helper]

[ 4917.122488] [     T48] =============================================

[ 5039.968890] [     T48] INFO: task check:66191 blocked for more than 1105 seconds.
[ 5039.971621] [     T48]       Not tainted 7.4.0-rc1 #487
[ 5039.973962] [     T48] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 5039.974377] [     T48] task:check           state:D stack:0     pid:66191 tgid:66191 ppid:23692  task_flags:0x400140 flags:0x00080000
[ 5039.974837] [     T48] Call Trace:
[ 5039.975271] [     T48]  <TASK>
[ 5039.975690] [     T48]  __schedule+0x1d90/0x6ea0
[ 5039.976159] [     T48]  ? __pfx___schedule+0x10/0x10
[ 5039.976625] [     T48]  ? lock_acquire+0x1c1/0x340
[ 5039.977093] [     T48]  ? schedule+0x24a/0x2e0
[ 5039.977523] [     T48]  ? lock_release+0x1b5/0x340
[ 5039.977911] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 5039.978365] [     T48]  schedule+0xe3/0x2e0
[ 5039.978805] [     T48]  schedule_timeout+0x1ac/0x290
[ 5039.979246] [     T48]  ? __pfx_schedule_timeout+0x10/0x10
[ 5039.979690] [     T48]  ? lockdep_hardirqs_on_prepare+0xd5/0x1b0
[ 5039.980120] [     T48]  ? trace_hardirqs_on+0x14/0x190
[ 5039.980560] [     T48]  wait_for_completion+0x17a/0x400
[ 5039.980980] [     T48]  ? __pfx_wait_for_completion+0x10/0x10
[ 5039.981390] [     T48]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 5039.981812] [     T48]  __nvmet_passthru_ctrl_disable+0xcf/0x320 [nvmet]
[ 5039.982303] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 5039.982805] [     T48]  nvmet_passthru_ctrl_disable+0x24/0x40 [nvmet]
[ 5039.983253] [     T48]  ? __kasan_kmalloc+0x96/0xb0
[ 5039.983701] [     T48]  nvmet_passthru_enable_store+0xa0/0x100 [nvmet]
[ 5039.984148] [     T48]  ? __pfx_nvmet_passthru_enable_store+0x10/0x10 [nvmet]
[ 5039.984588] [     T48]  ? configfs_write_iter+0x240/0x4a0
[ 5039.985000] [     T48]  configfs_write_iter+0x2ad/0x4a0
[ 5039.985406] [     T48]  ? __pfx_configfs_write_iter+0x10/0x10
[ 5039.985826] [     T48]  vfs_write+0x4b3/0xf40
[ 5039.986231] [     T48]  ? __pfx_vfs_write+0x10/0x10
[ 5039.986650] [     T48]  ? __lock_acquire+0x54a/0x2490
[ 5039.987060] [     T48]  ? lock_acquire+0x1b1/0x340
[ 5039.987474] [     T48]  ksys_write+0x112/0x250
[ 5039.987867] [     T48]  ? __pfx_ksys_write+0x10/0x10
[ 5039.988258] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 5039.988654] [     T48]  do_syscall_64+0xf2/0x690
[ 5039.989033] [     T48]  ? rcu_is_watching+0x12/0xb0
[ 5039.989405] [     T48]  ? do_syscall_64+0x2d/0x690
[ 5039.989820] [     T48]  ? trace_hardirqs_off_finish+0x139/0x180
[ 5039.990225] [     T48]  ? lockdep_hardirqs_off+0xaf/0xf0
[ 5039.990670] [     T48]  ? do_syscall_64+0x60/0x690
[ 5039.991070] [     T48]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 5039.991405] [     T48] RIP: 0033:0x7fb94423d54e
[ 5039.991801] [     T48] RSP: 002b:00007ffef5d375a0 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 5039.992163] [     T48] RAX: ffffffffffffffda RBX: 00007fb9443be580 RCX: 00007fb94423d54e
[ 5039.992547] [     T48] RDX: 0000000000000001 RSI: 000055c9e7267150 RDI: 0000000000000001
[ 5039.992919] [     T48] RBP: 00007ffef5d375b0 R08: 0000000000000000 R09: 0000000000000000
[ 5039.993270] [     T48] R10: 0000000000000000 R11: 0000000000000202 R12: 0000000000000001
[ 5039.993636] [     T48] R13: 0000000000000001 R14: 000055c9e7267150 R15: 0000000000000000
[ 5039.993986] [     T48]  </TASK>
[ 5039.994312] [     T48] 
                          Showing all locks held in the system:
[ 5039.994934] [     T48] locks held by pr/ttyS0/47: 2, last CPU#0:
[ 5039.995237] [     T48]  #0: ffffffff8b89ffd8 (console_srcu){....}-{0:0}, at: console_srcu_read_lock+0x45/0x60
[ 5039.995578] [     T48]  #1: ffffffffa2730278 (&port_lock_key){-.-.}-{3:3}, at: univ8250_console_device_lock+0x52/0xa0
[ 5039.995925] [     T48] locks held by khungtaskd/48: 1, last CPU#3:
[ 5039.996249] [     T48]  #0: ffffffff8b8b1e00 (rcu_read_lock){....}-{1:3}, at: debug_show_all_locks+0x39/0x180
[ 5039.996639] [     T48] locks held by check/66191: 3, on CPU#1:
[ 5039.996972] [     T48]  #0: ffff888142c0e478 (sb_writers#12){.+.+}-{0:0}, at: ksys_write+0x112/0x250
[ 5039.997331] [     T48]  #1: ffff888144737e80 (&buffer->mutex){+.+.}-{4:4}, at: configfs_write_iter+0x6d/0x4a0
[ 5039.997705] [     T48]  #2: ffff88813c09b070 (&p->frag_sem){++++}-{4:4}, at: configfs_write_iter+0x1db/0x4a0
[ 5039.998073] [     T48] locks held by kworker/1:1/67542: 4, on CPU#1:
[ 5039.998438] [     T48]  #0: ffff888100194148 ((wq_completion)events){+.+.}-{0:0}, at: process_one_work+0x1109/0x1760
[ 5039.998816] [     T48]  #1: ffff8881a7bbfd10 ((work_completion)(&helper->damage_work)){+.+.}-{0:0}, at: process_one_work+0x8c0/0x1760
[ 5039.999247] [     T48]  #2: ffff8881325dca70 (&helper->lock){+.+.}-{4:4}, at: drm_fb_helper_damage_work+0x175/0xac0 [drm_client_lib]
[ 5039.999695] [     T48]  #3: ffff8881333e0128 (&dev->master_mutex){+.+.}-{4:4}, at: drm_master_internal_acquire+0x1b/0x60 [drm]

[ 5040.000606] [     T48] =============================================

Re: [PATCH v2 2/2] nvmet: fix use-after-free in passthru I/O hotpath

[Nilay Shroff]

Hi Shin'ichiro,

On 11/2/26 4:19 PM, Shin'ichiro Kawasaki wrote:
> Hi, Nilay
> 
> I ran blktests with the kernel v7.4-rc1, and found the test case nvme/037 hangs
> when it runs for rdma transport and siw driver [1]. I bisected and found this
> patch as the commit 5e12d81874a1 is the trigger. If this han is a new issue, fix
> actions will be appreciated.

Thanks for report!

I tried recreating this issue on my system running nvme/037 multiple times but
I couldn't recreate it.

Looking through the code carefully, it seems that we release the passthru request
reference in __nvmet_req_complete() after ops->queue_repsonse() returns. However once
ops->queue_response() is called, the transport may complete and recycle the request
on another CPU. If the request is reused, its req->p.ref_held field may be reset
to false before __nvmet_req_complete() checks it in nvmet_put_passthru_ref().
Consequently, the passthru reference may not be released, leaving
wait_for_completion(&passthru->disable_done) in __nvmet_passthru_ctrl_disable()
blocked indefinitely.

Based on the above, I prepared a patch[1] to fix this issue. May I ask you to
validate this patch on your system? If this works for you then I'll send out a
formal patch.

[1]
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index a8171b88035c..714ac8add4c8 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -804,6 +804,7 @@ static void __nvmet_req_complete(struct nvmet_req *req, u16 status)
  {
  	struct nvmet_ns *ns = req->ns;
  	struct nvmet_pr_per_ctrl_ref *pc_ref = req->pc_ref;
+	struct nvmet_passthru *passthru = nvmet_passthru_ref_held(req);

  	if (!req->sq->sqhd_disabled)
  		nvmet_update_sq_head(req);
@@ -822,7 +823,8 @@ static void __nvmet_req_complete(struct nvmet_req *req, u16 status)
  	if (ns)
  		nvmet_put_namespace(ns);

-	nvmet_put_passthru_ref(req);
+	if (passthru)
+		nvmet_put_passthru_ref(passthru);
  }

  void nvmet_req_complete(struct nvmet_req *req, u16 status)
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index d7804bf91aaa..f8205fbfee6e 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -824,6 +824,15 @@ static inline struct nvmet_passthru *nvmet_subsys_passthru(
  	return subsys->passthru;
  }

+static inline struct nvmet_passthru *nvmet_passthru_ref_held(
+		struct nvmet_req *req)
+{
+	if (req->p.ref_held)
+		return nvmet_req_subsys(req)->passthru;
+
+	return NULL;
+}
+
  static inline bool nvmet_get_passthru_ref(struct nvmet_req *req)
  {
  	if (!percpu_ref_tryget_live(&nvmet_req_subsys(req)->passthru->ref))
@@ -833,13 +842,9 @@ static inline bool nvmet_get_passthru_ref(struct nvmet_req *req)
  	return true;
  }

-static inline void nvmet_put_passthru_ref(struct nvmet_req *req)
+static inline void nvmet_put_passthru_ref(struct nvmet_passthru *passthru)
  {
-	if (!req->p.ref_held)
-		return;
-
-	req->p.ref_held = false;
-	percpu_ref_put(&nvmet_req_subsys(req)->passthru->ref);
+	percpu_ref_put(&passthru->ref);
  }
  #else /* CONFIG_NVME_TARGET_PASSTHRU */
  static inline void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys)
@@ -865,11 +870,16 @@ static inline struct nvmet_passthru *nvmet_subsys_passthru(
  {
  	return NULL;
  }
+static inline struct nvmet_passthru *nvmet_passthru_ref_held(
+		struct nvmet_req *req)
+{
+	return NULL;
+}
  static inline bool nvmet_get_passthru_ref(struct nvmet_req *req)
  {
  	return false;
  }
-static inline void nvmet_put_passthru_ref(struct nvmet_req *req)
+static inline void nvmet_put_passthru_ref(struct nvmet_passthru *passthru)
  {
  }
  #endif /* CONFIG_NVME_TARGET_PASSTHRU */

Re: [PATCH v2 2/2] nvmet: fix use-after-free in passthru I/O hotpath

[Shin'ichiro Kawasaki]

On Nov 03, 2026 / 06:49, Nilay Shroff wrote:
[...]
> Based on the above, I prepared a patch[1] to fix this issue. May I ask you to
> validate this patch on your system? If this works for you then I'll send out a
> formal patch.
> 
> [1]

(snip)

Nilay, thank you for the action. I validated your patch, and confirmed it works!
The patch avoided the hang. I also run whole blktests in my environment and
observed no regression. When you post the patch, feel free to add,

Tested-by: Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com>

Re: [PATCH v2 0/2] nvmet: passthru cleanup and fixup I/O hotpath

[Keith Busch]

On Tue, Sep 29, 2026 at 04:56:46PM +0530, Nilay Shroff wrote:
> Hi,
> 
> This series addresses a race in the passthru I/O hotpath where
> concurrently disabling the passthru controller while I/Os are in flight
> can result in a use-after-free bug.
> 
> We were able to reproduce this bug with NVMe/TCP configured and by
> injecting an additional delay into the target-side I/O processing code.
> If the passthru controller is disabled while an I/O is still in flight,
> it results in the following kernel crash:

Thanks, applied to nvme-7.4.