[PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[Christoph Hellwig]

Currently bdex_statx is only called from the very high-level
vfs_statx_path function, and thus bypassing it for in-kernel calls
to vfs_getattr or vfs_getattr_nosec.

This breaks querying the block ѕize of the underlying device in the
loop driver and also is a pitfall for any other new kernel caller.

Move the call into the lowest level helper to ensure all callers get
the right results.

Fixes: 2d985f8c6b91 ("vfs: support STATX_DIOALIGN on block devices")
Fixes: f4774e92aab8 ("loop: take the file system minimum dio alignment into account")
Reported-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
 block/bdev.c           |  3 +--
 fs/stat.c              | 32 ++++++++++++++++++--------------
 include/linux/blkdev.h |  6 +++---
 3 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/block/bdev.c b/block/bdev.c
index 4844d1e27b6f..6a34179192c9 100644
--- a/block/bdev.c
+++ b/block/bdev.c
@@ -1272,8 +1272,7 @@ void sync_bdevs(bool wait)
 /*
  * Handle STATX_{DIOALIGN, WRITE_ATOMIC} for block devices.
  */
-void bdev_statx(struct path *path, struct kstat *stat,
-		u32 request_mask)
+void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
 {
 	struct inode *backing_inode;
 	struct block_device *bdev;
diff --git a/fs/stat.c b/fs/stat.c
index f13308bfdc98..3d9222807214 100644
--- a/fs/stat.c
+++ b/fs/stat.c
@@ -204,12 +204,25 @@ int vfs_getattr_nosec(const struct path *path, struct kstat *stat,
 				  STATX_ATTR_DAX);
 
 	idmap = mnt_idmap(path->mnt);
-	if (inode->i_op->getattr)
-		return inode->i_op->getattr(idmap, path, stat,
-					    request_mask,
-					    query_flags);
+	if (inode->i_op->getattr) {
+		int ret;
+
+		ret = inode->i_op->getattr(idmap, path, stat, request_mask,
+				query_flags);
+		if (ret)
+			return ret;
+	} else {
+		generic_fillattr(idmap, request_mask, inode, stat);
+	}
+
+	/*
+	 * If this is a block device inode, override the filesystem attributes
+	 * with the block device specific parameters that need to be obtained
+	 * from the bdev backing inode.
+	 */
+	if (S_ISBLK(stat->mode))
+		bdev_statx(path, stat, request_mask);
 
-	generic_fillattr(idmap, request_mask, inode, stat);
 	return 0;
 }
 EXPORT_SYMBOL(vfs_getattr_nosec);
@@ -295,15 +308,6 @@ static int vfs_statx_path(struct path *path, int flags, struct kstat *stat,
 	if (path_mounted(path))
 		stat->attributes |= STATX_ATTR_MOUNT_ROOT;
 	stat->attributes_mask |= STATX_ATTR_MOUNT_ROOT;
-
-	/*
-	 * If this is a block device inode, override the filesystem
-	 * attributes with the block device specific parameters that need to be
-	 * obtained from the bdev backing inode.
-	 */
-	if (S_ISBLK(stat->mode))
-		bdev_statx(path, stat, request_mask);
-
 	return 0;
 }
 
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index e39c45bc0a97..678dc38442bf 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -1685,7 +1685,7 @@ int sync_blockdev(struct block_device *bdev);
 int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend);
 int sync_blockdev_nowait(struct block_device *bdev);
 void sync_bdevs(bool wait);
-void bdev_statx(struct path *, struct kstat *, u32);
+void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask);
 void printk_all_partitions(void);
 int __init early_lookup_bdev(const char *pathname, dev_t *dev);
 #else
@@ -1703,8 +1703,8 @@ static inline int sync_blockdev_nowait(struct block_device *bdev)
 static inline void sync_bdevs(bool wait)
 {
 }
-static inline void bdev_statx(struct path *path, struct kstat *stat,
-				u32 request_mask)
+static inline void bdev_statx(const struct path *path, struct kstat *stat,
+		u32 request_mask)
 {
 }
 static inline void printk_all_partitions(void)
-- 
2.47.2

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[Christian Brauner]

On Thu, Apr 17, 2025 at 08:40:42AM +0200, Christoph Hellwig wrote:
> Currently bdex_statx is only called from the very high-level
> vfs_statx_path function, and thus bypassing it for in-kernel calls
> to vfs_getattr or vfs_getattr_nosec.
> 
> This breaks querying the block ѕize of the underlying device in the
> loop driver and also is a pitfall for any other new kernel caller.
> 
> Move the call into the lowest level helper to ensure all callers get
> the right results.
> 
> Fixes: 2d985f8c6b91 ("vfs: support STATX_DIOALIGN on block devices")
> Fixes: f4774e92aab8 ("loop: take the file system minimum dio alignment into account")
> Reported-by: Darrick J. Wong <djwong@kernel.org>
> Signed-off-by: Christoph Hellwig <hch@lst.de>
> ---

This looks fine but one thing below.

>  block/bdev.c           |  3 +--
>  fs/stat.c              | 32 ++++++++++++++++++--------------
>  include/linux/blkdev.h |  6 +++---
>  3 files changed, 22 insertions(+), 19 deletions(-)
> 
> diff --git a/block/bdev.c b/block/bdev.c
> index 4844d1e27b6f..6a34179192c9 100644
> --- a/block/bdev.c
> +++ b/block/bdev.c
> @@ -1272,8 +1272,7 @@ void sync_bdevs(bool wait)
>  /*
>   * Handle STATX_{DIOALIGN, WRITE_ATOMIC} for block devices.
>   */
> -void bdev_statx(struct path *path, struct kstat *stat,
> -		u32 request_mask)
> +void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
>  {
>  	struct inode *backing_inode;
>  	struct block_device *bdev;
> diff --git a/fs/stat.c b/fs/stat.c
> index f13308bfdc98..3d9222807214 100644
> --- a/fs/stat.c
> +++ b/fs/stat.c
> @@ -204,12 +204,25 @@ int vfs_getattr_nosec(const struct path *path, struct kstat *stat,
>  				  STATX_ATTR_DAX);
>  
>  	idmap = mnt_idmap(path->mnt);
> -	if (inode->i_op->getattr)
> -		return inode->i_op->getattr(idmap, path, stat,
> -					    request_mask,
> -					    query_flags);
> +	if (inode->i_op->getattr) {
> +		int ret;
> +
> +		ret = inode->i_op->getattr(idmap, path, stat, request_mask,
> +				query_flags);
> +		if (ret)
> +			return ret;
> +	} else {
> +		generic_fillattr(idmap, request_mask, inode, stat);
> +	}
> +
> +	/*
> +	 * If this is a block device inode, override the filesystem attributes
> +	 * with the block device specific parameters that need to be obtained
> +	 * from the bdev backing inode.
> +	 */
> +	if (S_ISBLK(stat->mode))
> +		bdev_statx(path, stat, request_mask);
>  
> -	generic_fillattr(idmap, request_mask, inode, stat);
>  	return 0;
>  }
>  EXPORT_SYMBOL(vfs_getattr_nosec);
> @@ -295,15 +308,6 @@ static int vfs_statx_path(struct path *path, int flags, struct kstat *stat,
>  	if (path_mounted(path))
>  		stat->attributes |= STATX_ATTR_MOUNT_ROOT;
>  	stat->attributes_mask |= STATX_ATTR_MOUNT_ROOT;
> -
> -	/*
> -	 * If this is a block device inode, override the filesystem
> -	 * attributes with the block device specific parameters that need to be
> -	 * obtained from the bdev backing inode.
> -	 */
> -	if (S_ISBLK(stat->mode))
> -		bdev_statx(path, stat, request_mask);

bdev_statx()
-> blkdev_get_no_open()
   {
           if (!inode && IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD)) {
                blk_request_module(dev);
                inode = ilookup(blockdev_superblock, dev);
                if (inode)
                        pr_warn_ratelimited("block device autoloading is deprecated and will be removed.\n");
        }

   }

So that means any unprivileged Schmock can trigger a module autoload
from statx() if that's enabled? It feels like this should really be
disabled for statx().

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[Christian Brauner]

On Thu, 17 Apr 2025 08:40:42 +0200, Christoph Hellwig wrote:
> Currently bdex_statx is only called from the very high-level
> vfs_statx_path function, and thus bypassing it for in-kernel calls
> to vfs_getattr or vfs_getattr_nosec.
> 
> This breaks querying the block ѕize of the underlying device in the
> loop driver and also is a pitfall for any other new kernel caller.
> 
> [...]

Applied to the vfs.fixes branch of the vfs/vfs.git tree.
Patches in the vfs.fixes branch should appear in linux-next soon.

Please report any outstanding bugs that were missed during review in a
new review to the original patch series allowing us to drop it.

It's encouraged to provide Acked-bys and Reviewed-bys even though the
patch has now been applied. If possible patch trailers will be updated.

Note that commit hashes shown below are subject to change due to rebase,
trailer updates or similar. If in doubt, please check the listed branch.

tree:   https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git
branch: vfs.fixes

[1/1] fs: move the bdex_statx call to vfs_getattr_nosec
      https://git.kernel.org/vfs/vfs/c/777d0961ff95

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[Darrick J. Wong]

On Thu, Apr 17, 2025 at 08:40:42AM +0200, Christoph Hellwig wrote:
> Currently bdex_statx is only called from the very high-level
> vfs_statx_path function, and thus bypassing it for in-kernel calls
> to vfs_getattr or vfs_getattr_nosec.
> 
> This breaks querying the block ѕize of the underlying device in the
> loop driver and also is a pitfall for any other new kernel caller.
> 
> Move the call into the lowest level helper to ensure all callers get
> the right results.
> 
> Fixes: 2d985f8c6b91 ("vfs: support STATX_DIOALIGN on block devices")
> Fixes: f4774e92aab8 ("loop: take the file system minimum dio alignment into account")
> Reported-by: Darrick J. Wong <djwong@kernel.org>
> Signed-off-by: Christoph Hellwig <hch@lst.de>

This appears to solve the problem as well.

Tested-by: "Darrick J. Wong" <djwong@kernel.org>

--D

> ---
>  block/bdev.c           |  3 +--
>  fs/stat.c              | 32 ++++++++++++++++++--------------
>  include/linux/blkdev.h |  6 +++---
>  3 files changed, 22 insertions(+), 19 deletions(-)
> 
> diff --git a/block/bdev.c b/block/bdev.c
> index 4844d1e27b6f..6a34179192c9 100644
> --- a/block/bdev.c
> +++ b/block/bdev.c
> @@ -1272,8 +1272,7 @@ void sync_bdevs(bool wait)
>  /*
>   * Handle STATX_{DIOALIGN, WRITE_ATOMIC} for block devices.
>   */
> -void bdev_statx(struct path *path, struct kstat *stat,
> -		u32 request_mask)
> +void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
>  {
>  	struct inode *backing_inode;
>  	struct block_device *bdev;
> diff --git a/fs/stat.c b/fs/stat.c
> index f13308bfdc98..3d9222807214 100644
> --- a/fs/stat.c
> +++ b/fs/stat.c
> @@ -204,12 +204,25 @@ int vfs_getattr_nosec(const struct path *path, struct kstat *stat,
>  				  STATX_ATTR_DAX);
>  
>  	idmap = mnt_idmap(path->mnt);
> -	if (inode->i_op->getattr)
> -		return inode->i_op->getattr(idmap, path, stat,
> -					    request_mask,
> -					    query_flags);
> +	if (inode->i_op->getattr) {
> +		int ret;
> +
> +		ret = inode->i_op->getattr(idmap, path, stat, request_mask,
> +				query_flags);
> +		if (ret)
> +			return ret;
> +	} else {
> +		generic_fillattr(idmap, request_mask, inode, stat);
> +	}
> +
> +	/*
> +	 * If this is a block device inode, override the filesystem attributes
> +	 * with the block device specific parameters that need to be obtained
> +	 * from the bdev backing inode.
> +	 */
> +	if (S_ISBLK(stat->mode))
> +		bdev_statx(path, stat, request_mask);
>  
> -	generic_fillattr(idmap, request_mask, inode, stat);
>  	return 0;
>  }
>  EXPORT_SYMBOL(vfs_getattr_nosec);
> @@ -295,15 +308,6 @@ static int vfs_statx_path(struct path *path, int flags, struct kstat *stat,
>  	if (path_mounted(path))
>  		stat->attributes |= STATX_ATTR_MOUNT_ROOT;
>  	stat->attributes_mask |= STATX_ATTR_MOUNT_ROOT;
> -
> -	/*
> -	 * If this is a block device inode, override the filesystem
> -	 * attributes with the block device specific parameters that need to be
> -	 * obtained from the bdev backing inode.
> -	 */
> -	if (S_ISBLK(stat->mode))
> -		bdev_statx(path, stat, request_mask);
> -
>  	return 0;
>  }
>  
> diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
> index e39c45bc0a97..678dc38442bf 100644
> --- a/include/linux/blkdev.h
> +++ b/include/linux/blkdev.h
> @@ -1685,7 +1685,7 @@ int sync_blockdev(struct block_device *bdev);
>  int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend);
>  int sync_blockdev_nowait(struct block_device *bdev);
>  void sync_bdevs(bool wait);
> -void bdev_statx(struct path *, struct kstat *, u32);
> +void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask);
>  void printk_all_partitions(void);
>  int __init early_lookup_bdev(const char *pathname, dev_t *dev);
>  #else
> @@ -1703,8 +1703,8 @@ static inline int sync_blockdev_nowait(struct block_device *bdev)
>  static inline void sync_bdevs(bool wait)
>  {
>  }
> -static inline void bdev_statx(struct path *path, struct kstat *stat,
> -				u32 request_mask)
> +static inline void bdev_statx(const struct path *path, struct kstat *stat,
> +		u32 request_mask)
>  {
>  }
>  static inline void printk_all_partitions(void)
> -- 
> 2.47.2
> 
> 

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[Shinichiro Kawasaki]

On Apr 17, 2025 / 08:40, Christoph Hellwig wrote:
> Currently bdex_statx is only called from the very high-level
> vfs_statx_path function, and thus bypassing it for in-kernel calls
> to vfs_getattr or vfs_getattr_nosec.
> 
> This breaks querying the block ѕize of the underlying device in the
> loop driver and also is a pitfall for any other new kernel caller.
> 
> Move the call into the lowest level helper to ensure all callers get
> the right results.

I ran blktests with the kernel v6.15-rc3, and found the test case md/001 hangs.
The hang is recreated in stable manner. I bisected and found this patch as the
commit 777d0961ff95 is the trigger. When I revert the commit from v6.15-rc3
kernel, the hang disappeared.

Actions for fix will be appreciated.

FYI, the kernel INFO messages recorded functions relevant to the trigger commit,
such as bdev_statx or vfs_getattr_nosec [1].


[1]

[ 3341.286639] [ T129061] run blktests md/001 at 2025-04-21 14:23:48
[ 3341.519196] [ T129079] brd: module loaded
[ 3341.629701] [ T129098] nvmet: adding nsid 1 to subsystem blktests-subsystem-1
[ 3341.637856] [ T129099] nvmet_tcp: enabling port 0 (127.0.0.1:4420)
[ 3341.677605] [  T80307] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-1 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349.
[ 3341.680557] [ T129106] nvme nvme1: Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.
[ 3341.683890] [ T129106] nvme nvme1: creating 4 I/O queues.
[ 3341.685462] [ T129106] nvme nvme1: failed to connect socket: -512
[ 3341.717282] [    T182] nvmet: Created nvm controller 1 for subsystem blktests-subsystem-1 for NQN nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349.
[ 3341.720494] [ T129106] nvme nvme1: Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.
[ 3341.722675] [ T129106] nvme nvme1: creating 4 I/O queues.
[ 3341.725909] [ T129106] nvme nvme1: mapped 4/0/0 default/read/poll queues.
[ 3341.728769] [ T129106] nvme nvme1: new ctrl: NQN "blktests-subsystem-1", addr 127.0.0.1:4420, hostnqn: nqn.2014-08.org.nvmexpress:uuid:0f01fb42-9f7f-4856-b0b3-51e60b8de349
[ 3347.648972] [ T129119] md/raid1:md127: active with 1 out of 2 mirrors
[ 3347.651175] [ T129119] md127: detected capacity change from 0 to 2093056
[ 3347.700481] [ T129124] md127: detected capacity change from 2093056 to 0
[ 3347.701652] [ T129124] md: md127 stopped.
[ 3347.728822] [ T129126] nvme nvme1: Removing ctrl: NQN "blktests-subsystem-1"
[ 3359.911067] [  T92330] nvmet: ctrl 1 keep-alive timer (5 seconds) expired!
[ 3359.914119] [  T92330] nvmet: ctrl 1 fatal error occurred!
[ 3565.221305] [     T46] INFO: task kdevtmpfs:40 blocked for more than 122 seconds.
[ 3565.222332] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3565.223197] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3565.224277] [     T46] task:kdevtmpfs       state:D stack:0     pid:40    tgid:40    ppid:2      task_flags:0x208140 flags:0x00004000
[ 3565.225452] [     T46] Call Trace:
[ 3565.226106] [     T46]  <TASK>
[ 3565.226708] [     T46]  __schedule+0x107c/0x5fa0
[ 3565.227416] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.228179] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.228886] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3565.229599] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.230320] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3565.231090] [     T46]  ? schedule+0x2be/0x390
[ 3565.231759] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.232474] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3565.233288] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.234013] [     T46]  schedule+0xde/0x390
[ 3565.234672] [     T46]  schedule_timeout+0x176/0x230
[ 3565.235389] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3565.236169] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.236867] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3565.237605] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.238310] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3565.239040] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3565.239770] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3565.240509] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3565.241240] [     T46]  ? flush_workqueue_prep_pwqs+0x333/0x440
[ 3565.242031] [     T46]  __flush_workqueue+0x373/0x10a0
[ 3565.242715] [     T46]  ? __pfx___might_resched+0x10/0x10
[ 3565.243417] [     T46]  ? kasan_save_track+0x10/0x30
[ 3565.244108] [     T46]  ? kasan_save_free_info+0x37/0x60
[ 3565.244782] [     T46]  ? __mutex_lock+0x538/0x1a40
[ 3565.245430] [     T46]  ? __pfx___flush_workqueue+0x10/0x10
[ 3565.246122] [     T46]  ? find_inode_fast+0x152/0x3f0
[ 3565.246755] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.247391] [     T46]  md_alloc+0x28/0xf10
[ 3565.247987] [     T46]  md_probe+0x25/0x70
[ 3565.248531] [     T46]  blk_probe_dev+0x108/0x180
[ 3565.249134] [     T46]  blk_request_module+0xe/0x60
[ 3565.249716] [     T46]  blkdev_get_no_open+0x5c/0xa0
[ 3565.250311] [     T46]  bdev_statx+0x98/0x550
[ 3565.250859] [     T46]  vfs_getattr_nosec+0x2be/0x3d0
[ 3565.251435] [     T46]  handle_remove+0x115/0x500
[ 3565.252010] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.252558] [     T46]  ? __pfx_handle_remove+0x10/0x10
[ 3565.253144] [     T46]  ? finish_task_switch.isra.0+0x1b9/0x860
[ 3565.253757] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.254312] [     T46]  ? __schedule+0x3099/0x5fa0
[ 3565.254858] [     T46]  ? __pfx_sched_clock_cpu+0x10/0x10
[ 3565.255438] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3565.256011] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3565.256554] [     T46]  ? devtmpfs_work_loop+0xb5/0x5c7
[ 3565.257113] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.257633] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.258150] [     T46]  ? devtmpfs_work_loop+0x55d/0x5c7
[ 3565.258687] [     T46]  devtmpfs_work_loop+0x4e9/0x5c7
[ 3565.259223] [     T46]  ? __pfx_devtmpfs_work_loop+0x10/0x10
[ 3565.259791] [     T46]  ? _raw_spin_unlock_irqrestore+0x35/0x60
[ 3565.260379] [     T46]  ? __pfx_do_raw_spin_lock+0x10/0x10
[ 3565.260959] [     T46]  ? devtmpfsd+0x19/0x30
[ 3565.261442] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3565.262046] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3565.262624] [     T46]  ? __pfx_devtmpfsd+0x10/0x10
[ 3565.263149] [     T46]  devtmpfsd+0x27/0x30
[ 3565.263618] [     T46]  kthread+0x3b0/0x770
[ 3565.264098] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3565.264599] [     T46]  ? ret_from_fork+0x17/0x70
[ 3565.265129] [     T46]  ? ret_from_fork+0x17/0x70
[ 3565.265639] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3565.266197] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3565.266710] [     T46]  ret_from_fork+0x30/0x70
[ 3565.267220] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3565.267732] [     T46]  ret_from_fork_asm+0x1a/0x30
[ 3565.268264] [     T46]  </TASK>
[ 3565.268732] [     T46] INFO: task kworker/1:1:91856 blocked for more than 122 seconds.
[ 3565.269468] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3565.270150] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3565.270985] [     T46] task:kworker/1:1     state:D stack:0     pid:91856 tgid:91856 ppid:2      task_flags:0x4288060 flags:0x00004000
[ 3565.272043] [     T46] Workqueue: md_misc mddev_delayed_delete
[ 3565.272658] [     T46] Call Trace:
[ 3565.273146] [     T46]  <TASK>
[ 3565.273568] [     T46]  __schedule+0x107c/0x5fa0
[ 3565.274117] [     T46]  ? unwind_next_frame+0x400/0x1f10
[ 3565.274697] [     T46]  ? ret_from_fork_asm+0x1a/0x30
[ 3565.275267] [     T46]  ? __pfx_stack_trace_consume_entry+0x10/0x10
[ 3565.275920] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3565.276489] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3565.277083] [     T46]  ? __pfx_do_raw_spin_lock+0x10/0x10
[ 3565.277674] [     T46]  ? schedule+0x2be/0x390
[ 3565.278208] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.278762] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.279320] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.279902] [     T46]  schedule+0xde/0x390
[ 3565.280418] [     T46]  schedule_timeout+0x176/0x230
[ 3565.281026] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3565.281617] [     T46]  ? llist_add_batch+0xb9/0x130
[ 3565.282190] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.282742] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3565.283330] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.283895] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3565.284466] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3565.285084] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3565.285682] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3565.286262] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.286825] [     T46]  devtmpfs_delete_node+0x142/0x1b0
[ 3565.287420] [     T46]  ? __pfx_devtmpfs_delete_node+0x10/0x10
[ 3565.288056] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.288593] [     T46]  ? __call_rcu_common.constprop.0+0x45f/0x1090
[ 3565.289260] [     T46]  ? trace_hardirqs_on+0x12/0x120
[ 3565.289854] [     T46]  ? kernfs_put.part.0+0x130/0x470
[ 3565.290441] [     T46]  device_del+0x6e2/0x9b0
[ 3565.290993] [     T46]  ? pm_runtime_set_memalloc_noio+0x8b/0x140
[ 3565.291633] [     T46]  ? __pfx_device_del+0x10/0x10
[ 3565.292206] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.292769] [     T46]  ? trace_hardirqs_on+0x12/0x120
[ 3565.293361] [     T46]  ? _raw_spin_unlock_irq+0x34/0x50
[ 3565.293975] [     T46]  del_gendisk+0x4f5/0xa20
[ 3565.294520] [     T46]  ? __pfx_del_gendisk+0x10/0x10
[ 3565.295198] [     T46]  ? up_write+0x1c6/0x500
[ 3565.295736] [     T46]  md_kobj_release+0x90/0xe0
[ 3565.296304] [     T46]  kobject_put+0x17b/0x4a0
[ 3565.296871] [     T46]  process_one_work+0x84f/0x1460
[ 3565.297466] [     T46]  ? __pfx_process_one_work+0x10/0x10
[ 3565.298100] [     T46]  ? assign_work+0x16c/0x240
[ 3565.298656] [     T46]  worker_thread+0x5ef/0xfd0
[ 3565.299197] [     T46]  ? __kthread_parkme+0xb4/0x200
[ 3565.299761] [     T46]  ? __pfx_worker_thread+0x10/0x10
[ 3565.300322] [     T46]  kthread+0x3b0/0x770
[ 3565.300813] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3565.301339] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.301883] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3565.302441] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.302991] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3565.303505] [     T46]  ret_from_fork+0x30/0x70
[ 3565.304028] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3565.304539] [     T46]  ret_from_fork_asm+0x1a/0x30
[ 3565.305087] [     T46]  </TASK>
[ 3565.305515] [     T46] INFO: task nvme:129126 blocked for more than 122 seconds.
[ 3565.306223] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3565.306879] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3565.307662] [     T46] task:nvme            state:D stack:0     pid:129126 tgid:129126 ppid:129061 task_flags:0x480100 flags:0x00024002
[ 3565.308665] [     T46] Call Trace:
[ 3565.309140] [     T46]  <TASK>
[ 3565.309574] [     T46]  __schedule+0x107c/0x5fa0
[ 3565.310119] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.310662] [     T46]  ? __pfx_stack_trace_consume_entry+0x10/0x10
[ 3565.311307] [     T46]  ? unwind_get_return_address+0x5a/0xa0
[ 3565.311927] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3565.312490] [     T46]  ? schedule+0x2be/0x390
[ 3565.313037] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.313587] [     T46]  ? stack_trace_save+0x8f/0xc0
[ 3565.314179] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.314706] [     T46]  schedule+0xde/0x390
[ 3565.315208] [     T46]  schedule_timeout+0x176/0x230
[ 3565.315777] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3565.316386] [     T46]  ? llist_add_batch+0xb9/0x130
[ 3565.316981] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.317554] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3565.318155] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.318716] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3565.319294] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3565.319911] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3565.320520] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3565.321120] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.321673] [     T46]  devtmpfs_delete_node+0x142/0x1b0
[ 3565.322270] [     T46]  ? __pfx_devtmpfs_delete_node+0x10/0x10
[ 3565.322904] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3565.323503] [     T46]  ? klist_release+0x2c9/0x460
[ 3565.324087] [     T46]  ? do_raw_spin_unlock+0x54/0x1e0
[ 3565.324667] [     T46]  ? kobject_put+0x5d/0x4a0
[ 3565.325224] [     T46]  device_del+0x6e2/0x9b0
[ 3565.325755] [     T46]  ? __pfx_device_del+0x10/0x10
[ 3565.326329] [     T46]  ? ktime_get_mono_fast_ns+0x81/0x360
[ 3565.326958] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3565.327586] [     T46]  cdev_device_del+0x19/0xf0
[ 3565.328141] [     T46]  nvme_ns_remove+0x2ab/0x740 [nvme_core]
[ 3565.328762] [     T46]  nvme_remove_namespaces+0x25d/0x3a0 [nvme_core]
[ 3565.329435] [     T46]  ? __pfx_nvme_remove_namespaces+0x10/0x10 [nvme_core]
[ 3565.330162] [     T46]  ? __pfx___might_resched+0x10/0x10
[ 3565.330748] [     T46]  ? __pfx_sysfs_kf_write+0x10/0x10
[ 3565.331337] [     T46]  nvme_do_delete_ctrl+0xf5/0x160 [nvme_core]
[ 3565.332006] [     T46]  nvme_delete_ctrl_sync.cold+0x8/0xd [nvme_core]
[ 3565.332672] [     T46]  nvme_sysfs_delete+0x92/0xb0 [nvme_core]
[ 3565.333317] [     T46]  kernfs_fop_write_iter+0x39f/0x5a0
[ 3565.333926] [     T46]  vfs_write+0x5f7/0xe90
[ 3565.334463] [     T46]  ? __pfx_vfs_write+0x10/0x10
[ 3565.335103] [     T46]  ksys_write+0xf5/0x1c0
[ 3565.335627] [     T46]  ? __pfx_ksys_write+0x10/0x10
[ 3565.336192] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.336736] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3565.337344] [     T46]  ? do_syscall_64+0x54/0x190
[ 3565.337883] [     T46]  do_syscall_64+0x93/0x190
[ 3565.338410] [     T46]  ? kmem_cache_free+0x393/0x580
[ 3565.338977] [     T46]  ? fput_close_sync+0x100/0x170
[ 3565.339519] [     T46]  ? fput_close_sync+0x100/0x170
[ 3565.340084] [     T46]  ? __pfx_fput_close_sync+0x10/0x10
[ 3565.340643] [     T46]  ? lock_release+0x217/0x2c0
[ 3565.341188] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3565.341774] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.342314] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3565.342914] [     T46]  ? trace_hardirqs_on_prepare+0xdb/0x120
[ 3565.343512] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3565.344124] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3565.344652] [     T46]  ? do_sys_openat2+0x108/0x180
[ 3565.345201] [     T46]  ? __pfx_do_sys_openat2+0x10/0x10
[ 3565.345757] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3565.346360] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3565.346902] [     T46]  ? __x64_sys_openat+0x104/0x1d0
[ 3565.347462] [     T46]  ? __pfx___x64_sys_openat+0x10/0x10
[ 3565.348056] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.348589] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3565.349200] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3565.349734] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3565.350339] [     T46]  ? trace_hardirqs_on_prepare+0xdb/0x120
[ 3565.350962] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3565.351555] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3565.352088] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3565.352635] [     T46]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 3565.353249] [     T46] RIP: 0033:0x7fedbaf01504
[ 3565.353760] [     T46] RSP: 002b:00007fff4c0f4858 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 3565.354536] [     T46] RAX: ffffffffffffffda RBX: 0000000000000003 RCX: 00007fedbaf01504
[ 3565.355287] [     T46] RDX: 0000000000000001 RSI: 00007fedbb050746 RDI: 0000000000000003
[ 3565.356037] [     T46] RBP: 00007fedbb050746 R08: 0000000000004000 R09: 00000000ffffffff
[ 3565.356770] [     T46] R10: 0000000000000000 R11: 0000000000000202 R12: 000000000e107e30
[ 3565.357515] [     T46] R13: 00007fff4c0f563b R14: 000000000e107e30 R15: 000000000e108730
[ 3565.358274] [     T46]  </TASK>
[ 3565.358743] [     T46] INFO: lockdep is turned off.
[ 3688.100058] [     T46] INFO: task kdevtmpfs:40 blocked for more than 245 seconds.
[ 3688.101052] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3688.101732] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3688.102546] [     T46] task:kdevtmpfs       state:D stack:0     pid:40    tgid:40    ppid:2      task_flags:0x208140 flags:0x00004000
[ 3688.103524] [     T46] Call Trace:
[ 3688.104003] [     T46]  <TASK>
[ 3688.104446] [     T46]  __schedule+0x107c/0x5fa0
[ 3688.105102] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.105659] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.106221] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3688.106780] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.107332] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3688.108088] [     T46]  ? schedule+0x2be/0x390
[ 3688.108611] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.109172] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3688.109744] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.110299] [     T46]  schedule+0xde/0x390
[ 3688.110828] [     T46]  schedule_timeout+0x176/0x230
[ 3688.111388] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3688.112153] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.112709] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3688.113304] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.113943] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3688.114515] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3688.115122] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3688.115718] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3688.116313] [     T46]  ? flush_workqueue_prep_pwqs+0x333/0x440
[ 3688.116970] [     T46]  __flush_workqueue+0x373/0x10a0
[ 3688.117543] [     T46]  ? __pfx___might_resched+0x10/0x10
[ 3688.118147] [     T46]  ? kasan_save_track+0x10/0x30
[ 3688.118710] [     T46]  ? kasan_save_free_info+0x37/0x60
[ 3688.119305] [     T46]  ? __mutex_lock+0x538/0x1a40
[ 3688.119941] [     T46]  ? __pfx___flush_workqueue+0x10/0x10
[ 3688.120541] [     T46]  ? find_inode_fast+0x152/0x3f0
[ 3688.121122] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.121682] [     T46]  md_alloc+0x28/0xf10
[ 3688.122208] [     T46]  md_probe+0x25/0x70
[ 3688.122710] [     T46]  blk_probe_dev+0x108/0x180
[ 3688.123266] [     T46]  blk_request_module+0xe/0x60
[ 3688.123988] [     T46]  blkdev_get_no_open+0x5c/0xa0
[ 3688.124550] [     T46]  bdev_statx+0x98/0x550
[ 3688.125083] [     T46]  vfs_getattr_nosec+0x2be/0x3d0
[ 3688.125641] [     T46]  handle_remove+0x115/0x500
[ 3688.126188] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.126728] [     T46]  ? __pfx_handle_remove+0x10/0x10
[ 3688.127296] [     T46]  ? finish_task_switch.isra.0+0x1b9/0x860
[ 3688.128084] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.128623] [     T46]  ? __schedule+0x3099/0x5fa0
[ 3688.129162] [     T46]  ? __pfx_sched_clock_cpu+0x10/0x10
[ 3688.129723] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3688.130282] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3688.130957] [     T46]  ? devtmpfs_work_loop+0xb5/0x5c7
[ 3688.131498] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.132020] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.132514] [     T46]  ? devtmpfs_work_loop+0x55d/0x5c7
[ 3688.133055] [     T46]  devtmpfs_work_loop+0x4e9/0x5c7
[ 3688.133574] [     T46]  ? __pfx_devtmpfs_work_loop+0x10/0x10
[ 3688.134135] [     T46]  ? _raw_spin_unlock_irqrestore+0x35/0x60
[ 3688.134706] [     T46]  ? __pfx_do_raw_spin_lock+0x10/0x10
[ 3688.135261] [     T46]  ? devtmpfsd+0x19/0x30
[ 3688.135811] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3688.136392] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3688.136987] [     T46]  ? __pfx_devtmpfsd+0x10/0x10
[ 3688.137492] [     T46]  devtmpfsd+0x27/0x30
[ 3688.137968] [     T46]  kthread+0x3b0/0x770
[ 3688.138431] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3688.139103] [     T46]  ? ret_from_fork+0x17/0x70
[ 3688.139601] [     T46]  ? ret_from_fork+0x17/0x70
[ 3688.140125] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3688.140653] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3688.141159] [     T46]  ret_from_fork+0x30/0x70
[ 3688.141640] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3688.142144] [     T46]  ret_from_fork_asm+0x1a/0x30
[ 3688.142647] [     T46]  </TASK>
[ 3688.143114] [     T46] INFO: task kworker/1:1:91856 blocked for more than 245 seconds.
[ 3688.143823] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3688.144445] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3688.145321] [     T46] task:kworker/1:1     state:D stack:0     pid:91856 tgid:91856 ppid:2      task_flags:0x4288060 flags:0x00004000
[ 3688.146310] [     T46] Workqueue: md_misc mddev_delayed_delete
[ 3688.147072] [     T46] Call Trace:
[ 3688.147516] [     T46]  <TASK>
[ 3688.147961] [     T46]  __schedule+0x107c/0x5fa0
[ 3688.148475] [     T46]  ? unwind_next_frame+0x400/0x1f10
[ 3688.149066] [     T46]  ? ret_from_fork_asm+0x1a/0x30
[ 3688.149603] [     T46]  ? __pfx_stack_trace_consume_entry+0x10/0x10
[ 3688.150228] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3688.150802] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3688.151351] [     T46]  ? __pfx_do_raw_spin_lock+0x10/0x10
[ 3688.151946] [     T46]  ? schedule+0x2be/0x390
[ 3688.152449] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.153025] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.153552] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.154097] [     T46]  schedule+0xde/0x390
[ 3688.154584] [     T46]  schedule_timeout+0x176/0x230
[ 3688.155136] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3688.155702] [     T46]  ? llist_add_batch+0xb9/0x130
[ 3688.156252] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.156820] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3688.157378] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.157999] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3688.158540] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3688.159130] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3688.159702] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3688.160264] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.160844] [     T46]  devtmpfs_delete_node+0x142/0x1b0
[ 3688.161406] [     T46]  ? __pfx_devtmpfs_delete_node+0x10/0x10
[ 3688.162025] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.162558] [     T46]  ? __call_rcu_common.constprop.0+0x45f/0x1090
[ 3688.163195] [     T46]  ? trace_hardirqs_on+0x12/0x120
[ 3688.163810] [     T46]  ? kernfs_put.part.0+0x130/0x470
[ 3688.164370] [     T46]  device_del+0x6e2/0x9b0
[ 3688.164925] [     T46]  ? pm_runtime_set_memalloc_noio+0x8b/0x140
[ 3688.165540] [     T46]  ? __pfx_device_del+0x10/0x10
[ 3688.166096] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.166636] [     T46]  ? trace_hardirqs_on+0x12/0x120
[ 3688.167205] [     T46]  ? _raw_spin_unlock_irq+0x34/0x50
[ 3688.167812] [     T46]  del_gendisk+0x4f5/0xa20
[ 3688.168340] [     T46]  ? __pfx_del_gendisk+0x10/0x10
[ 3688.168939] [     T46]  ? up_write+0x1c6/0x500
[ 3688.169457] [     T46]  md_kobj_release+0x90/0xe0
[ 3688.170025] [     T46]  kobject_put+0x17b/0x4a0
[ 3688.170551] [     T46]  process_one_work+0x84f/0x1460
[ 3688.171184] [     T46]  ? __pfx_process_one_work+0x10/0x10
[ 3688.171795] [     T46]  ? assign_work+0x16c/0x240
[ 3688.172329] [     T46]  worker_thread+0x5ef/0xfd0
[ 3688.172879] [     T46]  ? __kthread_parkme+0xb4/0x200
[ 3688.173424] [     T46]  ? __pfx_worker_thread+0x10/0x10
[ 3688.174011] [     T46]  kthread+0x3b0/0x770
[ 3688.174485] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3688.175008] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.175529] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3688.176083] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.176599] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3688.177119] [     T46]  ret_from_fork+0x30/0x70
[ 3688.177612] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3688.178127] [     T46]  ret_from_fork_asm+0x1a/0x30
[ 3688.178644] [     T46]  </TASK>
[ 3688.179094] [     T46] INFO: task nvme:129126 blocked for more than 245 seconds.
[ 3688.179803] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3688.180440] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3688.181224] [     T46] task:nvme            state:D stack:0     pid:129126 tgid:129126 ppid:129061 task_flags:0x480100 flags:0x00024002
[ 3688.182234] [     T46] Call Trace:
[ 3688.182686] [     T46]  <TASK>
[ 3688.183128] [     T46]  __schedule+0x107c/0x5fa0
[ 3688.183652] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.184201] [     T46]  ? __pfx_stack_trace_consume_entry+0x10/0x10
[ 3688.184914] [     T46]  ? unwind_get_return_address+0x5a/0xa0
[ 3688.185508] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3688.186068] [     T46]  ? schedule+0x2be/0x390
[ 3688.186583] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.187138] [     T46]  ? stack_trace_save+0x8f/0xc0
[ 3688.187684] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.188234] [     T46]  schedule+0xde/0x390
[ 3688.188735] [     T46]  schedule_timeout+0x176/0x230
[ 3688.189296] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3688.189959] [     T46]  ? llist_add_batch+0xb9/0x130
[ 3688.190503] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.191057] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3688.191610] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.192155] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3688.192694] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3688.193271] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3688.193878] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3688.194423] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.194976] [     T46]  devtmpfs_delete_node+0x142/0x1b0
[ 3688.195530] [     T46]  ? __pfx_devtmpfs_delete_node+0x10/0x10
[ 3688.196123] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3688.196669] [     T46]  ? klist_release+0x2c9/0x460
[ 3688.197213] [     T46]  ? do_raw_spin_unlock+0x54/0x1e0
[ 3688.197792] [     T46]  ? kobject_put+0x5d/0x4a0
[ 3688.198317] [     T46]  device_del+0x6e2/0x9b0
[ 3688.198863] [     T46]  ? __pfx_device_del+0x10/0x10
[ 3688.199406] [     T46]  ? ktime_get_mono_fast_ns+0x81/0x360
[ 3688.200058] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3688.200658] [     T46]  cdev_device_del+0x19/0xf0
[ 3688.201205] [     T46]  nvme_ns_remove+0x2ab/0x740 [nvme_core]
[ 3688.201859] [     T46]  nvme_remove_namespaces+0x25d/0x3a0 [nvme_core]
[ 3688.202510] [     T46]  ? __pfx_nvme_remove_namespaces+0x10/0x10 [nvme_core]
[ 3688.203270] [     T46]  ? __pfx___might_resched+0x10/0x10
[ 3688.203879] [     T46]  ? __pfx_sysfs_kf_write+0x10/0x10
[ 3688.204451] [     T46]  nvme_do_delete_ctrl+0xf5/0x160 [nvme_core]
[ 3688.205130] [     T46]  nvme_delete_ctrl_sync.cold+0x8/0xd [nvme_core]
[ 3688.205813] [     T46]  nvme_sysfs_delete+0x92/0xb0 [nvme_core]
[ 3688.206438] [     T46]  kernfs_fop_write_iter+0x39f/0x5a0
[ 3688.207046] [     T46]  vfs_write+0x5f7/0xe90
[ 3688.207566] [     T46]  ? __pfx_vfs_write+0x10/0x10
[ 3688.208131] [     T46]  ksys_write+0xf5/0x1c0
[ 3688.208645] [     T46]  ? __pfx_ksys_write+0x10/0x10
[ 3688.209209] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.209784] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3688.210373] [     T46]  ? do_syscall_64+0x54/0x190
[ 3688.210937] [     T46]  do_syscall_64+0x93/0x190
[ 3688.211448] [     T46]  ? kmem_cache_free+0x393/0x580
[ 3688.212063] [     T46]  ? fput_close_sync+0x100/0x170
[ 3688.212625] [     T46]  ? fput_close_sync+0x100/0x170
[ 3688.213203] [     T46]  ? __pfx_fput_close_sync+0x10/0x10
[ 3688.213790] [     T46]  ? lock_release+0x217/0x2c0
[ 3688.214308] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3688.214963] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.215484] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3688.216103] [     T46]  ? trace_hardirqs_on_prepare+0xdb/0x120
[ 3688.216685] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3688.217279] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3688.217836] [     T46]  ? do_sys_openat2+0x108/0x180
[ 3688.218369] [     T46]  ? __pfx_do_sys_openat2+0x10/0x10
[ 3688.218965] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3688.219546] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3688.220091] [     T46]  ? __x64_sys_openat+0x104/0x1d0
[ 3688.220632] [     T46]  ? __pfx___x64_sys_openat+0x10/0x10
[ 3688.221208] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.221768] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3688.222362] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3688.222937] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3688.223518] [     T46]  ? trace_hardirqs_on_prepare+0xdb/0x120
[ 3688.224114] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3688.224696] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3688.225230] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3688.225787] [     T46]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 3688.226384] [     T46] RIP: 0033:0x7fedbaf01504
[ 3688.226943] [     T46] RSP: 002b:00007fff4c0f4858 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 3688.227692] [     T46] RAX: ffffffffffffffda RBX: 0000000000000003 RCX: 00007fedbaf01504
[ 3688.228430] [     T46] RDX: 0000000000000001 RSI: 00007fedbb050746 RDI: 0000000000000003
[ 3688.229195] [     T46] RBP: 00007fedbb050746 R08: 0000000000004000 R09: 00000000ffffffff
[ 3688.229956] [     T46] R10: 0000000000000000 R11: 0000000000000202 R12: 000000000e107e30
[ 3688.230687] [     T46] R13: 00007fff4c0f563b R14: 000000000e107e30 R15: 000000000e108730
[ 3688.231429] [     T46]  </TASK>
[ 3688.232181] [     T46] INFO: lockdep is turned off.
[ 3810.978749] [     T46] INFO: task kdevtmpfs:40 blocked for more than 368 seconds.
[ 3810.981216] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3810.983118] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3810.985109] [     T46] task:kdevtmpfs       state:D stack:0     pid:40    tgid:40    ppid:2      task_flags:0x208140 flags:0x00004000
[ 3810.987369] [     T46] Call Trace:
[ 3810.988864] [     T46]  <TASK>
[ 3810.989909] [     T46]  __schedule+0x107c/0x5fa0
[ 3810.990439] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3810.990952] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3810.991388] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3810.991907] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3810.992387] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3810.992931] [     T46]  ? schedule+0x2be/0x390
[ 3810.993384] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3810.993915] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3810.994372] [     T46]  ? lock_release+0x217/0x2c0
[ 3810.994873] [     T46]  schedule+0xde/0x390
[ 3810.995386] [     T46]  schedule_timeout+0x176/0x230
[ 3810.995941] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3810.996422] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3810.996978] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3810.997449] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3810.997996] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3810.998463] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3810.998991] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3810.999474] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3811.000007] [     T46]  ? flush_workqueue_prep_pwqs+0x333/0x440
[ 3811.000501] [     T46]  __flush_workqueue+0x373/0x10a0
[ 3811.000990] [     T46]  ? __pfx___might_resched+0x10/0x10
[ 3811.001467] [     T46]  ? kasan_save_track+0x10/0x30
[ 3811.001978] [     T46]  ? kasan_save_free_info+0x37/0x60
[ 3811.002462] [     T46]  ? __mutex_lock+0x538/0x1a40
[ 3811.002967] [     T46]  ? __pfx___flush_workqueue+0x10/0x10
[ 3811.003445] [     T46]  ? find_inode_fast+0x152/0x3f0
[ 3811.003983] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.004451] [     T46]  md_alloc+0x28/0xf10
[ 3811.004940] [     T46]  md_probe+0x25/0x70
[ 3811.005377] [     T46]  blk_probe_dev+0x108/0x180
[ 3811.005886] [     T46]  blk_request_module+0xe/0x60
[ 3811.006363] [     T46]  blkdev_get_no_open+0x5c/0xa0
[ 3811.006872] [     T46]  bdev_statx+0x98/0x550
[ 3811.007391] [     T46]  vfs_getattr_nosec+0x2be/0x3d0
[ 3811.007907] [     T46]  handle_remove+0x115/0x500
[ 3811.008359] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.008866] [     T46]  ? __pfx_handle_remove+0x10/0x10
[ 3811.009420] [     T46]  ? finish_task_switch.isra.0+0x1b9/0x860
[ 3811.009955] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.010412] [     T46]  ? __schedule+0x3099/0x5fa0
[ 3811.010916] [     T46]  ? __pfx_sched_clock_cpu+0x10/0x10
[ 3811.011385] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3811.011911] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3811.012360] [     T46]  ? devtmpfs_work_loop+0xb5/0x5c7
[ 3811.012864] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.013356] [     T46]  ? lock_release+0x217/0x2c0
[ 3811.013832] [     T46]  ? devtmpfs_work_loop+0x55d/0x5c7
[ 3811.014327] [     T46]  devtmpfs_work_loop+0x4e9/0x5c7
[ 3811.014756] [     T46]  ? __pfx_devtmpfs_work_loop+0x10/0x10
[ 3811.015254] [     T46]  ? _raw_spin_unlock_irqrestore+0x35/0x60
[ 3811.015714] [     T46]  ? __pfx_do_raw_spin_lock+0x10/0x10
[ 3811.016206] [     T46]  ? devtmpfsd+0x19/0x30
[ 3811.016610] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3811.017086] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3811.017541] [     T46]  ? __pfx_devtmpfsd+0x10/0x10
[ 3811.017970] [     T46]  devtmpfsd+0x27/0x30
[ 3811.018373] [     T46]  kthread+0x3b0/0x770
[ 3811.018815] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3811.019280] [     T46]  ? ret_from_fork+0x17/0x70
[ 3811.019698] [     T46]  ? ret_from_fork+0x17/0x70
[ 3811.020138] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3811.020563] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3811.020985] [     T46]  ret_from_fork+0x30/0x70
[ 3811.021390] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3811.021845] [     T46]  ret_from_fork_asm+0x1a/0x30
[ 3811.022326] [     T46]  </TASK>
[ 3811.022731] [     T46] INFO: task kworker/1:1:91856 blocked for more than 368 seconds.
[ 3811.023350] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3811.023881] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3811.024493] [     T46] task:kworker/1:1     state:D stack:0     pid:91856 tgid:91856 ppid:2      task_flags:0x4288060 flags:0x00004000
[ 3811.025268] [     T46] Workqueue: md_misc mddev_delayed_delete
[ 3811.025780] [     T46] Call Trace:
[ 3811.026225] [     T46]  <TASK>
[ 3811.026598] [     T46]  __schedule+0x107c/0x5fa0
[ 3811.027047] [     T46]  ? unwind_next_frame+0x400/0x1f10
[ 3811.027504] [     T46]  ? ret_from_fork_asm+0x1a/0x30
[ 3811.027984] [     T46]  ? __pfx_stack_trace_consume_entry+0x10/0x10
[ 3811.028487] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3811.029016] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3811.029475] [     T46]  ? __pfx_do_raw_spin_lock+0x10/0x10
[ 3811.029997] [     T46]  ? schedule+0x2be/0x390
[ 3811.030435] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.030959] [     T46]  ? lock_release+0x217/0x2c0
[ 3811.031405] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.031906] [     T46]  schedule+0xde/0x390
[ 3811.032397] [     T46]  schedule_timeout+0x176/0x230
[ 3811.032898] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3811.033389] [     T46]  ? llist_add_batch+0xb9/0x130
[ 3811.033890] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.034346] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3811.034868] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.035385] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3811.035914] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3811.036387] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3811.036916] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3811.037380] [     T46]  ? lock_release+0x217/0x2c0
[ 3811.037907] [     T46]  devtmpfs_delete_node+0x142/0x1b0
[ 3811.038381] [     T46]  ? __pfx_devtmpfs_delete_node+0x10/0x10
[ 3811.038914] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.039374] [     T46]  ? __call_rcu_common.constprop.0+0x45f/0x1090
[ 3811.039940] [     T46]  ? trace_hardirqs_on+0x12/0x120
[ 3811.040409] [     T46]  ? kernfs_put.part.0+0x130/0x470
[ 3811.040970] [     T46]  device_del+0x6e2/0x9b0
[ 3811.041410] [     T46]  ? pm_runtime_set_memalloc_noio+0x8b/0x140
[ 3811.042003] [     T46]  ? __pfx_device_del+0x10/0x10
[ 3811.042469] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.042994] [     T46]  ? trace_hardirqs_on+0x12/0x120
[ 3811.043462] [     T46]  ? _raw_spin_unlock_irq+0x34/0x50
[ 3811.043990] [     T46]  del_gendisk+0x4f5/0xa20
[ 3811.044441] [     T46]  ? __pfx_del_gendisk+0x10/0x10
[ 3811.044958] [     T46]  ? up_write+0x1c6/0x500
[ 3811.045404] [     T46]  md_kobj_release+0x90/0xe0
[ 3811.045908] [     T46]  kobject_put+0x17b/0x4a0
[ 3811.046363] [     T46]  process_one_work+0x84f/0x1460
[ 3811.046881] [     T46]  ? __pfx_process_one_work+0x10/0x10
[ 3811.047364] [     T46]  ? assign_work+0x16c/0x240
[ 3811.047883] [     T46]  worker_thread+0x5ef/0xfd0
[ 3811.048422] [     T46]  ? __kthread_parkme+0xb4/0x200
[ 3811.048920] [     T46]  ? __pfx_worker_thread+0x10/0x10
[ 3811.049383] [     T46]  kthread+0x3b0/0x770
[ 3811.049857] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3811.050357] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.050849] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3811.051369] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.051859] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3811.052367] [     T46]  ret_from_fork+0x30/0x70
[ 3811.052848] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3811.053346] [     T46]  ret_from_fork_asm+0x1a/0x30
[ 3811.053832] [     T46]  </TASK>
[ 3811.054259] [     T46] INFO: task nvme:129126 blocked for more than 368 seconds.
[ 3811.054829] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3811.055339] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3811.055968] [     T46] task:nvme            state:D stack:0     pid:129126 tgid:129126 ppid:129061 task_flags:0x480100 flags:0x00024002
[ 3811.056668] [     T46] Call Trace:
[ 3811.057088] [     T46]  <TASK>
[ 3811.057478] [     T46]  __schedule+0x107c/0x5fa0
[ 3811.057976] [     T46]  ? lock_release+0x217/0x2c0
[ 3811.058427] [     T46]  ? __pfx_stack_trace_consume_entry+0x10/0x10
[ 3811.058978] [     T46]  ? unwind_get_return_address+0x5a/0xa0
[ 3811.059467] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3811.059996] [     T46]  ? schedule+0x2be/0x390
[ 3811.060432] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.060948] [     T46]  ? stack_trace_save+0x8f/0xc0
[ 3811.061404] [     T46]  ? lock_release+0x217/0x2c0
[ 3811.061938] [     T46]  schedule+0xde/0x390
[ 3811.062379] [     T46]  schedule_timeout+0x176/0x230
[ 3811.062896] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3811.063402] [     T46]  ? llist_add_batch+0xb9/0x130
[ 3811.063923] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.064377] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3811.064894] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.065420] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3811.065946] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3811.066419] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3811.066939] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3811.067398] [     T46]  ? lock_release+0x217/0x2c0
[ 3811.067891] [     T46]  devtmpfs_delete_node+0x142/0x1b0
[ 3811.068353] [     T46]  ? __pfx_devtmpfs_delete_node+0x10/0x10
[ 3811.068873] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3811.069412] [     T46]  ? klist_release+0x2c9/0x460
[ 3811.069949] [     T46]  ? do_raw_spin_unlock+0x54/0x1e0
[ 3811.070412] [     T46]  ? kobject_put+0x5d/0x4a0
[ 3811.070913] [     T46]  device_del+0x6e2/0x9b0
[ 3811.071344] [     T46]  ? __pfx_device_del+0x10/0x10
[ 3811.071848] [     T46]  ? ktime_get_mono_fast_ns+0x81/0x360
[ 3811.072392] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3811.072935] [     T46]  cdev_device_del+0x19/0xf0
[ 3811.073385] [     T46]  nvme_ns_remove+0x2ab/0x740 [nvme_core]
[ 3811.073938] [     T46]  nvme_remove_namespaces+0x25d/0x3a0 [nvme_core]
[ 3811.074454] [     T46]  ? __pfx_nvme_remove_namespaces+0x10/0x10 [nvme_core]
[ 3811.075060] [     T46]  ? __pfx___might_resched+0x10/0x10
[ 3811.075529] [     T46]  ? __pfx_sysfs_kf_write+0x10/0x10
[ 3811.076003] [     T46]  nvme_do_delete_ctrl+0xf5/0x160 [nvme_core]
[ 3811.076512] [     T46]  nvme_delete_ctrl_sync.cold+0x8/0xd [nvme_core]
[ 3811.077086] [     T46]  nvme_sysfs_delete+0x92/0xb0 [nvme_core]
[ 3811.077575] [     T46]  kernfs_fop_write_iter+0x39f/0x5a0
[ 3811.078049] [     T46]  vfs_write+0x5f7/0xe90
[ 3811.078491] [     T46]  ? __pfx_vfs_write+0x10/0x10
[ 3811.078974] [     T46]  ksys_write+0xf5/0x1c0
[ 3811.079411] [     T46]  ? __pfx_ksys_write+0x10/0x10
[ 3811.079905] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.080358] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3811.080877] [     T46]  ? do_syscall_64+0x54/0x190
[ 3811.081390] [     T46]  do_syscall_64+0x93/0x190
[ 3811.081860] [     T46]  ? kmem_cache_free+0x393/0x580
[ 3811.082373] [     T46]  ? fput_close_sync+0x100/0x170
[ 3811.082880] [     T46]  ? fput_close_sync+0x100/0x170
[ 3811.083334] [     T46]  ? __pfx_fput_close_sync+0x10/0x10
[ 3811.083830] [     T46]  ? lock_release+0x217/0x2c0
[ 3811.084319] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3811.084849] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.085350] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3811.085891] [     T46]  ? trace_hardirqs_on_prepare+0xdb/0x120
[ 3811.086369] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3811.086882] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3811.087395] [     T46]  ? do_sys_openat2+0x108/0x180
[ 3811.087880] [     T46]  ? __pfx_do_sys_openat2+0x10/0x10
[ 3811.088341] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3811.088875] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3811.089391] [     T46]  ? __x64_sys_openat+0x104/0x1d0
[ 3811.089911] [     T46]  ? __pfx___x64_sys_openat+0x10/0x10
[ 3811.090374] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.090864] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3811.091357] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3811.091858] [     T46]  ? syscall_exit_to_user_mode+0x8e/0x280
[ 3811.092359] [     T46]  ? trace_hardirqs_on_prepare+0xdb/0x120
[ 3811.092883] [     T46]  ? syscall_exit_to_user_mode+0x93/0x280
[ 3811.093383] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3811.093882] [     T46]  ? do_syscall_64+0x9f/0x190
[ 3811.094404] [     T46]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 3811.094973] [     T46] RIP: 0033:0x7fedbaf01504
[ 3811.095420] [     T46] RSP: 002b:00007fff4c0f4858 EFLAGS: 00000202 ORIG_RAX: 0000000000000001
[ 3811.096046] [     T46] RAX: ffffffffffffffda RBX: 0000000000000003 RCX: 00007fedbaf01504
[ 3811.096630] [     T46] RDX: 0000000000000001 RSI: 00007fedbb050746 RDI: 0000000000000003
[ 3811.097321] [     T46] RBP: 00007fedbb050746 R08: 0000000000004000 R09: 00000000ffffffff
[ 3811.098089] [     T46] R10: 0000000000000000 R11: 0000000000000202 R12: 000000000e107e30
[ 3811.099055] [     T46] R13: 00007fff4c0f563b R14: 000000000e107e30 R15: 000000000e108730
[ 3811.099823] [     T46]  </TASK>
[ 3811.100334] [     T46] INFO: lockdep is turned off.
[ 3933.856614] [     T46] INFO: task kdevtmpfs:40 blocked for more than 491 seconds.
[ 3933.857467] [     T46]       Tainted: G    B   W           6.15.0-rc3 #15
[ 3933.858156] [     T46] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 3933.859165] [     T46] task:kdevtmpfs       state:D stack:0     pid:40    tgid:40    ppid:2      task_flags:0x208140 flags:0x00004000
[ 3933.860620] [     T46] Call Trace:
[ 3933.861122] [     T46]  <TASK>
[ 3933.861575] [     T46]  __schedule+0x107c/0x5fa0
[ 3933.862257] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.862831] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.863397] [     T46]  ? __pfx___schedule+0x10/0x10
[ 3933.864129] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.864702] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3933.865291] [     T46]  ? schedule+0x2be/0x390
[ 3933.865822] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.866390] [     T46]  ? unwind_next_frame+0x3f6/0x1f10
[ 3933.867232] [     T46]  ? lock_release+0x217/0x2c0
[ 3933.867797] [     T46]  schedule+0xde/0x390
[ 3933.868329] [     T46]  schedule_timeout+0x176/0x230
[ 3933.868912] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3933.869527] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.870289] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3933.870930] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.871523] [     T46]  __wait_for_common+0x2ea/0x4c0
[ 3933.872251] [     T46]  ? __pfx_schedule_timeout+0x10/0x10
[ 3933.872889] [     T46]  ? __pfx___wait_for_common+0x10/0x10
[ 3933.873514] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3933.874186] [     T46]  ? flush_workqueue_prep_pwqs+0x333/0x440
[ 3933.874825] [     T46]  __flush_workqueue+0x373/0x10a0
[ 3933.875432] [     T46]  ? __pfx___might_resched+0x10/0x10
[ 3933.876080] [     T46]  ? kasan_save_track+0x10/0x30
[ 3933.876673] [     T46]  ? kasan_save_free_info+0x37/0x60
[ 3933.877267] [     T46]  ? __mutex_lock+0x538/0x1a40
[ 3933.877843] [     T46]  ? __pfx___flush_workqueue+0x10/0x10
[ 3933.878460] [     T46]  ? find_inode_fast+0x152/0x3f0
[ 3933.879104] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.879677] [     T46]  md_alloc+0x28/0xf10
[ 3933.880201] [     T46]  md_probe+0x25/0x70
[ 3933.880718] [     T46]  blk_probe_dev+0x108/0x180
[ 3933.881272] [     T46]  blk_request_module+0xe/0x60
[ 3933.881834] [     T46]  blkdev_get_no_open+0x5c/0xa0
[ 3933.882405] [     T46]  bdev_statx+0x98/0x550
[ 3933.882950] [     T46]  vfs_getattr_nosec+0x2be/0x3d0
[ 3933.883518] [     T46]  handle_remove+0x115/0x500
[ 3933.884067] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.884612] [     T46]  ? __pfx_handle_remove+0x10/0x10
[ 3933.885174] [     T46]  ? finish_task_switch.isra.0+0x1b9/0x860
[ 3933.885779] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.886325] [     T46]  ? __schedule+0x3099/0x5fa0
[ 3933.886873] [     T46]  ? __pfx_sched_clock_cpu+0x10/0x10
[ 3933.887445] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3933.888004] [     T46]  ? do_raw_spin_lock+0x129/0x260
[ 3933.888546] [     T46]  ? devtmpfs_work_loop+0xb5/0x5c7
[ 3933.889108] [     T46]  ? rcu_is_watching+0x11/0xb0
[ 3933.889628] [     T46]  ? lock_release+0x217/0x2c0
[ 3933.890134] [     T46]  ? devtmpfs_work_loop+0x55d/0x5c7
[ 3933.890676] [     T46]  devtmpfs_work_loop+0x4e9/0x5c7
[ 3933.891207] [     T46]  ? __pfx_devtmpfs_work_loop+0x10/0x10
[ 3933.891768] [     T46]  ? _raw_spin_unlock_irqrestore+0x35/0x60
[ 3933.892356] [     T46]  ? __pfx_do_raw_spin_lock+0x10/0x10
[ 3933.892924] [     T46]  ? devtmpfsd+0x19/0x30
[ 3933.893414] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3933.894004] [     T46]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[ 3933.894584] [     T46]  ? __pfx_devtmpfsd+0x10/0x10
[ 3933.895099] [     T46]  devtmpfsd+0x27/0x30
[ 3933.895571] [     T46]  kthread+0x3b0/0x770
[ 3933.896086] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3933.896596] [     T46]  ? ret_from_fork+0x17/0x70
[ 3933.897099] [     T46]  ? ret_from_fork+0x17/0x70
[ 3933.897600] [     T46]  ? _raw_spin_unlock_irq+0x24/0x50
[ 3933.898138] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3933.898642] [     T46]  ret_from_fork+0x30/0x70
[ 3933.899130] [     T46]  ? __pfx_kthread+0x10/0x10
[ 3933.899635] [     T46]  ret_from_fork_asm+0x1a/0x30
[ 3933.900148] [     T46]  </TASK>
[ 3933.900555] [     T46] Future hung task reports are suppressed, see sysctl kernel.hung_task_warnings
[ 3933.901497] [     T46] INFO: lockdep is turned off.

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[hch]

On Tue, Apr 22, 2025 at 05:03:19AM +0000, Shinichiro Kawasaki wrote:
> I ran blktests with the kernel v6.15-rc3, and found the test case md/001 hangs.
> The hang is recreated in stable manner. I bisected and found this patch as the
> commit 777d0961ff95 is the trigger. When I revert the commit from v6.15-rc3
> kernel, the hang disappeared.
> 
> Actions for fix will be appreciated.
> 
> FYI, the kernel INFO messages recorded functions relevant to the trigger commit,
> such as bdev_statx or vfs_getattr_nosec [1].

This should fix it:

diff --git a/block/bdev.c b/block/bdev.c
index 6a34179192c9..97d4c0ab1670 100644
--- a/block/bdev.c
+++ b/block/bdev.c
@@ -1274,18 +1274,23 @@ void sync_bdevs(bool wait)
  */
 void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
 {
-	struct inode *backing_inode;
 	struct block_device *bdev;
 
-	backing_inode = d_backing_inode(path->dentry);
-
 	/*
-	 * Note that backing_inode is the inode of a block device node file,
-	 * not the block device's internal inode.  Therefore it is *not* valid
-	 * to use I_BDEV() here; the block device has to be looked up by i_rdev
-	 * instead.
+	 * Note that d_backing_inode() returnsthe inode of a block device node
+	 * file, not the block device's internal inode.
+	 *
+	 * Therefore it is *not* valid to use I_BDEV() here; the block device
+	 * has to be looked up by i_rdev instead.
+	 *
+	 * Only do this lookup if actually needed to avoid the performance
+	 * overhead of the lookup, and to avoid injecting bdev lifetime issues
+	 * into devtmpfs.
 	 */
-	bdev = blkdev_get_no_open(backing_inode->i_rdev);
+	if (!(request_mask & (STATX_DIOALIGN | STATX_WRITE_ATOMIC)))
+		return;
+
+	bdev = blkdev_get_no_open(d_backing_inode(path->dentry)->i_rdev);
 	if (!bdev)
 		return;
 

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[Christian Brauner]

On Tue, Apr 22, 2025 at 07:51:49AM +0200, hch wrote:
> On Tue, Apr 22, 2025 at 05:03:19AM +0000, Shinichiro Kawasaki wrote:
> > I ran blktests with the kernel v6.15-rc3, and found the test case md/001 hangs.
> > The hang is recreated in stable manner. I bisected and found this patch as the
> > commit 777d0961ff95 is the trigger. When I revert the commit from v6.15-rc3
> > kernel, the hang disappeared.
> > 
> > Actions for fix will be appreciated.
> > 
> > FYI, the kernel INFO messages recorded functions relevant to the trigger commit,
> > such as bdev_statx or vfs_getattr_nosec [1].
> 
> This should fix it:

Can you send this as a proper patch so I can pick it up, please?
It needs to go upstream.

> 
> diff --git a/block/bdev.c b/block/bdev.c
> index 6a34179192c9..97d4c0ab1670 100644
> --- a/block/bdev.c
> +++ b/block/bdev.c
> @@ -1274,18 +1274,23 @@ void sync_bdevs(bool wait)
>   */
>  void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
>  {
> -	struct inode *backing_inode;
>  	struct block_device *bdev;
>  
> -	backing_inode = d_backing_inode(path->dentry);
> -
>  	/*
> -	 * Note that backing_inode is the inode of a block device node file,
> -	 * not the block device's internal inode.  Therefore it is *not* valid
> -	 * to use I_BDEV() here; the block device has to be looked up by i_rdev
> -	 * instead.
> +	 * Note that d_backing_inode() returnsthe inode of a block device node
> +	 * file, not the block device's internal inode.
> +	 *
> +	 * Therefore it is *not* valid to use I_BDEV() here; the block device
> +	 * has to be looked up by i_rdev instead.
> +	 *
> +	 * Only do this lookup if actually needed to avoid the performance
> +	 * overhead of the lookup, and to avoid injecting bdev lifetime issues
> +	 * into devtmpfs.
>  	 */
> -	bdev = blkdev_get_no_open(backing_inode->i_rdev);
> +	if (!(request_mask & (STATX_DIOALIGN | STATX_WRITE_ATOMIC)))
> +		return;
> +
> +	bdev = blkdev_get_no_open(d_backing_inode(path->dentry)->i_rdev);
>  	if (!bdev)
>  		return;
>  

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[hch]

On Tue, Apr 22, 2025 at 09:25:24AM +0200, Christian Brauner wrote:
> Can you send this as a proper patch so I can pick it up, please?
> It needs to go upstream.

I'll wait for a tested-by tag from at least on of the reports while
I'm running more tests.  The previous one was obviously a bit hurried
so I'm all for not rushing this round.

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[Christian Brauner]

On Tue, Apr 22, 2025 at 07:51:49AM +0200, hch wrote:
> On Tue, Apr 22, 2025 at 05:03:19AM +0000, Shinichiro Kawasaki wrote:
> > I ran blktests with the kernel v6.15-rc3, and found the test case md/001 hangs.
> > The hang is recreated in stable manner. I bisected and found this patch as the
> > commit 777d0961ff95 is the trigger. When I revert the commit from v6.15-rc3
> > kernel, the hang disappeared.
> > 
> > Actions for fix will be appreciated.
> > 
> > FYI, the kernel INFO messages recorded functions relevant to the trigger commit,
> > such as bdev_statx or vfs_getattr_nosec [1].
> 
> This should fix it:
> 
> diff --git a/block/bdev.c b/block/bdev.c
> index 6a34179192c9..97d4c0ab1670 100644
> --- a/block/bdev.c
> +++ b/block/bdev.c
> @@ -1274,18 +1274,23 @@ void sync_bdevs(bool wait)
>   */
>  void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
>  {
> -	struct inode *backing_inode;
>  	struct block_device *bdev;
>  
> -	backing_inode = d_backing_inode(path->dentry);
> -
>  	/*
> -	 * Note that backing_inode is the inode of a block device node file,
> -	 * not the block device's internal inode.  Therefore it is *not* valid
> -	 * to use I_BDEV() here; the block device has to be looked up by i_rdev
> -	 * instead.
> +	 * Note that d_backing_inode() returnsthe inode of a block device node
> +	 * file, not the block device's internal inode.
> +	 *
> +	 * Therefore it is *not* valid to use I_BDEV() here; the block device
> +	 * has to be looked up by i_rdev instead.
> +	 *
> +	 * Only do this lookup if actually needed to avoid the performance
> +	 * overhead of the lookup, and to avoid injecting bdev lifetime issues
> +	 * into devtmpfs.
>  	 */
> -	bdev = blkdev_get_no_open(backing_inode->i_rdev);
> +	if (!(request_mask & (STATX_DIOALIGN | STATX_WRITE_ATOMIC)))
> +		return;

This leaks the block device reference if blkdev_get_no_open() succeeds.

> +
> +	bdev = blkdev_get_no_open(d_backing_inode(path->dentry)->i_rdev);
>  	if (!bdev)
>  		return;
>  

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[hch]

On Tue, Apr 22, 2025 at 10:15:48AM +0200, Christian Brauner wrote:
> > -	bdev = blkdev_get_no_open(backing_inode->i_rdev);
> > +	if (!(request_mask & (STATX_DIOALIGN | STATX_WRITE_ATOMIC)))
> > +		return;
> 
> This leaks the block device reference if blkdev_get_no_open() succeeds.
> 
> > +
> > +	bdev = blkdev_get_no_open(d_backing_inode(path->dentry)->i_rdev);

The one removed by the patch above, or the one added after the check
below? :)

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[Christian Brauner]

On Tue, Apr 22, 2025 at 10:17:36AM +0200, hch wrote:
> On Tue, Apr 22, 2025 at 10:15:48AM +0200, Christian Brauner wrote:
> > > -	bdev = blkdev_get_no_open(backing_inode->i_rdev);
> > > +	if (!(request_mask & (STATX_DIOALIGN | STATX_WRITE_ATOMIC)))
> > > +		return;
> > 
> > This leaks the block device reference if blkdev_get_no_open() succeeds.
> > 
> > > +
> > > +	bdev = blkdev_get_no_open(d_backing_inode(path->dentry)->i_rdev);
> 
> The one removed by the patch above, or the one added after the check
> below? :)

Ah, I got confused by the inline diff.

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[hch]

Turns out this doesn't work.  We used to have the request_mask, but it
got removed in 25fbcd62d2e1 ("bdev: use bdev_io_min() for statx block
size") so that stat can expose the block device min I/O size in
st_blkdev, and as the blksize doesn't have it's own request_mask flag
is hard to special case.

So maybe the better question is why devtmpfs even calls into
vfs_getattr?  As far as I can tell handle_remove is only ever called on
the actual devtmpfs file system, so we don't need to go through the
VFS to query i_mode.  i.e. the patch should also fix the issue.  The
modify_change is probably not needed either, but for now I'm aiming
for the minimal fix.

diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c
index 6dd1a8860f1c..53fb0829eb7b 100644
--- a/drivers/base/devtmpfs.c
+++ b/drivers/base/devtmpfs.c
@@ -296,7 +296,7 @@ static int delete_path(const char *nodepath)
 	return err;
 }
 
-static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *stat)
+static int dev_mynode(struct device *dev, struct inode *inode)
 {
 	/* did we create it */
 	if (inode->i_private != &thread)
@@ -304,13 +304,13 @@ static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *sta
 
 	/* does the dev_t match */
 	if (is_blockdev(dev)) {
-		if (!S_ISBLK(stat->mode))
+		if (!S_ISBLK(inode->i_mode))
 			return 0;
 	} else {
-		if (!S_ISCHR(stat->mode))
+		if (!S_ISCHR(inode->i_mode))
 			return 0;
 	}
-	if (stat->rdev != dev->devt)
+	if (inode->i_rdev != dev->devt)
 		return 0;
 
 	/* ours */
@@ -321,8 +321,7 @@ static int handle_remove(const char *nodename, struct device *dev)
 {
 	struct path parent;
 	struct dentry *dentry;
-	struct kstat stat;
-	struct path p;
+	struct inode *inode;
 	int deleted = 0;
 	int err;
 
@@ -330,11 +329,8 @@ static int handle_remove(const char *nodename, struct device *dev)
 	if (IS_ERR(dentry))
 		return PTR_ERR(dentry);
 
-	p.mnt = parent.mnt;
-	p.dentry = dentry;
-	err = vfs_getattr(&p, &stat, STATX_TYPE | STATX_MODE,
-			  AT_STATX_SYNC_AS_STAT);
-	if (!err && dev_mynode(dev, d_inode(dentry), &stat)) {
+	inode = d_inode(dentry);
+	if (dev_mynode(dev, inode)) {
 		struct iattr newattrs;
 		/*
 		 * before unlinking this node, reset permissions
@@ -342,7 +338,7 @@ static int handle_remove(const char *nodename, struct device *dev)
 		 */
 		newattrs.ia_uid = GLOBAL_ROOT_UID;
 		newattrs.ia_gid = GLOBAL_ROOT_GID;
-		newattrs.ia_mode = stat.mode & ~0777;
+		newattrs.ia_mode = inode->i_mode & ~0777;
 		newattrs.ia_valid =
 			ATTR_UID|ATTR_GID|ATTR_MODE;
 		inode_lock(d_inode(dentry));

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[Christian Brauner]

On Tue, Apr 22, 2025 at 04:15:05PM +0200, hch wrote:
> Turns out this doesn't work.  We used to have the request_mask, but it
> got removed in 25fbcd62d2e1 ("bdev: use bdev_io_min() for statx block
> size") so that stat can expose the block device min I/O size in
> st_blkdev, and as the blksize doesn't have it's own request_mask flag
> is hard to special case.
> 
> So maybe the better question is why devtmpfs even calls into
> vfs_getattr?  As far as I can tell handle_remove is only ever called on
> the actual devtmpfs file system, so we don't need to go through the
> VFS to query i_mode.  i.e. the patch should also fix the issue.  The

Hm, yes. Just looked at the history and it dates back to Kay's original
devtmpfs patch.

> modify_change is probably not needed either, but for now I'm aiming
> for the minimal fix.
> 
> diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c
> index 6dd1a8860f1c..53fb0829eb7b 100644
> --- a/drivers/base/devtmpfs.c
> +++ b/drivers/base/devtmpfs.c
> @@ -296,7 +296,7 @@ static int delete_path(const char *nodepath)
>  	return err;
>  }
>  
> -static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *stat)
> +static int dev_mynode(struct device *dev, struct inode *inode)
>  {
>  	/* did we create it */
>  	if (inode->i_private != &thread)

It seems off that there are deletion requests coming in for a files that
weren't created by devtmpfsd. But maybe that can somehow happen.

> @@ -304,13 +304,13 @@ static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *sta
>  
>  	/* does the dev_t match */
>  	if (is_blockdev(dev)) {
> -		if (!S_ISBLK(stat->mode))
> +		if (!S_ISBLK(inode->i_mode))
>  			return 0;
>  	} else {
> -		if (!S_ISCHR(stat->mode))
> +		if (!S_ISCHR(inode->i_mode))
>  			return 0;
>  	}
> -	if (stat->rdev != dev->devt)
> +	if (inode->i_rdev != dev->devt)
>  		return 0;
>  
>  	/* ours */
> @@ -321,8 +321,7 @@ static int handle_remove(const char *nodename, struct device *dev)
>  {
>  	struct path parent;
>  	struct dentry *dentry;
> -	struct kstat stat;
> -	struct path p;
> +	struct inode *inode;
>  	int deleted = 0;
>  	int err;
>  
> @@ -330,11 +329,8 @@ static int handle_remove(const char *nodename, struct device *dev)
>  	if (IS_ERR(dentry))
>  		return PTR_ERR(dentry);
>  
> -	p.mnt = parent.mnt;
> -	p.dentry = dentry;
> -	err = vfs_getattr(&p, &stat, STATX_TYPE | STATX_MODE,
> -			  AT_STATX_SYNC_AS_STAT);
> -	if (!err && dev_mynode(dev, d_inode(dentry), &stat)) {
> +	inode = d_inode(dentry);
> +	if (dev_mynode(dev, inode)) {
>  		struct iattr newattrs;
>  		/*
>  		 * before unlinking this node, reset permissions
> @@ -342,7 +338,7 @@ static int handle_remove(const char *nodename, struct device *dev)
>  		 */
>  		newattrs.ia_uid = GLOBAL_ROOT_UID;
>  		newattrs.ia_gid = GLOBAL_ROOT_GID;
> -		newattrs.ia_mode = stat.mode & ~0777;
> +		newattrs.ia_mode = inode->i_mode & ~0777;
>  		newattrs.ia_valid =
>  			ATTR_UID|ATTR_GID|ATTR_MODE;
>  		inode_lock(d_inode(dentry));

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[Shinichiro Kawasaki]

On Apr 22, 2025 / 16:15, hch wrote:
> Turns out this doesn't work.  We used to have the request_mask, but it
> got removed in 25fbcd62d2e1 ("bdev: use bdev_io_min() for statx block
> size") so that stat can expose the block device min I/O size in
> st_blkdev, and as the blksize doesn't have it's own request_mask flag
> is hard to special case.
> 
> So maybe the better question is why devtmpfs even calls into
> vfs_getattr?  As far as I can tell handle_remove is only ever called on
> the actual devtmpfs file system, so we don't need to go through the
> VFS to query i_mode.  i.e. the patch should also fix the issue.  The
> modify_change is probably not needed either, but for now I'm aiming
> for the minimal fix.
> 
> diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c
> index 6dd1a8860f1c..53fb0829eb7b 100644
> --- a/drivers/base/devtmpfs.c
> +++ b/drivers/base/devtmpfs.c
> @@ -296,7 +296,7 @@ static int delete_path(const char *nodepath)
>  	return err;
>  }
>  
> -static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *stat)
> +static int dev_mynode(struct device *dev, struct inode *inode)
>  {
>  	/* did we create it */
>  	if (inode->i_private != &thread)
> @@ -304,13 +304,13 @@ static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *sta
>  
>  	/* does the dev_t match */
>  	if (is_blockdev(dev)) {
> -		if (!S_ISBLK(stat->mode))
> +		if (!S_ISBLK(inode->i_mode))
>  			return 0;
>  	} else {
> -		if (!S_ISCHR(stat->mode))
> +		if (!S_ISCHR(inode->i_mode))
>  			return 0;
>  	}
> -	if (stat->rdev != dev->devt)
> +	if (inode->i_rdev != dev->devt)
>  		return 0;
>  
>  	/* ours */
> @@ -321,8 +321,7 @@ static int handle_remove(const char *nodename, struct device *dev)
>  {
>  	struct path parent;
>  	struct dentry *dentry;
> -	struct kstat stat;
> -	struct path p;
> +	struct inode *inode;
>  	int deleted = 0;
>  	int err;
>  
> @@ -330,11 +329,8 @@ static int handle_remove(const char *nodename, struct device *dev)
>  	if (IS_ERR(dentry))
>  		return PTR_ERR(dentry);
>  
> -	p.mnt = parent.mnt;
> -	p.dentry = dentry;
> -	err = vfs_getattr(&p, &stat, STATX_TYPE | STATX_MODE,
> -			  AT_STATX_SYNC_AS_STAT);
> -	if (!err && dev_mynode(dev, d_inode(dentry), &stat)) {
> +	inode = d_inode(dentry);
> +	if (dev_mynode(dev, inode)) {
>  		struct iattr newattrs;
>  		/*
>  		 * before unlinking this node, reset permissions
> @@ -342,7 +338,7 @@ static int handle_remove(const char *nodename, struct device *dev)
>  		 */
>  		newattrs.ia_uid = GLOBAL_ROOT_UID;
>  		newattrs.ia_gid = GLOBAL_ROOT_GID;
> -		newattrs.ia_mode = stat.mode & ~0777;
> +		newattrs.ia_mode = inode->i_mode & ~0777;
>  		newattrs.ia_valid =
>  			ATTR_UID|ATTR_GID|ATTR_MODE;
>  		inode_lock(d_inode(dentry));

I evaluated this new fix also, and confirmed it avoids the hang. Good.
I also ran whole blktests and observed no other regression. Thanks.

Tested-by: Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com>

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[Jain, Ayush]

On 4/22/2025 7:45 PM, hch wrote:
> Turns out this doesn't work.  We used to have the request_mask, but it
> got removed in 25fbcd62d2e1 ("bdev: use bdev_io_min() for statx block
> size") so that stat can expose the block device min I/O size in
> st_blkdev, and as the blksize doesn't have it's own request_mask flag
> is hard to special case.
> 
> So maybe the better question is why devtmpfs even calls into
> vfs_getattr?  As far as I can tell handle_remove is only ever called on
> the actual devtmpfs file system, so we don't need to go through the
> VFS to query i_mode.  i.e. the patch should also fix the issue.  The
> modify_change is probably not needed either, but for now I'm aiming
> for the minimal fix.
> 
> diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c
> index 6dd1a8860f1c..53fb0829eb7b 100644
> --- a/drivers/base/devtmpfs.c
> +++ b/drivers/base/devtmpfs.c
> @@ -296,7 +296,7 @@ static int delete_path(const char *nodepath)
>  	return err;
>  }
>  
> -static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *stat)
> +static int dev_mynode(struct device *dev, struct inode *inode)
>  {
>  	/* did we create it */
>  	if (inode->i_private != &thread)
> @@ -304,13 +304,13 @@ static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *sta
>  
>  	/* does the dev_t match */
>  	if (is_blockdev(dev)) {
> -		if (!S_ISBLK(stat->mode))
> +		if (!S_ISBLK(inode->i_mode))
>  			return 0;
>  	} else {
> -		if (!S_ISCHR(stat->mode))
> +		if (!S_ISCHR(inode->i_mode))
>  			return 0;
>  	}
> -	if (stat->rdev != dev->devt)
> +	if (inode->i_rdev != dev->devt)
>  		return 0;
>  
>  	/* ours */
> @@ -321,8 +321,7 @@ static int handle_remove(const char *nodename, struct device *dev)
>  {
>  	struct path parent;
>  	struct dentry *dentry;
> -	struct kstat stat;
> -	struct path p;
> +	struct inode *inode;
>  	int deleted = 0;
>  	int err;
>  
> @@ -330,11 +329,8 @@ static int handle_remove(const char *nodename, struct device *dev)
>  	if (IS_ERR(dentry))
>  		return PTR_ERR(dentry);
>  
> -	p.mnt = parent.mnt;
> -	p.dentry = dentry;
> -	err = vfs_getattr(&p, &stat, STATX_TYPE | STATX_MODE,
> -			  AT_STATX_SYNC_AS_STAT);
> -	if (!err && dev_mynode(dev, d_inode(dentry), &stat)) {
> +	inode = d_inode(dentry);
> +	if (dev_mynode(dev, inode)) {
>  		struct iattr newattrs;
>  		/*
>  		 * before unlinking this node, reset permissions
> @@ -342,7 +338,7 @@ static int handle_remove(const char *nodename, struct device *dev)
>  		 */
>  		newattrs.ia_uid = GLOBAL_ROOT_UID;
>  		newattrs.ia_gid = GLOBAL_ROOT_GID;
> -		newattrs.ia_mode = stat.mode & ~0777;
> +		newattrs.ia_mode = inode->i_mode & ~0777;
>  		newattrs.ia_valid =
>  			ATTR_UID|ATTR_GID|ATTR_MODE;
>  		inode_lock(d_inode(dentry));


Thank you for the quick fix, this fixes blk devices hang issues on AMD
EPYC systems as well.

Tested-by: Ayush Jain <Ayush.jain3@amd.com>

Thanks,
Ayush

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[Shinichiro Kawasaki]

On Apr 22, 2025 / 07:51, hch wrote:
> On Tue, Apr 22, 2025 at 05:03:19AM +0000, Shinichiro Kawasaki wrote:
> > I ran blktests with the kernel v6.15-rc3, and found the test case md/001 hangs.
> > The hang is recreated in stable manner. I bisected and found this patch as the
> > commit 777d0961ff95 is the trigger. When I revert the commit from v6.15-rc3
> > kernel, the hang disappeared.
> > 
> > Actions for fix will be appreciated.
> > 
> > FYI, the kernel INFO messages recorded functions relevant to the trigger commit,
> > such as bdev_statx or vfs_getattr_nosec [1].
> 
> This should fix it:
> 
> diff --git a/block/bdev.c b/block/bdev.c
> index 6a34179192c9..97d4c0ab1670 100644
> --- a/block/bdev.c
> +++ b/block/bdev.c
> @@ -1274,18 +1274,23 @@ void sync_bdevs(bool wait)
>   */
>  void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
>  {
> -	struct inode *backing_inode;
>  	struct block_device *bdev;
>  
> -	backing_inode = d_backing_inode(path->dentry);
> -
>  	/*
> -	 * Note that backing_inode is the inode of a block device node file,
> -	 * not the block device's internal inode.  Therefore it is *not* valid
> -	 * to use I_BDEV() here; the block device has to be looked up by i_rdev
> -	 * instead.
> +	 * Note that d_backing_inode() returnsthe inode of a block device node
> +	 * file, not the block device's internal inode.
> +	 *
> +	 * Therefore it is *not* valid to use I_BDEV() here; the block device
> +	 * has to be looked up by i_rdev instead.
> +	 *
> +	 * Only do this lookup if actually needed to avoid the performance
> +	 * overhead of the lookup, and to avoid injecting bdev lifetime issues
> +	 * into devtmpfs.
>  	 */
> -	bdev = blkdev_get_no_open(backing_inode->i_rdev);
> +	if (!(request_mask & (STATX_DIOALIGN | STATX_WRITE_ATOMIC)))
> +		return;
> +
> +	bdev = blkdev_get_no_open(d_backing_inode(path->dentry)->i_rdev);
>  	if (!bdev)
>  		return;
>  

Thank you for the quick action. I applied this patch on top of the kernel
v6.15-rc3 and confirmed the hang disappears. Good. I also ran whole blktests
and observed no other regression. When you post the fix as a formal patch,
feel free to add,

Tested-by: Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com>

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[Heiko Carstens]

On Tue, Apr 22, 2025 at 07:51:49AM +0200, hch wrote:
> On Tue, Apr 22, 2025 at 05:03:19AM +0000, Shinichiro Kawasaki wrote:
> > I ran blktests with the kernel v6.15-rc3, and found the test case md/001 hangs.
> > The hang is recreated in stable manner. I bisected and found this patch as the
> > commit 777d0961ff95 is the trigger. When I revert the commit from v6.15-rc3
> > kernel, the hang disappeared.
> > 
> > Actions for fix will be appreciated.
> > 
> > FYI, the kernel INFO messages recorded functions relevant to the trigger commit,
> > such as bdev_statx or vfs_getattr_nosec [1].
> 
> This should fix it:

FWIW, I was also about to report the same problem. Any reboot with dm
targets hangs. Your patch fixes it also for me.

Tested-by: Heiko Carstens <hca@linux.ibm.com>

Re: [PATCH] fs: move the bdex_statx call to vfs_getattr_nosec

[Christoph Hellwig]

On Fri, Apr 25, 2025 at 03:32:55PM +0200, Nikita Dubrovskii wrote:
> Hi all,
> 
> We're seeing a boot failure on first boot in Fedora CoreOS starting
> with kernel 6.15.0-0.rc3, tracked here:
> https://github.com/coreos/fedora-coreos-tracker/issues/1936

This should be fixed by:

https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git/commit/?h=vfs.fixes&id=e079d7c4db5cba1e8a315dc93030dfb6c7b49459