[PATCH 0/5] btrfs: zoned: fix two deadlocks when runing fstests

[PATCH 0/5] btrfs: zoned: fix two deadlocks when runing fstests

[Johannes Thumshirn]

Fix two distinct deadlocks when running fstests:

- The first one happens primarily (but not only) on btrfs/187, Where we
  bunch up dirty metadata and then on umount try to submit it. But at
  the time of submission the block-group is already gone leaving stale
  folios behind that can't be written out. Patch 2 fixes it by clearing
  the stale buffers, patch 1 is a preparation patch for patch 2.

- The second one is a double lock of fs_info->ordered_extent_mutex when
  doing zone finish. It creates a copy of the roots that need to be
  finished and then does an unlocked version of
  btrfs_wait_ordered_roots() btrfs_wait_ordered_roots_nolock() on that
  copy.

For both issues debugging was done LLM assisted and I attributed it in
the respective fix patches (not the prep patches).

Johannes Thumshirn (5):
  btrfs: factor out dirty-clearing part of btrfs_clear_buffer_dirty
  btrfs: zoned: drop stale dirty btree buffers at close_ctree()
  btrfs: walk waited ordered extents in place
  btrfs: walk waited ordered roots in place
  btrfs: zoned: avoid ordered_operations_mutex when finishing a zone

 fs/btrfs/disk-io.c      |   4 ++
 fs/btrfs/extent_io.c    |  57 ++++++++++++------
 fs/btrfs/extent_io.h    |   1 +
 fs/btrfs/ordered-data.c | 129 +++++++++++++++++++++++++++++++++-------
 fs/btrfs/ordered-data.h |   4 ++
 fs/btrfs/zoned.c        |   8 ++-
 6 files changed, 159 insertions(+), 44 deletions(-)

-- 
2.54.0

[PATCH 1/5] btrfs: factor out dirty-clearing part of btrfs_clear_buffer_dirty

[Johannes Thumshirn]

Split the dirty-clearing part of btrfs_clear_buffer_dirty() into a helper so
a later caller can reuse it while bypassing the zoned zero-out deferral.
No functional change.

Signed-off-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
---
 fs/btrfs/extent_io.c | 47 +++++++++++++++++++++++++-------------------
 1 file changed, 27 insertions(+), 20 deletions(-)

diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 647b109ef61b..4dd8fc5ad247 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -3788,6 +3788,32 @@ void free_extent_buffer_stale(struct extent_buffer *eb)
 	release_extent_buffer(eb);
 }
 
+static void __btrfs_clear_buffer_dirty(struct extent_buffer *eb)
+{
+	struct btrfs_fs_info *fs_info = eb->fs_info;
+
+	if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags))
+		return;
+
+	buffer_tree_clear_mark(eb, PAGECACHE_TAG_DIRTY);
+	percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -(s64)eb->len,
+				 fs_info->dirty_metadata_batch);
+
+	for (int i = 0; i < num_extent_folios(eb); i++) {
+		struct folio *folio = eb->folios[i];
+		bool last;
+
+		if (!folio_test_dirty(folio))
+			continue;
+		folio_lock(folio);
+		last = btrfs_meta_folio_clear_and_test_dirty(folio, eb);
+		if (last)
+			btrfs_clear_folio_dirty_tag(folio);
+		folio_unlock(folio);
+	}
+	WARN_ON(refcount_read(&eb->refs) == 0);
+}
+
 void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
 			      struct extent_buffer *eb)
 {
@@ -3812,26 +3838,7 @@ void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
 		return;
 	}
 
-	if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags))
-		return;
-
-	buffer_tree_clear_mark(eb, PAGECACHE_TAG_DIRTY);
-	percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -(s64)eb->len,
-				 fs_info->dirty_metadata_batch);
-
-	for (int i = 0; i < num_extent_folios(eb); i++) {
-		struct folio *folio = eb->folios[i];
-		bool last;
-
-		if (!folio_test_dirty(folio))
-			continue;
-		folio_lock(folio);
-		last = btrfs_meta_folio_clear_and_test_dirty(folio, eb);
-		if (last)
-			btrfs_clear_folio_dirty_tag(folio);
-		folio_unlock(folio);
-	}
-	WARN_ON(refcount_read(&eb->refs) == 0);
+	__btrfs_clear_buffer_dirty(eb);
 }
 
 void set_extent_buffer_dirty(struct extent_buffer *eb)
-- 
2.54.0

Re: [PATCH 1/5] btrfs: factor out dirty-clearing part of btrfs_clear_buffer_dirty

[Qu Wenruo]

在 2026/7/16 21:26, Johannes Thumshirn 写道:
> Split the dirty-clearing part of btrfs_clear_buffer_dirty() into a helper so
> a later caller can reuse it while bypassing the zoned zero-out deferral.
> No functional change.

I'd prefer fold this patch into the user.

> 
> Signed-off-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
> ---
>   fs/btrfs/extent_io.c | 47 +++++++++++++++++++++++++-------------------
>   1 file changed, 27 insertions(+), 20 deletions(-)
> 
> diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
> index 647b109ef61b..4dd8fc5ad247 100644
> --- a/fs/btrfs/extent_io.c
> +++ b/fs/btrfs/extent_io.c
> @@ -3788,6 +3788,32 @@ void free_extent_buffer_stale(struct extent_buffer *eb)
>   	release_extent_buffer(eb);
>   }
>   
> +static void __btrfs_clear_buffer_dirty(struct extent_buffer *eb)

Not sure if we still allow double underline function names.

Thanks,
Qu

> +{
> +	struct btrfs_fs_info *fs_info = eb->fs_info;
> +
> +	if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags))
> +		return;
> +
> +	buffer_tree_clear_mark(eb, PAGECACHE_TAG_DIRTY);
> +	percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -(s64)eb->len,
> +				 fs_info->dirty_metadata_batch);
> +
> +	for (int i = 0; i < num_extent_folios(eb); i++) {
> +		struct folio *folio = eb->folios[i];
> +		bool last;
> +
> +		if (!folio_test_dirty(folio))
> +			continue;
> +		folio_lock(folio);
> +		last = btrfs_meta_folio_clear_and_test_dirty(folio, eb);
> +		if (last)
> +			btrfs_clear_folio_dirty_tag(folio);
> +		folio_unlock(folio);
> +	}
> +	WARN_ON(refcount_read(&eb->refs) == 0);
> +}
> +
>   void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
>   			      struct extent_buffer *eb)
>   {
> @@ -3812,26 +3838,7 @@ void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
>   		return;
>   	}
>   
> -	if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags))
> -		return;
> -
> -	buffer_tree_clear_mark(eb, PAGECACHE_TAG_DIRTY);
> -	percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -(s64)eb->len,
> -				 fs_info->dirty_metadata_batch);
> -
> -	for (int i = 0; i < num_extent_folios(eb); i++) {
> -		struct folio *folio = eb->folios[i];
> -		bool last;
> -
> -		if (!folio_test_dirty(folio))
> -			continue;
> -		folio_lock(folio);
> -		last = btrfs_meta_folio_clear_and_test_dirty(folio, eb);
> -		if (last)
> -			btrfs_clear_folio_dirty_tag(folio);
> -		folio_unlock(folio);
> -	}
> -	WARN_ON(refcount_read(&eb->refs) == 0);
> +	__btrfs_clear_buffer_dirty(eb);
>   }
>   
>   void set_extent_buffer_dirty(struct extent_buffer *eb)

[PATCH 2/5] btrfs: zoned: drop stale dirty btree buffers at close_ctree()

[Johannes Thumshirn]

On a zoned filesystem btrfs_clear_buffer_dirty() keeps a freed-but-dirty
tree block dirty (EXTENT_BUFFER_ZONED_ZEROOUT) so its zero-out keeps the
zone write pointer moving. A block left ahead of the block group's
meta_write_pointer (e.g. a tree-log block freed before being written) can
never be written by btree_writepages() and survives to unmount.

close_ctree() stops the endio workqueues before the final iput() of the
btree inode but frees the block groups only afterwards. Once the block
group is gone btrfs_check_meta_write_pointer() no longer defers the buffer,
so the iput() submits it for writeback; its completion can no longer be
queued on the destroyed endio_meta_workers and umount hangs on writeback
that never finishes.

btrfs_free_block_groups() has to stay after btrfs_stop_all_workers()
(see also commit 5cdd7db6c5c9), so instead drop the dirty state of such
stale buffers in invalidate_and_check_btree_folios(), which still runs
while the workqueues are alive.

Assisted-by: LLM (debugging, commit message)
Signed-off-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
---
 fs/btrfs/disk-io.c   |  4 ++++
 fs/btrfs/extent_io.c | 12 ++++++++++++
 fs/btrfs/extent_io.h |  1 +
 3 files changed, 17 insertions(+)

diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index acff40469589..b66525701b33 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -3317,6 +3317,10 @@ static void invalidate_and_check_btree_folios(struct btrfs_fs_info *fs_info)
 		if (test_bit(EXTENT_BUFFER_READING, &eb->bflags))
 			wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING,
 				       TASK_UNINTERRUPTIBLE);
+
+		if (btrfs_is_zoned(fs_info) &&
+		    test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags))
+			btrfs_clear_stale_buffer_dirty(eb);
 		/*
 		 * The refs threshold is 2, one held by us at the beginning
 		 * of the loop, one for the ownership in the buffer tree.
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 4dd8fc5ad247..480d4ec8ca7f 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -3841,6 +3841,18 @@ void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
 	__btrfs_clear_buffer_dirty(eb);
 }
 
+/*
+ * Drop the dirty state of a stale, already-freed metadata buffer, bypassing
+ * the zoned EXTENT_BUFFER_ZONED_ZEROOUT deferral of btrfs_clear_buffer_dirty().
+ */
+void btrfs_clear_stale_buffer_dirty(struct extent_buffer *eb)
+{
+	btrfs_tree_lock(eb);
+	clear_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags);
+	__btrfs_clear_buffer_dirty(eb);
+	btrfs_tree_unlock(eb);
+}
+
 void set_extent_buffer_dirty(struct extent_buffer *eb)
 {
 	bool was_dirty;
diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h
index 869925337699..290d6cff6ddb 100644
--- a/fs/btrfs/extent_io.h
+++ b/fs/btrfs/extent_io.h
@@ -393,6 +393,7 @@ void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
 				  u32 bits_to_clear, unsigned long page_ops);
 void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
 			      struct extent_buffer *buf);
+void btrfs_clear_stale_buffer_dirty(struct extent_buffer *eb);
 
 static inline void btrfs_clear_folio_dirty_tag(struct folio *folio)
 {
-- 
2.54.0

Re: [PATCH 2/5] btrfs: zoned: drop stale dirty btree buffers at close_ctree()

[Qu Wenruo]

在 2026/7/16 21:26, Johannes Thumshirn 写道:
> On a zoned filesystem btrfs_clear_buffer_dirty() keeps a freed-but-dirty
> tree block dirty (EXTENT_BUFFER_ZONED_ZEROOUT) so its zero-out keeps the
> zone write pointer moving. A block left ahead of the block group's
> meta_write_pointer (e.g. a tree-log block freed before being written) can
> never be written by btree_writepages() and survives to unmount.
> 
> close_ctree() stops the endio workqueues before the final iput() of the
> btree inode but frees the block groups only afterwards. Once the block
> group is gone btrfs_check_meta_write_pointer() no longer defers the buffer,
> so the iput() submits it for writeback; its completion can no longer be
> queued on the destroyed endio_meta_workers and umount hangs on writeback
> that never finishes.

Can we just trigger a btree inode write back for zoned cases?

That sounds much simpler and less hacky.

Thanks,
Qu
> 
> btrfs_free_block_groups() has to stay after btrfs_stop_all_workers()
> (see also commit 5cdd7db6c5c9), so instead drop the dirty state of such
> stale buffers in invalidate_and_check_btree_folios(), which still runs
> while the workqueues are alive.
> 
> Assisted-by: LLM (debugging, commit message)
> Signed-off-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
> ---
>   fs/btrfs/disk-io.c   |  4 ++++
>   fs/btrfs/extent_io.c | 12 ++++++++++++
>   fs/btrfs/extent_io.h |  1 +
>   3 files changed, 17 insertions(+)
> 
> diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
> index acff40469589..b66525701b33 100644
> --- a/fs/btrfs/disk-io.c
> +++ b/fs/btrfs/disk-io.c
> @@ -3317,6 +3317,10 @@ static void invalidate_and_check_btree_folios(struct btrfs_fs_info *fs_info)
>   		if (test_bit(EXTENT_BUFFER_READING, &eb->bflags))
>   			wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING,
>   				       TASK_UNINTERRUPTIBLE);
> +
> +		if (btrfs_is_zoned(fs_info) &&
> +		    test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags))
> +			btrfs_clear_stale_buffer_dirty(eb);
>   		/*
>   		 * The refs threshold is 2, one held by us at the beginning
>   		 * of the loop, one for the ownership in the buffer tree.
> diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
> index 4dd8fc5ad247..480d4ec8ca7f 100644
> --- a/fs/btrfs/extent_io.c
> +++ b/fs/btrfs/extent_io.c
> @@ -3841,6 +3841,18 @@ void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
>   	__btrfs_clear_buffer_dirty(eb);
>   }
>   
> +/*
> + * Drop the dirty state of a stale, already-freed metadata buffer, bypassing
> + * the zoned EXTENT_BUFFER_ZONED_ZEROOUT deferral of btrfs_clear_buffer_dirty().
> + */
> +void btrfs_clear_stale_buffer_dirty(struct extent_buffer *eb)
> +{
> +	btrfs_tree_lock(eb);
> +	clear_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags);
> +	__btrfs_clear_buffer_dirty(eb);
> +	btrfs_tree_unlock(eb);
> +}
> +
>   void set_extent_buffer_dirty(struct extent_buffer *eb)
>   {
>   	bool was_dirty;
> diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h
> index 869925337699..290d6cff6ddb 100644
> --- a/fs/btrfs/extent_io.h
> +++ b/fs/btrfs/extent_io.h
> @@ -393,6 +393,7 @@ void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
>   				  u32 bits_to_clear, unsigned long page_ops);
>   void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
>   			      struct extent_buffer *buf);
> +void btrfs_clear_stale_buffer_dirty(struct extent_buffer *eb);
>   
>   static inline void btrfs_clear_folio_dirty_tag(struct folio *folio)
>   {

Re: [PATCH 2/5] btrfs: zoned: drop stale dirty btree buffers at close_ctree()

[Johannes Thumshirn]

On 17.07.26 00:49, Qu Wenruo wrote:
> 在 2026/7/16 21:26, Johannes Thumshirn 写道:
>> On a zoned filesystem btrfs_clear_buffer_dirty() keeps a freed-but-dirty
>> tree block dirty (EXTENT_BUFFER_ZONED_ZEROOUT) so its zero-out keeps the
>> zone write pointer moving. A block left ahead of the block group's
>> meta_write_pointer (e.g. a tree-log block freed before being written) can
>> never be written by btree_writepages() and survives to unmount.
>>
>> close_ctree() stops the endio workqueues before the final iput() of the
>> btree inode but frees the block groups only afterwards. Once the block
>> group is gone btrfs_check_meta_write_pointer() no longer defers the 
>> buffer,
>> so the iput() submits it for writeback; its completion can no longer be
>> queued on the destroyed endio_meta_workers and umount hangs on writeback
>> that never finishes.
> 
> Can we just trigger a btree inode write back for zoned cases?
> 
> That sounds much simpler and less hacky.

Let me see what I can do.

Re: [PATCH 2/5] btrfs: zoned: drop stale dirty btree buffers at close_ctree()

[Johannes Thumshirn]

On 17.07.26 00:49, Qu Wenruo wrote:
> 
> 
> 在 2026/7/16 21:26, Johannes Thumshirn 写道:
>> On a zoned filesystem btrfs_clear_buffer_dirty() keeps a freed-but-dirty
>> tree block dirty (EXTENT_BUFFER_ZONED_ZEROOUT) so its zero-out keeps the
>> zone write pointer moving. A block left ahead of the block group's
>> meta_write_pointer (e.g. a tree-log block freed before being written) can
>> never be written by btree_writepages() and survives to unmount.
>>
>> close_ctree() stops the endio workqueues before the final iput() of the
>> btree inode but frees the block groups only afterwards. Once the block
>> group is gone btrfs_check_meta_write_pointer() no longer defers the 
>> buffer,
>> so the iput() submits it for writeback; its completion can no longer be
>> queued on the destroyed endio_meta_workers and umount hangs on writeback
>> that never finishes.
> 
> Can we just trigger a btree inode write back for zoned cases?
> 
> That sounds much simpler and less hacky.
I tried freeing the stale ebs in btrfs_reset_unused_block_groups() 
before actually resetting the bg, but then the WARN() in 
invalidate_and_check_btree_folios() triggered again. So that doesn't 
work sorry.

Re: [PATCH 2/5] btrfs: zoned: drop stale dirty btree buffers at close_ctree()

[Qu Wenruo]

在 2026/7/17 23:47, Johannes Thumshirn 写道:
> On 17.07.26 00:49, Qu Wenruo wrote:
>>
>>
>> 在 2026/7/16 21:26, Johannes Thumshirn 写道:
>>> On a zoned filesystem btrfs_clear_buffer_dirty() keeps a freed-but-dirty
>>> tree block dirty (EXTENT_BUFFER_ZONED_ZEROOUT) so its zero-out keeps the
>>> zone write pointer moving. A block left ahead of the block group's
>>> meta_write_pointer (e.g. a tree-log block freed before being written) 
>>> can
>>> never be written by btree_writepages() and survives to unmount.
>>>
>>> close_ctree() stops the endio workqueues before the final iput() of the
>>> btree inode but frees the block groups only afterwards. Once the block
>>> group is gone btrfs_check_meta_write_pointer() no longer defers the 
>>> buffer,
>>> so the iput() submits it for writeback; its completion can no longer be
>>> queued on the destroyed endio_meta_workers and umount hangs on writeback
>>> that never finishes.
>>
>> Can we just trigger a btree inode write back for zoned cases?
>>
>> That sounds much simpler and less hacky.
> I tried freeing the stale ebs in btrfs_reset_unused_block_groups() 
> before actually resetting the bg, but then the WARN() in 
> invalidate_and_check_btree_folios() triggered again. So that doesn't 
> work sorry.
> 

Sorry I'm not familiar with zoned metadata writeback.
But it still doesn't sound correctly to me.

To me, this seems to be a problem that btrfs_check_meta_write_pointer() 
can skip some ebs from being written back, and in that case those ebs 
should be properly handled (e.g. clearing dirty etc) but that's not the 
case.

Yes, btrfs_clear_buffer_dirty() will not clear the EXTENT_BUFFER_DIRTY 
flag, but only set EXTENT_BUFFER_ZEROOUT flag.

But still that dirty eb will go through writeback, as they still have 
the folio dirty flag and page cache tags, being properly submitted and 
got its EXTENT_BUFFER_DIRTY cleared, in the regular 
lock_extent_buffer_for_io() path, then go through the write_one_eb() to 
be submitted.


However if btrfs_check_meta_write_pointer() returns non-zero values, we 
skip the writeback for that eb completely, so that eb stays dirty forever.

Shouldn't we do something to clean up those ebs in the first place?

So to me, this seems to be a problem in the zoned metadata writeback 
behavior, just being exposed by the newly introduced sanity checks.

Thanks,
Qu

[PATCH 3/5] btrfs: walk waited ordered extents in place

[Johannes Thumshirn]

btrfs_wait_ordered_extents() splices root->ordered_extents onto a private
list while it flushes and waits, making the ordered extents temporarily
invisible on the root. Walk the list in place instead, moving each entry to
the tail as it is processed and guarding against re-queueing with the new
BTRFS_ORDERED_FLUSHING flag.

No functional change for current callers; this lets a future lock-free
waiter observe all in-flight ordered extents.

Signed-off-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
---
 fs/btrfs/ordered-data.c | 35 +++++++++++++++++++++--------------
 fs/btrfs/ordered-data.h |  2 ++
 2 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c
index b32d4eabe0ab..11aa7e8efe04 100644
--- a/fs/btrfs/ordered-data.c
+++ b/fs/btrfs/ordered-data.c
@@ -751,6 +751,13 @@ static void btrfs_run_ordered_extent_work(struct btrfs_work *work)
 	complete(&ordered->completion);
 }
 
+static bool ordered_in_range(const struct btrfs_ordered_extent *ordered,
+			     u64 range_start, u64 range_end)
+{
+	return !(range_end <= ordered->disk_bytenr ||
+		 ordered->disk_bytenr + ordered->disk_num_bytes <= range_start);
+}
+
 /*
  * Wait for all the ordered extents in a root. Use @bg as range or do whole
  * range if it's NULL.
@@ -759,13 +766,12 @@ u64 btrfs_wait_ordered_extents(struct btrfs_root *root, u64 nr,
 			       const struct btrfs_block_group *bg)
 {
 	struct btrfs_fs_info *fs_info = root->fs_info;
-	LIST_HEAD(splice);
-	LIST_HEAD(skipped);
 	LIST_HEAD(works);
 	struct btrfs_ordered_extent *ordered, *next;
 	u64 count = 0;
 	u64 range_start, range_len;
 	u64 range_end;
+	u64 remaining;
 
 	if (bg) {
 		range_start = bg->start;
@@ -778,20 +784,22 @@ u64 btrfs_wait_ordered_extents(struct btrfs_root *root, u64 nr,
 
 	mutex_lock(&root->ordered_extent_mutex);
 	spin_lock(&root->ordered_extent_lock);
-	list_splice_init(&root->ordered_extents, &splice);
-	while (!list_empty(&splice) && nr) {
-		ordered = list_first_entry(&splice, struct btrfs_ordered_extent,
+	remaining = root->nr_ordered_extents;
+	while (remaining && nr && !list_empty(&root->ordered_extents)) {
+		remaining--;
+		ordered = list_first_entry(&root->ordered_extents,
+					   struct btrfs_ordered_extent,
 					   root_extent_list);
+		list_move_tail(&ordered->root_extent_list,
+			       &root->ordered_extents);
 
-		if (range_end <= ordered->disk_bytenr ||
-		    ordered->disk_bytenr + ordered->disk_num_bytes <= range_start) {
-			list_move_tail(&ordered->root_extent_list, &skipped);
-			cond_resched_lock(&root->ordered_extent_lock);
+		if (!ordered_in_range(ordered, range_start, range_end))
+			continue;
+
+		/* Already queued to 'works' by this call, skip on revisit. */
+		if (test_and_set_bit(BTRFS_ORDERED_FLUSHING, &ordered->flags))
 			continue;
-		}
 
-		list_move_tail(&ordered->root_extent_list,
-			       &root->ordered_extents);
 		refcount_inc(&ordered->refs);
 		spin_unlock(&root->ordered_extent_lock);
 
@@ -806,13 +814,12 @@ u64 btrfs_wait_ordered_extents(struct btrfs_root *root, u64 nr,
 		count++;
 		spin_lock(&root->ordered_extent_lock);
 	}
-	list_splice_tail(&skipped, &root->ordered_extents);
-	list_splice_tail(&splice, &root->ordered_extents);
 	spin_unlock(&root->ordered_extent_lock);
 
 	list_for_each_entry_safe(ordered, next, &works, work_list) {
 		list_del_init(&ordered->work_list);
 		wait_for_completion(&ordered->completion);
+		clear_bit(BTRFS_ORDERED_FLUSHING, &ordered->flags);
 		btrfs_put_ordered_extent(ordered);
 		cond_resched();
 	}
diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h
index 8d5d5ba1e02f..f4d0675bb9c3 100644
--- a/fs/btrfs/ordered-data.h
+++ b/fs/btrfs/ordered-data.h
@@ -63,6 +63,8 @@ enum {
 	BTRFS_ORDERED_LOGGED_CSUM,
 	/* We wait for this extent to complete in the current transaction. */
 	BTRFS_ORDERED_PENDING,
+	/* Flush work is queued by btrfs_wait_ordered_extents(), avoid requeue. */
+	BTRFS_ORDERED_FLUSHING,
 
 	/*
 	 * Different types for ordered extents, one and only one of these types
-- 
2.54.0

[PATCH 4/5] btrfs: walk waited ordered roots in place

[Johannes Thumshirn]

Like the previous change for btrfs_wait_ordered_extents(), walk
fs_info->ordered_roots in place instead of splicing it onto a private list,
so the roots with pending ordered extents stay visible while we wait.

No functional change intended.

Signed-off-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
---
 fs/btrfs/ordered-data.c | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c
index 11aa7e8efe04..ddd21aa5c640 100644
--- a/fs/btrfs/ordered-data.c
+++ b/fs/btrfs/ordered-data.c
@@ -836,19 +836,22 @@ void btrfs_wait_ordered_roots(struct btrfs_fs_info *fs_info, u64 nr,
 			      const struct btrfs_block_group *bg)
 {
 	struct btrfs_root *root;
-	LIST_HEAD(splice);
 	u64 done;
+	u64 remaining;
 
 	mutex_lock(&fs_info->ordered_operations_mutex);
 	spin_lock(&fs_info->ordered_root_lock);
-	list_splice_init(&fs_info->ordered_roots, &splice);
-	while (!list_empty(&splice) && nr) {
-		root = list_first_entry(&splice, struct btrfs_root,
-					ordered_root);
+	remaining = 0;
+	list_for_each_entry(root, &fs_info->ordered_roots, ordered_root)
+		remaining++;
+	while (remaining && nr && !list_empty(&fs_info->ordered_roots)) {
+		remaining--;
+		root = list_first_entry(&fs_info->ordered_roots,
+					struct btrfs_root, ordered_root);
+		list_move_tail(&root->ordered_root, &fs_info->ordered_roots);
 		root = btrfs_grab_root(root);
-		BUG_ON(!root);
-		list_move_tail(&root->ordered_root,
-			       &fs_info->ordered_roots);
+		if (!root)
+			continue;
 		spin_unlock(&fs_info->ordered_root_lock);
 
 		done = btrfs_wait_ordered_extents(root, nr, bg);
@@ -859,7 +862,6 @@ void btrfs_wait_ordered_roots(struct btrfs_fs_info *fs_info, u64 nr,
 
 		spin_lock(&fs_info->ordered_root_lock);
 	}
-	list_splice_tail(&splice, &fs_info->ordered_roots);
 	spin_unlock(&fs_info->ordered_root_lock);
 	mutex_unlock(&fs_info->ordered_operations_mutex);
 }
-- 
2.54.0

[PATCH 5/5] btrfs: zoned: avoid ordered_operations_mutex when finishing a zone

[Johannes Thumshirn]

On a zoned filesystem with no free active zones, a data allocation finishes a
block group to make room:

  cow_file_range -> btrfs_reserve_extent -> find_free_extent ->
  btrfs_zone_finish_one_bg -> do_zone_finish ->
  btrfs_wait_ordered_roots -> mutex_lock(&fs_info->ordered_operations_mutex)

This can be reached from an ordered extent flush worker
(btrfs_run_ordered_extent_work) which is run and waited for by
btrfs_wait_ordered_roots() while it holds ordered_operations_mutex:

  btrfs_async_reclaim_metadata_space -> flush_space ->
  btrfs_wait_ordered_roots -> mutex_lock(&fs_info->ordered_operations_mutex)
                           -> wait for the flush work to complete

The flush worker then blocks on the same, non-recursive mutex the reclaim
task holds while waiting for that very worker, so the filesystem deadlocks.
It was observed as a hang in generic/048:

  INFO: task kworker/u36:8 blocked for more than 120 seconds.
   btrfs_wait_ordered_roots -> flush_space -> btrfs_async_reclaim_metadata_space
  INFO: task kworker/u34:6 is blocked on a mutex likely owned by kworker/u36:8
   btrfs_wait_ordered_roots -> do_zone_finish -> btrfs_zone_finish_one_bg ->
   find_free_extent -> cow_file_range

The cycle has existed since data block group allocation started finishing
zones (commit 393f646e34c1 ("btrfs: zoned: finish least available block
group on data bg allocation")).

Add btrfs_wait_ordered_roots_nolock(), which waits for a block group's
ordered extents using the multi-waiter safe ordered->wait bit and never
takes ordered_operations_mutex or ordered_extent_mutex, and use it from
do_zone_finish(). The block group is already read-only there, so no new
ordered extents can appear in its range while we wait. This relies on the
preceding two patches keeping the ordered extent and root lists visible
(they no longer splice the lists away) so the walk cannot miss an in-flight
ordered extent.

Assisted-by: LLM (debugging, commit message)
Signed-off-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
---
 fs/btrfs/ordered-data.c | 74 +++++++++++++++++++++++++++++++++++++++++
 fs/btrfs/ordered-data.h |  2 ++
 fs/btrfs/zoned.c        |  8 +++--
 3 files changed, 82 insertions(+), 2 deletions(-)

diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c
index ddd21aa5c640..12102fc76696 100644
--- a/fs/btrfs/ordered-data.c
+++ b/fs/btrfs/ordered-data.c
@@ -866,6 +866,80 @@ void btrfs_wait_ordered_roots(struct btrfs_fs_info *fs_info, u64 nr,
 	mutex_unlock(&fs_info->ordered_operations_mutex);
 }
 
+/*
+ * Wait for all ordered extents intersecting @bg to finish without taking
+ * ordered_operations_mutex or ordered_extent_mutex, so it is safe to call from
+ * the zone-finish path (which can run inside the ordered extent flush worker
+ * that a btrfs_wait_ordered_roots() caller is waiting for). @bg must be
+ * read-only, so its set of ordered extents only shrinks while we wait.
+ */
+void btrfs_wait_ordered_roots_nolock(struct btrfs_fs_info *fs_info,
+				     const struct btrfs_block_group *bg)
+{
+	const u64 range_start = bg->start;
+	const u64 range_end = bg->start + bg->length;
+	struct btrfs_root **roots;
+	struct btrfs_root *root;
+	int nr_roots = 0;
+	int cap = 0;
+	int i;
+
+	spin_lock(&fs_info->ordered_root_lock);
+	cap = list_count_nodes(&fs_info->ordered_roots);
+	spin_unlock(&fs_info->ordered_root_lock);
+	if (!cap)
+		return;
+
+	roots = kvmalloc_array(cap, sizeof(*roots), GFP_NOFS);
+	if (!roots) {
+		btrfs_wait_ordered_roots(fs_info, U64_MAX, bg);
+		return;
+	}
+
+	spin_lock(&fs_info->ordered_root_lock);
+	list_for_each_entry(root, &fs_info->ordered_roots, ordered_root) {
+		if (nr_roots >= cap)
+			break;
+		if (!btrfs_grab_root(root))
+			continue;
+		roots[nr_roots++] = root;
+	}
+	spin_unlock(&fs_info->ordered_root_lock);
+
+	for (i = 0; i < nr_roots; i++) {
+		root = roots[i];
+
+		while (true) {
+			struct btrfs_ordered_extent *found = NULL;
+			struct btrfs_ordered_extent *ordered;
+
+			spin_lock(&root->ordered_extent_lock);
+			list_for_each_entry(ordered, &root->ordered_extents,
+					    root_extent_list) {
+				if (test_bit(BTRFS_ORDERED_COMPLETE,
+					     &ordered->flags))
+					continue;
+				if (!ordered_in_range(ordered, range_start,
+						      range_end))
+					continue;
+				found = ordered;
+				refcount_inc(&found->refs);
+				break;
+			}
+			spin_unlock(&root->ordered_extent_lock);
+
+			if (!found)
+				break;
+
+			btrfs_start_ordered_extent(found);
+			btrfs_put_ordered_extent(found);
+			cond_resched();
+		}
+		btrfs_put_root(root);
+	}
+	kvfree(roots);
+}
+
 /*
  * Start IO and wait for a given ordered extent to finish.
  *
diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h
index f4d0675bb9c3..d8280a7bccb1 100644
--- a/fs/btrfs/ordered-data.h
+++ b/fs/btrfs/ordered-data.h
@@ -220,6 +220,8 @@ u64 btrfs_wait_ordered_extents(struct btrfs_root *root, u64 nr,
 			       const struct btrfs_block_group *bg);
 void btrfs_wait_ordered_roots(struct btrfs_fs_info *fs_info, u64 nr,
 			      const struct btrfs_block_group *bg);
+void btrfs_wait_ordered_roots_nolock(struct btrfs_fs_info *fs_info,
+				     const struct btrfs_block_group *bg);
 void btrfs_lock_and_flush_ordered_range(struct btrfs_inode *inode, u64 start,
 					u64 end,
 					struct extent_state **cached_state);
diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c
index a016cb471beb..3a3ef96dcc8b 100644
--- a/fs/btrfs/zoned.c
+++ b/fs/btrfs/zoned.c
@@ -2585,8 +2585,12 @@ static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_writ
 
 		/* Ensure all writes in this block group finish */
 		btrfs_wait_block_group_reservations(block_group);
-		/* No need to wait for NOCOW writers. Zoned mode does not allow that */
-		btrfs_wait_ordered_roots(fs_info, U64_MAX, block_group);
+		/*
+		 * No need to wait for NOCOW writers. Zoned mode does not allow
+		 * that. Use the lock-free variant as we can be called from the
+		 * allocator inside an ordered extent flush worker.
+		 */
+		btrfs_wait_ordered_roots_nolock(fs_info, block_group);
 		/* Wait for extent buffers to be written. */
 		if (is_metadata)
 			wait_eb_writebacks(block_group);
-- 
2.54.0