[PATCH blktests] tests/md: add test 005 for raid1 deadlock under I/O, limit changes and recovery

[PATCH blktests] tests/md: add test 005 for raid1 deadlock under I/O, limit changes and recovery

[Jack Wang]

From: Jack Wang <jinpu.wang@cloud.ionos.com>

Add a regression test for a deadlock reported against raid1 with an
internal bitmap, where concurrent I/O, queue limit updates
(max_sectors_kb) and device fail/remove/add (recovery) cycles can
deadlock.

The test creates a raid1 array with an internal bitmap on two brd
devices, drives it with fio while repeatedly toggling
max_sectors_kb and cycling one member through fail/remove/add, and
flags a likely deadlock if any of these steps fail to make progress
within a bounded timeout.

Link: https://lore.kernel.org/linux-raid/CAMGffE=heGA3y8FjQ0Sm1jj-kd-=H9Y54WozKASSEZhc9UNKjA@mail.gmail.com/T/#t
Signed-off-by: Jack Wang <jinpu.wang@cloud.ionos.com>
---
 tests/md/005     | 142 +++++++++++++++++++++++++++++++++++++++++++++++
 tests/md/005.out |   2 +
 2 files changed, 144 insertions(+)
 create mode 100755 tests/md/005
 create mode 100644 tests/md/005.out

diff --git a/tests/md/005 b/tests/md/005
new file mode 100755
index 000000000000..5493d0edc933
--- /dev/null
+++ b/tests/md/005
@@ -0,0 +1,142 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2026 Jack Wang
+#
+# Regression test for a deadlock reported against raid1 with an internal
+# bitmap, where concurrent I/O, queue limit updates (max_sectors_kb) and
+# device fail/remove/add cycles (recovery) can deadlock.
+#
+# See: https://lore.kernel.org/linux-raid/CAMGffE=heGA3y8FjQ0Sm1jj-kd-=H9Y54WozKASSEZhc9UNKjA@mail.gmail.com/T/#t
+
+. tests/md/rc
+. common/brd
+. common/fio
+
+DESCRIPTION="test for raid1 deadlock with internal bitmap under I/O, queue limit changes and fail/remove/add"
+
+requires() {
+	_have_fio
+	_have_driver raid1
+	_have_brd
+}
+
+# Resolve the real md device node (e.g. "md126") backing /dev/ram0 via
+# sysfs. This is more reliable than the mdadm-requested /dev/md/<name>
+# symlink, whose creation depends on udev rules/timing that can vary
+# across systems.
+_resolve_md_dev() {
+	local holder
+
+	for holder in /sys/class/block/ram0/holders/*; do
+		[[ -e $holder ]] || continue
+		basename "$(readlink -f "$holder")"
+		return 0
+	done
+
+	return 1
+}
+
+_toggle_max_sectors_kb() {
+	local sysfs_attr="/sys/block/${1}/queue/max_sectors_kb"
+
+	while [[ -e "$sysfs_attr" ]]; do
+		echo 128 > "$sysfs_attr" 2>/dev/null
+	done
+}
+
+# Wait up to $2 seconds for background pid $1 to exit. Returns 1 only if
+# it is still running once the deadline is hit (a real hang); the pid's
+# own exit status is irrelevant here and is not propagated.
+_wait_pid_timeout() {
+	local pid=$1
+	local secs=$2
+	local waited=0
+
+	while kill -0 "$pid" 2>/dev/null; do
+		if ((waited >= secs)); then
+			return 1
+		fi
+		sleep 1
+		((waited++))
+	done
+
+	wait "$pid" 2>/dev/null
+	return 0
+}
+
+# Repeatedly fail/remove/add a member while I/O is ongoing, racing
+# recovery against I/O submission and queue limit updates. mdadm --wait
+# returns failure when there was nothing to wait for (e.g. a fail/remove/
+# add cycle that didn't actually trigger recovery), which is expected and
+# not itself a sign of trouble here; a real hang is instead caught by the
+# overall `timeout` this is invoked under.
+_fail_remove_add_loop() {
+	local md_dev=$1
+	local i
+
+	for ((i = 0; i < 10; i++)); do
+		mdadm "/dev/${md_dev}" --fail /dev/ram0
+		mdadm "/dev/${md_dev}" --remove /dev/ram0
+		mdadm "/dev/${md_dev}" --add /dev/ram0
+		mdadm --wait "/dev/${md_dev}"
+	done
+}
+export -f _fail_remove_add_loop
+
+test() {
+	echo "Running ${TEST_NAME}"
+
+	local md_dev
+	local toggle_pid
+	local fio_pid
+
+	if ! _init_brd rd_size=262144 rd_nr=2; then
+		return 1
+	fi
+
+	mdadm --quiet --create /dev/md/blktests_md --force -e 1.2 \
+		--assume-clean --level=1 --bitmap=internal --raid-devices=2 \
+		/dev/ram0 /dev/ram1 >> "$FULL" 2>&1
+
+	if ! md_dev=$(_resolve_md_dev); then
+		echo "failed to resolve md device for /dev/ram0"
+		_cleanup_brd
+		return 1
+	fi
+
+	# Keep writing to max_sectors_kb while I/O and recovery are ongoing,
+	# this races queue limit updates against I/O submission and md
+	# reconfiguration.
+	_toggle_max_sectors_kb "$md_dev" &
+	toggle_pid=$!
+
+	fio --direct=1 --rw=randrw --ioengine=libaio --iodepth=32 --numjobs=4 \
+		--time_based=1 --runtime=60 --filename="/dev/${md_dev}" \
+		--name=repro >> "$FULL" 2>&1 &
+	fio_pid=$!
+
+	# Give fio a moment to start driving I/O so the array is in
+	# read-write mode before failing/adding a member below.
+	sleep 2
+
+	# shellcheck disable=SC2016
+	if ! timeout 120 bash -c '_fail_remove_add_loop "$1"' _ "$md_dev" \
+		>> "$FULL" 2>&1; then
+		echo "fail/remove/add loop did not complete in time, possible deadlock"
+	fi
+
+	if ! _wait_pid_timeout "$fio_pid" 90; then
+		echo "fio did not complete in time, possible deadlock"
+		kill -9 "$fio_pid" 2>/dev/null
+	fi
+
+	kill "$toggle_pid" 2>/dev/null
+	wait "$toggle_pid" 2>/dev/null
+
+	if ! timeout 30 mdadm --quiet --stop "/dev/${md_dev}" >> "$FULL" 2>&1; then
+		echo "mdadm --stop timed out, possible deadlock"
+	fi
+	_cleanup_brd
+
+	echo "Test complete"
+}
diff --git a/tests/md/005.out b/tests/md/005.out
new file mode 100644
index 000000000000..d5ca24713e54
--- /dev/null
+++ b/tests/md/005.out
@@ -0,0 +1,2 @@
+Running md/005
+Test complete
-- 
2.43.0