[PATCH v3] tests/nvme: Add admin-passthru+reset race test

[PATCH v3] tests/nvme: Add admin-passthru+reset race test

[Jonathan Derrick]

Adds a test which runs many formats and controller resets in parallel.
The intent is to expose timing holes in the controller state machine
which will lead to hung task timeouts and the controller becoming
unavailable.

Reported by https://bugzilla.kernel.org/show_bug.cgi?id=216354

Signed-off-by: Jonathan Derrick <jonathan.derrick@linux.dev>
---
v3:
I noticed I couldn't rely on checking state within the loop because it was
constantly on 'resetting', so I moved the last_live declaration to update when
a reset completes.

I updated the kernel in my QEMU instance to origin/nvme-6.2 (it was on a
previous origin/nvme-6.2), and applied Klaus' format fix to QEMU. This doesn't
crash my QEMU anymore, but now gets stuck in 'connecting'.

 tests/nvme/047     | 137 +++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/047.out |   2 +
 2 files changed, 139 insertions(+)
 create mode 100755 tests/nvme/047
 create mode 100644 tests/nvme/047.out

diff --git a/tests/nvme/047 b/tests/nvme/047
new file mode 100755
index 0000000..76b7fdf
--- /dev/null
+++ b/tests/nvme/047
@@ -0,0 +1,137 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2022 Jonathan Derrick <jonathan.derrick@linux.dev>
+#
+# Test nvme reset controller during admin passthru
+#
+# Regression for issue reported by
+# https://bugzilla.kernel.org/show_bug.cgi?id=216354
+#
+# Simpler form:
+# for i in {1..50}; do
+#	nvme format -f /dev/nvme0n1 &
+#	echo 1 > /sys/block/nvme0n1/device/reset_controller &
+# done
+
+. tests/nvme/rc
+
+#restrict test to nvme-pci only
+nvme_trtype=pci
+
+DESCRIPTION="test nvme reset controller during admin passthru"
+QUICK=1
+CAN_BE_ZONED=1
+
+RUN_TIME=300
+RESET_MISSING=true
+RESET_DEAD=true
+
+requires() {
+	_nvme_requires
+}
+
+device_requires() {
+	_require_test_dev_is_nvme
+}
+
+test_device() {
+	echo "Running ${TEST_NAME}"
+
+	local pdev
+	local blkdev
+	local ctrldev
+	local sysfs
+	local max_timeout
+	local timeout
+	local timeleft
+	local start
+	local i
+
+	pdev="$(_get_pci_dev_from_blkdev)"
+	blkdev="${TEST_DEV_SYSFS##*/}"
+	ctrldev="$(echo "$blkdev" | grep -Eo 'nvme[0-9]+')"
+	sysfs="/sys/block/$blkdev/device"
+	max_timeout=$(cat /proc/sys/kernel/hung_task_timeout_secs)
+	timeout=$((max_timeout * 3 / 4))
+
+	get_state() {
+		state=$(cat "$sysfs/state" 2> /dev/null)
+		if [[ -n "$state" ]]; then
+			echo "$state"
+		else
+			echo "unknown"
+		fi
+	}
+
+	tmp=$(mktemp /tmp/blk_tmp_XXXXXX)
+	lock=$(mktemp /tmp/blk_lock_XXXXXX)
+	exec 47>"$lock"
+	update_live() {
+		flock -s 047
+		date "+%s" > "$tmp"
+		flock -u 47
+	}
+
+	last_live() {
+		flock -s 47
+		cat "$tmp"
+		flock -u 47
+	}
+
+	now() {
+		date "+%s"
+	}
+
+	sleep 5
+
+	update_live
+	start=$(now)
+	while [[ $(($(now) - start)) -le $RUN_TIME ]]; do
+		# Failure case appears to stack up formats while controller is resetting/connecting
+		if [[ $(pgrep -cf "nvme format") -lt 100 ]]; then
+			for ((i=0; i<100; i++)); do
+				nvme format -f "$TEST_DEV" &
+				( echo 1 > "$sysfs/reset_controller" && update_live; ) &
+			done &> /dev/null
+		fi
+
+		# Might have failed probe, so reset and continue test
+		if [[ $(($(now) - $(last_live))) -gt 10 ]]; then
+			if [[ (! -c "/dev/$ctrldev" && "$RESET_MISSING" == true) ||
+			      ("$(get_state)" == "dead" && "$RESET_DEAD" == true) ]]; then
+				{
+					echo 1 > /sys/bus/pci/devices/"$pdev"/remove
+					echo 1 > /sys/bus/pci/rescan
+				} &
+
+				timeleft=$((max_timeout - timeout))
+				sleep $((timeleft < 30 ? timeleft : 30))
+				if [[ ! -c "/dev/$ctrldev" ]]; then
+					echo "/dev/$ctrldev missing"
+					echo "failed to reset $ctrldev's pcie device $pdev"
+					break
+				fi
+				sleep 5
+				continue
+			fi
+		fi
+
+		if [[ $(($(now) - $(last_live))) -gt $timeout ]]; then
+			if [[ ! -c "/dev/$ctrldev" ]]; then
+				echo "/dev/$ctrldev missing"
+				break
+			fi
+
+			# Assume the controller is hung and unrecoverable
+			echo "nvme controller hung ($(get_state))"
+			break
+		fi
+	done
+
+	if [[ ! -c "/dev/$ctrldev" || "$(get_state)" != "live" ]]; then
+		echo "nvme still not live after $(($(now) - $(last_live))) seconds!"
+	fi
+	udevadm settle
+
+	echo "Test complete"
+}
diff --git a/tests/nvme/047.out b/tests/nvme/047.out
new file mode 100644
index 0000000..915d0a2
--- /dev/null
+++ b/tests/nvme/047.out
@@ -0,0 +1,2 @@
+Running nvme/047
+Test complete
-- 
2.27.0