[PATCH blktests 3/5] nvme/071: test I/O queue limits per queue type

[Daniel Wagner <dwagner@suse.de>]

From: Daniel Wagner <wagi@kernel.org>

nvmf_nr_io_queues() limits each queue type to the number of online CPUs
when asking the controller for I/O queues, but nvmf_set_io_queues()
used to hand out the granted queues without these limits. With
nr_write_queues larger than the number of online CPUs, the default
queues got more queues than there are CPUs, some hardware contexts had
no CPU mapped and connecting their I/O queues failed with -EXDEV.

Connect with nr_io_queues set to the number of online CPUs and twice as
many write and poll queues, check that every hardware context has a CPU
mapped and that no queue type has more hardware contexts than online
CPUs, and run verified I/O.

The problem is fixed by kernel commit XXX ("nvme-fabrics: limit each
queue type to the number of online CPUs").

Signed-off-by: Daniel Wagner <wagi@kernel.org>
---
 tests/nvme/071     | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/071.out |   2 ++
 2 files changed, 106 insertions(+)

diff --git a/tests/nvme/071 b/tests/nvme/071
new file mode 100755
index 000000000000..2ac5dd761876
--- /dev/null
+++ b/tests/nvme/071
@@ -0,0 +1,104 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (c) 2026 SUSE LLC and contributors
+#
+# Test that no queue type gets more I/O queues than there are online CPUs.
+# nvmf_nr_io_queues limits each queue type to the number of online CPUs when
+# asking for I/O queues, but nvmf_set_io_queues used to hand out the granted
+# queues without these limits. With nr_write_queues larger than the number of
+# online CPUs, the default queues got more queues than CPUs, some hardware
+# contexts had no CPU mapped and connecting their I/O queues failed.
+#
+# The bug was introduced for nvme-tcp by commit 6486199378a5 ("nvme-tcp: fix
+# queue mapping when queue count is limited") and for nvme-rdma by commit
+# a249d3066de6 ("nvme-fabrics: add queue setup helpers").
+#
+# Regression test for commit XXX ("nvme-fabrics: limit each queue type to the
+# number of online CPUs").
+
+. tests/nvme/rc
+
+DESCRIPTION="Test I/O queue limits per queue type"
+QUICK=1
+
+requires() {
+	_nvme_requires
+	_have_fio
+	_have_loop
+	_have_debugfs
+	_require_nvme_trtype tcp rdma
+}
+
+set_conditions() {
+	_set_nvme_trtype "$@"
+}
+
+# Check that every hardware context has a software context mapped and that no
+# queue type has more hardware contexts than there are online CPUs.
+check_hctxs() {
+	local nvmedev="$1"
+	local nr_cpus="$2"
+	local disk
+	local hctx
+	local type
+	local -A nr_type=()
+
+	disk=$(basename /sys/class/nvme/"${nvmedev}"/nvme*n*)
+	for hctx in /sys/block/"${disk}"/mq/*; do
+		if ! compgen -G "${hctx}/cpu*[0-9]" > /dev/null; then
+			echo "FAIL: hctx $(basename "${hctx}") has no CPU mapped"
+		fi
+	done
+
+	for hctx in /sys/kernel/debug/block/"${disk}"/hctx*; do
+		type=$(cat "${hctx}/type")
+		nr_type[${type}]=$((${nr_type[${type}]:-0} + 1))
+	done
+	for type in "${!nr_type[@]}"; do
+		if [[ "${nr_type[${type}]}" -gt "${nr_cpus}" ]]; then
+			echo "FAIL: ${nr_type[${type}]} ${type} queues for ${nr_cpus} CPUs"
+		fi
+	done
+}
+
+test() {
+	echo "Running ${TEST_NAME}"
+
+	local cfs_path="${NVMET_CFS}/subsystems/${def_subsysnqn}"
+	local nr_cpus
+	local nvmedev
+	local ns
+
+	_setup_nvmet
+
+	nr_cpus=$(getconf _NPROCESSORS_ONLN)
+
+	_nvmet_target_setup
+
+	# The bug only shows if the target grants a queue per online CPU for
+	# each queue type.
+	if [[ -f "${cfs_path}/attr_qid_max" ]] &&
+	   [[ $(cat "${cfs_path}/attr_qid_max") -lt $((3 * nr_cpus)) ]]; then
+		SKIP_REASONS+=("target supports less than $((3 * nr_cpus)) I/O queues")
+		_nvmet_target_cleanup
+		return 1
+	fi
+
+	if _nvme_connect_subsys --nr-io-queues "${nr_cpus}" \
+				--nr-write-queues $((2 * nr_cpus)) \
+				--nr-poll-queues $((2 * nr_cpus)); then
+		_nvmf_wait_for_state "${def_subsysnqn}" "live" || echo FAIL
+		nvmedev=$(_find_nvme_dev "${def_subsysnqn}")
+		check_hctxs "${nvmedev}" "${nr_cpus}"
+
+		ns=$(_find_nvme_ns "${def_subsys_uuid}")
+		_run_fio_verify_io --size=32m --filename="/dev/${ns}" ||
+			echo "FAIL: fio verify"
+
+		_nvme_disconnect_subsys
+	fi
+
+	_nvmet_target_cleanup
+
+	echo "Test complete"
+}
diff --git a/tests/nvme/071.out b/tests/nvme/071.out
new file mode 100644
index 000000000000..146809b91e0a
--- /dev/null
+++ b/tests/nvme/071.out
@@ -0,0 +1,2 @@
+Running nvme/071
+Test complete

-- 
2.56.0