[PATCH blktests] nvme/081: test the per-queue io_cpu sysfs attribute

[PATCH blktests] nvme/081: test the per-queue io_cpu sysfs attribute

[Saravanan D]

nvme-tcp exposes /sys/class/nvme/nvmeX/queues/<qid>/io_cpu and managed,
so a control plane can choose the CPU that runs each I/O queue's socket
work [1]. Test that

  - io_cpu reads back a written CPU and rejects invalid values
  - managed is read only and turns 0 once io_cpu is written
  - an assignment survives a controller reset
  - the queue directories follow the queue count as attr_qid_max
    shrinks and grows it
  - removing the controller while io_cpu is written completes

The test skips when the kernel does not have the attribute.

[1] https://lore.kernel.org/linux-nvme/cover.1794015622.git.saravanand@crusoe.ai/

Assisted-by: Claude:claude-opus-5-5 [Claude Code]
Signed-off-by: Saravanan D <saravanand@crusoe.ai>
---
This is the nvme/077 test posted with v7 [2], renumbered because other
pending blktests patches use nvme/076 to nvme/080. The test itself is
unchanged.

[2] https://lore.kernel.org/all/20261104140721.4489-1-saravanand@crusoe.ai/

 tests/nvme/081     | 185 +++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/081.out |   2 +
 2 files changed, 187 insertions(+)
 create mode 100755 tests/nvme/081
 create mode 100644 tests/nvme/081.out

diff --git a/tests/nvme/081 b/tests/nvme/081
new file mode 100755
index 0000000..0b4bce5
--- /dev/null
+++ b/tests/nvme/081
@@ -0,0 +1,185 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2026 Saravanan D <saravanand@crusoe.ai>
+#
+# Test the per-queue io_cpu sysfs attribute of nvme-tcp
+
+. tests/nvme/rc
+
+DESCRIPTION="test the per-queue io_cpu sysfs attribute"
+
+requires() {
+	_nvme_requires
+	_have_loop
+	_require_nvme_trtype tcp
+	_require_min_cpus 2
+}
+
+set_conditions() {
+	_set_nvme_trtype "$@"
+}
+
+# last CPU in the online list, and the first CPU id past the possible ones
+last_online_cpu() {
+	awk -F'[-,]' '{ print $NF }' /sys/devices/system/cpu/online
+}
+
+first_impossible_cpu() {
+	echo $(( $(awk -F'[-,]' '{ print $NF }' /sys/devices/system/cpu/possible) + 1 ))
+}
+
+nr_queue_dirs() {
+	local nvmedev="$1"
+
+	find "/sys/class/nvme/${nvmedev}/queues" -mindepth 1 -maxdepth 1 \
+		-type d 2>/dev/null | wc -l
+}
+
+expect_attr() {
+	local file="$1"
+	local expected="$2"
+	local value
+
+	value=$(cat "${file}")
+	if [[ "${value}" != "${expected}" ]]; then
+		echo "${file}: read ${value}, expected ${expected}"
+		return 1
+	fi
+	return 0
+}
+
+test_read_write() {
+	local qdir="$1"
+	local cpu="$2"
+	local bad
+
+	expect_attr "${qdir}/1/managed" 1
+
+	echo "${cpu}" > "${qdir}/1/io_cpu"
+	expect_attr "${qdir}/1/io_cpu" "${cpu}"
+	expect_attr "${qdir}/1/managed" 0
+
+	for bad in "$(first_impossible_cpu)" -2 abc 0x1; do
+		if { echo "${bad}" > "${qdir}/1/io_cpu"; } 2>/dev/null; then
+			echo "io_cpu accepted ${bad}"
+		fi
+	done
+	expect_attr "${qdir}/1/io_cpu" "${cpu}"
+
+	if { echo 1 > "${qdir}/1/managed"; } 2>/dev/null; then
+		echo "managed is writable"
+	fi
+
+	echo unbound > "${qdir}/2/io_cpu"
+	expect_attr "${qdir}/2/io_cpu" -1
+	echo -1 > "${qdir}/2/io_cpu"
+	expect_attr "${qdir}/2/io_cpu" -1
+	expect_attr "${qdir}/2/managed" 0
+}
+
+test_reset() {
+	local nvmedev="$1"
+	local qdir="$2"
+	local cpu="$3"
+
+	echo 1 > "/sys/class/nvme/${nvmedev}/reset_controller"
+	_nvmf_wait_for_state "${def_subsysnqn}" "live" || return 1
+
+	expect_attr "${qdir}/1/io_cpu" "${cpu}"
+	expect_attr "${qdir}/1/managed" 0
+	expect_attr "${qdir}/2/io_cpu" -1
+	expect_attr "${qdir}/2/managed" 0
+}
+
+set_qid_max() {
+	local nvmedev="$1"
+	local qid_max="$2"
+	local queue_count_file="/sys/class/nvme-fabrics/ctl/${nvmedev}/queue_count"
+	local retries=10
+
+	echo "${qid_max}" > \
+		"${NVMET_CFS}/subsystems/${def_subsysnqn}/attr_qid_max"
+	while [[ "$(cat "${queue_count_file}")" -ne $((qid_max + 1)) ]]; do
+		if (( retries-- == 0 )); then
+			echo "queue count did not change to $((qid_max + 1))"
+			return 1
+		fi
+		sleep 1
+	done
+	_nvmf_wait_for_state "${def_subsysnqn}" "live" || return 1
+
+	if [[ "$(nr_queue_dirs "${nvmedev}")" -ne "${qid_max}" ]]; then
+		echo "$(nr_queue_dirs "${nvmedev}") queue directories," \
+			"expected ${qid_max}"
+		return 1
+	fi
+	return 0
+}
+
+io_cpu_writer() {
+	local qdir="$1"
+	local cpu="$2"
+	local file
+
+	while true; do
+		for file in "${qdir}"/*/io_cpu; do
+			{ echo "${cpu}" > "${file}"; } 2>/dev/null
+			{ echo unbound > "${file}"; } 2>/dev/null
+		done
+	done
+}
+
+test() {
+	echo "Running ${TEST_NAME}"
+
+	_setup_nvmet
+
+	local cfs_path="${NVMET_CFS}/subsystems/${def_subsysnqn}"
+	local skipped=false
+	local nvmedev qdir cpu writer
+
+	_nvmet_target_setup --blkdev file
+
+	_nvme_connect_subsys --keep-alive-tmo 1 --reconnect-delay 2
+	_nvmf_wait_for_state "${def_subsysnqn}" "live"
+	nvmedev=$(_find_nvme_dev "${def_subsysnqn}")
+	qdir="/sys/class/nvme/${nvmedev}/queues"
+	cpu=$(last_online_cpu)
+
+	if [[ ! -f "${qdir}/1/io_cpu" ]]; then
+		SKIP_REASONS+=("missing per-queue io_cpu attribute")
+		skipped=true
+	elif [[ "$(nr_queue_dirs "${nvmedev}")" -lt 2 ]]; then
+		SKIP_REASONS+=("controller has fewer than 2 I/O queues")
+		skipped=true
+	else
+		test_read_write "${qdir}" "${cpu}"
+		test_reset "${nvmedev}" "${qdir}" "${cpu}"
+
+		if [[ -f "${cfs_path}/attr_qid_max" ]]; then
+			set_qid_max "${nvmedev}" 1
+			expect_attr "${qdir}/1/io_cpu" "${cpu}"
+			set_qid_max "${nvmedev}" 2
+		fi
+
+		# remove the controller while its queue directories are written
+		io_cpu_writer "${qdir}" "${cpu}" &
+		writer=$!
+		sleep 1
+	fi
+
+	_nvme_disconnect_subsys
+
+	if [[ -n "${writer}" ]]; then
+		kill "${writer}"
+		wait "${writer}" 2>/dev/null
+	fi
+
+	_nvmet_target_cleanup
+
+	if [[ "${skipped}" = true ]] ; then
+		return 1
+	fi
+
+	echo "Test complete"
+}
diff --git a/tests/nvme/081.out b/tests/nvme/081.out
new file mode 100644
index 0000000..5deb638
--- /dev/null
+++ b/tests/nvme/081.out
@@ -0,0 +1,2 @@
+Running nvme/081
+Test complete
-- 
2.56.0

Re: [PATCH blktests] nvme/081: test the per-queue io_cpu sysfs attribute

[Nilay Shroff]

On 11/7/26 3:23 PM, Saravanan D wrote:
> +test() {
> +	echo "Running ${TEST_NAME}"
> +
> +	_setup_nvmet
> +
> +	local cfs_path="${NVMET_CFS}/subsystems/${def_subsysnqn}"
> +	local skipped=false
> +	local nvmedev qdir cpu writer
> +
> +	_nvmet_target_setup --blkdev file
> +
> +	_nvme_connect_subsys --keep-alive-tmo 1 --reconnect-delay 2

How about adding connection option "--nr-io-queues 2"? If we add
this option then we may not need to have the requirement of
_require_min_cpus 2. I think, we shall be able to run this test
also on the system with only one cpu.

> +	_nvmf_wait_for_state "${def_subsysnqn}" "live"
> +	nvmedev=$(_find_nvme_dev "${def_subsysnqn}")
> +	qdir="/sys/class/nvme/${nvmedev}/queues"
> +	cpu=$(last_online_cpu)
> +
Can we also add one more test here which tests whether "$qdir"
exists or not?

> +	if [[ ! -f "${qdir}/1/io_cpu" ]]; then
> +		SKIP_REASONS+=("missing per-queue io_cpu attribute")
> +		skipped=true
> +	elif [[ "$(nr_queue_dirs "${nvmedev}")" -lt 2 ]]; then
> +		SKIP_REASONS+=("controller has fewer than 2 I/O queues")
> +		skipped=true

I think using --nr-io-queues option would also replace the above relative
comparison with an exact comparison based on num of queues selected.

> +	else
> +		test_read_write "${qdir}" "${cpu}"
> +		test_reset "${nvmedev}" "${qdir}" "${cpu}"

The reset test could be updated to ensure that number of queues
remains unchanged after controller reset. Current test_reset()
verifies that the io_cpu and managed attributes retain their expected
values after reset. However, it doesn't explicitly verify that the
queue directories were recreated with the same count. I would capture
the queue_count before reset and compare it afterward as well as
ensure that queue_count matches with nr_queue_dirs.

Thanks,
--Nilay

Re: [PATCH blktests] nvme/081: test the per-queue io_cpu sysfs attribute

[Saravanan D]

On Mon, Nov 09, 2026 at 07:13:27PM +0530, Nilay Shroff wrote:
> > +	_nvme_connect_subsys --keep-alive-tmo 1 --reconnect-delay 2
>
> How about adding connection option "--nr-io-queues 2"? If we add
> this option then we may not need to have the requirement of
> _require_min_cpus 2. I think, we shall be able to run this test
> also on the system with only one cpu.

I added --nr-io-queues 2, which pins the queue count. The test still
needs _require_min_cpus 2 though. nvmf_parse_options() caps
nr_io_queues at num_online_cpus(), so a single CPU host still gets one
I/O queue, and the test needs queues 1 and 2.

> Can we also add one more test here which tests whether "$qdir"
> exists or not?

Yes. v2 checks the queues directory and the io_cpu attribute
separately, with a skip reason for each, so a kernel that has only the
first patch of the series reports which part is missing.

> I think using --nr-io-queues option would also replace the above relative
> comparison with an exact comparison based on num of queues selected.

Yes. The test now expects exactly two queue directories after connect,
and a different count fails the test instead of skipping it.

> The reset test could be updated to ensure that number of queues
> remains unchanged after controller reset. Current test_reset()
> verifies that the io_cpu and managed attributes retain their expected
> values after reset. However, it doesn't explicitly verify that the
> queue directories were recreated with the same count. I would capture
> the queue_count before reset and compare it afterward as well as
> ensure that queue_count matches with nr_queue_dirs.

v2 reads queue_count before the reset and checks that it is unchanged
afterwards. queue_count includes the admin queue, so the test then
checks that the number of queue directories is queue_count - 1.

I will send v2 in reply to this mail. Thanks also for reviewing both
kernel patches.

Thanks,
Saravanan D.

[PATCH blktests v2] nvme/081: test the per-queue io_cpu sysfs attribute

[Saravanan D]

nvme-tcp exposes /sys/class/nvme/nvmeX/queues/<qid>/io_cpu and managed,
so a control plane can choose the CPU that runs each I/O queue's socket
work [1]. Test that

  - io_cpu reads back a written CPU and rejects invalid values
  - managed is read only and turns 0 once io_cpu is written
  - an assignment and the queue directories survive a controller reset
  - the queue directories follow the queue count as attr_qid_max
    shrinks and grows it
  - removing the controller while io_cpu is written completes

The test skips when the kernel does not have the queues/ directory or
the io_cpu attribute.

[1] https://lore.kernel.org/linux-nvme/cover.1794015622.git.saravanand@crusoe.ai/

Assisted-by: Claude:claude-opus-5-5 [Claude Code]
Signed-off-by: Saravanan D <saravanand@crusoe.ai>
---
Changes since v1 [2]:
- Connect with --nr-io-queues 2 and expect exactly two queue
  directories (Nilay).
- Skip with separate reasons when the queues directory or the io_cpu
  attribute is missing (Nilay).
- Check that queue_count is unchanged across a controller reset and
  that the queue directories match it (Nilay).

[2] https://lore.kernel.org/all/20261107095350.98072-1-saravanand@crusoe.ai/

 tests/nvme/081     | 208 +++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/081.out |   2 +
 2 files changed, 210 insertions(+)
 create mode 100755 tests/nvme/081
 create mode 100644 tests/nvme/081.out

diff --git a/tests/nvme/081 b/tests/nvme/081
new file mode 100755
index 0000000..0bda988
--- /dev/null
+++ b/tests/nvme/081
@@ -0,0 +1,208 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2026 Saravanan D <saravanand@crusoe.ai>
+#
+# Test the per-queue io_cpu sysfs attribute of nvme-tcp
+
+. tests/nvme/rc
+
+DESCRIPTION="test the per-queue io_cpu sysfs attribute"
+
+requires() {
+	_nvme_requires
+	_have_loop
+	_require_nvme_trtype tcp
+	_require_min_cpus 2
+}
+
+set_conditions() {
+	_set_nvme_trtype "$@"
+}
+
+# last CPU in the online list, and the first CPU id past the possible ones
+last_online_cpu() {
+	awk -F'[-,]' '{ print $NF }' /sys/devices/system/cpu/online
+}
+
+first_impossible_cpu() {
+	echo $(( $(awk -F'[-,]' '{ print $NF }' /sys/devices/system/cpu/possible) + 1 ))
+}
+
+nr_queue_dirs() {
+	local nvmedev="$1"
+
+	find "/sys/class/nvme/${nvmedev}/queues" -mindepth 1 -maxdepth 1 \
+		-type d 2>/dev/null | wc -l
+}
+
+# queue_count includes the admin queue
+queue_count() {
+	cat "/sys/class/nvme-fabrics/ctl/$1/queue_count"
+}
+
+expect_queue_dirs() {
+	local nvmedev="$1"
+	local expected="$2"
+	local nr
+
+	nr=$(nr_queue_dirs "${nvmedev}")
+	if [[ "${nr}" -ne "${expected}" ]]; then
+		echo "${nr} queue directories, expected ${expected}"
+		return 1
+	fi
+	return 0
+}
+
+expect_attr() {
+	local file="$1"
+	local expected="$2"
+	local value
+
+	value=$(cat "${file}")
+	if [[ "${value}" != "${expected}" ]]; then
+		echo "${file}: read ${value}, expected ${expected}"
+		return 1
+	fi
+	return 0
+}
+
+test_read_write() {
+	local qdir="$1"
+	local cpu="$2"
+	local bad
+
+	expect_attr "${qdir}/1/managed" 1
+
+	echo "${cpu}" > "${qdir}/1/io_cpu"
+	expect_attr "${qdir}/1/io_cpu" "${cpu}"
+	expect_attr "${qdir}/1/managed" 0
+
+	for bad in "$(first_impossible_cpu)" -2 abc 0x1; do
+		if { echo "${bad}" > "${qdir}/1/io_cpu"; } 2>/dev/null; then
+			echo "io_cpu accepted ${bad}"
+		fi
+	done
+	expect_attr "${qdir}/1/io_cpu" "${cpu}"
+
+	if { echo 1 > "${qdir}/1/managed"; } 2>/dev/null; then
+		echo "managed is writable"
+	fi
+
+	echo unbound > "${qdir}/2/io_cpu"
+	expect_attr "${qdir}/2/io_cpu" -1
+	echo -1 > "${qdir}/2/io_cpu"
+	expect_attr "${qdir}/2/io_cpu" -1
+	expect_attr "${qdir}/2/managed" 0
+}
+
+test_reset() {
+	local nvmedev="$1"
+	local qdir="$2"
+	local cpu="$3"
+	local count
+
+	count=$(queue_count "${nvmedev}")
+	echo 1 > "/sys/class/nvme/${nvmedev}/reset_controller"
+	_nvmf_wait_for_state "${def_subsysnqn}" "live" || return 1
+
+	if [[ "$(queue_count "${nvmedev}")" -ne "${count}" ]]; then
+		echo "queue count $(queue_count "${nvmedev}") after reset," \
+			"expected ${count}"
+	fi
+	expect_queue_dirs "${nvmedev}" $((count - 1))
+	expect_attr "${qdir}/1/io_cpu" "${cpu}"
+	expect_attr "${qdir}/1/managed" 0
+	expect_attr "${qdir}/2/io_cpu" -1
+	expect_attr "${qdir}/2/managed" 0
+}
+
+set_qid_max() {
+	local nvmedev="$1"
+	local qid_max="$2"
+	local retries=10
+
+	echo "${qid_max}" > \
+		"${NVMET_CFS}/subsystems/${def_subsysnqn}/attr_qid_max"
+	while [[ "$(queue_count "${nvmedev}")" -ne $((qid_max + 1)) ]]; do
+		if (( retries-- == 0 )); then
+			echo "queue count did not change to $((qid_max + 1))"
+			return 1
+		fi
+		sleep 1
+	done
+	_nvmf_wait_for_state "${def_subsysnqn}" "live" || return 1
+
+	expect_queue_dirs "${nvmedev}" "${qid_max}"
+}
+
+io_cpu_writer() {
+	local qdir="$1"
+	local cpu="$2"
+	local file
+
+	while true; do
+		for file in "${qdir}"/*/io_cpu; do
+			{ echo "${cpu}" > "${file}"; } 2>/dev/null
+			{ echo unbound > "${file}"; } 2>/dev/null
+		done
+	done
+}
+
+test() {
+	echo "Running ${TEST_NAME}"
+
+	_setup_nvmet
+
+	local cfs_path="${NVMET_CFS}/subsystems/${def_subsysnqn}"
+	local skipped=false
+	local nvmedev qdir cpu writer
+
+	_nvmet_target_setup --blkdev file
+
+	# fabrics caps nr_io_queues at the online CPU count, hence the
+	# _require_min_cpus 2
+	_nvme_connect_subsys --keep-alive-tmo 1 --reconnect-delay 2 \
+		--nr-io-queues 2
+	_nvmf_wait_for_state "${def_subsysnqn}" "live"
+	nvmedev=$(_find_nvme_dev "${def_subsysnqn}")
+	qdir="/sys/class/nvme/${nvmedev}/queues"
+	cpu=$(last_online_cpu)
+
+	if [[ ! -d "${qdir}" ]]; then
+		SKIP_REASONS+=("missing per-queue sysfs directories")
+		skipped=true
+	elif [[ ! -f "${qdir}/1/io_cpu" ]]; then
+		SKIP_REASONS+=("missing per-queue io_cpu attribute")
+		skipped=true
+	else
+		expect_queue_dirs "${nvmedev}" 2
+		test_read_write "${qdir}" "${cpu}"
+		test_reset "${nvmedev}" "${qdir}" "${cpu}"
+
+		if [[ -f "${cfs_path}/attr_qid_max" ]]; then
+			set_qid_max "${nvmedev}" 1
+			expect_attr "${qdir}/1/io_cpu" "${cpu}"
+			set_qid_max "${nvmedev}" 2
+		fi
+
+		# remove the controller while its queue directories are written
+		io_cpu_writer "${qdir}" "${cpu}" &
+		writer=$!
+		sleep 1
+	fi
+
+	_nvme_disconnect_subsys
+
+	if [[ -n "${writer}" ]]; then
+		kill "${writer}"
+		wait "${writer}" 2>/dev/null
+	fi
+
+	_nvmet_target_cleanup
+
+	if [[ "${skipped}" = true ]] ; then
+		return 1
+	fi
+
+	echo "Test complete"
+}
diff --git a/tests/nvme/081.out b/tests/nvme/081.out
new file mode 100644
index 0000000..5deb638
--- /dev/null
+++ b/tests/nvme/081.out
@@ -0,0 +1,2 @@
+Running nvme/081
+Test complete
-- 
2.56.0

Re: [PATCH blktests v2] nvme/081: test the per-queue io_cpu sysfs attribute

[Nilay Shroff]

On 11/10/26 5:20 PM, Saravanan D wrote:
> nvme-tcp exposes/sys/class/nvme/nvmeX/queues/<qid>/io_cpu and managed,
> so a control plane can choose the CPU that runs each I/O queue's socket
> work [1]. Test that
> 
>    - io_cpu reads back a written CPU and rejects invalid values
>    - managed is read only and turns 0 once io_cpu is written
>    - an assignment and the queue directories survive a controller reset
>    - the queue directories follow the queue count as attr_qid_max
>      shrinks and grows it
>    - removing the controller while io_cpu is written completes
> 
> The test skips when the kernel does not have the queues/ directory or
> the io_cpu attribute.
> 
> [1]https://lore.kernel.org/linux-nvme/cover.1794015622.git.saravanand@crusoe.ai/
> 
> Assisted-by: Claude:claude-opus-5-5 [Claude Code]
> Signed-off-by: Saravanan D<saravanand@crusoe.ai>

Looks good to me:

Reviewed-by: Nilay Shroff <nilay@linux.ibm.com>