Linux block layer
 help / color / mirror / Atom feed
From: Bart Van Assche <bvanassche@acm.org>
To: Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com>
Cc: linux-block@vger.kernel.org, Bart Van Assche <bvanassche@acm.org>
Subject: [PATCH blktests] nbd/006: Measure NBD IOPS with a null_blk backing device
Date: Tue, 17 Nov 2026 15:06:13 -0800	[thread overview]
Message-ID: <20261117230613.3989814-1-bvanassche@acm.org> (raw)

Add a test that measures the maximum number of random read and random
write IOPS that the NBD driver can achieve. Such a test is useful to
detect performance regressions in the NBD driver and to evaluate NBD
performance optimizations.

Minimize the overhead outside the NBD driver as follows:
- Back the NBD export with a 1 GiB null_blk device that is tuned for
  maximum IOPS: no memory backing, immediate completion, a 4 KiB logical
  block size, one hardware queue per CPU and a hardware queue depth of
  1024. Keep the device small because nbd-server opens the exported
  device without O_DIRECT and hence all NBD I/O passes through the page
  cache of the null_blk device.
- Select the "none" I/O scheduler for the null_blk device and disable
  readahead such that each 4 KiB NBD read results in a single 4 KiB read
  of the null_blk device.
- Connect nbd-client and nbd-server over a Unix domain socket if
  nbd-client supports the -unix option. Fall back to a TCP connection
  over the loopback interface otherwise.
- Use one NBD connection per CPU, with a maximum of 16 connections.
- Measure the IOPS with fio using the io_uring I/O engine, the "none" I/O
  scheduler for the NBD device, direct I/O, 4 KiB requests, a queue depth
  of 64 and one job per NBD connection. Run a random read pass and a
  random write pass that each take half of the test time and report the
  results via the "read iops" and "write iops" performance fields.

Note: since nbd-server performs buffered writes, the write IOPS mainly
reflect how fast nbd-server can dirty pages in the page cache.

Fail the test if configuring the null_blk request queue fails or if
nbd-server does not start listening in time.

Signed-off-by: Bart Van Assche <bvanassche@acm.org>
---
 tests/nbd/006     | 159 ++++++++++++++++++++++++++++++++++++++++++++++
 tests/nbd/006.out |   2 +
 2 files changed, 161 insertions(+)
 create mode 100755 tests/nbd/006
 create mode 100644 tests/nbd/006.out

diff --git a/tests/nbd/006 b/tests/nbd/006
new file mode 100755
index 000000000000..cd3e705f4b2f
--- /dev/null
+++ b/tests/nbd/006
@@ -0,0 +1,159 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2026 Google LLC
+#
+# Measure the random read and random write IOPS of an NBD device that is backed
+# by a null_blk device. Use a Unix domain socket for the NBD connections if
+# supported by nbd-client and nbd-server and TCP otherwise.
+
+. tests/nbd/rc
+. common/null_blk
+
+DESCRIPTION="measure NBD read and write IOPS with null_blk as backing device"
+TIMED=1
+
+requires() {
+	_have_nbd
+	_have_null_blk
+	_have_fio
+}
+
+# Whether nbd-client supports connecting over a Unix domain socket. The
+# nbd-server "unixsock" configuration option has been introduced in the same
+# nbd release as the nbd-client -unix option.
+_nbd_have_unix_socket() {
+	nbd-client --help 2>&1 | grep -q -- '-unix'
+}
+
+test() {
+	local nullb=nullb1 nullb_dev=/dev/nullb1 nbd_dev=/dev/nbd0
+	local sock="${TMPDIR}/nbd.sock" pid_file="${TMPDIR}/nbd.pid"
+	local -i nr_cpus nr_conns i
+	local -a connect_args
+	local use_unix=0
+
+	echo "Running ${TEST_NAME}"
+
+	nr_cpus=$(nproc)
+	nr_conns=$((nr_cpus < 16 ? nr_cpus : 16))
+
+	# Load null_blk without creating any device and next create a single
+	# null_blk device via configfs that is tuned for maximum IOPS: no
+	# memory backing, immediate completion and one hardware queue per CPU.
+	if ! _init_null_blk nr_devices=0; then
+		echo "Loading null_blk failed"
+		return 1
+	fi
+	local nullb_params=(
+		blocksize=4096
+		completion_nsec=0
+		hw_queue_depth=1024
+		irqmode=0
+		memory_backed=0
+		queue_mode=2
+		size=1024	# MiB
+		submit_queues="$nr_cpus"
+		power=1
+	)
+	if ! _configure_null_blk "$nullb" "${nullb_params[@]}"; then
+		echo "Configuring null_blk failed"
+		_exit_null_blk
+		return 1
+	fi
+	# nbd-server opens the exported device without O_DIRECT. Disable
+	# readahead such that each NBD read causes a single 4 KiB read of the
+	# null_blk device.
+	if ! _write_queue_sched none "/sys/block/${nullb}/queue/scheduler" ||
+	   ! grep -q '\[none\]' "/sys/block/${nullb}/queue/scheduler" ||
+	   ! echo 0 > "/sys/block/${nullb}/queue/read_ahead_kb"; then
+		echo "Configuring the ${nullb} request queue failed"
+		_exit_null_blk
+		return 1
+	fi
+
+	_nbd_have_unix_socket && use_unix=1
+
+	{
+		echo "[generic]"
+		echo "allowlist=true"
+		if ((use_unix)); then
+			echo "unixsock=${sock}"
+		fi
+		echo "[export]"
+		echo "exportname=${nullb_dev}"
+	} > "${TMPDIR}/nbd.conf"
+	cat "${TMPDIR}/nbd.conf" >> "$FULL"
+
+	if ! nbd-server -p "$pid_file" -C "${TMPDIR}/nbd.conf" \
+	     >> "$FULL" 2>&1; then
+		echo "Starting nbd-server failed"
+		_exit_null_blk
+		return 1
+	fi
+
+	if ((use_unix)); then
+		echo "Using Unix domain socket ${sock}" >> "$FULL"
+		connect_args=(-unix "$sock")
+		for ((i = 0; i < 100; i++)); do
+			[[ -S "$sock" ]] && break
+			sleep .1
+		done
+	else
+		echo "Using TCP socket" >> "$FULL"
+		connect_args=(localhost)
+		for ((i = 0; i < 100; i++)); do
+			nbd-client -l localhost >> "$FULL" 2>&1 && break
+			sleep .1
+		done
+	fi
+	if ((i >= 100)); then
+		echo "Timeout while waiting for nbd-server"
+		kill -SIGTERM "$(cat "$pid_file")"
+		_exit_null_blk
+		return 1
+	fi
+
+	# Use -connections instead of -C because some nbd-client versions
+	# do not declare that -C takes an argument and pass a NULL pointer
+	# to strtol() if -C is used.
+	if ! nbd-client -L -N export -connections "$nr_conns" -b 4096 \
+	     "${connect_args[@]}" "$nbd_dev" >> "$FULL" 2>&1 ||
+	   ! _wait_for_nbd_connect; then
+		echo "Connecting to nbd-server failed"
+		kill -SIGTERM "$(cat "$pid_file")"
+		_exit_null_blk
+		return 1
+	fi
+	udevadm settle
+
+	# _run_fio passes --group_reporting=1 and --runtime="$TIMEOUT" to fio.
+	: "${TIMEOUT:=60}"
+	_divide_timeout 2
+
+	local fio_params=(
+		--bs=4k
+		--direct=1
+		--filename="$nbd_dev"
+		--iodepth=64
+		--ioengine=io_uring
+		--ioscheduler=none
+		--norandommap
+		--numjobs="$nr_conns"
+		--time_based=1
+	)
+	local rw
+	for rw in read write; do
+		FIO_PERF_FIELDS=("${rw} iops")
+		_fio_perf --name="nbd-${rw}-iops" --rw="rand${rw}" \
+			  "${fio_params[@]}"
+	done
+
+	nbd-client -d "$nbd_dev" >> "$FULL" 2>&1
+	_wait_for_nbd_disconnect
+	kill -SIGTERM "$(cat "$pid_file")"
+	rm -f "$pid_file" "$sock"
+	udevadm settle
+	_exit_null_blk
+
+	echo "Test complete"
+}
diff --git a/tests/nbd/006.out b/tests/nbd/006.out
new file mode 100644
index 000000000000..acc8af5bfb2e
--- /dev/null
+++ b/tests/nbd/006.out
@@ -0,0 +1,2 @@
+Running nbd/006
+Test complete

                 reply	other threads:[~2026-11-17 23:06 UTC|newest]

Thread overview: [no followups] expand[flat|nested]  mbox.gz  Atom feed

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261117230613.3989814-1-bvanassche@acm.org \
    --to=bvanassche@acm.org \
    --cc=linux-block@vger.kernel.org \
    --cc=shinichiro.kawasaki@wdc.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox