From: Bart Van Assche <bvanassche@acm.org>
To: Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com>
Cc: linux-block@vger.kernel.org, Bart Van Assche <bvanassche@acm.org>
Subject: [PATCH blktests] nbd/006: Measure NBD IOPS with a null_blk backing device
Date: Tue, 17 Nov 2026 15:06:13 -0800 [thread overview]
Message-ID: <20261117230613.3989814-1-bvanassche@acm.org> (raw)
Add a test that measures the maximum number of random read and random
write IOPS that the NBD driver can achieve. Such a test is useful to
detect performance regressions in the NBD driver and to evaluate NBD
performance optimizations.
Minimize the overhead outside the NBD driver as follows:
- Back the NBD export with a 1 GiB null_blk device that is tuned for
maximum IOPS: no memory backing, immediate completion, a 4 KiB logical
block size, one hardware queue per CPU and a hardware queue depth of
1024. Keep the device small because nbd-server opens the exported
device without O_DIRECT and hence all NBD I/O passes through the page
cache of the null_blk device.
- Select the "none" I/O scheduler for the null_blk device and disable
readahead such that each 4 KiB NBD read results in a single 4 KiB read
of the null_blk device.
- Connect nbd-client and nbd-server over a Unix domain socket if
nbd-client supports the -unix option. Fall back to a TCP connection
over the loopback interface otherwise.
- Use one NBD connection per CPU, with a maximum of 16 connections.
- Measure the IOPS with fio using the io_uring I/O engine, the "none" I/O
scheduler for the NBD device, direct I/O, 4 KiB requests, a queue depth
of 64 and one job per NBD connection. Run a random read pass and a
random write pass that each take half of the test time and report the
results via the "read iops" and "write iops" performance fields.
Note: since nbd-server performs buffered writes, the write IOPS mainly
reflect how fast nbd-server can dirty pages in the page cache.
Fail the test if configuring the null_blk request queue fails or if
nbd-server does not start listening in time.
Signed-off-by: Bart Van Assche <bvanassche@acm.org>
---
tests/nbd/006 | 159 ++++++++++++++++++++++++++++++++++++++++++++++
tests/nbd/006.out | 2 +
2 files changed, 161 insertions(+)
create mode 100755 tests/nbd/006
create mode 100644 tests/nbd/006.out
diff --git a/tests/nbd/006 b/tests/nbd/006
new file mode 100755
index 000000000000..cd3e705f4b2f
--- /dev/null
+++ b/tests/nbd/006
@@ -0,0 +1,159 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2026 Google LLC
+#
+# Measure the random read and random write IOPS of an NBD device that is backed
+# by a null_blk device. Use a Unix domain socket for the NBD connections if
+# supported by nbd-client and nbd-server and TCP otherwise.
+
+. tests/nbd/rc
+. common/null_blk
+
+DESCRIPTION="measure NBD read and write IOPS with null_blk as backing device"
+TIMED=1
+
+requires() {
+ _have_nbd
+ _have_null_blk
+ _have_fio
+}
+
+# Whether nbd-client supports connecting over a Unix domain socket. The
+# nbd-server "unixsock" configuration option has been introduced in the same
+# nbd release as the nbd-client -unix option.
+_nbd_have_unix_socket() {
+ nbd-client --help 2>&1 | grep -q -- '-unix'
+}
+
+test() {
+ local nullb=nullb1 nullb_dev=/dev/nullb1 nbd_dev=/dev/nbd0
+ local sock="${TMPDIR}/nbd.sock" pid_file="${TMPDIR}/nbd.pid"
+ local -i nr_cpus nr_conns i
+ local -a connect_args
+ local use_unix=0
+
+ echo "Running ${TEST_NAME}"
+
+ nr_cpus=$(nproc)
+ nr_conns=$((nr_cpus < 16 ? nr_cpus : 16))
+
+ # Load null_blk without creating any device and next create a single
+ # null_blk device via configfs that is tuned for maximum IOPS: no
+ # memory backing, immediate completion and one hardware queue per CPU.
+ if ! _init_null_blk nr_devices=0; then
+ echo "Loading null_blk failed"
+ return 1
+ fi
+ local nullb_params=(
+ blocksize=4096
+ completion_nsec=0
+ hw_queue_depth=1024
+ irqmode=0
+ memory_backed=0
+ queue_mode=2
+ size=1024 # MiB
+ submit_queues="$nr_cpus"
+ power=1
+ )
+ if ! _configure_null_blk "$nullb" "${nullb_params[@]}"; then
+ echo "Configuring null_blk failed"
+ _exit_null_blk
+ return 1
+ fi
+ # nbd-server opens the exported device without O_DIRECT. Disable
+ # readahead such that each NBD read causes a single 4 KiB read of the
+ # null_blk device.
+ if ! _write_queue_sched none "/sys/block/${nullb}/queue/scheduler" ||
+ ! grep -q '\[none\]' "/sys/block/${nullb}/queue/scheduler" ||
+ ! echo 0 > "/sys/block/${nullb}/queue/read_ahead_kb"; then
+ echo "Configuring the ${nullb} request queue failed"
+ _exit_null_blk
+ return 1
+ fi
+
+ _nbd_have_unix_socket && use_unix=1
+
+ {
+ echo "[generic]"
+ echo "allowlist=true"
+ if ((use_unix)); then
+ echo "unixsock=${sock}"
+ fi
+ echo "[export]"
+ echo "exportname=${nullb_dev}"
+ } > "${TMPDIR}/nbd.conf"
+ cat "${TMPDIR}/nbd.conf" >> "$FULL"
+
+ if ! nbd-server -p "$pid_file" -C "${TMPDIR}/nbd.conf" \
+ >> "$FULL" 2>&1; then
+ echo "Starting nbd-server failed"
+ _exit_null_blk
+ return 1
+ fi
+
+ if ((use_unix)); then
+ echo "Using Unix domain socket ${sock}" >> "$FULL"
+ connect_args=(-unix "$sock")
+ for ((i = 0; i < 100; i++)); do
+ [[ -S "$sock" ]] && break
+ sleep .1
+ done
+ else
+ echo "Using TCP socket" >> "$FULL"
+ connect_args=(localhost)
+ for ((i = 0; i < 100; i++)); do
+ nbd-client -l localhost >> "$FULL" 2>&1 && break
+ sleep .1
+ done
+ fi
+ if ((i >= 100)); then
+ echo "Timeout while waiting for nbd-server"
+ kill -SIGTERM "$(cat "$pid_file")"
+ _exit_null_blk
+ return 1
+ fi
+
+ # Use -connections instead of -C because some nbd-client versions
+ # do not declare that -C takes an argument and pass a NULL pointer
+ # to strtol() if -C is used.
+ if ! nbd-client -L -N export -connections "$nr_conns" -b 4096 \
+ "${connect_args[@]}" "$nbd_dev" >> "$FULL" 2>&1 ||
+ ! _wait_for_nbd_connect; then
+ echo "Connecting to nbd-server failed"
+ kill -SIGTERM "$(cat "$pid_file")"
+ _exit_null_blk
+ return 1
+ fi
+ udevadm settle
+
+ # _run_fio passes --group_reporting=1 and --runtime="$TIMEOUT" to fio.
+ : "${TIMEOUT:=60}"
+ _divide_timeout 2
+
+ local fio_params=(
+ --bs=4k
+ --direct=1
+ --filename="$nbd_dev"
+ --iodepth=64
+ --ioengine=io_uring
+ --ioscheduler=none
+ --norandommap
+ --numjobs="$nr_conns"
+ --time_based=1
+ )
+ local rw
+ for rw in read write; do
+ FIO_PERF_FIELDS=("${rw} iops")
+ _fio_perf --name="nbd-${rw}-iops" --rw="rand${rw}" \
+ "${fio_params[@]}"
+ done
+
+ nbd-client -d "$nbd_dev" >> "$FULL" 2>&1
+ _wait_for_nbd_disconnect
+ kill -SIGTERM "$(cat "$pid_file")"
+ rm -f "$pid_file" "$sock"
+ udevadm settle
+ _exit_null_blk
+
+ echo "Test complete"
+}
diff --git a/tests/nbd/006.out b/tests/nbd/006.out
new file mode 100644
index 000000000000..acc8af5bfb2e
--- /dev/null
+++ b/tests/nbd/006.out
@@ -0,0 +1,2 @@
+Running nbd/006
+Test complete
reply other threads:[~2026-11-17 23:06 UTC|newest]
Thread overview: [no followups] expand[flat|nested] mbox.gz Atom feed
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261117230613.3989814-1-bvanassche@acm.org \
--to=bvanassche@acm.org \
--cc=linux-block@vger.kernel.org \
--cc=shinichiro.kawasaki@wdc.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox