* [RFC v2 3/3] tools/ufq_iosched: add BPF example scheduler and build scaffolding
From: Kaitao cheng @ 2026-05-03 3:56 UTC (permalink / raw)
To: axboe, ast, daniel, andrii, martin.lau, eddyz87, memxor, song,
yonghong.song, jolsa, john.fastabend
Cc: bpf, linux-block, linux-kernel, Kaitao Cheng
In-Reply-To: <20260503035623.28771-1-kaitao.cheng@linux.dev>
From: Kaitao Cheng <chengkaitao@kylinos.cn>
Add ufq_iosched as a simple example for the UFQ block I/O scheduler,
In the ufq_simple example, we implement the eBPF struct_ops hooks the
kernel exposes so we can exercise and validate the behavior and stability
of the kernel UFQ scheduling framework. The Makefile and directory
layout are modeled after sched_ext.
This mirrors the sched_ext examples pattern so developers can experiment
with user-defined queueing policies on top of IOSCHED_UFQ.
Signed-off-by: Kaitao Cheng <chengkaitao@kylinos.cn>
---
tools/ufq_iosched/.gitignore | 2 +
tools/ufq_iosched/Makefile | 262 ++++++++
tools/ufq_iosched/README.md | 145 +++++
.../include/bpf-compat/gnu/stubs.h | 12 +
tools/ufq_iosched/include/ufq/common.bpf.h | 75 +++
tools/ufq_iosched/include/ufq/common.h | 90 +++
tools/ufq_iosched/include/ufq/simple_stat.h | 23 +
tools/ufq_iosched/ufq_simple.bpf.c | 604 ++++++++++++++++++
tools/ufq_iosched/ufq_simple.c | 120 ++++
9 files changed, 1333 insertions(+)
create mode 100644 tools/ufq_iosched/.gitignore
create mode 100644 tools/ufq_iosched/Makefile
create mode 100644 tools/ufq_iosched/README.md
create mode 100644 tools/ufq_iosched/include/bpf-compat/gnu/stubs.h
create mode 100644 tools/ufq_iosched/include/ufq/common.bpf.h
create mode 100644 tools/ufq_iosched/include/ufq/common.h
create mode 100644 tools/ufq_iosched/include/ufq/simple_stat.h
create mode 100644 tools/ufq_iosched/ufq_simple.bpf.c
create mode 100644 tools/ufq_iosched/ufq_simple.c
diff --git a/tools/ufq_iosched/.gitignore b/tools/ufq_iosched/.gitignore
new file mode 100644
index 000000000000..d6264fe1c8cd
--- /dev/null
+++ b/tools/ufq_iosched/.gitignore
@@ -0,0 +1,2 @@
+tools/
+build/
diff --git a/tools/ufq_iosched/Makefile b/tools/ufq_iosched/Makefile
new file mode 100644
index 000000000000..7dc37d9172aa
--- /dev/null
+++ b/tools/ufq_iosched/Makefile
@@ -0,0 +1,262 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2026 KylinSoft Corporation.
+# Copyright (c) 2026 Kaitao Cheng <chengkaitao@kylinos.cn>
+include ../build/Build.include
+include ../scripts/Makefile.arch
+include ../scripts/Makefile.include
+
+all: all_targets
+
+ifneq ($(LLVM),)
+ifneq ($(filter %/,$(LLVM)),)
+LLVM_PREFIX := $(LLVM)
+else ifneq ($(filter -%,$(LLVM)),)
+LLVM_SUFFIX := $(LLVM)
+endif
+
+CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi
+CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu
+CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl
+CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu
+CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu
+CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu
+CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu
+CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu
+CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu
+CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH))
+
+ifeq ($(CROSS_COMPILE),)
+ifeq ($(CLANG_TARGET_FLAGS),)
+$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk)
+else
+CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS)
+endif # CLANG_TARGET_FLAGS
+else
+CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%))
+endif # CROSS_COMPILE
+
+CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as
+else
+CC := $(CROSS_COMPILE)gcc
+endif # LLVM
+
+CURDIR := $(abspath .)
+TOOLSDIR := $(abspath ..)
+LIBDIR := $(TOOLSDIR)/lib
+BPFDIR := $(LIBDIR)/bpf
+TOOLSINCDIR := $(TOOLSDIR)/include
+BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool
+APIDIR := $(TOOLSINCDIR)/uapi
+GENDIR := $(abspath ../../include/generated)
+GENHDR := $(GENDIR)/autoconf.h
+
+ifeq ($(O),)
+OUTPUT_DIR := $(CURDIR)/build
+else
+OUTPUT_DIR := $(O)/build
+endif # O
+OBJ_DIR := $(OUTPUT_DIR)/obj
+INCLUDE_DIR := $(OUTPUT_DIR)/include
+BPFOBJ_DIR := $(OBJ_DIR)/libbpf
+UFQOBJ_DIR := $(OBJ_DIR)/ufq_iosched
+BINDIR := $(OUTPUT_DIR)/bin
+BPFOBJ := $(BPFOBJ_DIR)/libbpf.a
+ifneq ($(CROSS_COMPILE),)
+HOST_BUILD_DIR := $(OBJ_DIR)/host/obj
+HOST_OUTPUT_DIR := $(OBJ_DIR)/host
+HOST_INCLUDE_DIR := $(HOST_OUTPUT_DIR)/include
+else
+HOST_BUILD_DIR := $(OBJ_DIR)
+HOST_OUTPUT_DIR := $(OUTPUT_DIR)
+HOST_INCLUDE_DIR := $(INCLUDE_DIR)
+endif
+HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a
+RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids
+DEFAULT_BPFTOOL := $(HOST_OUTPUT_DIR)/sbin/bpftool
+
+VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \
+ $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \
+ ../../vmlinux \
+ /sys/kernel/btf/vmlinux \
+ /boot/vmlinux-$(shell uname -r)
+VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS))))
+ifeq ($(VMLINUX_BTF),)
+$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)")
+endif
+
+BPFTOOL ?= $(DEFAULT_BPFTOOL)
+
+ifneq ($(wildcard $(GENHDR)),)
+ GENFLAGS := -DHAVE_GENHDR
+endif
+
+CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \
+ -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \
+ -I$(TOOLSINCDIR) -I$(APIDIR) -I$(CURDIR)/include
+
+# Silence some warnings when compiled with clang
+ifneq ($(LLVM),)
+CFLAGS += -Wno-unused-command-line-argument
+endif
+
+LDFLAGS += -lelf -lz -lpthread
+
+IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null | \
+ grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__')
+
+# Get Clang's default includes on this system, as opposed to those seen by
+# '-target bpf'. This fixes "missing" files on some architectures/distros,
+# such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc.
+#
+# Use '-idirafter': Don't interfere with include mechanics except where the
+# build would have failed anyways.
+define get_sys_includes
+$(shell $(1) -v -E - </dev/null 2>&1 \
+ | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \
+$(shell $(1) -dM -E - </dev/null | grep '__riscv_xlen ' | awk '{printf("-D__riscv_xlen=%d -D__BITS_PER_LONG=%d", $$3, $$3)}')
+endef
+
+BPF_CFLAGS = -g -D__TARGET_ARCH_$(SRCARCH) \
+ $(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian) \
+ -I$(CURDIR)/include -I$(CURDIR)/include/bpf-compat \
+ -I$(INCLUDE_DIR) -I$(APIDIR) \
+ -I../../include \
+ $(call get_sys_includes,$(CLANG)) \
+ -Wall -Wno-compare-distinct-pointer-types \
+ -Wno-microsoft-anon-tag \
+ -fms-extensions \
+ -O2 -mcpu=v3
+
+# sort removes libbpf duplicates when not cross-building
+MAKE_DIRS := $(sort $(OBJ_DIR)/libbpf $(HOST_BUILD_DIR)/libbpf \
+ $(HOST_BUILD_DIR)/bpftool $(HOST_BUILD_DIR)/resolve_btfids \
+ $(INCLUDE_DIR) $(UFQOBJ_DIR) $(BINDIR))
+
+$(MAKE_DIRS):
+ $(call msg,MKDIR,,$@)
+ $(Q)mkdir -p $@
+
+ifneq ($(CROSS_COMPILE),)
+$(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \
+ $(APIDIR)/linux/bpf.h \
+ | $(OBJ_DIR)/libbpf
+ $(Q)$(MAKE) $(submake_extras) CROSS_COMPILE=$(CROSS_COMPILE) \
+ -C $(BPFDIR) OUTPUT=$(OBJ_DIR)/libbpf/ \
+ EXTRA_CFLAGS='-g -O0 -fPIC' \
+ LDFLAGS="$(LDFLAGS)" \
+ DESTDIR=$(OUTPUT_DIR) prefix= all install_headers
+endif
+
+$(HOST_BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \
+ $(APIDIR)/linux/bpf.h \
+ | $(HOST_BUILD_DIR)/libbpf
+ $(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) \
+ OUTPUT=$(HOST_BUILD_DIR)/libbpf/ \
+ ARCH= CROSS_COMPILE= CC="$(HOSTCC)" LD=$(HOSTLD) \
+ EXTRA_CFLAGS='-g -O0 -fPIC' \
+ DESTDIR=$(HOST_OUTPUT_DIR) prefix= all install_headers
+
+$(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \
+ $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/bpftool
+ $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \
+ ARCH= CROSS_COMPILE= CC="$(HOSTCC)" LD=$(HOSTLD) \
+ EXTRA_CFLAGS='-g -O0' \
+ OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \
+ LIBBPF_OUTPUT=$(HOST_BUILD_DIR)/libbpf/ \
+ LIBBPF_DESTDIR=$(HOST_OUTPUT_DIR)/ \
+ prefix= DESTDIR=$(HOST_OUTPUT_DIR)/ install-bin
+
+$(INCLUDE_DIR)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL) | $(INCLUDE_DIR)
+ifeq ($(VMLINUX_H),)
+ $(call msg,GEN,,$@)
+ $(Q)$(BPFTOOL) btf dump file $(VMLINUX_BTF) format c > $@
+else
+ $(call msg,CP,,$@)
+ $(Q)cp "$(VMLINUX_H)" $@
+endif
+
+$(UFQOBJ_DIR)/%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h include/ufq/*.h \
+ | $(BPFOBJ) $(UFQOBJ_DIR)
+ $(call msg,CLNG-BPF,,$(notdir $@))
+ $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@
+
+$(INCLUDE_DIR)/%.bpf.skel.h: $(UFQOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BPFTOOL)
+ $(eval sched=$(notdir $@))
+ $(call msg,GEN-SKEL,,$(sched))
+ $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $<
+ $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o)
+ $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o)
+ $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o)
+ $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(subst .bpf.skel.h,,$(sched)) > $@
+ $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(subst .bpf.skel.h,,$(sched)) > $(@:.skel.h=.subskel.h)
+
+UFQ_COMMON_DEPS := include/ufq/common.h include/ufq/simple_stat.h | $(BINDIR)
+
+c-sched-targets = ufq_simple
+
+$(addprefix $(BINDIR)/,$(c-sched-targets)): \
+ $(BINDIR)/%: \
+ $(filter-out %.bpf.c,%.c) \
+ $(INCLUDE_DIR)/%.bpf.skel.h \
+ $(UFQ_COMMON_DEPS)
+ $(eval sched=$(notdir $@))
+ $(CC) $(CFLAGS) -c $(sched).c -o $(UFQOBJ_DIR)/$(sched).o
+ $(CC) -o $@ $(UFQOBJ_DIR)/$(sched).o $(BPFOBJ) $(LDFLAGS)
+
+$(c-sched-targets): %: $(BINDIR)/%
+
+install: all
+ $(Q)mkdir -p $(DESTDIR)/usr/local/bin/
+ $(Q)cp $(BINDIR)/* $(DESTDIR)/usr/local/bin/
+
+clean:
+ rm -rf $(OUTPUT_DIR) $(HOST_OUTPUT_DIR)
+ rm -f *.o *.bpf.o *.bpf.skel.h *.bpf.subskel.h
+ rm -f $(c-sched-targets)
+
+help:
+ @echo 'Building targets'
+ @echo '================'
+ @echo ''
+ @echo ' all - Compile all schedulers'
+ @echo ''
+ @echo 'Alternatively, you may compile individual schedulers:'
+ @echo ''
+ @printf ' %s\n' $(c-sched-targets)
+ @echo ''
+ @echo 'For any scheduler build target, you may specify an alternative'
+ @echo 'build output path with the O= environment variable. For example:'
+ @echo ''
+ @echo ' O=/tmp/ufq_iosched make all'
+ @echo ''
+ @echo 'will compile all schedulers, and emit the build artifacts to'
+ @echo '/tmp/ufq_iosched/build.'
+ @echo ''
+ @echo ''
+ @echo 'Installing targets'
+ @echo '=================='
+ @echo ''
+ @echo ' install - Compile and install all schedulers to /usr/bin.'
+ @echo ' You may specify the DESTDIR= environment variable'
+ @echo ' to indicate a prefix for /usr/bin. For example:'
+ @echo ''
+ @echo ' DESTDIR=/tmp/ufq_iosched make install'
+ @echo ''
+ @echo ' will build the schedulers in CWD/build, and'
+ @echo ' install the schedulers to /tmp/ufq_iosched/usr/bin.'
+ @echo ''
+ @echo ''
+ @echo 'Cleaning targets'
+ @echo '================'
+ @echo ''
+ @echo ' clean - Remove all generated files'
+
+all_targets: $(c-sched-targets)
+
+.PHONY: all all_targets $(c-sched-targets) clean help
+
+# delete failed targets
+.DELETE_ON_ERROR:
+
+# keep intermediate (.bpf.skel.h, .bpf.o, etc) targets
+.SECONDARY:
diff --git a/tools/ufq_iosched/README.md b/tools/ufq_iosched/README.md
new file mode 100644
index 000000000000..1fa305fb576e
--- /dev/null
+++ b/tools/ufq_iosched/README.md
@@ -0,0 +1,145 @@
+UFQ IOSCHED EXAMPLE SCHEDULERS
+============================
+
+# Introduction
+
+This directory contains a simple example of the ufq IO scheduler. It is meant
+to illustrate the different kinds of IO schedulers you can build with ufq;
+new schedulers will be added as the project evolves across releases.
+
+# Compiling the examples
+
+There are a few toolchain dependencies for compiling the example schedulers.
+
+## Toolchain dependencies
+
+1. clang >= 16.0.0
+
+The schedulers are BPF programs, and therefore must be compiled with clang. gcc
+is actively working on adding a BPF backend compiler as well, but are still
+missing some features such as BTF type tags which are necessary for using
+kptrs.
+
+2. pahole >= 1.25
+
+You may need pahole in order to generate BTF from DWARF.
+
+3. rust >= 1.85.0
+
+Rust schedulers uses features present in the rust toolchain >= 1.85.0. You
+should be able to use the stable build from rustup.
+
+There are other requirements as well, such as make, but these are the main /
+non-trivial ones.
+
+## Compiling the kernel
+
+In order to run a ufq scheduler, you'll have to run a kernel compiled
+with the patches in this repository, and with a minimum set of necessary
+Kconfig options:
+
+```
+CONFIG_BPF=y
+CONFIG_IOSCHED_UFQ=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_BPF_JIT=y
+CONFIG_DEBUG_INFO_BTF=y
+```
+
+It's also recommended that you also include the following Kconfig options:
+
+```
+CONFIG_BPF_JIT_ALWAYS_ON=y
+CONFIG_BPF_JIT_DEFAULT_ON=y
+CONFIG_PAHOLE_HAS_SPLIT_BTF=y
+CONFIG_PAHOLE_HAS_BTF_TAG=y
+```
+
+There is a `Kconfig` file in this directory whose contents you can append to
+your local `.config` file, as long as there are no conflicts with any existing
+options in the file.
+
+## Getting a vmlinux.h file
+
+You may notice that most of the example schedulers include a "vmlinux.h" file.
+This is a large, auto-generated header file that contains all of the types
+defined in some vmlinux binary that was compiled with
+[BTF](https://docs.kernel.org/bpf/btf.html) (i.e. with the BTF-related Kconfig
+options specified above).
+
+The header file is created using `bpftool`, by passing it a vmlinux binary
+compiled with BTF as follows:
+
+```bash
+$ bpftool btf dump file /path/to/vmlinux format c > vmlinux.h
+```
+
+`bpftool` analyzes all of the BTF encodings in the binary, and produces a
+header file that can be included by BPF programs to access those types. For
+example, using vmlinux.h allows a scheduler to access fields defined directly
+in vmlinux
+
+The scheduler build system will generate this vmlinux.h file as part of the
+scheduler build pipeline. It looks for a vmlinux file in the following
+dependency order:
+
+1. If the O= environment variable is defined, at `$O/vmlinux`
+2. If the KBUILD_OUTPUT= environment variable is defined, at
+ `$KBUILD_OUTPUT/vmlinux`
+3. At `../../vmlinux` (i.e. at the root of the kernel tree where you're
+ compiling the schedulers)
+4. `/sys/kernel/btf/vmlinux`
+5. `/boot/vmlinux-$(uname -r)`
+
+In other words, if you have compiled a kernel in your local repo, its vmlinux
+file will be used to generate vmlinux.h. Otherwise, it will be the vmlinux of
+the kernel you're currently running on. This means that if you're running on a
+kernel with ufq support, you may not need to compile a local kernel at
+all.
+
+### Aside on CO-RE
+
+One of the cooler features of BPF is that it supports
+[CO-RE](https://nakryiko.com/posts/bpf-core-reference-guide/) (Compile Once Run
+Everywhere). This feature allows you to reference fields inside of structs with
+types defined internal to the kernel, and not have to recompile if you load the
+BPF program on a different kernel with the field at a different offset.
+
+## Compiling the schedulers
+
+Once you have your toolchain setup, and a vmlinux that can be used to generate
+a full vmlinux.h file, you can compile the schedulers using `make`. The build
+vendors libbpf and bpftool under `build/`, then compiles BPF objects with
+`clang` and the userspace loader (for example `ufq_simple.c`) with `gcc`.
+
+```bash
+$ make -j($nproc)
+```
+
+## ufq_simple
+
+A simple IO scheduler that provides an example of a minimal ufq scheduler.
+Populates commonly used kernel-exposed BPF interfaces for testing the UFQ
+scheduler framework in the kernel.
+
+### bpftool feature detection (`llvm`, `libcap`, `libbfd`)
+
+While building bpftool, you may see lines similar to:
+
+```
+Auto-detecting system features:
+... clang-bpf-co-re: [ on ]
+... llvm: [ OFF ]
+... libcap: [ OFF ]
+... libbfd: [ OFF ]
+```
+
+This matches a typical minimal build environment: CO-RE support is on, while
+bpftool's optional links to LLVM (for some disassembly paths), libcap, and
+libbfd are off. **`llvm: [ OFF ]` is not a build failure** for these examples;
+the scheduler build can still complete (libbpf static library, bpftool,
+`vmlinux.h` generation, BPF `.o` files, and final `ufq_simple` link with `gcc`).
+
+If `libcap` or `libbfd` show `[ OFF ]`, you only miss optional bpftool features
+that depend on those libraries; you do not need them for compiling and loading
+the schedulers in this directory.
diff --git a/tools/ufq_iosched/include/bpf-compat/gnu/stubs.h b/tools/ufq_iosched/include/bpf-compat/gnu/stubs.h
new file mode 100644
index 000000000000..f200ac0f6ccd
--- /dev/null
+++ b/tools/ufq_iosched/include/bpf-compat/gnu/stubs.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Dummy gnu/stubs.h. clang can end up including /usr/include/gnu/stubs.h when
+ * compiling BPF files although its content doesn't play any role. The file in
+ * turn includes stubs-64.h or stubs-32.h depending on whether __x86_64__ is
+ * defined. When compiling a BPF source, __x86_64__ isn't set and thus
+ * stubs-32.h is selected. However, the file is not there if the system doesn't
+ * have 32bit glibc devel package installed leading to a build failure.
+ *
+ * The problem is worked around by making this file available in the include
+ * search paths before the system one when building BPF.
+ */
diff --git a/tools/ufq_iosched/include/ufq/common.bpf.h b/tools/ufq_iosched/include/ufq/common.bpf.h
new file mode 100644
index 000000000000..515821ba5b01
--- /dev/null
+++ b/tools/ufq_iosched/include/ufq/common.bpf.h
@@ -0,0 +1,75 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 KylinSoft Corporation.
+ * Copyright (c) 2026 Kaitao Cheng <chengkaitao@kylinos.cn>
+ */
+#ifndef __UFQ_COMMON_BPF_H
+#define __UFQ_COMMON_BPF_H
+
+#ifdef LSP
+#define __bpf__
+#include "../vmlinux/vmlinux.h"
+#else
+#include "vmlinux.h"
+#endif
+
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include <bpf/bpf_core_read.h>
+#include <asm-generic/errno.h>
+#include "simple_stat.h"
+
+#define BPF_STRUCT_OPS(name, args...) \
+ SEC("struct_ops/" #name) BPF_PROG(name, ##args)
+
+/* Define struct ufq_iosched_ops for .struct_ops.link in the BPF object */
+#define UFQ_OPS_DEFINE(__name, ...) \
+ SEC(".struct_ops.link") \
+ struct ufq_iosched_ops __name = { \
+ __VA_ARGS__, \
+ }
+
+/* list and rbtree */
+#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node)))
+
+struct request *bpf_request_acquire(struct request *rq) __ksym;
+void bpf_request_release(struct request *rq) __ksym;
+bool bpf_request_bio_try_merge(struct request *rq, struct bio *bio,
+ unsigned int nr_segs) __ksym;
+struct request *bpf_request_try_merge(struct request *rq, struct request *next) __ksym;
+
+void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym;
+void bpf_obj_drop_impl(void *kptr, void *meta) __ksym;
+
+#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL))
+#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL)
+
+int bpf_list_push_front_impl(struct bpf_list_head *head,
+ struct bpf_list_node *node,
+ void *meta, __u64 off) __ksym;
+#define bpf_list_push_front(head, node) bpf_list_push_front_impl(head, node, NULL, 0)
+
+int bpf_list_push_back_impl(struct bpf_list_head *head,
+ struct bpf_list_node *node,
+ void *meta, __u64 off) __ksym;
+#define bpf_list_push_back(head, node) bpf_list_push_back_impl(head, node, NULL, 0)
+
+struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym;
+struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym;
+bool bpf_list_empty(struct bpf_list_head *head) __ksym;
+struct bpf_list_node *bpf_list_del(struct bpf_list_head *head,
+ struct bpf_list_node *node) __ksym;
+
+struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root,
+ struct bpf_rb_node *node) __ksym;
+int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node,
+ bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b),
+ void *meta, __u64 off) __ksym;
+#define bpf_rbtree_add(head, node, less) bpf_rbtree_add_impl(head, node, less, NULL, 0)
+
+struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym;
+
+void *bpf_refcount_acquire_impl(void *kptr, void *meta) __ksym;
+#define bpf_refcount_acquire(kptr) bpf_refcount_acquire_impl(kptr, NULL)
+
+#endif /* __UFQ_COMMON_BPF_H */
diff --git a/tools/ufq_iosched/include/ufq/common.h b/tools/ufq_iosched/include/ufq/common.h
new file mode 100644
index 000000000000..a1e3a07844c4
--- /dev/null
+++ b/tools/ufq_iosched/include/ufq/common.h
@@ -0,0 +1,90 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 KylinSoft Corporation.
+ * Copyright (c) 2026 Kaitao Cheng <chengkaitao@kylinos.cn>
+ */
+#ifndef __UFQ_IOSCHED_COMMON_H
+#define __UFQ_IOSCHED_COMMON_H
+
+#ifdef __KERNEL__
+#error "Should not be included by BPF programs"
+#endif
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <errno.h>
+#include <bpf/bpf.h>
+#include "simple_stat.h"
+
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+typedef int8_t s8;
+typedef int16_t s16;
+typedef int32_t s32;
+typedef int64_t s64;
+
+#define UFQ_ERR(__fmt, ...) \
+ do { \
+ fprintf(stderr, "[UFQ_ERR] %s:%d", __FILE__, __LINE__); \
+ if (errno) \
+ fprintf(stderr, " (%s)\n", strerror(errno)); \
+ else \
+ fprintf(stderr, "\n"); \
+ fprintf(stderr, __fmt __VA_OPT__(,) __VA_ARGS__); \
+ fprintf(stderr, "\n"); \
+ \
+ exit(EXIT_FAILURE); \
+ } while (0)
+
+#define UFQ_ERR_IF(__cond, __fmt, ...) \
+ do { \
+ if (__cond) \
+ UFQ_ERR((__fmt) __VA_OPT__(,) __VA_ARGS__); \
+ } while (0)
+
+/*
+ * struct ufq_iosched_ops can grow over time. With common.bpf.h::UFQ_OPS_DEFINE()
+ * and UFQ_OPS_LOAD()/UFQ_OPS_ATTACH(), libbpf performs struct_ops attachment.
+ */
+#define UFQ_OPS_OPEN(__ops_name, __ufq_name) ({ \
+ struct __ufq_name *__skel; \
+ \
+ __skel = __ufq_name##__open(); \
+ UFQ_ERR_IF(!__skel, "Could not open " #__ufq_name); \
+ (void)__skel->maps.__ops_name; \
+ __skel; \
+})
+
+#define UFQ_OPS_LOAD(__skel, __ops_name, __ufq_name) ({ \
+ (void)(__skel)->maps.__ops_name; \
+ UFQ_ERR_IF(__ufq_name##__load((__skel)), "Failed to load skel"); \
+})
+
+/*
+ * New versions of bpftool emit additional link placeholders for BPF maps,
+ * and set up BPF skeleton so libbpf can auto-attach BPF maps (v1.5+). Old
+ * libbpf ignores those links. Disable autoattach on newer libbpf to avoid
+ * attaching twice when we attach struct_ops explicitly.
+ */
+#if LIBBPF_MAJOR_VERSION > 1 || \
+ (LIBBPF_MAJOR_VERSION == 1 && LIBBPF_MINOR_VERSION >= 5)
+#define __UFQ_OPS_DISABLE_AUTOATTACH(__skel, __ops_name) \
+ bpf_map__set_autoattach((__skel)->maps.__ops_name, false)
+#else
+#define __UFQ_OPS_DISABLE_AUTOATTACH(__skel, __ops_name) do {} while (0)
+#endif
+
+#define UFQ_OPS_ATTACH(__skel, __ops_name, __ufq_name) ({ \
+ struct bpf_link *__link; \
+ __UFQ_OPS_DISABLE_AUTOATTACH(__skel, __ops_name); \
+ UFQ_ERR_IF(__ufq_name##__attach((__skel)), "Failed to attach skel"); \
+ __link = bpf_map__attach_struct_ops((__skel)->maps.__ops_name); \
+ UFQ_ERR_IF(!__link, "Failed to attach struct_ops"); \
+ __link; \
+})
+
+#endif /* __UFQ_IOSCHED_COMMON_H */
diff --git a/tools/ufq_iosched/include/ufq/simple_stat.h b/tools/ufq_iosched/include/ufq/simple_stat.h
new file mode 100644
index 000000000000..c26d0ea57a69
--- /dev/null
+++ b/tools/ufq_iosched/include/ufq/simple_stat.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 KylinSoft Corporation.
+ * Copyright (c) 2026 Kaitao Cheng <chengkaitao@kylinos.cn>
+ */
+#ifndef __UFQ_SIMPLE_STAT_H
+#define __UFQ_SIMPLE_STAT_H
+
+enum ufq_simp_stat_index {
+ UFQ_SIMP_INSERT_CNT,
+ UFQ_SIMP_INSERT_SIZE,
+ UFQ_SIMP_DISPATCH_CNT,
+ UFQ_SIMP_DISPATCH_SIZE,
+ UFQ_SIMP_RQMERGE_CNT,
+ UFQ_SIMP_RQMERGE_SIZE,
+ UFQ_SIMP_BIOMERGE_CNT,
+ UFQ_SIMP_BIOMERGE_SIZE,
+ UFQ_SIMP_FINISH_CNT,
+ UFQ_SIMP_FINISH_SIZE,
+ UFQ_SIMP_STAT_MAX,
+};
+
+#endif /* __UFQ_SIMPLE_STAT_H */
diff --git a/tools/ufq_iosched/ufq_simple.bpf.c b/tools/ufq_iosched/ufq_simple.bpf.c
new file mode 100644
index 000000000000..81954e73068a
--- /dev/null
+++ b/tools/ufq_iosched/ufq_simple.bpf.c
@@ -0,0 +1,604 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 KylinSoft Corporation.
+ * Copyright (c) 2026 Kaitao Cheng <chengkaitao@kylinos.cn>
+ */
+#include <ufq/common.bpf.h>
+
+char _license[] SEC("license") = "GPL";
+
+#define UFQ_DISK_SUM 20
+#define BLK_MQ_INSERT_AT_HEAD 0x01
+#define REQ_OP_MASK ((1 << 8) - 1)
+#define SECTOR_SHIFT 9
+#define UFQ_LOOP_MAX 100
+
+struct {
+ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+ __uint(key_size, sizeof(u32));
+ __uint(value_size, sizeof(u64));
+ __uint(max_entries, UFQ_SIMP_STAT_MAX);
+} stats SEC(".maps");
+
+enum ufq_simp_data_dir {
+ UFQ_SIMP_READ,
+ UFQ_SIMP_WRITE,
+ UFQ_SIMP_DIR_COUNT
+};
+
+struct queue_list_node {
+ struct bpf_list_node node;
+ struct request __kptr * req;
+};
+
+struct sort_tree_node {
+ struct bpf_refcount ref;
+ struct bpf_rb_node rb_node;
+ u64 key;
+ struct request __kptr * req;
+};
+
+struct ufq_simple_data {
+ struct bpf_spin_lock lock;
+ struct bpf_rb_root sort_tree_read __contains(sort_tree_node, rb_node);
+ struct bpf_rb_root sort_tree_write __contains(sort_tree_node, rb_node);
+ struct bpf_list_head dispatch __contains(queue_list_node, node);
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_HASH);
+ __uint(max_entries, UFQ_DISK_SUM);
+ __type(key, s32);
+ __type(value, struct ufq_simple_data);
+} ufq_map SEC(".maps");
+
+static void stat_add(u32 idx, u32 val)
+{
+ u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx);
+
+ if (cnt_p)
+ (*cnt_p) += val;
+}
+
+static void stat_sub(u32 idx, u32 val)
+{
+ u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx);
+
+ if (cnt_p)
+ (*cnt_p) -= val;
+}
+
+static bool sort_tree_less(struct bpf_rb_node *a, const struct bpf_rb_node *b)
+{
+ struct sort_tree_node *node_a, *node_b;
+
+ node_a = container_of(a, struct sort_tree_node, rb_node);
+ node_b = container_of(b, struct sort_tree_node, rb_node);
+
+ return node_a->key < node_b->key;
+}
+
+static struct ufq_simple_data *dd_init_sched(struct request_queue *q)
+{
+ struct ufq_simple_data ufq_sd = {}, *ufq_sp;
+ int ret, id = q->id;
+
+ bpf_printk("ufq_simple init sched!");
+ ret = bpf_map_update_elem(&ufq_map, &id, &ufq_sd, BPF_NOEXIST);
+ if (ret && ret != -EEXIST) {
+ bpf_printk("ufq_simple/init_sched: update ufq_map err %d", ret);
+ return NULL;
+ }
+
+ ufq_sp = bpf_map_lookup_elem(&ufq_map, &id);
+ if (!ufq_sp) {
+ bpf_printk("ufq_simple/init_sched: lookup queue id %d in ufq_map failed", id);
+ return NULL;
+ }
+
+ return ufq_sp;
+}
+
+int BPF_STRUCT_OPS(ufq_simple_init_sched, struct request_queue *q)
+{
+ if (dd_init_sched(q))
+ return 0;
+ else
+ return -EPERM;
+}
+
+int BPF_STRUCT_OPS(ufq_simple_exit_sched, struct request_queue *q)
+{
+ int id = q->id;
+
+ bpf_printk("ufq_simple exit sched!");
+ bpf_map_delete_elem(&ufq_map, &id);
+ return 0;
+}
+
+int BPF_STRUCT_OPS(ufq_simple_insert_req, struct request_queue *q,
+ struct request *rq, blk_insert_t flags,
+ struct list_head *freeq)
+{
+ struct ufq_simple_data *ufq_sd;
+ struct queue_list_node *qnode;
+ struct sort_tree_node *snode;
+ int id = q->id, ret = 0;
+ struct request *acquired, *old;
+ enum ufq_simp_data_dir dir = ((rq->cmd_flags & REQ_OP_MASK) & 1) ?
+ UFQ_SIMP_WRITE : UFQ_SIMP_READ;
+
+ ufq_sd = bpf_map_lookup_elem(&ufq_map, &id);
+ if (!ufq_sd) {
+ ufq_sd = dd_init_sched(q);
+ if (!ufq_sd) {
+ bpf_printk("ufq_simple/insert_req: dd_init_sched failed");
+ return -EPERM;
+ }
+ }
+
+ if (flags & BLK_MQ_INSERT_AT_HEAD) {
+ /* create queue_list_node */
+ qnode = bpf_obj_new(typeof(*qnode));
+ if (!qnode) {
+ bpf_printk("ufq_simple/insert_req: qnode alloc failed");
+ return -ENOMEM;
+ }
+
+ acquired = bpf_request_acquire(rq);
+ if (!acquired) {
+ bpf_obj_drop(qnode);
+ bpf_printk("ufq_simple/head-insert_req: request_acquire failed");
+ return -EPERM;
+ }
+
+ /* Set request for queue_list_node */
+ old = bpf_kptr_xchg(&qnode->req, acquired);
+ if (old)
+ bpf_request_release(old);
+
+ /* Add queue_list_node to dispatch list */
+ bpf_spin_lock(&ufq_sd->lock);
+ ret = bpf_list_push_back(&ufq_sd->dispatch, &qnode->node);
+ bpf_spin_unlock(&ufq_sd->lock);
+ } else {
+ /* create sort_tree_node */
+ snode = bpf_obj_new(typeof(*snode));
+ if (!snode) {
+ bpf_printk("ufq_simple/insert_req: sort_tree_node alloc failed");
+ return -ENOMEM;
+ }
+
+ /* Use request's starting sector as sort key */
+ snode->key = rq->__sector;
+
+ /*
+ * Acquire request reference again for sort_tree_node (each node
+ * needs independent reference)
+ */
+ acquired = bpf_request_acquire(rq);
+ if (!acquired) {
+ bpf_obj_drop(snode);
+ bpf_printk("ufq_simple/insert_req: bpf_request_acquire failed");
+ return -EPERM;
+ }
+
+ /* Set request for sort_tree_node */
+ old = bpf_kptr_xchg(&snode->req, acquired);
+ if (old)
+ bpf_request_release(old);
+
+ /* Add sort_tree_node to red-black tree */
+ bpf_spin_lock(&ufq_sd->lock);
+ if (dir == UFQ_SIMP_READ)
+ bpf_rbtree_add(&ufq_sd->sort_tree_read, &snode->rb_node, sort_tree_less);
+ else
+ bpf_rbtree_add(&ufq_sd->sort_tree_write, &snode->rb_node, sort_tree_less);
+ bpf_spin_unlock(&ufq_sd->lock);
+ }
+
+ if (!ret) {
+ stat_add(UFQ_SIMP_INSERT_CNT, 1);
+ stat_add(UFQ_SIMP_INSERT_SIZE, rq->__data_len);
+ }
+ return ret;
+}
+
+struct request *BPF_STRUCT_OPS(ufq_simple_dispatch_req, struct request_queue *q)
+{
+ struct request *rq = NULL;
+ struct bpf_list_node *list_node;
+ struct bpf_rb_node *rb_node = NULL;
+ struct queue_list_node *qnode;
+ struct sort_tree_node *snode;
+ struct ufq_simple_data *ufq_sd;
+ int id = q->id;
+
+ ufq_sd = bpf_map_lookup_elem(&ufq_map, &id);
+ if (!ufq_sd) {
+ bpf_printk("ufq_simple/dispatch_req: ufq_map lookup %d failed", id);
+ return NULL;
+ }
+
+ bpf_spin_lock(&ufq_sd->lock);
+ list_node = bpf_list_pop_front(&ufq_sd->dispatch);
+
+ if (list_node) {
+ qnode = container_of(list_node, struct queue_list_node, node);
+ rq = bpf_kptr_xchg(&qnode->req, NULL);
+ bpf_spin_unlock(&ufq_sd->lock);
+ bpf_obj_drop(qnode);
+ } else {
+ rb_node = bpf_rbtree_first(&ufq_sd->sort_tree_read);
+ if (rb_node) {
+ rb_node = bpf_rbtree_remove(&ufq_sd->sort_tree_read, rb_node);
+ } else {
+ rb_node = bpf_rbtree_first(&ufq_sd->sort_tree_write);
+ if (rb_node)
+ rb_node = bpf_rbtree_remove(&ufq_sd->sort_tree_write, rb_node);
+ }
+
+ if (!rb_node) {
+ bpf_spin_unlock(&ufq_sd->lock);
+ goto out;
+ }
+
+ snode = container_of(rb_node, struct sort_tree_node, rb_node);
+
+ /* Get request from sort_tree_node (this will be returned) */
+ rq = bpf_kptr_xchg(&snode->req, NULL);
+ bpf_spin_unlock(&ufq_sd->lock);
+ bpf_obj_drop(snode);
+ }
+ if (!rq)
+ bpf_printk("ufq_simple/dispatch_req: no request to dispatch");
+
+out:
+ if (rq) {
+ stat_add(UFQ_SIMP_DISPATCH_CNT, 1);
+ stat_add(UFQ_SIMP_DISPATCH_SIZE, rq->__data_len);
+ }
+
+ return rq;
+}
+
+bool BPF_STRUCT_OPS(ufq_simple_has_req, struct request_queue *q, int rqs_count)
+{
+ struct ufq_simple_data *ufq_sd;
+ bool has;
+ int id = q->id;
+
+ ufq_sd = bpf_map_lookup_elem(&ufq_map, &id);
+ if (!ufq_sd) {
+ bpf_printk("ufq_simple/has_req: ufq_map lookup %d failed", id);
+ return false;
+ }
+
+ bpf_spin_lock(&ufq_sd->lock);
+ has = !bpf_list_empty(&ufq_sd->dispatch) ||
+ bpf_rbtree_root(&ufq_sd->sort_tree_read) ||
+ bpf_rbtree_root(&ufq_sd->sort_tree_write);
+ bpf_spin_unlock(&ufq_sd->lock);
+
+ return has;
+}
+
+void BPF_STRUCT_OPS(ufq_simple_finish_req, struct request *rq)
+{
+ if (rq) {
+ stat_add(UFQ_SIMP_FINISH_CNT, 1);
+ stat_add(UFQ_SIMP_FINISH_SIZE, (u64)rq->stats_sectors << SECTOR_SHIFT);
+ }
+}
+
+struct request *BPF_STRUCT_OPS(ufq_simple_merge_req, struct request_queue *q,
+ struct request *rq, int *type)
+{
+ struct sort_tree_node *snode = NULL;
+ sector_t rq_start, rq_end, other_start, other_end;
+ enum elv_merge mt = ELEVATOR_NO_MERGE;
+ struct bpf_rb_node *rb_node = NULL;
+ struct blk_mq_ctx *targ_mq_ctx;
+ struct blk_mq_hw_ctx *targ_mq_hctx;
+ struct ufq_simple_data *ufq_sd;
+ struct request *targ = NULL;
+ enum ufq_simp_data_dir dir;
+ struct bpf_rb_root *tree;
+ int id = q->id;
+ int count = 0;
+
+ *type = ELEVATOR_NO_MERGE;
+ dir = ((rq->cmd_flags & REQ_OP_MASK) & 1) ? UFQ_SIMP_WRITE : UFQ_SIMP_READ;
+ ufq_sd = bpf_map_lookup_elem(&ufq_map, &id);
+ if (!ufq_sd)
+ return NULL;
+
+ /* Calculate current request position and end */
+ rq_start = rq->__sector;
+ rq_end = rq_start + (rq->__data_len >> SECTOR_SHIFT);
+
+ if (dir == UFQ_SIMP_READ)
+ tree = &ufq_sd->sort_tree_read;
+ else
+ tree = &ufq_sd->sort_tree_write;
+
+ bpf_spin_lock(&ufq_sd->lock);
+ rb_node = bpf_rbtree_root(tree);
+ if (!rb_node) {
+ bpf_spin_unlock(&ufq_sd->lock);
+ return NULL;
+ }
+
+ while (mt == ELEVATOR_NO_MERGE && rb_node && count < UFQ_LOOP_MAX) {
+ count++;
+ snode = container_of(rb_node, struct sort_tree_node, rb_node);
+ targ = bpf_kptr_xchg(&snode->req, NULL);
+ if (!targ)
+ break;
+
+ other_start = targ->__sector;
+ other_end = other_start + (targ->__data_len >> SECTOR_SHIFT);
+ targ_mq_ctx = targ->mq_ctx;
+ targ_mq_hctx = targ->mq_hctx;
+
+ targ = bpf_kptr_xchg(&snode->req, targ);
+ if (targ) {
+ bpf_spin_unlock(&ufq_sd->lock);
+ bpf_request_release(targ);
+ return NULL;
+ }
+
+ if (rq_start > other_end)
+ rb_node = bpf_rbtree_right(tree, rb_node);
+ else if (rq_end < other_start)
+ rb_node = bpf_rbtree_left(tree, rb_node);
+ else if (rq_end == other_start)
+ mt = ELEVATOR_FRONT_MERGE;
+ else if (other_end == rq_start)
+ mt = ELEVATOR_BACK_MERGE;
+ else
+ break;
+
+ if (mt) {
+ if (rq->mq_ctx != targ_mq_ctx || rq->mq_hctx != targ_mq_hctx) {
+ mt = ELEVATOR_NO_MERGE;
+ break;
+ }
+
+ rb_node = bpf_rbtree_remove(tree, rb_node);
+ if (rb_node) {
+ snode = container_of(rb_node,
+ struct sort_tree_node, rb_node);
+ targ = bpf_kptr_xchg(&snode->req, NULL);
+ bpf_spin_unlock(&ufq_sd->lock);
+ if (targ) {
+ *type = mt;
+ stat_add(UFQ_SIMP_RQMERGE_CNT, 1);
+ stat_add(UFQ_SIMP_RQMERGE_SIZE, targ->__data_len);
+ stat_sub(UFQ_SIMP_INSERT_CNT, 1);
+ stat_sub(UFQ_SIMP_INSERT_SIZE, targ->__data_len);
+ }
+
+ bpf_obj_drop(snode);
+ } else {
+ bpf_spin_unlock(&ufq_sd->lock);
+ *type = ELEVATOR_NO_MERGE;
+ }
+ return targ;
+ }
+ }
+ bpf_spin_unlock(&ufq_sd->lock);
+
+ return NULL;
+}
+
+static struct request *merge_bio_left_unlock(struct ufq_simple_data *ufq_sd,
+ struct bpf_rb_root *tree,
+ struct sort_tree_node *snode,
+ struct request *cand)
+{
+ sector_t cand_start, left_start, left_end;
+ struct request *free = NULL;
+ struct request *left_rq = NULL;
+ struct sort_tree_node *left_node = NULL;
+ struct bpf_rb_node *tmp, *removed = NULL;
+
+ cand_start = cand->__sector;
+ tmp = bpf_rbtree_left(tree, &snode->rb_node);
+ if (!tmp)
+ goto end;
+
+ left_node = container_of(tmp, struct sort_tree_node, rb_node);
+ if (!left_node)
+ goto end;
+
+ left_rq = bpf_kptr_xchg(&left_node->req, NULL);
+ if (!left_rq)
+ goto end;
+
+ left_start = left_rq->__sector;
+ left_end = left_start + (left_rq->__data_len >> SECTOR_SHIFT);
+
+ if (left_end == cand_start)
+ free = bpf_request_try_merge(left_rq, cand);
+
+ if (free == cand) {
+ removed = bpf_rbtree_remove(tree, &snode->rb_node);
+ left_rq = bpf_kptr_xchg(&left_node->req, left_rq);
+ bpf_spin_unlock(&ufq_sd->lock);
+ if (removed) {
+ struct sort_tree_node *drop = container_of(removed,
+ struct sort_tree_node, rb_node);
+ bpf_obj_drop(drop);
+ }
+ stat_add(UFQ_SIMP_RQMERGE_CNT, 1);
+ stat_add(UFQ_SIMP_RQMERGE_SIZE, free->__data_len);
+
+ if (left_rq)
+ bpf_request_release(left_rq);
+
+ return free;
+ }
+
+ left_rq = bpf_kptr_xchg(&left_node->req, left_rq);
+
+end:
+ cand = bpf_kptr_xchg(&snode->req, cand);
+ bpf_spin_unlock(&ufq_sd->lock);
+ if (left_rq)
+ bpf_request_release(left_rq);
+
+ if (cand)
+ bpf_request_release(cand);
+
+ return NULL;
+}
+
+static struct request *merge_bio_right_unlock(struct ufq_simple_data *ufq_sd,
+ struct bpf_rb_root *tree,
+ struct sort_tree_node *snode,
+ struct request *cand)
+{
+ sector_t cand_end, right_start;
+ struct request *free = NULL;
+ struct request *right_rq = NULL;
+ struct sort_tree_node *right_node = NULL;
+ struct bpf_rb_node *right_rb, *removed = NULL;
+
+ cand_end = cand->__sector + (cand->__data_len >> SECTOR_SHIFT);
+
+ right_rb = bpf_rbtree_right(tree, &snode->rb_node);
+ if (!right_rb)
+ goto end;
+
+ right_node = container_of(right_rb, struct sort_tree_node, rb_node);
+ if (!right_node)
+ goto end;
+
+ right_rq = bpf_kptr_xchg(&right_node->req, NULL);
+ if (!right_rq)
+ goto end;
+
+ right_start = right_rq->__sector;
+ if (cand_end == right_start)
+ free = bpf_request_try_merge(cand, right_rq);
+
+ if (free == right_rq) {
+ removed = bpf_rbtree_remove(tree, right_rb);
+ cand = bpf_kptr_xchg(&snode->req, cand);
+ bpf_spin_unlock(&ufq_sd->lock);
+ if (removed) {
+ struct sort_tree_node *drop = container_of(removed,
+ struct sort_tree_node, rb_node);
+ bpf_obj_drop(drop);
+ }
+ stat_add(UFQ_SIMP_RQMERGE_CNT, 1);
+ stat_add(UFQ_SIMP_RQMERGE_SIZE, free->__data_len);
+
+ if (cand)
+ bpf_request_release(cand);
+
+ return free;
+ }
+
+ right_rq = bpf_kptr_xchg(&right_node->req, right_rq);
+
+end:
+ cand = bpf_kptr_xchg(&snode->req, cand);
+ bpf_spin_unlock(&ufq_sd->lock);
+ if (right_rq)
+ bpf_request_release(right_rq);
+
+ if (cand)
+ bpf_request_release(cand);
+
+ return NULL;
+}
+
+struct request *BPF_STRUCT_OPS(ufq_simple_merge_bio,
+ struct request_queue *q, struct bio *bio,
+ unsigned int nr_segs, bool *merged)
+{
+ sector_t start, end, cand_start, cand_end;
+ struct request *cand = NULL, *old, *free = NULL;
+ struct sort_tree_node *snode = NULL;
+ struct bpf_rb_node *rb_node = NULL;
+ struct ufq_simple_data *ufq_sd;
+ enum ufq_simp_data_dir dir;
+ int id = q->id, count = 0;
+ struct bpf_rb_root *tree;
+
+ if (!merged)
+ return NULL;
+ start = bio->bi_iter.bi_sector;
+ end = start + (bio->bi_iter.bi_size >> SECTOR_SHIFT);
+ dir = ((bio->bi_opf & REQ_OP_MASK) & 1) ? UFQ_SIMP_WRITE : UFQ_SIMP_READ;
+ ufq_sd = bpf_map_lookup_elem(&ufq_map, &id);
+ if (!ufq_sd)
+ return NULL;
+
+ bpf_spin_lock(&ufq_sd->lock);
+ if (dir == UFQ_SIMP_READ)
+ tree = &ufq_sd->sort_tree_read;
+ else
+ tree = &ufq_sd->sort_tree_write;
+
+ rb_node = bpf_rbtree_root(tree);
+ while (rb_node && count < UFQ_LOOP_MAX) {
+ count++;
+ snode = container_of(rb_node, struct sort_tree_node, rb_node);
+ cand = bpf_kptr_xchg(&snode->req, NULL);
+ if (!cand)
+ break;
+
+ cand_start = cand->__sector;
+ cand_end = cand_start + (cand->__data_len >> SECTOR_SHIFT);
+
+ if (end < cand_start) {
+ rb_node = bpf_rbtree_left(tree, rb_node);
+ } else if (start > cand_end) {
+ rb_node = bpf_rbtree_right(tree, rb_node);
+ } else if (cand_start == end) {
+ if (bpf_request_bio_try_merge(cand, bio, nr_segs)) {
+ *merged = true;
+ free = merge_bio_left_unlock(ufq_sd, tree, snode, cand);
+ stat_add(UFQ_SIMP_BIOMERGE_CNT, 1);
+ stat_add(UFQ_SIMP_BIOMERGE_SIZE, bio->bi_iter.bi_size);
+ return free;
+ }
+ rb_node = NULL;
+ } else if (cand_end == start) {
+ if (bpf_request_bio_try_merge(cand, bio, nr_segs)) {
+ *merged = true;
+ free = merge_bio_right_unlock(ufq_sd, tree, snode, cand);
+ stat_add(UFQ_SIMP_BIOMERGE_CNT, 1);
+ stat_add(UFQ_SIMP_BIOMERGE_SIZE, bio->bi_iter.bi_size);
+ return free;
+ }
+ rb_node = NULL;
+ } else {
+ rb_node = NULL;
+ }
+
+ old = bpf_kptr_xchg(&snode->req, cand);
+ if (old) {
+ bpf_spin_unlock(&ufq_sd->lock);
+ bpf_request_release(old);
+ return NULL;
+ }
+ }
+
+ bpf_spin_unlock(&ufq_sd->lock);
+ return NULL;
+}
+
+UFQ_OPS_DEFINE(ufq_simple_ops,
+ .init_sched = (void *)ufq_simple_init_sched,
+ .exit_sched = (void *)ufq_simple_exit_sched,
+ .insert_req = (void *)ufq_simple_insert_req,
+ .dispatch_req = (void *)ufq_simple_dispatch_req,
+ .has_req = (void *)ufq_simple_has_req,
+ .finish_req = (void *)ufq_simple_finish_req,
+ .merge_req = (void *)ufq_simple_merge_req,
+ .merge_bio = (void *)ufq_simple_merge_bio,
+ .name = "ufq_simple");
diff --git a/tools/ufq_iosched/ufq_simple.c b/tools/ufq_iosched/ufq_simple.c
new file mode 100644
index 000000000000..0e7b018699a2
--- /dev/null
+++ b/tools/ufq_iosched/ufq_simple.c
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 KylinSoft Corporation.
+ * Copyright (c) 2026 Kaitao Cheng <chengkaitao@kylinos.cn>
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <signal.h>
+#include <libgen.h>
+#include <bpf/bpf.h>
+#include <ufq/common.h>
+#include "ufq_simple.bpf.skel.h"
+
+const char help_fmt[] =
+"A simple ufq scheduler.\n"
+"\n"
+"Usage: %s [-v] [-d] [-h]\n"
+"\n"
+" -v Print version\n"
+" -d Print libbpf debug messages\n"
+" -h Display this help and exit\n";
+
+#define UFQ_SIMPLE_VERSION "0.1.0"
+#define TIME_INTERVAL 3
+static bool verbose;
+static volatile int exit_req;
+__u64 old_stats[UFQ_SIMP_STAT_MAX];
+
+static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
+{
+ if (level == LIBBPF_DEBUG && !verbose)
+ return 0;
+ return vfprintf(stderr, format, args);
+}
+
+static void sigint_handler(int simple)
+{
+ exit_req = 1;
+}
+
+static void read_stats(struct ufq_simple *skel, __u64 *stats)
+{
+ int nr_cpus = libbpf_num_possible_cpus();
+ __u64 cnts[UFQ_SIMP_STAT_MAX][nr_cpus];
+ __u32 idx;
+
+ memset(stats, 0, sizeof(stats[0]) * UFQ_SIMP_STAT_MAX);
+
+ for (idx = 0; idx < UFQ_SIMP_STAT_MAX; idx++) {
+ int ret, cpu;
+
+ ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats),
+ &idx, cnts[idx]);
+ if (ret < 0)
+ continue;
+ for (cpu = 0; cpu < nr_cpus; cpu++)
+ stats[idx] += cnts[idx][cpu];
+ }
+}
+
+int main(int argc, char **argv)
+{
+ struct ufq_simple *skel;
+ struct bpf_link *link;
+ __u32 opt;
+
+ libbpf_set_print(libbpf_print_fn);
+ signal(SIGINT, sigint_handler);
+ signal(SIGTERM, sigint_handler);
+
+ skel = UFQ_OPS_OPEN(ufq_simple_ops, ufq_simple);
+
+ while ((opt = getopt(argc, argv, "vdh")) != -1) {
+ switch (opt) {
+ case 'v':
+ printf("ufq_simple version: %s\n", UFQ_SIMPLE_VERSION);
+ return 0;
+ case 'd':
+ verbose = true;
+ break;
+ default:
+ fprintf(stderr, help_fmt, basename(argv[0]));
+ return opt != 'h';
+ }
+ }
+
+ UFQ_OPS_LOAD(skel, ufq_simple_ops, ufq_simple);
+ link = UFQ_OPS_ATTACH(skel, ufq_simple_ops, ufq_simple);
+
+ printf("ufq_simple loop ...\n");
+ while (!exit_req) {
+ __u64 stats[UFQ_SIMP_STAT_MAX];
+
+ printf("--------------------------------\n");
+ read_stats(skel, stats);
+ printf("bps:%lluk iops:%llu\n",
+ (stats[UFQ_SIMP_FINISH_SIZE] -
+ old_stats[UFQ_SIMP_FINISH_SIZE]) / 1024 / TIME_INTERVAL,
+ (stats[UFQ_SIMP_FINISH_CNT] -
+ old_stats[UFQ_SIMP_FINISH_CNT]) / TIME_INTERVAL);
+ printf("(insert: cnt=%llu size=%llu)\n",
+ stats[UFQ_SIMP_INSERT_CNT], stats[UFQ_SIMP_INSERT_SIZE]);
+ printf("(rqmerge: cnt=%llu size=%llu) (biomerge: cnt=%llu size=%llu)\n",
+ stats[UFQ_SIMP_RQMERGE_CNT], stats[UFQ_SIMP_RQMERGE_SIZE],
+ stats[UFQ_SIMP_BIOMERGE_CNT], stats[UFQ_SIMP_BIOMERGE_SIZE]);
+ printf("(dispatch: cnt=%llu size=%llu) (finish: cnt=%llu size=%llu)\n",
+ stats[UFQ_SIMP_DISPATCH_CNT], stats[UFQ_SIMP_DISPATCH_SIZE],
+ stats[UFQ_SIMP_FINISH_CNT], stats[UFQ_SIMP_FINISH_SIZE]);
+ memcpy(old_stats, stats, sizeof(old_stats));
+ sleep(TIME_INTERVAL);
+ }
+
+ printf("ufq_simple loop exit ...\n");
+ bpf_link__destroy(link);
+ ufq_simple__destroy(skel);
+
+ return 0;
+}
--
2.50.1 (Apple Git-155)
^ permalink raw reply related
* [RFC v2 2/3] block: Introduce the UFQ I/O scheduler
From: Kaitao cheng @ 2026-05-03 3:56 UTC (permalink / raw)
To: axboe, ast, daniel, andrii, martin.lau, eddyz87, memxor, song,
yonghong.song, jolsa, john.fastabend
Cc: bpf, linux-block, linux-kernel, Kaitao Cheng
In-Reply-To: <20260503035623.28771-1-kaitao.cheng@linux.dev>
From: Kaitao Cheng <chengkaitao@kylinos.cn>
Introduce IOSCHED_UFQ, a blk-mq elevator ("ufq: User-programmable
Flexible Queueing") whose policy is supplied by an eBPF program via
struct_ops (insert, dispatch, merge, finish, etc.).
When no eBPF program is attached, the UFQ I/O scheduler uses a simple,
per-ctx queueing policy (similar to none). After an eBPF program is
attached, the user-defined scheduling policy replaces UFQ’s built-in
queueing policy, while per-ctx queues remain available as a fallback
mechanism.
Signed-off-by: Kaitao Cheng <chengkaitao@kylinos.cn>
---
block/Kconfig.iosched | 8 +
block/Makefile | 1 +
block/blk-merge.c | 28 +-
block/blk-mq.c | 8 +-
block/blk-mq.h | 2 +-
block/blk.h | 5 +
block/ufq-bpfops.c | 241 ++++++++++++++++
block/ufq-iosched.c | 640 ++++++++++++++++++++++++++++++++++++++++++
block/ufq-iosched.h | 64 +++++
block/ufq-kfunc.c | 131 +++++++++
10 files changed, 1115 insertions(+), 13 deletions(-)
create mode 100644 block/ufq-bpfops.c
create mode 100644 block/ufq-iosched.c
create mode 100644 block/ufq-iosched.h
create mode 100644 block/ufq-kfunc.c
diff --git a/block/Kconfig.iosched b/block/Kconfig.iosched
index 27f11320b8d1..56afc425cc52 100644
--- a/block/Kconfig.iosched
+++ b/block/Kconfig.iosched
@@ -44,4 +44,12 @@ config BFQ_CGROUP_DEBUG
Enable some debugging help. Currently it exports additional stat
files in a cgroup which can be useful for debugging.
+config IOSCHED_UFQ
+ tristate "UFQ I/O scheduler"
+ default y
+ help
+ The UFQ I/O scheduler is a programmable I/O scheduler. When
+ enabled, an out-of-kernel I/O scheduler based on eBPF can be
+ designed to interact with it, leveraging its customizable
+ hooks to redefine I/O scheduling policies.
endmenu
diff --git a/block/Makefile b/block/Makefile
index 7dce2e44276c..a58ea7384b7a 100644
--- a/block/Makefile
+++ b/block/Makefile
@@ -24,6 +24,7 @@ obj-$(CONFIG_MQ_IOSCHED_DEADLINE) += mq-deadline.o
obj-$(CONFIG_MQ_IOSCHED_KYBER) += kyber-iosched.o
bfq-y := bfq-iosched.o bfq-wf2q.o bfq-cgroup.o
obj-$(CONFIG_IOSCHED_BFQ) += bfq.o
+obj-$(CONFIG_IOSCHED_UFQ) += ufq-iosched.o ufq-bpfops.o ufq-kfunc.o
obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o t10-pi.o \
bio-integrity-auto.o bio-integrity-fs.o
diff --git a/block/blk-merge.c b/block/blk-merge.c
index fcf09325b22e..7a98dc75a06f 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -774,8 +774,8 @@ u8 bio_seg_gap(struct request_queue *q, struct bio *prev, struct bio *next,
* For non-mq, this has to be called with the request spinlock acquired.
* For mq with scheduling, the appropriate queue wide lock should be held.
*/
-static struct request *attempt_merge(struct request_queue *q,
- struct request *req, struct request *next)
+static struct request *attempt_merge(struct request_queue *q, struct request *req,
+ struct request *next, bool nohash)
{
if (!rq_mergeable(req) || !rq_mergeable(next))
return NULL;
@@ -842,7 +842,7 @@ static struct request *attempt_merge(struct request_queue *q,
req->__data_len += blk_rq_bytes(next);
- if (!blk_discard_mergable(req))
+ if (!nohash && !blk_discard_mergable(req))
elv_merge_requests(q, req, next);
blk_crypto_rq_put_keyslot(next);
@@ -868,7 +868,7 @@ static struct request *attempt_back_merge(struct request_queue *q,
struct request *next = elv_latter_request(q, rq);
if (next)
- return attempt_merge(q, rq, next);
+ return attempt_merge(q, rq, next, false);
return NULL;
}
@@ -879,11 +879,17 @@ static struct request *attempt_front_merge(struct request_queue *q,
struct request *prev = elv_former_request(q, rq);
if (prev)
- return attempt_merge(q, prev, rq);
+ return attempt_merge(q, prev, rq, false);
return NULL;
}
+struct request *bpf_attempt_merge(struct request_queue *q, struct request *rq,
+ struct request *next)
+{
+ return attempt_merge(q, rq, next, true);
+}
+
/*
* Try to merge 'next' into 'rq'. Return true if the merge happened, false
* otherwise. The caller is responsible for freeing 'next' if the merge
@@ -892,7 +898,7 @@ static struct request *attempt_front_merge(struct request_queue *q,
bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
struct request *next)
{
- return attempt_merge(q, rq, next);
+ return attempt_merge(q, rq, next, false);
}
bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
@@ -1035,11 +1041,11 @@ static enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
return BIO_MERGE_FAILED;
}
-static enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q,
- struct request *rq,
- struct bio *bio,
- unsigned int nr_segs,
- bool sched_allow_merge)
+enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q,
+ struct request *rq,
+ struct bio *bio,
+ unsigned int nr_segs,
+ bool sched_allow_merge)
{
if (!blk_rq_merge_ok(rq, bio))
return BIO_MERGE_NONE;
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 4c5c16cce4f8..bebc1306d8fd 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -796,7 +796,7 @@ static void blk_mq_finish_request(struct request *rq)
}
}
-static void __blk_mq_free_request(struct request *rq)
+void __blk_mq_free_request(struct request *rq)
{
struct request_queue *q = rq->q;
struct blk_mq_ctx *ctx = rq->mq_ctx;
@@ -1844,6 +1844,12 @@ static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr,
if (list_empty(&ctx->rq_lists[type]))
sbitmap_clear_bit(sb, bitnr);
}
+
+ if (dispatch_data->rq) {
+ dispatch_data->rq->rq_flags |= RQF_STARTED;
+ if (hctx->queue->last_merge == dispatch_data->rq)
+ hctx->queue->last_merge = NULL;
+ }
spin_unlock(&ctx->lock);
return !dispatch_data->rq;
diff --git a/block/blk-mq.h b/block/blk-mq.h
index aa15d31aaae9..3f85cae7bf57 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -56,7 +56,7 @@ void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list);
struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
struct blk_mq_ctx *start);
void blk_mq_put_rq_ref(struct request *rq);
-
+void __blk_mq_free_request(struct request *rq);
/*
* Internal helpers for allocating/freeing the request map
*/
diff --git a/block/blk.h b/block/blk.h
index ec4674cdf2ea..51adc4cdcee4 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -313,6 +313,9 @@ enum bio_merge_status {
enum bio_merge_status bio_attempt_back_merge(struct request *req,
struct bio *bio, unsigned int nr_segs);
+enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q,
+ struct request *rq, struct bio *bio, unsigned int nr_segs,
+ bool sched_allow_merge);
bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
unsigned int nr_segs);
bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
@@ -444,6 +447,8 @@ static inline unsigned get_max_segment_size(const struct queue_limits *lim,
int ll_back_merge_fn(struct request *req, struct bio *bio,
unsigned int nr_segs);
+struct request *bpf_attempt_merge(struct request_queue *q, struct request *rq,
+ struct request *next);
bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
struct request *next);
unsigned int blk_recalc_rq_segments(struct request *rq);
diff --git a/block/ufq-bpfops.c b/block/ufq-bpfops.c
new file mode 100644
index 000000000000..1c3c62e6c47e
--- /dev/null
+++ b/block/ufq-bpfops.c
@@ -0,0 +1,241 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 KylinSoft Corporation.
+ * Copyright (c) 2026 Kaitao Cheng <chengkaitao@kylinos.cn>
+ */
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/bpf_verifier.h>
+#include <linux/bpf.h>
+#include <linux/btf.h>
+#include <linux/btf_ids.h>
+#include <linux/string.h>
+#include <linux/wait.h>
+#include <linux/rcupdate.h>
+#include "ufq-iosched.h"
+
+struct ufq_iosched_ops ufq_ops;
+static atomic_t ufq_bpfops_enabled;
+static atomic_t ufq_bpfops_users;
+static DECLARE_WAIT_QUEUE_HEAD(ufq_bpfops_wq);
+
+const struct ufq_iosched_ops *ufq_bpfops_tryget(void)
+{
+ if (!atomic_read(&ufq_bpfops_enabled))
+ return NULL;
+
+ atomic_inc(&ufq_bpfops_users);
+ /*
+ * Pairs with disable path flipping ufq_bpfops_enabled to make sure no
+ * callback runs after teardown starts.
+ */
+ smp_mb__after_atomic();
+
+ if (unlikely(!atomic_read(&ufq_bpfops_enabled))) {
+ if (atomic_dec_and_test(&ufq_bpfops_users))
+ wake_up_all(&ufq_bpfops_wq);
+ return NULL;
+ }
+
+ return &ufq_ops;
+}
+
+void ufq_bpfops_put(void)
+{
+ if (atomic_dec_and_test(&ufq_bpfops_users))
+ wake_up_all(&ufq_bpfops_wq);
+}
+
+static const struct bpf_func_proto *
+bpf_ufq_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
+{
+ return bpf_base_func_proto(func_id, prog);
+}
+
+static bool bpf_ufq_is_valid_access(int off, int size,
+ enum bpf_access_type type,
+ const struct bpf_prog *prog,
+ struct bpf_insn_access_aux *info)
+{
+ if (type != BPF_READ)
+ return false;
+ if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
+ return false;
+ if (off % size != 0)
+ return false;
+
+ /*
+ * btf_ctx_access() treats pointers that are not "pointer to struct"
+ * as scalars (no reg_type), so loading pointers like merge_req()'s
+ * int *type or merge_bio()'s bool *merged from ctx leaves a SCALAR
+ * and stores through them fail verification. Model both as writable
+ * buffers.
+ */
+ if (size == sizeof(__u64) && prog->aux->attach_func_name &&
+ ((!strcmp(prog->aux->attach_func_name, "merge_req") && off == 16) ||
+ (!strcmp(prog->aux->attach_func_name, "merge_bio") && off == 24))) {
+ if (!btf_ctx_access(off, size, type, prog, info))
+ return false;
+ info->reg_type = PTR_TO_BUF;
+ return true;
+ }
+
+ return btf_ctx_access(off, size, type, prog, info);
+}
+
+static const struct bpf_verifier_ops bpf_ufq_verifier_ops = {
+ .get_func_proto = bpf_ufq_get_func_proto,
+ .is_valid_access = bpf_ufq_is_valid_access,
+};
+
+static int bpf_ufq_init_member(const struct btf_type *t,
+ const struct btf_member *member,
+ void *kdata, const void *udata)
+{
+ const struct ufq_iosched_ops *uops = udata;
+ struct ufq_iosched_ops *ops = kdata;
+ u32 moff = __btf_member_bit_offset(t, member) / 8;
+ int ret;
+
+ switch (moff) {
+ case offsetof(struct ufq_iosched_ops, name):
+ ret = bpf_obj_name_cpy(ops->name, uops->name,
+ sizeof(ops->name));
+ if (ret < 0)
+ return ret;
+ if (ret == 0)
+ return -EINVAL;
+ return 1;
+ /* other var adding .... */
+ }
+
+ return 0;
+}
+
+static int bpf_ufq_check_member(const struct btf_type *t,
+ const struct btf_member *member,
+ const struct bpf_prog *prog)
+{
+ return 0;
+}
+
+static int bpf_ufq_enable(void *ops)
+{
+ ufq_ops = *(struct ufq_iosched_ops *)ops;
+ atomic_set(&ufq_bpfops_enabled, 1);
+ return 0;
+}
+
+static void bpf_ufq_disable(struct ufq_iosched_ops *ops)
+{
+ atomic_set(&ufq_bpfops_enabled, 0);
+ wait_event(ufq_bpfops_wq, !atomic_read(&ufq_bpfops_users));
+ memset(&ufq_ops, 0, sizeof(ufq_ops));
+}
+
+static int bpf_ufq_reg(void *kdata, struct bpf_link *link)
+{
+ return ufq_prepare_bpf_attach(bpf_ufq_enable, kdata);
+}
+
+static void bpf_ufq_unreg(void *kdata, struct bpf_link *link)
+{
+ bpf_ufq_disable(kdata);
+ ufq_kick_all_hw_queues();
+}
+
+static int bpf_ufq_init(struct btf *btf)
+{
+ return 0;
+}
+
+static int bpf_ufq_update(void *kdata, void *old_kdata, struct bpf_link *link)
+{
+ /*
+ * UFQ does not support live-updating an already-attached BPF scheduler:
+ * partial failure during callback setup (e.g. init_sched) would be hard
+ * to reason about, and update can race with unregister/teardown.
+ */
+ return -EOPNOTSUPP;
+}
+
+static int bpf_ufq_validate(void *kdata)
+{
+ return 0;
+}
+
+static int init_sched_stub(struct request_queue *q)
+{
+ return -EPERM;
+}
+
+static int exit_sched_stub(struct request_queue *q)
+{
+ return -EPERM;
+}
+
+static int insert_req_stub(struct request_queue *q, struct request *rq,
+ blk_insert_t flags)
+{
+ return 0;
+}
+
+static struct request *dispatch_req_stub(struct request_queue *q)
+{
+ return NULL;
+}
+
+static bool has_req_stub(struct request_queue *q, int rqs_count)
+{
+ return rqs_count > 0;
+}
+
+static void finish_req_stub(struct request *rq)
+{
+}
+
+static struct request *merge_req_stub(struct request_queue *q, struct request *rq,
+ int *type)
+{
+ *type = ELEVATOR_NO_MERGE;
+ return NULL;
+}
+
+static struct request *merge_bio_stub(struct request_queue *q, struct bio *bio,
+ unsigned int nr_segs, bool *merged)
+{
+ if (merged)
+ *merged = false;
+
+ return NULL;
+}
+
+static struct ufq_iosched_ops __bpf_ops_ufq_ops = {
+ .init_sched = init_sched_stub,
+ .exit_sched = exit_sched_stub,
+ .insert_req = insert_req_stub,
+ .dispatch_req = dispatch_req_stub,
+ .has_req = has_req_stub,
+ .merge_req = merge_req_stub,
+ .finish_req = finish_req_stub,
+ .merge_bio = merge_bio_stub,
+};
+
+static struct bpf_struct_ops bpf_iosched_ufq_ops = {
+ .verifier_ops = &bpf_ufq_verifier_ops,
+ .reg = bpf_ufq_reg,
+ .unreg = bpf_ufq_unreg,
+ .check_member = bpf_ufq_check_member,
+ .init_member = bpf_ufq_init_member,
+ .init = bpf_ufq_init,
+ .update = bpf_ufq_update,
+ .validate = bpf_ufq_validate,
+ .name = "ufq_iosched_ops",
+ .owner = THIS_MODULE,
+ .cfi_stubs = &__bpf_ops_ufq_ops
+};
+
+int bpf_ufq_ops_init(void)
+{
+ return register_bpf_struct_ops(&bpf_iosched_ufq_ops, ufq_iosched_ops);
+}
diff --git a/block/ufq-iosched.c b/block/ufq-iosched.c
new file mode 100644
index 000000000000..ebbb63e0ef51
--- /dev/null
+++ b/block/ufq-iosched.c
@@ -0,0 +1,640 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 KylinSoft Corporation.
+ * Copyright (c) 2026 Kaitao Cheng <chengkaitao@kylinos.cn>
+ */
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/blkdev.h>
+#include <linux/bio.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/init.h>
+#include <linux/compiler.h>
+#include <linux/sbitmap.h>
+#include <linux/workqueue.h>
+
+#include <trace/events/block.h>
+
+#include "elevator.h"
+#include "blk.h"
+#include "blk-mq.h"
+#include "blk-mq-sched.h"
+#include "blk-mq-debugfs.h"
+#include "ufq-iosched.h"
+
+static DEFINE_MUTEX(ufq_active_queues_lock);
+static LIST_HEAD(ufq_active_queues);
+
+enum ufq_priv_state {
+ UFQ_PRIV_NOT_IN_SCHED = 0,
+ UFQ_PRIV_IN_BPF = 1,
+ UFQ_PRIV_IN_UFQ = 2,
+ UFQ_PRIV_IN_SCHED = 3,
+};
+
+static struct request *ufq_dispatch_request(struct blk_mq_hw_ctx *hctx)
+{
+ struct ufq_data *ufq = hctx->queue->elevator->elevator_data;
+ const struct ufq_iosched_ops *ops;
+ struct blk_mq_ctx *ctx;
+ struct request *rq = NULL;
+ unsigned short idx;
+
+ ops = ufq_bpfops_tryget();
+ if (ops && ops->dispatch_req) {
+ rq = ops->dispatch_req(hctx->queue);
+ if (!rq) {
+ atomic_inc(&ufq->ops_stats.dispatch_null_count);
+ ufq_bpfops_put();
+ return NULL;
+ }
+ ufq_bpfops_put();
+
+ /* The BPF insert_req callback bumps the request's reference
+ * count; dispatch_req returns that same request with an extra
+ * reference held. The kernel must put that reference here,
+ * and the request's refcount is always greater than zero at
+ * this point.
+ */
+ if (WARN_ON_ONCE(req_ref_put_and_test(rq))) {
+ __blk_mq_free_request(rq);
+ return NULL;
+ }
+
+ ctx = rq->mq_ctx;
+ spin_lock(&ctx->lock);
+ if (unlikely(blk_mq_rq_state(rq) != MQ_RQ_IDLE ||
+ (rq->rq_flags & RQF_STARTED) ||
+ list_empty(&rq->queuelist))) {
+ spin_unlock(&ctx->lock);
+ return NULL;
+ }
+ list_del_init(&rq->queuelist);
+ rq->rq_flags |= RQF_STARTED;
+ if (hctx->queue->last_merge == rq)
+ hctx->queue->last_merge = NULL;
+ if (list_empty(&ctx->rq_lists[rq->mq_hctx->type]))
+ sbitmap_clear_bit(&rq->mq_hctx->ctx_map,
+ ctx->index_hw[rq->mq_hctx->type]);
+ spin_unlock(&ctx->lock);
+ atomic_inc(&ufq->ops_stats.dispatch_ok_count);
+ atomic64_add(blk_rq_sectors(rq), &ufq->ops_stats.dispatch_ok_sectors);
+ rq->elv.priv[0] = (void *)((uintptr_t)rq->elv.priv[0]
+ & ~UFQ_PRIV_IN_UFQ);
+ } else {
+ if (ops)
+ ufq_bpfops_put();
+ ctx = READ_ONCE(hctx->dispatch_from);
+ rq = blk_mq_dequeue_from_ctx(hctx, ctx);
+ if (rq) {
+ idx = rq->mq_ctx->index_hw[hctx->type];
+ if (++idx == hctx->nr_ctx)
+ idx = 0;
+ WRITE_ONCE(hctx->dispatch_from, hctx->ctxs[idx]);
+ }
+ }
+
+ if (rq)
+ atomic_dec(&ufq->rqs_count);
+ return rq;
+}
+
+/*
+ * Called by __blk_mq_alloc_request(). The shallow_depth value set by this
+ * function is used by __blk_mq_get_tag().
+ */
+static void ufq_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data)
+{
+ struct ufq_data *ufq = data->q->elevator->elevator_data;
+
+ /* Do not throttle synchronous reads. */
+ if (op_is_sync(opf) && !op_is_write(opf))
+ return;
+
+ /*
+ * Throttle asynchronous requests and writes such that these requests
+ * do not block the allocation of synchronous requests.
+ */
+ data->shallow_depth = ufq->async_depth;
+}
+
+static void ufq_depth_updated(struct request_queue *q)
+{
+ struct ufq_data *ufq = q->elevator->elevator_data;
+
+ ufq->async_depth = q->nr_requests;
+ q->async_depth = q->nr_requests;
+ blk_mq_set_min_shallow_depth(q, 1);
+}
+
+static int ufq_init_sched(struct request_queue *q, struct elevator_queue *eq)
+{
+ const struct ufq_iosched_ops *ops;
+ struct ufq_data *ufq;
+
+ ufq = kzalloc_node(sizeof(*ufq), GFP_KERNEL, q->node);
+ if (!ufq)
+ return -ENOMEM;
+
+ eq->elevator_data = ufq;
+ ufq->q = q;
+ INIT_LIST_HEAD(&ufq->active_node);
+
+ blk_queue_flag_set(QUEUE_FLAG_SQ_SCHED, q);
+ q->elevator = eq;
+
+ q->async_depth = q->nr_requests;
+ ufq->async_depth = q->nr_requests;
+
+ ops = ufq_bpfops_tryget();
+ if (ops) {
+ if (ops->init_sched)
+ ops->init_sched(q);
+ ufq_bpfops_put();
+ }
+
+ mutex_lock(&ufq_active_queues_lock);
+ list_add_tail(&ufq->active_node, &ufq_active_queues);
+ mutex_unlock(&ufq_active_queues_lock);
+
+ ufq_depth_updated(q);
+ return 0;
+}
+
+static void ufq_exit_sched(struct elevator_queue *e)
+{
+ const struct ufq_iosched_ops *ops;
+ struct ufq_data *ufq = e->elevator_data;
+
+ ops = ufq_bpfops_tryget();
+ if (ops) {
+ if (ops->exit_sched)
+ ops->exit_sched(ufq->q);
+ ufq_bpfops_put();
+ }
+
+ mutex_lock(&ufq_active_queues_lock);
+ if (!list_empty(&ufq->active_node))
+ list_del_init(&ufq->active_node);
+ mutex_unlock(&ufq_active_queues_lock);
+
+ WARN_ON_ONCE(atomic_read(&ufq->rqs_count));
+
+ kfree(ufq);
+ e->elevator_data = NULL;
+}
+
+void ufq_kick_all_hw_queues(void)
+{
+ struct ufq_data *ufq;
+
+ mutex_lock(&ufq_active_queues_lock);
+ list_for_each_entry(ufq, &ufq_active_queues, active_node)
+ blk_mq_run_hw_queues(ufq->q, true);
+ mutex_unlock(&ufq_active_queues_lock);
+}
+
+static int ufq_drain_ctx_rqs(struct ufq_data *ufq)
+{
+ struct request_queue *q = ufq->q;
+ unsigned long deadline = jiffies + 8 * HZ;
+
+ while (atomic_read(&ufq->rqs_count) > 0 && time_before(jiffies, deadline)) {
+ blk_mq_run_hw_queues(q, false);
+ if (atomic_read(&ufq->rqs_count) > 0) {
+ struct blk_mq_hw_ctx *hctx;
+ unsigned long i;
+
+ queue_for_each_hw_ctx(q, hctx, i)
+ flush_delayed_work(&hctx->run_work);
+ flush_delayed_work(&q->requeue_work);
+ }
+ cond_resched();
+ }
+
+ if (atomic_read(&ufq->rqs_count) > 0) {
+ pr_warn_ratelimited("ufq: drain timeout (%d rqs) before BPF attach\n",
+ atomic_read(&ufq->rqs_count));
+ return -EBUSY;
+ }
+ return 0;
+}
+
+/*
+ * Mirror elevator_change(): freeze each queue, cancel mq dispatch work,
+ * then drain software-ctx requests while BPF callbacks are still off.
+ * @enable runs with all those queues still frozen so new ctx backlog cannot
+ * race ahead of turning BPF dispatch on.
+ */
+int ufq_prepare_bpf_attach(int (*enable)(void *kdata), void *kdata)
+{
+ struct ufq_data *ufq;
+ unsigned int memflags;
+ int frozen = 0, ret = 0;
+
+ mutex_lock(&ufq_active_queues_lock);
+ if (list_empty(&ufq_active_queues)) {
+ mutex_unlock(&ufq_active_queues_lock);
+ return enable(kdata);
+ }
+
+ memflags = memalloc_noio_save();
+ list_for_each_entry(ufq, &ufq_active_queues, active_node) {
+ blk_mq_freeze_queue_nomemsave(ufq->q);
+ blk_mq_cancel_work_sync(ufq->q);
+ frozen++;
+ ret = ufq_drain_ctx_rqs(ufq);
+ if (ret)
+ goto unfreeze;
+ }
+
+ ret = enable(kdata);
+unfreeze:
+ list_for_each_entry(ufq, &ufq_active_queues, active_node) {
+ if (!frozen--)
+ break;
+ blk_mq_unfreeze_queue_nomemrestore(ufq->q);
+ }
+ memalloc_noio_restore(memflags);
+ mutex_unlock(&ufq_active_queues_lock);
+ return ret;
+}
+
+static bool ufq_bio_merge(struct request_queue *q, struct bio *bio,
+ unsigned int nr_segs)
+{
+ struct ufq_data *ufq = q->elevator->elevator_data;
+ const struct ufq_iosched_ops *ops;
+ struct request *rq = NULL, *last;
+ enum bio_merge_status mstat;
+ struct blk_mq_ctx *ctx;
+ bool ret = false;
+
+ /*
+ * Levels of merges:
+ * nomerges: No merges at all attempted
+ * noxmerges: Only simple one-hit cache try
+ * merges: All merge tries attempted
+ */
+ if (blk_queue_nomerges(q) || !bio_mergeable(bio))
+ return false;
+
+ last = q->last_merge;
+ if (last) {
+ ctx = last->mq_ctx;
+ spin_lock(&ctx->lock);
+ if (last == q->last_merge && !list_empty(&last->queuelist)
+ && elv_bio_merge_ok(last, bio)) {
+ mstat = blk_attempt_bio_merge(q, last, bio, nr_segs, true);
+ if (mstat == BIO_MERGE_OK) {
+ spin_unlock(&ctx->lock);
+ atomic_inc(&ufq->ops_stats.merge_bio_ok_count);
+ atomic64_add(bio->bi_iter.bi_size >> SECTOR_SHIFT,
+ &ufq->ops_stats.merge_bio_ok_sectors);
+ return true;
+ }
+ if (mstat == BIO_MERGE_FAILED) {
+ spin_unlock(&ctx->lock);
+ return false;
+ }
+ }
+ spin_unlock(&ctx->lock);
+ }
+
+ if (blk_queue_noxmerges(q))
+ return false;
+
+ ops = ufq_bpfops_tryget();
+ if (ops) {
+ if (ops->merge_bio) {
+ rq = ops->merge_bio(q, bio, nr_segs, &ret);
+ if (ret) {
+ atomic_inc(&ufq->ops_stats.merge_bio_ok_count);
+ atomic64_add(bio->bi_iter.bi_size >> SECTOR_SHIFT,
+ &ufq->ops_stats.merge_bio_ok_sectors);
+ } else {
+ ufq_bpfops_put();
+ return false;
+ }
+
+ if (rq) {
+ ufq_bpfops_put();
+ spin_lock(&rq->mq_ctx->lock);
+ if (!list_empty(&rq->queuelist)) {
+ list_del_init(&rq->queuelist);
+ atomic_dec(&ufq->rqs_count);
+ }
+ spin_unlock(&rq->mq_ctx->lock);
+ blk_mq_free_request(rq);
+ atomic_inc(&ufq->ops_stats.merge_request_ok_count);
+ atomic64_add(bio->bi_iter.bi_size >> SECTOR_SHIFT,
+ &ufq->ops_stats.merge_request_ok_sectors);
+ return ret;
+ }
+ }
+ ufq_bpfops_put();
+ }
+
+ return ret;
+}
+
+static enum elv_merge ufq_try_insert_merge(struct request_queue *q,
+ struct request **new)
+{
+ const struct ufq_iosched_ops *ops;
+ struct request *target = NULL, *free = NULL, *last, *rq = *new;
+ struct ufq_data *ufq = q->elevator->elevator_data;
+ enum elv_merge type = ELEVATOR_NO_MERGE;
+ int merge_type = ELEVATOR_NO_MERGE;
+
+ if (!rq_mergeable(rq))
+ return ELEVATOR_NO_MERGE;
+
+ if (blk_queue_nomerges(q))
+ return ELEVATOR_NO_MERGE;
+
+ last = q->last_merge;
+ if (last) {
+ spin_lock(&last->mq_ctx->lock);
+ if (last == q->last_merge && !list_empty(&last->queuelist)
+ && bpf_attempt_merge(q, last, rq)) {
+ spin_unlock(&last->mq_ctx->lock);
+ type = ELEVATOR_BACK_MERGE;
+ free = rq;
+ *new = NULL;
+ goto end;
+ }
+ spin_unlock(&last->mq_ctx->lock);
+ }
+
+ if (blk_queue_noxmerges(q))
+ return ELEVATOR_NO_MERGE;
+
+ ops = ufq_bpfops_tryget();
+ if (ops && ops->merge_req) {
+ target = ops->merge_req(q, rq, &merge_type);
+ type = (enum elv_merge)merge_type;
+ }
+
+ if (target && WARN_ON_ONCE(req_ref_put_and_test(target))) {
+ __blk_mq_free_request(target);
+ ufq_bpfops_put();
+ return ELEVATOR_NO_MERGE;
+ }
+
+ if (type == ELEVATOR_NO_MERGE || !target) {
+ if (ops)
+ ufq_bpfops_put();
+ return ELEVATOR_NO_MERGE;
+ } else if (type == ELEVATOR_FRONT_MERGE) {
+ if (rq->mq_ctx != target->mq_ctx || rq->mq_hctx != target->mq_hctx)
+ goto rollback;
+ spin_lock(&target->mq_ctx->lock);
+ free = bpf_attempt_merge(q, rq, target);
+ if (!free) {
+ spin_unlock(&target->mq_ctx->lock);
+ pr_err("ufq-iosched: front merge failed\n");
+ goto rollback;
+ }
+ rq->elv.priv[0] = (void *)((uintptr_t)rq->elv.priv[0]
+ | UFQ_PRIV_IN_UFQ);
+ list_replace_init(&target->queuelist, &rq->queuelist);
+ rq->fifo_time = target->fifo_time;
+ q->last_merge = rq;
+ } else if (type == ELEVATOR_BACK_MERGE) {
+ spin_lock(&target->mq_ctx->lock);
+ free = bpf_attempt_merge(q, target, rq);
+ if (!free) {
+ spin_unlock(&target->mq_ctx->lock);
+ pr_err("ufq-iosched: back merge failed\n");
+ goto rollback;
+ }
+ *new = target;
+ q->last_merge = target;
+ }
+
+ spin_unlock(&target->mq_ctx->lock);
+ if (ops)
+ ufq_bpfops_put();
+end:
+ atomic_inc(&ufq->ops_stats.merge_request_ok_count);
+ atomic64_add(blk_rq_sectors(free), &ufq->ops_stats.merge_request_ok_sectors);
+ blk_mq_free_request(free);
+ return type;
+
+rollback:
+ if (ops) {
+ if (ops->insert_req && ops->insert_req(q, target, 0)) {
+ atomic_inc(&ufq->ops_stats.insert_err_count);
+ pr_err("ufq-iosched: rollback insert_req error\n");
+ }
+ ufq_bpfops_put();
+ }
+
+ return ELEVATOR_NO_MERGE;
+}
+
+static void ufq_insert_requests(struct blk_mq_hw_ctx *hctx,
+ struct list_head *list,
+ blk_insert_t flags)
+{
+ struct request_queue *q = hctx->queue;
+ struct ufq_data *ufq = q->elevator->elevator_data;
+ const struct ufq_iosched_ops *ops;
+ struct blk_mq_ctx *ctx;
+ enum elv_merge type;
+ int bit, ret = 0;
+
+ ops = ufq_bpfops_tryget();
+
+ while (!list_empty(list)) {
+ struct request *rq;
+
+ rq = list_first_entry(list, struct request, queuelist);
+ list_del_init(&rq->queuelist);
+
+ type = ufq_try_insert_merge(q, &rq);
+ if (type == ELEVATOR_NO_MERGE) {
+ rq->fifo_time = jiffies;
+ ctx = rq->mq_ctx;
+ rq->elv.priv[0] = (void *)((uintptr_t)rq->elv.priv[0]
+ | UFQ_PRIV_IN_UFQ);
+ spin_lock(&ctx->lock);
+ if (flags & BLK_MQ_INSERT_AT_HEAD)
+ list_add(&rq->queuelist, &ctx->rq_lists[hctx->type]);
+ else
+ list_add_tail(&rq->queuelist,
+ &ctx->rq_lists[hctx->type]);
+
+ bit = ctx->index_hw[hctx->type];
+ if (!sbitmap_test_bit(&hctx->ctx_map, bit))
+ sbitmap_set_bit(&hctx->ctx_map, bit);
+ q->last_merge = rq;
+ spin_unlock(&ctx->lock);
+ atomic_inc(&ufq->rqs_count);
+ }
+
+ if (ops && rq && ops->insert_req) {
+ rq->elv.priv[0] = (void *)((uintptr_t)rq->elv.priv[0]
+ | UFQ_PRIV_IN_BPF);
+ ret = ops->insert_req(q, rq, flags);
+ if (ret) {
+ atomic_inc(&ufq->ops_stats.insert_err_count);
+ pr_err("ufq-iosched: bpf insert_req error (%d)\n", ret);
+ } else {
+ atomic_inc(&ufq->ops_stats.insert_ok_count);
+ atomic64_add(blk_rq_sectors(rq), &ufq->ops_stats.insert_ok_sectors);
+ }
+ }
+ }
+
+ if (ops)
+ ufq_bpfops_put();
+}
+
+static void ufq_prepare_request(struct request *rq)
+{
+ rq->elv.priv[0] = (void *)(uintptr_t)UFQ_PRIV_NOT_IN_SCHED;
+}
+
+static void ufq_finish_request(struct request *rq)
+{
+ const struct ufq_iosched_ops *ops;
+ struct ufq_data *ufq = rq->q->elevator->elevator_data;
+
+ /*
+ * The block layer core may call ufq_finish_request() without having
+ * called ufq_insert_requests(). Skip requests that bypassed I/O
+ * scheduling.
+ */
+ if (!((uintptr_t)rq->elv.priv[0] & UFQ_PRIV_IN_BPF))
+ return;
+
+ ops = ufq_bpfops_tryget();
+ if (ops) {
+ if (ops->finish_req)
+ ops->finish_req(rq);
+ ufq_bpfops_put();
+ }
+
+ atomic_inc(&ufq->ops_stats.finish_ok_count);
+ atomic64_add(blk_rq_stats_sectors(rq), &ufq->ops_stats.finish_ok_sectors);
+}
+
+static bool ufq_has_work(struct blk_mq_hw_ctx *hctx)
+{
+ const struct ufq_iosched_ops *ops;
+ struct ufq_data *ufq = hctx->queue->elevator->elevator_data;
+ int rqs_count = atomic_read(&ufq->rqs_count);
+
+ ops = ufq_bpfops_tryget();
+ if (!ops)
+ return rqs_count > 0;
+
+ if (ops->has_req)
+ rqs_count = ops->has_req(hctx->queue, rqs_count);
+ ufq_bpfops_put();
+ return rqs_count > 0;
+}
+
+#ifdef CONFIG_BLK_DEBUG_FS
+static int ufq_ops_stats_show(void *data, struct seq_file *m)
+{
+ struct request_queue *q = data;
+ struct ufq_data *ufq = q->elevator->elevator_data;
+ struct ufq_ops_stats *s = &ufq->ops_stats;
+
+ /* for debug */
+ seq_printf(m, "dispatch_ok_count %d\n",
+ atomic_read(&s->dispatch_ok_count));
+ seq_printf(m, "dispatch_ok_sectors %lld\n",
+ (long long)atomic64_read(&s->dispatch_ok_sectors));
+ seq_printf(m, "dispatch_null_count %d\n",
+ atomic_read(&s->dispatch_null_count));
+ seq_printf(m, "insert_ok_count %d\n",
+ atomic_read(&s->insert_ok_count));
+ seq_printf(m, "insert_ok_sectors %lld\n",
+ (long long)atomic64_read(&s->insert_ok_sectors));
+ seq_printf(m, "insert_err_count %d\n",
+ atomic_read(&s->insert_err_count));
+ seq_printf(m, "merge_req_ok_count %d\n",
+ atomic_read(&s->merge_request_ok_count));
+ seq_printf(m, "merge_req_ok_sectors %lld\n",
+ (long long)atomic64_read(&s->merge_request_ok_sectors));
+ seq_printf(m, "merge_bio_ok_count %d\n",
+ atomic_read(&s->merge_bio_ok_count));
+ seq_printf(m, "merge_bio_ok_sectors %lld\n",
+ (long long)atomic64_read(&s->merge_bio_ok_sectors));
+ seq_printf(m, "finish_ok_count %d\n",
+ atomic_read(&s->finish_ok_count));
+ seq_printf(m, "finish_ok_sectors %lld\n",
+ (long long)atomic64_read(&s->finish_ok_sectors));
+ return 0;
+}
+
+static const struct blk_mq_debugfs_attr ufq_iosched_debugfs_attrs[] = {
+ {"ops_stats", 0400, ufq_ops_stats_show},
+ {},
+};
+#endif
+
+static struct elevator_type ufq_iosched_mq = {
+ .ops = {
+ .depth_updated = ufq_depth_updated,
+ .limit_depth = ufq_limit_depth,
+ .insert_requests = ufq_insert_requests,
+ .dispatch_request = ufq_dispatch_request,
+ .prepare_request = ufq_prepare_request,
+ .finish_request = ufq_finish_request,
+ .bio_merge = ufq_bio_merge,
+ .has_work = ufq_has_work,
+ .init_sched = ufq_init_sched,
+ .exit_sched = ufq_exit_sched,
+ },
+
+#ifdef CONFIG_BLK_DEBUG_FS
+ .queue_debugfs_attrs = ufq_iosched_debugfs_attrs,
+#endif
+ .elevator_name = "ufq",
+ .elevator_alias = "ufq_iosched",
+ .elevator_owner = THIS_MODULE,
+};
+MODULE_ALIAS("ufq-iosched");
+
+static int __init ufq_init(void)
+{
+ int ret;
+
+ ret = elv_register(&ufq_iosched_mq);
+ if (ret)
+ return ret;
+
+ ret = bpf_ufq_kfunc_init();
+ if (ret) {
+ pr_err("ufq-iosched: Failed to register kfunc sets (%d)\n", ret);
+ elv_unregister(&ufq_iosched_mq);
+ return ret;
+ }
+
+ ret = bpf_ufq_ops_init();
+ if (ret) {
+ pr_err("ufq-iosched: Failed to register struct_ops (%d)\n", ret);
+ elv_unregister(&ufq_iosched_mq);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void __exit ufq_exit(void)
+{
+ elv_unregister(&ufq_iosched_mq);
+}
+
+module_init(ufq_init);
+module_exit(ufq_exit);
+
+MODULE_AUTHOR("Kaitao Cheng <chengkaitao@kylinos.cn>");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("User-programmable Flexible Queueing");
diff --git a/block/ufq-iosched.h b/block/ufq-iosched.h
new file mode 100644
index 000000000000..26a9c1708e8b
--- /dev/null
+++ b/block/ufq-iosched.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 KylinSoft Corporation.
+ * Copyright (c) 2026 Kaitao Cheng <chengkaitao@kylinos.cn>
+ */
+#ifndef _BLOCK_UFQ_IOSCHED_H
+#define _BLOCK_UFQ_IOSCHED_H
+
+#include <linux/types.h>
+#include "elevator.h"
+#include "blk-mq.h"
+
+#ifndef BPF_IOSCHED_NAME_MAX
+#define BPF_IOSCHED_NAME_MAX 16
+#endif
+
+/* For testing and debugging */
+struct ufq_ops_stats {
+ atomic_t dispatch_ok_count;
+ atomic64_t dispatch_ok_sectors;
+ atomic_t dispatch_null_count;
+ atomic_t insert_ok_count;
+ atomic64_t insert_ok_sectors;
+ atomic_t insert_err_count;
+ atomic_t merge_request_ok_count;
+ atomic64_t merge_request_ok_sectors;
+ atomic_t merge_bio_ok_count;
+ atomic64_t merge_bio_ok_sectors;
+ atomic_t finish_ok_count;
+ atomic64_t finish_ok_sectors;
+};
+
+struct ufq_iosched_ops {
+ int (*init_sched)(struct request_queue *q);
+ int (*exit_sched)(struct request_queue *q);
+ bool (*has_req)(struct request_queue *q, int rqs_count);
+ int (*insert_req)(struct request_queue *q, struct request *rq,
+ blk_insert_t flags);
+ void (*finish_req)(struct request *rq);
+ struct request *(*merge_req)(struct request_queue *q, struct request *rq,
+ int *type);
+ struct request *(*merge_bio)(struct request_queue *q, struct bio *bio,
+ unsigned int nr_segs, bool *merged);
+ struct request *(*dispatch_req)(struct request_queue *q);
+ char name[BPF_IOSCHED_NAME_MAX];
+};
+
+struct ufq_data {
+ struct request_queue *q;
+ u32 async_depth;
+ atomic_t rqs_count;
+ struct list_head active_node;
+ struct ufq_ops_stats ops_stats;
+};
+
+const struct ufq_iosched_ops *ufq_bpfops_tryget(void);
+void ufq_bpfops_put(void);
+void ufq_kick_all_hw_queues(void);
+int ufq_prepare_bpf_attach(int (*enable)(void *kdata), void *kdata);
+
+int bpf_ufq_ops_init(void);
+int bpf_ufq_kfunc_init(void);
+
+#endif /* _BLOCK_UFQ_IOSCHED_H */
diff --git a/block/ufq-kfunc.c b/block/ufq-kfunc.c
new file mode 100644
index 000000000000..c799eeffcba0
--- /dev/null
+++ b/block/ufq-kfunc.c
@@ -0,0 +1,131 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 KylinSoft Corporation.
+ * Copyright (c) 2026 Kaitao Cheng <chengkaitao@kylinos.cn>
+ */
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/bpf_verifier.h>
+#include <linux/bpf.h>
+#include <linux/btf.h>
+#include <linux/btf_ids.h>
+#include <trace/events/block.h>
+#include "blk.h"
+#include "ufq-iosched.h"
+
+__bpf_kfunc_start_defs();
+
+__bpf_kfunc struct request *bpf_request_acquire(struct request *rq)
+{
+ if (req_ref_inc_not_zero(rq))
+ return rq;
+ return NULL;
+}
+
+__bpf_kfunc void bpf_request_release(struct request *rq)
+{
+ if (req_ref_put_and_test(rq))
+ __blk_mq_free_request(rq);
+}
+
+__bpf_kfunc bool bpf_request_bio_try_merge(struct request *rq, struct bio *bio,
+ unsigned int nr_segs)
+{
+ struct blk_mq_ctx *ctx;
+ bool merged;
+
+ if (!rq || !bio)
+ return false;
+
+ ctx = rq->mq_ctx;
+ if (!ctx || !rq->q || !bio->bi_bdev || !bio->bi_bdev->bd_disk ||
+ bio->bi_bdev->bd_disk->queue != rq->q)
+ return false;
+
+ spin_lock(&ctx->lock);
+ merged = blk_attempt_bio_merge(rq->q, rq, bio, nr_segs, true) == BIO_MERGE_OK;
+ spin_unlock(&ctx->lock);
+
+ return merged;
+}
+
+__bpf_kfunc struct request *bpf_request_try_merge(struct request *rq, struct request *next)
+{
+ struct blk_mq_ctx *ctx;
+ struct ufq_data *ufq;
+ struct request *free;
+
+ if (!rq || !next || !rq->q || rq->q != next->q)
+ return NULL;
+
+ ufq = rq->q->elevator->elevator_data;
+ if (!ufq)
+ return NULL;
+
+ if (rq->mq_ctx != next->mq_ctx || rq->mq_hctx != next->mq_hctx)
+ return NULL;
+
+ ctx = rq->mq_ctx;
+ if (!ctx)
+ return NULL;
+
+ spin_lock(&ctx->lock);
+ free = bpf_attempt_merge(rq->q, rq, next);
+ if (free) {
+ if (rq->q->last_merge == free)
+ rq->q->last_merge = NULL;
+ list_del_init(&free->queuelist);
+ atomic_dec(&ufq->rqs_count);
+ }
+ spin_unlock(&ctx->lock);
+
+ return free;
+}
+
+__bpf_kfunc_end_defs();
+
+#if defined(CONFIG_X86_KERNEL_IBT)
+static const void * const __used __section(".discard.ibt_endbr_noseal")
+__ibt_noseal_bpf_request_release = (void *)bpf_request_release;
+#endif
+
+BTF_KFUNCS_START(ufq_kfunc_set_ops)
+BTF_ID_FLAGS(func, bpf_request_acquire, KF_ACQUIRE | KF_RET_NULL)
+BTF_ID_FLAGS(func, bpf_request_release, KF_RELEASE)
+BTF_ID_FLAGS(func, bpf_request_bio_try_merge, KF_SPIN_LOCK)
+BTF_ID_FLAGS(func, bpf_request_try_merge, KF_SPIN_LOCK)
+BTF_KFUNCS_END(ufq_kfunc_set_ops)
+
+static const struct btf_kfunc_id_set bpf_ufq_kfunc_set = {
+ .owner = THIS_MODULE,
+ .set = &ufq_kfunc_set_ops,
+};
+
+BTF_ID_LIST(bpf_ufq_dtor_kfunc_ids)
+BTF_ID(struct, request)
+BTF_ID(func, bpf_request_release)
+
+int bpf_ufq_kfunc_init(void)
+{
+ int ret;
+ const struct btf_id_dtor_kfunc bpf_ufq_dtor_kfunc[] = {
+ {
+ .btf_id = bpf_ufq_dtor_kfunc_ids[0],
+ .kfunc_btf_id = bpf_ufq_dtor_kfunc_ids[1]
+ },
+ };
+
+ ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_ufq_kfunc_set);
+ if (ret)
+ return ret;
+ ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &bpf_ufq_kfunc_set);
+ if (ret)
+ return ret;
+ ret = register_btf_id_dtor_kfuncs(bpf_ufq_dtor_kfunc,
+ ARRAY_SIZE(bpf_ufq_dtor_kfunc),
+ THIS_MODULE);
+ if (ret)
+ return ret;
+
+ return 0;
+}
--
2.50.1 (Apple Git-155)
^ permalink raw reply related
* [RFC v2 0/3] block: Introduce a BPF-based I/O scheduler
From: Kaitao cheng @ 2026-05-03 3:56 UTC (permalink / raw)
To: axboe, ast, daniel, andrii, martin.lau, eddyz87, memxor, song,
yonghong.song, jolsa, john.fastabend
Cc: bpf, linux-block, linux-kernel, Kaitao cheng
I have been working on adding a new BPF-based I/O scheduler. It has both
kernel and user-space parts. In kernel space, using per-ctx, I implemented
a simple elevator that exposes a set of BPF hooks. The goal is to move the
policy side of I/O scheduling out of the kernel and into user space, which
should greatly increase flexibility and applicability. To verify that the
whole stack works end to end, I wrote a simple BPF example program. I am
calling this feature the UFQ (User-programmable Flexible Queueing) I/O
scheduler.
This patch depends on new BPF functionality that I have already posted to
the BPF community but that is not yet in mainline. Details are in the
thread:
https://lore.kernel.org/bpf/20260427165906.84420-1-kaitao.cheng@linux.dev/
To try it, you need to apply the patches from that series first.
I am looking for community feedback on whether this direction and the
implementation approach make sense, and what else we should
consider.
todo:
1. More thorough testing
2. Split the code into multiple sub-patches
3. Identify concrete use cases
Changes in v2:
- Remove bpf_request_put (Alexei Starovoitov)
- Update the UFQ README (Miguel Ojeda)
- Add bio merge support
- Fix synchronization issues during UFQ scheduler transitions
Link to v1:
https://lore.kernel.org/bpf/20260327114741.91500-1-pilgrimtao@gmail.com/
Kaitao Cheng (3):
bpf: Add KF_SPIN_LOCK flag for kfuncs under bpf_spin_lock
block: Introduce the UFQ I/O scheduler
tools/ufq_iosched: add BPF example scheduler and build scaffolding
block/Kconfig.iosched | 8 +
block/Makefile | 1 +
block/blk-merge.c | 28 +-
block/blk-mq.c | 8 +-
block/blk-mq.h | 2 +-
block/blk.h | 5 +
block/ufq-bpfops.c | 241 +++++++
block/ufq-iosched.c | 640 ++++++++++++++++++
block/ufq-iosched.h | 64 ++
block/ufq-kfunc.c | 131 ++++
include/linux/btf.h | 1 +
kernel/bpf/verifier.c | 20 +-
tools/ufq_iosched/.gitignore | 2 +
tools/ufq_iosched/Makefile | 262 +++++++
tools/ufq_iosched/README.md | 145 ++++
.../include/bpf-compat/gnu/stubs.h | 12 +
tools/ufq_iosched/include/ufq/common.bpf.h | 75 ++
tools/ufq_iosched/include/ufq/common.h | 90 +++
tools/ufq_iosched/include/ufq/simple_stat.h | 23 +
tools/ufq_iosched/ufq_simple.bpf.c | 604 +++++++++++++++++
tools/ufq_iosched/ufq_simple.c | 120 ++++
21 files changed, 2464 insertions(+), 18 deletions(-)
create mode 100644 block/ufq-bpfops.c
create mode 100644 block/ufq-iosched.c
create mode 100644 block/ufq-iosched.h
create mode 100644 block/ufq-kfunc.c
create mode 100644 tools/ufq_iosched/.gitignore
create mode 100644 tools/ufq_iosched/Makefile
create mode 100644 tools/ufq_iosched/README.md
create mode 100644 tools/ufq_iosched/include/bpf-compat/gnu/stubs.h
create mode 100644 tools/ufq_iosched/include/ufq/common.bpf.h
create mode 100644 tools/ufq_iosched/include/ufq/common.h
create mode 100644 tools/ufq_iosched/include/ufq/simple_stat.h
create mode 100644 tools/ufq_iosched/ufq_simple.bpf.c
create mode 100644 tools/ufq_iosched/ufq_simple.c
--
2.50.1 (Apple Git-155)
^ permalink raw reply
* [RFC v2 1/3] bpf: Add KF_SPIN_LOCK flag for kfuncs under bpf_spin_lock
From: Kaitao cheng @ 2026-05-03 3:56 UTC (permalink / raw)
To: axboe, ast, daniel, andrii, martin.lau, eddyz87, memxor, song,
yonghong.song, jolsa, john.fastabend
Cc: bpf, linux-block, linux-kernel, Kaitao Cheng
In-Reply-To: <20260503035623.28771-1-kaitao.cheng@linux.dev>
From: Kaitao Cheng <chengkaitao@kylinos.cn>
Introduce the KF_SPIN_LOCK kfunc metadata flag in BTF so kfuncs may be
explicitly marked as safe to call while holding bpf_spin_lock.
Allow kfuncs defined in kernel modules to be marked with KF_SPIN_LOCK.
Signed-off-by: Kaitao Cheng <chengkaitao@kylinos.cn>
---
include/linux/btf.h | 1 +
kernel/bpf/verifier.c | 20 +++++++++++++++-----
2 files changed, 16 insertions(+), 5 deletions(-)
diff --git a/include/linux/btf.h b/include/linux/btf.h
index c82d0d689059..08d8a023ffe6 100644
--- a/include/linux/btf.h
+++ b/include/linux/btf.h
@@ -79,6 +79,7 @@
#define KF_ARENA_ARG1 (1 << 14) /* kfunc takes an arena pointer as its first argument */
#define KF_ARENA_ARG2 (1 << 15) /* kfunc takes an arena pointer as its second argument */
#define KF_IMPLICIT_ARGS (1 << 16) /* kfunc has implicit arguments supplied by the verifier */
+#define KF_SPIN_LOCK (1 << 17) /* kfunc is allowed inside bpf_spin_lock-ed region */
/*
* Tag marking a kernel function as a kfunc. This is meant to minimize the
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 4d78d834c609..cffba6a714f3 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -11403,11 +11403,21 @@ static bool is_bpf_stream_kfunc(u32 btf_id)
btf_id == special_kfunc_list[KF_bpf_stream_print_stack];
}
-static bool kfunc_spin_allowed(u32 btf_id)
+static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 offset)
{
- return is_bpf_graph_api_kfunc(btf_id) || is_bpf_iter_num_api_kfunc(btf_id) ||
- is_bpf_res_spin_lock_kfunc(btf_id) || is_bpf_arena_kfunc(btf_id) ||
- is_bpf_stream_kfunc(btf_id);
+ struct bpf_kfunc_meta kfunc;
+ int err;
+
+ if (is_bpf_graph_api_kfunc(func_id) || is_bpf_iter_num_api_kfunc(func_id) ||
+ is_bpf_res_spin_lock_kfunc(func_id) || is_bpf_arena_kfunc(func_id) ||
+ is_bpf_stream_kfunc(func_id))
+ return true;
+
+ err = fetch_kfunc_meta(env, func_id, offset, &kfunc);
+ if (err || !kfunc.flags)
+ return false;
+
+ return *kfunc.flags & KF_SPIN_LOCK;
}
static bool is_sync_callback_calling_kfunc(u32 btf_id)
@@ -17025,7 +17035,7 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
insn->imm != BPF_FUNC_spin_unlock &&
insn->imm != BPF_FUNC_kptr_xchg) ||
(insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&
- (insn->off != 0 || !kfunc_spin_allowed(insn->imm)))) {
+ !kfunc_spin_allowed(env, insn->imm, insn->off))) {
verbose(env,
"function calls are not allowed while holding a lock\n");
return -EINVAL;
--
2.50.1 (Apple Git-155)
^ permalink raw reply related
* Re: [PATCH] block/blk-iolatency: Add the processing flow of the chained bio in the QoS and define the related types to solve the problem of incorrect inflight processing in the QoS. The usage of the done_split_bio abstract function in the blk-iolatency project.
From: kernel test robot @ 2026-05-03 0:29 UTC (permalink / raw)
To: Li kunyu, axboe, tj, josef
Cc: llvm, oe-kbuild-all, linux-block, linux-kernel, Li kunyu
In-Reply-To: <20260429094148.2394-1-likunyu10@163.com>
Hi Li,
kernel test robot noticed the following build errors:
[auto build test ERROR on axboe/for-next]
[also build test ERROR on linus/master v7.1-rc1 next-20260430]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Li-kunyu/block-blk-iolatency-Add-the-processing-flow-of-the-chained-bio-in-the-QoS-and-define-the-related-types-to-solve-the-prob/20260502-071718
base: https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git for-next
patch link: https://lore.kernel.org/r/20260429094148.2394-1-likunyu10%40163.com
patch subject: [PATCH] block/blk-iolatency: Add the processing flow of the chained bio in the QoS and define the related types to solve the problem of incorrect inflight processing in the QoS. The usage of the done_split_bio abstract function in the blk-iolatency project.
config: s390-allnoconfig (https://download.01.org/0day-ci/archive/20260503/202605030844.RgfP9Jsd-lkp@intel.com/config)
compiler: clang version 23.0.0git (https://github.com/llvm/llvm-project 5bac06718f502014fade905512f1d26d578a18f3)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260503/202605030844.RgfP9Jsd-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202605030844.RgfP9Jsd-lkp@intel.com/
All errors (new ones prefixed by >>):
>> block/blk-merge.c:159:17: error: use of undeclared identifier 'split'
159 | bio_set_flag(split, BIO_QOS_CHAIN_CHILD);
| ^~~~~
1 error generated.
vim +/split +159 block/blk-merge.c
142
143 static struct bio *bio_submit_split(struct bio *bio, int split_sectors)
144 {
145 if (unlikely(split_sectors < 0)) {
146 bio->bi_status = errno_to_blk_status(split_sectors);
147 bio_endio(bio);
148 return NULL;
149 }
150
151 if (split_sectors) {
152 bio = bio_submit_split_bioset(bio, split_sectors,
153 &bio->bi_bdev->bd_disk->bio_split);
154 if (bio) {
155 bio->bi_opf |= REQ_NOMERGE;
156 /* Fix the issue where the inflight statistics
157 * of the chained bio in the QoS are incorrect.
158 */
> 159 bio_set_flag(split, BIO_QOS_CHAIN_CHILD);
160 }
161 }
162
163 return bio;
164 }
165
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply
* [PATCH] block: restore mempool reserves for non-block
From: Carlos Llamas @ 2026-05-03 0:17 UTC (permalink / raw)
To: Christoph Hellwig, Jens Axboe, Martin K. Petersen,
Chaitanya Kulkarni
Cc: kernel-team, linux-kernel, Carlos Llamas, open list:BLOCK LAYER
Commit b520c4eef83d ("block: split bio_alloc_bioset more clearly into a
fast and slowpath") prevents non-blocking allocations from falling back
to mempool_alloc() after the initial slab allocation has failed. This
was based on the assumption that mempool_alloc() would simply retry the
slab and fail again (for non-block).
However, mempool_alloc() also attempts to dip into the pool reserves,
even for non-blocking requests, and this option is no longer available
after adding the early fail. This was noticed through the SCSI generic
module which calls blk_rq_map_user_io() with GFP_ATOMIC.
Remove the check to allow mempool reserves to be used for non-blocking
requests, restoring the previous behavior.
Fixes: b520c4eef83d ("block: split bio_alloc_bioset more clearly into a fast and slowpath")
Signed-off-by: Carlos Llamas <cmllamas@google.com>
---
block/bio.c | 7 -------
1 file changed, 7 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index b8972dba68a0..e8f4934593b3 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -576,13 +576,6 @@ struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
}
if (unlikely(!bio)) {
- /*
- * Give up if we are not allow to sleep as non-blocking mempool
- * allocations just go back to the slab allocation.
- */
- if (!(saved_gfp & __GFP_DIRECT_RECLAIM))
- return NULL;
-
punt_bios_to_rescuer(bs);
/*
--
2.54.0.545.g6539524ca2-goog
^ permalink raw reply related
* Re: [PATCH] block/blk-iolatency: Add the processing flow of the chained bio in the QoS and define the related types to solve the problem of incorrect inflight processing in the QoS. The usage of the done_split_bio abstract function in the blk-iolatency project.
From: kernel test robot @ 2026-05-02 23:34 UTC (permalink / raw)
To: Li kunyu, axboe, tj, josef
Cc: oe-kbuild-all, linux-block, linux-kernel, Li kunyu
In-Reply-To: <20260429094148.2394-1-likunyu10@163.com>
Hi Li,
kernel test robot noticed the following build errors:
[auto build test ERROR on axboe/for-next]
[also build test ERROR on linus/master v7.1-rc1 next-20260430]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Li-kunyu/block-blk-iolatency-Add-the-processing-flow-of-the-chained-bio-in-the-QoS-and-define-the-related-types-to-solve-the-prob/20260502-071718
base: https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git for-next
patch link: https://lore.kernel.org/r/20260429094148.2394-1-likunyu10%40163.com
patch subject: [PATCH] block/blk-iolatency: Add the processing flow of the chained bio in the QoS and define the related types to solve the problem of incorrect inflight processing in the QoS. The usage of the done_split_bio abstract function in the blk-iolatency project.
config: riscv-allnoconfig (https://download.01.org/0day-ci/archive/20260503/202605030725.SpXiNjpU-lkp@intel.com/config)
compiler: riscv64-linux-gcc (GCC) 15.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260503/202605030725.SpXiNjpU-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202605030725.SpXiNjpU-lkp@intel.com/
All errors (new ones prefixed by >>):
block/blk-merge.c: In function 'bio_submit_split':
>> block/blk-merge.c:159:38: error: 'split' undeclared (first use in this function); did you mean 'sg_split'?
159 | bio_set_flag(split, BIO_QOS_CHAIN_CHILD);
| ^~~~~
| sg_split
block/blk-merge.c:159:38: note: each undeclared identifier is reported only once for each function it appears in
vim +159 block/blk-merge.c
142
143 static struct bio *bio_submit_split(struct bio *bio, int split_sectors)
144 {
145 if (unlikely(split_sectors < 0)) {
146 bio->bi_status = errno_to_blk_status(split_sectors);
147 bio_endio(bio);
148 return NULL;
149 }
150
151 if (split_sectors) {
152 bio = bio_submit_split_bioset(bio, split_sectors,
153 &bio->bi_bdev->bd_disk->bio_split);
154 if (bio) {
155 bio->bi_opf |= REQ_NOMERGE;
156 /* Fix the issue where the inflight statistics
157 * of the chained bio in the QoS are incorrect.
158 */
> 159 bio_set_flag(split, BIO_QOS_CHAIN_CHILD);
160 }
161 }
162
163 return bio;
164 }
165
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply
* Re: [PATCH v12 10/13] blk-mq: use hk cpus only when isolcpus=io_queue is enabled
From: Aaron Tomlin @ 2026-05-02 21:25 UTC (permalink / raw)
To: axboe, kbusch, hch, sagi, mst
Cc: aacraid, James.Bottomley, martin.petersen, liyihang9,
kashyap.desai, sumit.saxena, shivasharan.srikanteshwara,
chandrakanth.patil, sathya.prakash, sreekanth.reddy,
suganath-prabu.subramani, ranjan.kumar, jinpu.wang, tglx, mingo,
peterz, juri.lelli, vincent.guittot, akpm, maz, ruanjinjie,
bigeasy, yphbchou0911, wagi, frederic, longman, chenridong, hare,
kch, ming.lei, tom.leiming, steve, sean, chjohnst, neelx, mproche,
nick.lange, marco.crivellari, linux-block, linux-kernel,
virtualization, linux-nvme, linux-scsi, megaraidlinux.pdl,
mpi3mr-linuxdrv.pdl, MPT-FusionLinux.pdl
In-Reply-To: <20260422185215.100929-11-atomlin@atomlin.com>
[-- Attachment #1: Type: text/plain, Size: 3734 bytes --]
On Wed, Apr 22, 2026 at 02:52:12PM -0400, Aaron Tomlin wrote:
> From: Daniel Wagner <wagi@kernel.org>
> +static void blk_mq_map_fallback(struct blk_mq_queue_map *qmap)
> +{
> + unsigned int cpu;
> +
> + /*
> + * Map all CPUs to the first hctx to ensure at least one online
> + * CPU is serving it.
> + */
> + for_each_possible_cpu(cpu)
> + qmap->mq_map[cpu] = 0;
> +}
I suspect we should use 'qmap->mq_map[cpu] = qmap->queue_offset' to respect
the specified map's boundaries. For instance, secondary maps may not start
at zero.
> void blk_mq_map_queues(struct blk_mq_queue_map *qmap)
> {
> - const struct cpumask *masks;
> + struct cpumask *masks __free(kfree) = NULL;
> + const struct cpumask *constraint;
> unsigned int queue, cpu, nr_masks;
> + cpumask_var_t active_hctx;
[ ... ]
> + /* Map CPUs to the hardware contexts (hctx) */
> + masks = group_mask_cpus_evenly(qmap->nr_queues, constraint, &nr_masks);
> + if (!masks)
> + goto free_fallback;
According to Documentation/dev-tools/checkpatch.rst, pointers with the
'__free' attribute should be declared at the place of use and
initialisation. However, I suspect we should not use traditional goto error
unwinding with scope-based cleanups in the same function. I propose we drop
the use of __free and free 'masks' during the success code path.
> for (queue = 0; queue < qmap->nr_queues; queue++) {
> - for_each_cpu(cpu, &masks[queue % nr_masks])
> + unsigned int idx = (qmap->queue_offset + queue) % nr_masks;
> +
> + for_each_cpu(cpu, &masks[idx]) {
> qmap->mq_map[cpu] = qmap->queue_offset + queue;
> +
> + if (cpu_online(cpu))
> + cpumask_set_cpu(queue, active_hctx);
> + }
> }
[ ... ]
> +
> + /* Map any unassigned CPU evenly to the hardware contexts (hctx) */
> + queue = cpumask_first(active_hctx);
> + for_each_cpu_andnot(cpu, cpu_possible_mask, constraint) {
> + qmap->mq_map[cpu] = qmap->queue_offset + queue;
> + queue = cpumask_next_wrap(queue, active_hctx);
> + }
> +
> + if (!blk_mq_validate(qmap, active_hctx))
> + goto free_fallback;
> +
There is a potential out-of-bounds write vulnerability here.
The variable active_hctx (of type cpumask_var_t), after the call
zalloc_cpumask_var(), the Linux kernel would have allocated exactly enough
bits to represent the number of CPUs the system is configured to support
(nr_cpumask_bits). For example, nr_cpumask_bits could be set to 16.
Now, in the above loop, we are using active_hctx to track hardware queues
(qmap->nr_queues), passing queue as the index into cpumask_set_cpu().
A modern, high-end NVMe drive can expose 128 hardware queues.
If we plug that drive into a machine with only 16 CPUs:
1. zalloc_cpumask_var() allocates 16 bits (perhaps rounded up to a
standard word/slab size).
2. The loop iterates up to queue = 127.
3. cpumask_set_cpu(127, active_hctx) blindly writes to the 127th bit,
drastically overshooting the allocated memory and corrupting
adjacent slab memory in the kernel heap.
Because active_hctx is tracking hardware queues and not CPUs, we must not
use the cpumask API. Instead, switch it to the kernel's standard bitmap
API so the exact bit-length based on qmap->nr_queues can be dynamically
allocated.
Additionally, if all CPUs in the generated masks happen to be offline, the
bitmap will be empty. In that scenario, the bit-finding function will
return the size of the array, causing the unassigned CPU loop to map those
CPUs out-of-bounds. Checking if the bitmap is empty before that loop would
prevent this.
I will address the above in the next series iteration.
Kind regards,
--
Aaron Tomlin
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply
* Re: [Lsf-pc] [LSF/MM/BPF TOPIC] A block level, active-active replication solution
From: Keith Busch @ 2026-05-02 11:41 UTC (permalink / raw)
To: Matthew Wilcox; +Cc: Haris Iqbal, lsf-pc, linux-block, Jia Li
In-Reply-To: <afUDV3ue45-shCNU@casper.infradead.org>
On Fri, May 01, 2026 at 08:47:35PM +0100, Matthew Wilcox wrote:
> On Tue, Feb 03, 2026 at 04:09:59PM +0100, Haris Iqbal via Lsf-pc wrote:
> > We are working on a pair of kernel modules which would offer a new
> > replication solution in the Linux kernel. It would be a block level,
> > active-active replication solution for RDMA transport.
>
> Why is active-active a good idea?
>
> With an active-passive solution, network traffic is directed to the
> active node. Over time at some point we get close to saturating the
> link and performance drops. At that point, human intervention will
> occur and the network link will be upgraded.
>
> With an active-active solution, traffic goes to each node. At smoe point
> each link will be about 75% utilised and we won't see any performance
> problems. But then a node goes down and all of a sudden the remaining
> node is being hit with 150% of the link capacity. There's no gradual
> degradation here; the whole solution just goes down.
Maybe I'm out of touch with reality, but I could swear active-passive
setups are often configured such that the passive node for one resource
is the active node for another. That would also suffer the same link
capacity issues you're describing.
^ permalink raw reply
* Re: [RFC PATCH v2 0/4] mm/zsmalloc: reduce zs_free() latency on swap release path
From: Nhat Pham @ 2026-05-02 7:21 UTC (permalink / raw)
To: Wenchao Hao
Cc: Yosry Ahmed, Andrew Morton, Chengming Zhou, Jens Axboe,
Johannes Weiner, Minchan Kim, Sergey Senozhatsky, linux-block,
linux-kernel, linux-mm, Barry Song, Xueyuan Chen, Wenchao Hao
In-Reply-To: <CAOptpSMQyyb+q-VJdJ0RzNkjaer1CTePM5WkwEj_7cAO4aUToQ@mail.gmail.com>
On Tue, Apr 28, 2026 at 2:51 PM Wenchao Hao <haowenchao22@gmail.com> wrote:
>
> On Tue, Apr 28, 2026 at 2:17 AM Yosry Ahmed <yosry@kernel.org> wrote:
> >
> > On Sat, Apr 25, 2026 at 9:13 PM Wenchao Hao <haowenchao22@gmail.com> wrote:
> > >
> > > On Tue, Apr 21, 2026 at 8:16 PM Wenchao Hao <haowenchao22@gmail.com> wrote:
> > > >
> > > > Swap freeing can be expensive when unmapping a VMA containing
> > > > many swap entries. This has been reported to significantly
> > > > delay memory reclamation during Android's low-memory killing,
> > > > especially when multiple processes are terminated to free
> > > > memory, with slot_free() accounting for more than 80% of
> > > > the total cost of freeing swap entries.
> > > >
> > > > Two earlier attempts by Lei and Zhiguo added a new thread in the mm core
> > > > to asynchronously collect and free swap entries [1][2], but the
> > > > design itself is fairly complex.
> > > >
> > > Hi Nhat, Kairui, Barry, Xueyuan,
> > >
> > > Thanks for the review. I agree with the direction and have some ideas for
> > > an alternative approach.
> > >
> > > My approach: first eliminate pool->lock from zs_free() itself, then defer
> > > free to per-cpu buffers with a lockless handoff, and finally reduce
> > > class->lock overhead during drain by exploiting natural class locality.
> > > Achieving both per-cpu and per-class is difficult, so the class->lock
> > > optimization is a compromise — but one that works well in practice.
> > >
> > > 1. Encode class_idx in obj to eliminate pool->lock
> > >
> > > OBJ_INDEX_BITS is over-provisioned on 64-bit. For example on arm64
> > > (chain_size=8): OBJ_INDEX_BITS=24 but only 10 bits are actually needed
> > > for obj_idx, leaving 14 spare bits.
> > > We can split OBJ_INDEX into class_idx + obj_idx:
> > >
> > > obj: [PFN | class_idx (OBJ_CLASS_BITS) | obj_idx (OBJ_IDX_BITS)]
> > >
> > > OBJ_CLASS_BITS is computed dynamically as `ilog2(ZS_SIZE_CLASSES - 1) + 1`
> > > (8 bits for 4K pages, 9 for 64K).
> > > Since class_idx is invariant across migration (only PFN changes), zs_free()
> > > can extract class_idx locklessly, then acquire class->lock and re-read obj for a
> > > stable PFN. No pool->lock needed.
> >
> > How much of the benefit do we get with just these locking improvements
> > without having to defer any of the freeing work?
> >
>
> Hi Yosry,
>
> Thanks for the review. Great question — we tested exactly this.
>
> With only the class_idx-in-obj encoding (eliminating pool->lock from
> zs_free, no deferred freeing), we measured on two platforms.
>
> Test: each process independently mmap 256MB, write data, madvise
> MADV_PAGEOUT to swap out via zram (lzo-rle), then concurrent munmap.
>
> Raspberry Pi 4B (4-core ARM64 Cortex-A72):
>
> mode Base ClassIdx-only Speedup
> single 59.0ms 56.0ms 1.05x
> multi 2p 94.6ms 66.7ms 1.42x
> multi 4p 202.9ms 110.6ms 1.83x
>
> x86 physical machine (4-core Intel i7-12700, 2 rounds averaged):
>
> mode Base ClassIdx-only Speedup
> single 11.7ms 9.8ms 1.19x
> multi 2p 24.1ms 17.2ms 1.40x
> multi 4p 63.0ms 45.3ms 1.39x
Oh man, you are eliminating pool lock here right? This would help my
other patch series a lot too :)
https://lore.kernel.org/all/CAKEwX=M5YpR0cQrryX_y4pm_BuxyUWZ_8MbhWodwbf1Fe=gzew@mail.gmail.com/
https://lore.kernel.org/all/CAKEwX=PkFiP+u+ThrzjTKBi+usQf2uuhTZcfB2BNNA8RboOFDQ@mail.gmail.com/
Well, the deferred freeing would completely move that contention out
of the way lol. But this would benefit all users, regardless of
whether we're deferring the free step or not (for instance, this will
reduce contention between page fault and compaction, IIUC?) I feel
like you'll get some good numbers testing in a system with compaction
and THP enabled, with lots of swap activities. Which is... a lot of
server setup :)
If the deferred freeing is too controversial, this smells like
something that should be upstreamed independently.
>
> Single-process shows modest improvement. With multiple processes,
> each read_lock/read_unlock atomically modifies the shared rwlock
> reader count, and the cost of these atomic operations increases
> with more CPUs accessing the same cacheline concurrently.
> Eliminating pool->lock removes this overhead entirely.
>
> This only works on 64-bit systems where OBJ_INDEX_BITS has enough
> spare bits to fit class_idx. 32-bit systems don't have the room.
> I'm still working on the compile-time gating to properly enable
> this based on architecture and page size configuration.
/*
* The pool->lock protects the race with zpage's migration
* so it's safe to get the page from handle.
*/
read_lock(&pool->lock);
obj = handle_to_obj(handle);
obj_to_zpdesc(obj, &f_zpdesc);
zspage = get_zspage(f_zpdesc);
class = zspage_class(pool, zspage);
spin_lock(&class->lock);
read_unlock(&pool->lock);
It's basically just this blob right?
>
> > As others have pointed out, I don't want to just defer expensive work
> > without understanding why it's expensive and running into limitations
> > about why it cannot be improved without deferring.
>
> For the deferred freeing part: the class_idx-in-obj optimization
> addresses the multi-process scenario where concurrent atomic
> operations on pool->lock become expensive, but does not help
> single-process munmap. Deferred freeing moves the entire zs_free
> cost (including class->lock and zspage freeing) off the munmap
> hot path, which benefits even single-process workloads. The two
> optimizations are complementary.
+1 :)
^ permalink raw reply
* Re: [PATCH] block/blk-iolatency: Add the processing flow of the chained bio in the QoS and define the related types to solve the problem of incorrect inflight processing in the QoS. The usage of the done_split_bio abstract function in the blk-iolatency project.
From: kernel test robot @ 2026-05-02 0:19 UTC (permalink / raw)
To: Li kunyu, axboe, tj, josef
Cc: oe-kbuild-all, linux-block, linux-kernel, Li kunyu
In-Reply-To: <20260429092920.2124-1-likunyu10@163.com>
Hi Li,
kernel test robot noticed the following build errors:
[auto build test ERROR on axboe/for-next]
[also build test ERROR on next-20260430]
[cannot apply to linus/master v6.16-rc1]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Li-kunyu/block-blk-iolatency-Add-the-processing-flow-of-the-chained-bio-in-the-QoS-and-define-the-related-types-to-solve-the-prob/20260501-153918
base: https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git for-next
patch link: https://lore.kernel.org/r/20260429092920.2124-1-likunyu10%40163.com
patch subject: [PATCH] block/blk-iolatency: Add the processing flow of the chained bio in the QoS and define the related types to solve the problem of incorrect inflight processing in the QoS. The usage of the done_split_bio abstract function in the blk-iolatency project.
config: x86_64-rhel-9.4 (https://download.01.org/0day-ci/archive/20260502/202605020222.x7HELUvP-lkp@intel.com/config)
compiler: gcc-14 (Debian 14.2.0-19) 14.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260502/202605020222.x7HELUvP-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202605020222.x7HELUvP-lkp@intel.com/
All errors (new ones prefixed by >>):
block/blk-merge.c: In function 'bio_submit_split':
>> block/blk-merge.c:159:38: error: 'split' undeclared (first use in this function); did you mean 'sg_split'?
159 | bio_set_flag(split, BIO_QOS_CHAIN_CHILD);
| ^~~~~
| sg_split
block/blk-merge.c:159:38: note: each undeclared identifier is reported only once for each function it appears in
>> block/blk-merge.c:165:20: error: invalid storage class for function '__bio_split_discard'
165 | static struct bio *__bio_split_discard(struct bio *bio,
| ^~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:214:28: error: invalid storage class for function 'blk_boundary_sectors'
214 | static inline unsigned int blk_boundary_sectors(const struct queue_limits *lim,
| ^~~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:235:24: error: invalid storage class for function 'get_max_io_size'
235 | static inline unsigned get_max_io_size(struct bio *bio,
| ^~~~~~~~~~~~~~~
>> block/blk-merge.c:288:13: error: invalid storage class for function 'bvec_split_segs'
288 | static bool bvec_split_segs(const struct queue_limits *lim,
| ^~~~~~~~~~~~~~~
>> block/blk-merge.c:314:21: error: invalid storage class for function 'bio_split_alignment'
314 | static unsigned int bio_split_alignment(struct bio *bio,
| ^~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:322:28: error: invalid storage class for function 'bvec_seg_gap'
322 | static inline unsigned int bvec_seg_gap(struct bio_vec *bvprv,
| ^~~~~~~~~~~~
In file included from include/linux/linkage.h:7,
from include/linux/kernel.h:18,
from block/blk-merge.c:5:
>> block/blk-merge.c:425:19: error: non-static declaration of 'bio_split_io_at' follows static declaration
425 | EXPORT_SYMBOL_GPL(bio_split_io_at);
| ^~~~~~~~~~~~~~~
include/linux/export.h:76:28: note: in definition of macro '__EXPORT_SYMBOL'
76 | extern typeof(sym) sym; \
| ^~~
include/linux/export.h:90:41: note: in expansion of macro '_EXPORT_SYMBOL'
90 | #define EXPORT_SYMBOL_GPL(sym) _EXPORT_SYMBOL(sym, "GPL")
| ^~~~~~~~~~~~~~
block/blk-merge.c:425:1: note: in expansion of macro 'EXPORT_SYMBOL_GPL'
425 | EXPORT_SYMBOL_GPL(bio_split_io_at);
| ^~~~~~~~~~~~~~~~~
block/blk-merge.c:341:5: note: previous definition of 'bio_split_io_at' with type 'int(struct bio *, const struct queue_limits *, unsigned int *, unsigned int, unsigned int)'
341 | int bio_split_io_at(struct bio *bio, const struct queue_limits *lim,
| ^~~~~~~~~~~~~~~
>> block/blk-merge.c:491:15: error: non-static declaration of 'bio_split_to_limits' follows static declaration
491 | EXPORT_SYMBOL(bio_split_to_limits);
| ^~~~~~~~~~~~~~~~~~~
include/linux/export.h:76:28: note: in definition of macro '__EXPORT_SYMBOL'
76 | extern typeof(sym) sym; \
| ^~~
include/linux/export.h:89:41: note: in expansion of macro '_EXPORT_SYMBOL'
89 | #define EXPORT_SYMBOL(sym) _EXPORT_SYMBOL(sym, "")
| ^~~~~~~~~~~~~~
block/blk-merge.c:491:1: note: in expansion of macro 'EXPORT_SYMBOL'
491 | EXPORT_SYMBOL(bio_split_to_limits);
| ^~~~~~~~~~~~~
block/blk-merge.c:485:13: note: previous definition of 'bio_split_to_limits' with type 'struct bio *(struct bio *)'
485 | struct bio *bio_split_to_limits(struct bio *bio)
| ^~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:526:28: error: invalid storage class for function 'blk_rq_get_max_sectors'
526 | static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
| ^~~~~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:548:19: error: invalid storage class for function 'll_new_hw_segment'
548 | static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
| ^~~~~~~~~~~~~~~~~
>> block/blk-merge.c:597:12: error: invalid storage class for function 'll_front_merge_fn'
597 | static int ll_front_merge_fn(struct request *req, struct bio *bio,
| ^~~~~~~~~~~~~~~~~
>> block/blk-merge.c:616:13: error: invalid storage class for function 'req_attempt_discard_merge'
616 | static bool req_attempt_discard_merge(struct request_queue *q, struct request *req,
| ^~~~~~~~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:634:12: error: invalid storage class for function 'll_merge_requests_fn'
634 | static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
| ^~~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:677:13: error: invalid storage class for function 'blk_rq_set_mixed_merge'
677 | static void blk_rq_set_mixed_merge(struct request *rq)
| ^~~~~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:698:25: error: invalid storage class for function 'bio_failfast'
698 | static inline blk_opf_t bio_failfast(const struct bio *bio)
| ^~~~~~~~~~~~
>> block/blk-merge.c:711:20: error: invalid storage class for function 'blk_update_mixed_merge'
711 | static inline void blk_update_mixed_merge(struct request *req,
| ^~~~~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:725:13: error: invalid storage class for function 'blk_account_io_merge_request'
725 | static void blk_account_io_merge_request(struct request *req)
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:736:23: error: invalid storage class for function 'blk_try_req_merge'
736 | static enum elv_merge blk_try_req_merge(struct request *req,
| ^~~~~~~~~~~~~~~~~
>> block/blk-merge.c:747:13: error: invalid storage class for function 'blk_atomic_write_mergeable_rq_bio'
747 | static bool blk_atomic_write_mergeable_rq_bio(struct request *rq,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:753:13: error: invalid storage class for function 'blk_atomic_write_mergeable_rqs'
753 | static bool blk_atomic_write_mergeable_rqs(struct request *rq,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:781:24: error: invalid storage class for function 'attempt_merge'
781 | static struct request *attempt_merge(struct request_queue *q,
| ^~~~~~~~~~~~~
block/blk-merge.c:869:24: error: invalid storage class for function 'attempt_back_merge'
869 | static struct request *attempt_back_merge(struct request_queue *q,
| ^~~~~~~~~~~~~~~~~~
block/blk-merge.c:880:24: error: invalid storage class for function 'attempt_front_merge'
880 | static struct request *attempt_front_merge(struct request_queue *q,
| ^~~~~~~~~~~~~~~~~~~
block/blk-merge.c:939:13: error: invalid storage class for function 'blk_account_io_merge_bio'
939 | static void blk_account_io_merge_bio(struct request *req)
| ^~~~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:979:30: error: invalid storage class for function 'bio_attempt_front_merge'
979 | static enum bio_merge_status bio_attempt_front_merge(struct request *req,
| ^~~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:1017:30: error: invalid storage class for function 'bio_attempt_discard_merge'
1017 | static enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
| ^~~~~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:1042:30: error: invalid storage class for function 'blk_attempt_bio_merge'
1042 | static enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q,
| ^~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:1143:19: error: non-static declaration of 'blk_bio_list_merge' follows static declaration
1143 | EXPORT_SYMBOL_GPL(blk_bio_list_merge);
| ^~~~~~~~~~~~~~~~~~
include/linux/export.h:76:28: note: in definition of macro '__EXPORT_SYMBOL'
76 | extern typeof(sym) sym; \
| ^~~
include/linux/export.h:90:41: note: in expansion of macro '_EXPORT_SYMBOL'
90 | #define EXPORT_SYMBOL_GPL(sym) _EXPORT_SYMBOL(sym, "GPL")
| ^~~~~~~~~~~~~~
block/blk-merge.c:1143:1: note: in expansion of macro 'EXPORT_SYMBOL_GPL'
1143 | EXPORT_SYMBOL_GPL(blk_bio_list_merge);
| ^~~~~~~~~~~~~~~~~
block/blk-merge.c:1120:6: note: previous definition of 'blk_bio_list_merge' with type 'bool(struct request_queue *, struct list_head *, struct bio *, unsigned int)' {aka '_Bool(struct request_queue *, struct list_head *, struct bio *, unsigned int)'}
1120 | bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
| ^~~~~~~~~~~~~~~~~~
block/blk-merge.c:1175:19: error: non-static declaration of 'blk_mq_sched_try_merge' follows static declaration
1175 | EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
| ^~~~~~~~~~~~~~~~~~~~~~
include/linux/export.h:76:28: note: in definition of macro '__EXPORT_SYMBOL'
76 | extern typeof(sym) sym; \
| ^~~
include/linux/export.h:90:41: note: in expansion of macro '_EXPORT_SYMBOL'
90 | #define EXPORT_SYMBOL_GPL(sym) _EXPORT_SYMBOL(sym, "GPL")
| ^~~~~~~~~~~~~~
block/blk-merge.c:1175:1: note: in expansion of macro 'EXPORT_SYMBOL_GPL'
1175 | EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
| ^~~~~~~~~~~~~~~~~
block/blk-merge.c:1145:6: note: previous definition of 'blk_mq_sched_try_merge' with type 'bool(struct request_queue *, struct bio *, unsigned int, struct request **)' {aka '_Bool(struct request_queue *, struct bio *, unsigned int, struct request **)'}
1145 | bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
| ^~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:1175:1: error: expected declaration or statement at end of input
1175 | EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
| ^~~~~~~~~~~~~~~~~
block/blk-merge.c: At top level:
block/blk-merge.c:1089:6: warning: 'blk_attempt_plug_merge' defined but not used [-Wunused-function]
1089 | bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
| ^~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:896:6: warning: 'blk_attempt_req_merge' defined but not used [-Wunused-function]
896 | bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
| ^~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:493:14: warning: 'blk_recalc_rq_segments' defined but not used [-Wunused-function]
493 | unsigned int blk_recalc_rq_segments(struct request *rq)
| ^~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:454:13: warning: 'bio_split_write_zeroes' defined but not used [-Wunused-function]
454 | struct bio *bio_split_write_zeroes(struct bio *bio,
| ^~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:442:13: warning: 'bio_split_zone_append' defined but not used [-Wunused-function]
442 | struct bio *bio_split_zone_append(struct bio *bio,
| ^~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:427:13: warning: 'bio_split_rw' defined but not used [-Wunused-function]
427 | struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim,
| ^~~~~~~~~~~~
block/blk-merge.c:201:13: warning: 'bio_split_discard' defined but not used [-Wunused-function]
201 | struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim,
| ^~~~~~~~~~~~~~~~~
vim +159 block/blk-merge.c
142
143 static struct bio *bio_submit_split(struct bio *bio, int split_sectors)
144 {
145 if (unlikely(split_sectors < 0)) {
146 bio->bi_status = errno_to_blk_status(split_sectors);
147 bio_endio(bio);
148 return NULL;
149 }
150
151 if (split_sectors) {
152 bio = bio_submit_split_bioset(bio, split_sectors,
153 &bio->bi_bdev->bd_disk->bio_split);
154 if (bio) {
155 bio->bi_opf |= REQ_NOMERGE;
156 /* Fix the issue where the inflight statistics
157 * of the chained bio in the QoS are incorrect.
158 */
> 159 bio_set_flag(split, BIO_QOS_CHAIN_CHILD);
160 }
161
162 return bio;
163 }
164
> 165 static struct bio *__bio_split_discard(struct bio *bio,
166 const struct queue_limits *lim, unsigned *nsegs,
167 unsigned int max_sectors)
168 {
169 unsigned int max_discard_sectors, granularity;
170 sector_t tmp;
171 unsigned split_sectors;
172
173 *nsegs = 1;
174
175 granularity = max(lim->discard_granularity >> 9, 1U);
176
177 max_discard_sectors = min(max_sectors, bio_allowed_max_sectors(lim));
178 max_discard_sectors -= max_discard_sectors % granularity;
179 if (unlikely(!max_discard_sectors))
180 return bio;
181
182 if (bio_sectors(bio) <= max_discard_sectors)
183 return bio;
184
185 split_sectors = max_discard_sectors;
186
187 /*
188 * If the next starting sector would be misaligned, stop the discard at
189 * the previous aligned sector.
190 */
191 tmp = bio->bi_iter.bi_sector + split_sectors -
192 ((lim->discard_alignment >> 9) % granularity);
193 tmp = sector_div(tmp, granularity);
194
195 if (split_sectors > tmp)
196 split_sectors -= tmp;
197
198 return bio_submit_split(bio, split_sectors);
199 }
200
201 struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim,
202 unsigned *nsegs)
203 {
204 unsigned int max_sectors;
205
206 if (bio_op(bio) == REQ_OP_SECURE_ERASE)
207 max_sectors = lim->max_secure_erase_sectors;
208 else
209 max_sectors = lim->max_discard_sectors;
210
211 return __bio_split_discard(bio, lim, nsegs, max_sectors);
212 }
213
> 214 static inline unsigned int blk_boundary_sectors(const struct queue_limits *lim,
215 bool is_atomic)
216 {
217 /*
218 * chunk_sectors must be a multiple of atomic_write_boundary_sectors if
219 * both non-zero.
220 */
221 if (is_atomic && lim->atomic_write_boundary_sectors)
222 return lim->atomic_write_boundary_sectors;
223
224 return lim->chunk_sectors;
225 }
226
227 /*
228 * Return the maximum number of sectors from the start of a bio that may be
229 * submitted as a single request to a block device. If enough sectors remain,
230 * align the end to the physical block size. Otherwise align the end to the
231 * logical block size. This approach minimizes the number of non-aligned
232 * requests that are submitted to a block device if the start of a bio is not
233 * aligned to a physical block boundary.
234 */
> 235 static inline unsigned get_max_io_size(struct bio *bio,
236 const struct queue_limits *lim)
237 {
238 unsigned pbs = lim->physical_block_size >> SECTOR_SHIFT;
239 unsigned lbs = lim->logical_block_size >> SECTOR_SHIFT;
240 bool is_atomic = bio->bi_opf & REQ_ATOMIC;
241 unsigned boundary_sectors = blk_boundary_sectors(lim, is_atomic);
242 unsigned max_sectors, start, end;
243
244 /*
245 * We ignore lim->max_sectors for atomic writes because it may less
246 * than the actual bio size, which we cannot tolerate.
247 */
248 if (bio_op(bio) == REQ_OP_WRITE_ZEROES)
249 max_sectors = lim->max_write_zeroes_sectors;
250 else if (is_atomic)
251 max_sectors = lim->atomic_write_max_sectors;
252 else
253 max_sectors = lim->max_sectors;
254
255 if (boundary_sectors) {
256 max_sectors = min(max_sectors,
257 blk_boundary_sectors_left(bio->bi_iter.bi_sector,
258 boundary_sectors));
259 }
260
261 start = bio->bi_iter.bi_sector & (pbs - 1);
262 end = (start + max_sectors) & ~(pbs - 1);
263 if (end > start)
264 return end - start;
265 return max_sectors & ~(lbs - 1);
266 }
267
268 /**
269 * bvec_split_segs - verify whether or not a bvec should be split in the middle
270 * @lim: [in] queue limits to split based on
271 * @bv: [in] bvec to examine
272 * @nsegs: [in,out] Number of segments in the bio being built. Incremented
273 * by the number of segments from @bv that may be appended to that
274 * bio without exceeding @max_segs
275 * @bytes: [in,out] Number of bytes in the bio being built. Incremented
276 * by the number of bytes from @bv that may be appended to that
277 * bio without exceeding @max_bytes
278 * @max_segs: [in] upper bound for *@nsegs
279 * @max_bytes: [in] upper bound for *@bytes
280 *
281 * When splitting a bio, it can happen that a bvec is encountered that is too
282 * big to fit in a single segment and hence that it has to be split in the
283 * middle. This function verifies whether or not that should happen. The value
284 * %true is returned if and only if appending the entire @bv to a bio with
285 * *@nsegs segments and *@sectors sectors would make that bio unacceptable for
286 * the block driver.
287 */
> 288 static bool bvec_split_segs(const struct queue_limits *lim,
289 const struct bio_vec *bv, unsigned *nsegs, unsigned *bytes,
290 unsigned max_segs, unsigned max_bytes)
291 {
292 unsigned max_len = max_bytes - *bytes;
293 unsigned len = min(bv->bv_len, max_len);
294 unsigned total_len = 0;
295 unsigned seg_size = 0;
296
297 while (len && *nsegs < max_segs) {
298 seg_size = get_max_segment_size(lim, bvec_phys(bv) + total_len, len);
299
300 (*nsegs)++;
301 total_len += seg_size;
302 len -= seg_size;
303
304 if ((bv->bv_offset + total_len) & lim->virt_boundary_mask)
305 break;
306 }
307
308 *bytes += total_len;
309
310 /* tell the caller to split the bvec if it is too big to fit */
311 return len > 0 || bv->bv_len > max_len;
312 }
313
> 314 static unsigned int bio_split_alignment(struct bio *bio,
315 const struct queue_limits *lim)
316 {
317 if (op_is_write(bio_op(bio)) && lim->zone_write_granularity)
318 return lim->zone_write_granularity;
319 return lim->logical_block_size;
320 }
321
> 322 static inline unsigned int bvec_seg_gap(struct bio_vec *bvprv,
323 struct bio_vec *bv)
324 {
325 return bv->bv_offset | (bvprv->bv_offset + bvprv->bv_len);
326 }
327
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply
* Re: [PATCH] block/blk-iolatency: Add the processing flow of the chained bio in the QoS and define the related types to solve the problem of incorrect inflight processing in the QoS. The usage of the done_split_bio abstract function in the blk-iolatency project.
From: kernel test robot @ 2026-05-01 21:47 UTC (permalink / raw)
To: Li kunyu, axboe, tj, josef
Cc: oe-kbuild-all, linux-block, linux-kernel, Li kunyu
In-Reply-To: <20260429092920.2124-1-likunyu10@163.com>
Hi Li,
kernel test robot noticed the following build warnings:
[auto build test WARNING on axboe/for-next]
[also build test WARNING on next-20260430]
[cannot apply to linus/master v6.16-rc1]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Li-kunyu/block-blk-iolatency-Add-the-processing-flow-of-the-chained-bio-in-the-QoS-and-define-the-related-types-to-solve-the-prob/20260501-153918
base: https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git for-next
patch link: https://lore.kernel.org/r/20260429092920.2124-1-likunyu10%40163.com
patch subject: [PATCH] block/blk-iolatency: Add the processing flow of the chained bio in the QoS and define the related types to solve the problem of incorrect inflight processing in the QoS. The usage of the done_split_bio abstract function in the blk-iolatency project.
config: x86_64-rhel-9.4 (https://download.01.org/0day-ci/archive/20260501/202605012326.wuozpBQs-lkp@intel.com/config)
compiler: gcc-14 (Debian 14.2.0-19) 14.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260501/202605012326.wuozpBQs-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202605012326.wuozpBQs-lkp@intel.com/
All warnings (new ones prefixed by >>):
include/linux/export.h:89:41: note: in expansion of macro '_EXPORT_SYMBOL'
89 | #define EXPORT_SYMBOL(sym) _EXPORT_SYMBOL(sym, "")
| ^~~~~~~~~~~~~~
block/blk-merge.c:491:1: note: in expansion of macro 'EXPORT_SYMBOL'
491 | EXPORT_SYMBOL(bio_split_to_limits);
| ^~~~~~~~~~~~~
block/blk-merge.c:485:13: note: previous definition of 'bio_split_to_limits' with type 'struct bio *(struct bio *)'
485 | struct bio *bio_split_to_limits(struct bio *bio)
| ^~~~~~~~~~~~~~~~~~~
block/blk-merge.c:526:28: error: invalid storage class for function 'blk_rq_get_max_sectors'
526 | static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
| ^~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:548:19: error: invalid storage class for function 'll_new_hw_segment'
548 | static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
| ^~~~~~~~~~~~~~~~~
block/blk-merge.c:597:12: error: invalid storage class for function 'll_front_merge_fn'
597 | static int ll_front_merge_fn(struct request *req, struct bio *bio,
| ^~~~~~~~~~~~~~~~~
block/blk-merge.c:616:13: error: invalid storage class for function 'req_attempt_discard_merge'
616 | static bool req_attempt_discard_merge(struct request_queue *q, struct request *req,
| ^~~~~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:634:12: error: invalid storage class for function 'll_merge_requests_fn'
634 | static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
| ^~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:677:13: error: invalid storage class for function 'blk_rq_set_mixed_merge'
677 | static void blk_rq_set_mixed_merge(struct request *rq)
| ^~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:698:25: error: invalid storage class for function 'bio_failfast'
698 | static inline blk_opf_t bio_failfast(const struct bio *bio)
| ^~~~~~~~~~~~
block/blk-merge.c:711:20: error: invalid storage class for function 'blk_update_mixed_merge'
711 | static inline void blk_update_mixed_merge(struct request *req,
| ^~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:725:13: error: invalid storage class for function 'blk_account_io_merge_request'
725 | static void blk_account_io_merge_request(struct request *req)
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:736:23: error: invalid storage class for function 'blk_try_req_merge'
736 | static enum elv_merge blk_try_req_merge(struct request *req,
| ^~~~~~~~~~~~~~~~~
block/blk-merge.c:747:13: error: invalid storage class for function 'blk_atomic_write_mergeable_rq_bio'
747 | static bool blk_atomic_write_mergeable_rq_bio(struct request *rq,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:753:13: error: invalid storage class for function 'blk_atomic_write_mergeable_rqs'
753 | static bool blk_atomic_write_mergeable_rqs(struct request *rq,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:781:24: error: invalid storage class for function 'attempt_merge'
781 | static struct request *attempt_merge(struct request_queue *q,
| ^~~~~~~~~~~~~
block/blk-merge.c:869:24: error: invalid storage class for function 'attempt_back_merge'
869 | static struct request *attempt_back_merge(struct request_queue *q,
| ^~~~~~~~~~~~~~~~~~
block/blk-merge.c:880:24: error: invalid storage class for function 'attempt_front_merge'
880 | static struct request *attempt_front_merge(struct request_queue *q,
| ^~~~~~~~~~~~~~~~~~~
block/blk-merge.c:939:13: error: invalid storage class for function 'blk_account_io_merge_bio'
939 | static void blk_account_io_merge_bio(struct request *req)
| ^~~~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:979:30: error: invalid storage class for function 'bio_attempt_front_merge'
979 | static enum bio_merge_status bio_attempt_front_merge(struct request *req,
| ^~~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:1017:30: error: invalid storage class for function 'bio_attempt_discard_merge'
1017 | static enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
| ^~~~~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:1042:30: error: invalid storage class for function 'blk_attempt_bio_merge'
1042 | static enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q,
| ^~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:1143:19: error: non-static declaration of 'blk_bio_list_merge' follows static declaration
1143 | EXPORT_SYMBOL_GPL(blk_bio_list_merge);
| ^~~~~~~~~~~~~~~~~~
include/linux/export.h:76:28: note: in definition of macro '__EXPORT_SYMBOL'
76 | extern typeof(sym) sym; \
| ^~~
include/linux/export.h:90:41: note: in expansion of macro '_EXPORT_SYMBOL'
90 | #define EXPORT_SYMBOL_GPL(sym) _EXPORT_SYMBOL(sym, "GPL")
| ^~~~~~~~~~~~~~
block/blk-merge.c:1143:1: note: in expansion of macro 'EXPORT_SYMBOL_GPL'
1143 | EXPORT_SYMBOL_GPL(blk_bio_list_merge);
| ^~~~~~~~~~~~~~~~~
block/blk-merge.c:1120:6: note: previous definition of 'blk_bio_list_merge' with type 'bool(struct request_queue *, struct list_head *, struct bio *, unsigned int)' {aka '_Bool(struct request_queue *, struct list_head *, struct bio *, unsigned int)'}
1120 | bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
| ^~~~~~~~~~~~~~~~~~
block/blk-merge.c:1175:19: error: non-static declaration of 'blk_mq_sched_try_merge' follows static declaration
1175 | EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
| ^~~~~~~~~~~~~~~~~~~~~~
include/linux/export.h:76:28: note: in definition of macro '__EXPORT_SYMBOL'
76 | extern typeof(sym) sym; \
| ^~~
include/linux/export.h:90:41: note: in expansion of macro '_EXPORT_SYMBOL'
90 | #define EXPORT_SYMBOL_GPL(sym) _EXPORT_SYMBOL(sym, "GPL")
| ^~~~~~~~~~~~~~
block/blk-merge.c:1175:1: note: in expansion of macro 'EXPORT_SYMBOL_GPL'
1175 | EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
| ^~~~~~~~~~~~~~~~~
block/blk-merge.c:1145:6: note: previous definition of 'blk_mq_sched_try_merge' with type 'bool(struct request_queue *, struct bio *, unsigned int, struct request **)' {aka '_Bool(struct request_queue *, struct bio *, unsigned int, struct request **)'}
1145 | bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
| ^~~~~~~~~~~~~~~~~~~~~~
block/blk-merge.c:1175:1: error: expected declaration or statement at end of input
1175 | EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
| ^~~~~~~~~~~~~~~~~
block/blk-merge.c: At top level:
>> block/blk-merge.c:1089:6: warning: 'blk_attempt_plug_merge' defined but not used [-Wunused-function]
1089 | bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
| ^~~~~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:896:6: warning: 'blk_attempt_req_merge' defined but not used [-Wunused-function]
896 | bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
| ^~~~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:493:14: warning: 'blk_recalc_rq_segments' defined but not used [-Wunused-function]
493 | unsigned int blk_recalc_rq_segments(struct request *rq)
| ^~~~~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:454:13: warning: 'bio_split_write_zeroes' defined but not used [-Wunused-function]
454 | struct bio *bio_split_write_zeroes(struct bio *bio,
| ^~~~~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:442:13: warning: 'bio_split_zone_append' defined but not used [-Wunused-function]
442 | struct bio *bio_split_zone_append(struct bio *bio,
| ^~~~~~~~~~~~~~~~~~~~~
>> block/blk-merge.c:427:13: warning: 'bio_split_rw' defined but not used [-Wunused-function]
427 | struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim,
| ^~~~~~~~~~~~
>> block/blk-merge.c:201:13: warning: 'bio_split_discard' defined but not used [-Wunused-function]
201 | struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim,
| ^~~~~~~~~~~~~~~~~
vim +/blk_attempt_plug_merge +1089 block/blk-merge.c
5e84ea3a9c662d Jens Axboe 2011-03-21 890
fd2ef39cc9a6b9 Jan Kara 2021-06-23 891 /*
fd2ef39cc9a6b9 Jan Kara 2021-06-23 892 * Try to merge 'next' into 'rq'. Return true if the merge happened, false
fd2ef39cc9a6b9 Jan Kara 2021-06-23 893 * otherwise. The caller is responsible for freeing 'next' if the merge
fd2ef39cc9a6b9 Jan Kara 2021-06-23 894 * happened.
fd2ef39cc9a6b9 Jan Kara 2021-06-23 895 */
fd2ef39cc9a6b9 Jan Kara 2021-06-23 @896 bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
5e84ea3a9c662d Jens Axboe 2011-03-21 897 struct request *next)
5e84ea3a9c662d Jens Axboe 2011-03-21 898 {
fd2ef39cc9a6b9 Jan Kara 2021-06-23 899 return attempt_merge(q, rq, next);
5e84ea3a9c662d Jens Axboe 2011-03-21 900 }
050c8ea80e3e90 Tejun Heo 2012-02-08 901
050c8ea80e3e90 Tejun Heo 2012-02-08 902 bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
050c8ea80e3e90 Tejun Heo 2012-02-08 903 {
e2a60da74fc821 Martin K. Petersen 2012-09-18 904 if (!rq_mergeable(rq) || !bio_mergeable(bio))
050c8ea80e3e90 Tejun Heo 2012-02-08 905 return false;
050c8ea80e3e90 Tejun Heo 2012-02-08 906
288dab8a35a0bd Christoph Hellwig 2016-06-09 907 if (req_op(rq) != bio_op(bio))
f31dc1cd490539 Martin K. Petersen 2012-09-18 908 return false;
f31dc1cd490539 Martin K. Petersen 2012-09-18 909
6b2b04590b51aa Tejun Heo 2022-03-14 910 if (!blk_cgroup_mergeable(rq, bio))
6b2b04590b51aa Tejun Heo 2022-03-14 911 return false;
4eaf99beadcefb Martin K. Petersen 2014-09-26 912 if (blk_integrity_merge_bio(rq->q, rq, bio) == false)
050c8ea80e3e90 Tejun Heo 2012-02-08 913 return false;
a892c8d52c0228 Satya Tangirala 2020-05-14 914 if (!bio_crypt_rq_ctx_compatible(rq, bio))
a892c8d52c0228 Satya Tangirala 2020-05-14 915 return false;
61952bb73486ff Christoph Hellwig 2024-11-12 916 if (rq->bio->bi_write_hint != bio->bi_write_hint)
449813515d3e5e Bart Van Assche 2024-02-02 917 return false;
5006f85ea23ea0 Christoph Hellwig 2025-05-06 918 if (rq->bio->bi_write_stream != bio->bi_write_stream)
5006f85ea23ea0 Christoph Hellwig 2025-05-06 919 return false;
6975c1a486a404 Christoph Hellwig 2024-11-12 920 if (rq->bio->bi_ioprio != bio->bi_ioprio)
668ffc03418bc7 Damien Le Moal 2018-11-20 921 return false;
9da3d1e912f395 John Garry 2024-06-20 922 if (blk_atomic_write_mergeable_rq_bio(rq, bio) == false)
9da3d1e912f395 John Garry 2024-06-20 923 return false;
9da3d1e912f395 John Garry 2024-06-20 924
050c8ea80e3e90 Tejun Heo 2012-02-08 925 return true;
050c8ea80e3e90 Tejun Heo 2012-02-08 926 }
050c8ea80e3e90 Tejun Heo 2012-02-08 927
34fe7c05400663 Christoph Hellwig 2017-02-08 928 enum elv_merge blk_try_merge(struct request *rq, struct bio *bio)
050c8ea80e3e90 Tejun Heo 2012-02-08 929 {
caebce24f6a7f8 Jens Axboe 2025-11-18 930 if (blk_discard_mergable(rq))
caebce24f6a7f8 Jens Axboe 2025-11-18 931 return ELEVATOR_DISCARD_MERGE;
caebce24f6a7f8 Jens Axboe 2025-11-18 932 else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector)
050c8ea80e3e90 Tejun Heo 2012-02-08 933 return ELEVATOR_BACK_MERGE;
4f024f3797c43c Kent Overstreet 2013-10-11 934 else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector)
050c8ea80e3e90 Tejun Heo 2012-02-08 935 return ELEVATOR_FRONT_MERGE;
050c8ea80e3e90 Tejun Heo 2012-02-08 936 return ELEVATOR_NO_MERGE;
050c8ea80e3e90 Tejun Heo 2012-02-08 937 }
8e756373d7c8eb Baolin Wang 2020-08-28 938
8e756373d7c8eb Baolin Wang 2020-08-28 939 static void blk_account_io_merge_bio(struct request *req)
8e756373d7c8eb Baolin Wang 2020-08-28 940 {
e3569ecae44daa Jens Axboe 2024-10-03 941 if (req->rq_flags & RQF_IO_STAT) {
8e756373d7c8eb Baolin Wang 2020-08-28 942 part_stat_lock();
8e756373d7c8eb Baolin Wang 2020-08-28 943 part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
8e756373d7c8eb Baolin Wang 2020-08-28 944 part_stat_unlock();
8e756373d7c8eb Baolin Wang 2020-08-28 945 }
e3569ecae44daa Jens Axboe 2024-10-03 946 }
8e756373d7c8eb Baolin Wang 2020-08-28 947
dd850ff3eee428 Damien Le Moal 2024-04-08 948 enum bio_merge_status bio_attempt_back_merge(struct request *req,
eda5cc997abd21 Christoph Hellwig 2020-10-06 949 struct bio *bio, unsigned int nr_segs)
8e756373d7c8eb Baolin Wang 2020-08-28 950 {
3ce6a115980c01 Ming Lei 2023-02-09 951 const blk_opf_t ff = bio_failfast(bio);
8e756373d7c8eb Baolin Wang 2020-08-28 952
8e756373d7c8eb Baolin Wang 2020-08-28 953 if (!ll_back_merge_fn(req, bio, nr_segs))
7d7ca7c5269bec Baolin Wang 2020-08-28 954 return BIO_MERGE_FAILED;
8e756373d7c8eb Baolin Wang 2020-08-28 955
e8a676d61c07ec Christoph Hellwig 2020-12-03 956 trace_block_bio_backmerge(bio);
8e756373d7c8eb Baolin Wang 2020-08-28 957 rq_qos_merge(req->q, req, bio);
8e756373d7c8eb Baolin Wang 2020-08-28 958
8e756373d7c8eb Baolin Wang 2020-08-28 959 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
8e756373d7c8eb Baolin Wang 2020-08-28 960 blk_rq_set_mixed_merge(req);
8e756373d7c8eb Baolin Wang 2020-08-28 961
3ce6a115980c01 Ming Lei 2023-02-09 962 blk_update_mixed_merge(req, bio, false);
3ce6a115980c01 Ming Lei 2023-02-09 963
dd291d77cc90eb Damien Le Moal 2024-04-08 964 if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING)
dd291d77cc90eb Damien Le Moal 2024-04-08 965 blk_zone_write_plug_bio_merged(bio);
dd291d77cc90eb Damien Le Moal 2024-04-08 966
2f6b2565d43cdb Keith Busch 2025-10-14 967 req->phys_gap_bit = bio_seg_gap(req->q, req->biotail, bio,
2f6b2565d43cdb Keith Busch 2025-10-14 968 req->phys_gap_bit);
8e756373d7c8eb Baolin Wang 2020-08-28 969 req->biotail->bi_next = bio;
8e756373d7c8eb Baolin Wang 2020-08-28 970 req->biotail = bio;
8e756373d7c8eb Baolin Wang 2020-08-28 971 req->__data_len += bio->bi_iter.bi_size;
8e756373d7c8eb Baolin Wang 2020-08-28 972
8e756373d7c8eb Baolin Wang 2020-08-28 973 bio_crypt_free_ctx(bio);
8e756373d7c8eb Baolin Wang 2020-08-28 974
8e756373d7c8eb Baolin Wang 2020-08-28 975 blk_account_io_merge_bio(req);
7d7ca7c5269bec Baolin Wang 2020-08-28 976 return BIO_MERGE_OK;
8e756373d7c8eb Baolin Wang 2020-08-28 977 }
8e756373d7c8eb Baolin Wang 2020-08-28 978
eda5cc997abd21 Christoph Hellwig 2020-10-06 979 static enum bio_merge_status bio_attempt_front_merge(struct request *req,
eda5cc997abd21 Christoph Hellwig 2020-10-06 980 struct bio *bio, unsigned int nr_segs)
8e756373d7c8eb Baolin Wang 2020-08-28 981 {
3ce6a115980c01 Ming Lei 2023-02-09 982 const blk_opf_t ff = bio_failfast(bio);
8e756373d7c8eb Baolin Wang 2020-08-28 983
dd291d77cc90eb Damien Le Moal 2024-04-08 984 /*
dd291d77cc90eb Damien Le Moal 2024-04-08 985 * A front merge for writes to sequential zones of a zoned block device
dd291d77cc90eb Damien Le Moal 2024-04-08 986 * can happen only if the user submitted writes out of order. Do not
dd291d77cc90eb Damien Le Moal 2024-04-08 987 * merge such write to let it fail.
dd291d77cc90eb Damien Le Moal 2024-04-08 988 */
dd291d77cc90eb Damien Le Moal 2024-04-08 989 if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING)
dd291d77cc90eb Damien Le Moal 2024-04-08 990 return BIO_MERGE_FAILED;
dd291d77cc90eb Damien Le Moal 2024-04-08 991
8e756373d7c8eb Baolin Wang 2020-08-28 992 if (!ll_front_merge_fn(req, bio, nr_segs))
7d7ca7c5269bec Baolin Wang 2020-08-28 993 return BIO_MERGE_FAILED;
8e756373d7c8eb Baolin Wang 2020-08-28 994
e8a676d61c07ec Christoph Hellwig 2020-12-03 995 trace_block_bio_frontmerge(bio);
8e756373d7c8eb Baolin Wang 2020-08-28 996 rq_qos_merge(req->q, req, bio);
8e756373d7c8eb Baolin Wang 2020-08-28 997
8e756373d7c8eb Baolin Wang 2020-08-28 998 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
8e756373d7c8eb Baolin Wang 2020-08-28 999 blk_rq_set_mixed_merge(req);
8e756373d7c8eb Baolin Wang 2020-08-28 1000
3ce6a115980c01 Ming Lei 2023-02-09 1001 blk_update_mixed_merge(req, bio, true);
3ce6a115980c01 Ming Lei 2023-02-09 1002
2f6b2565d43cdb Keith Busch 2025-10-14 1003 req->phys_gap_bit = bio_seg_gap(req->q, bio, req->bio,
2f6b2565d43cdb Keith Busch 2025-10-14 1004 req->phys_gap_bit);
8e756373d7c8eb Baolin Wang 2020-08-28 1005 bio->bi_next = req->bio;
8e756373d7c8eb Baolin Wang 2020-08-28 1006 req->bio = bio;
8e756373d7c8eb Baolin Wang 2020-08-28 1007
8e756373d7c8eb Baolin Wang 2020-08-28 1008 req->__sector = bio->bi_iter.bi_sector;
8e756373d7c8eb Baolin Wang 2020-08-28 1009 req->__data_len += bio->bi_iter.bi_size;
8e756373d7c8eb Baolin Wang 2020-08-28 1010
8e756373d7c8eb Baolin Wang 2020-08-28 1011 bio_crypt_do_front_merge(req, bio);
8e756373d7c8eb Baolin Wang 2020-08-28 1012
8e756373d7c8eb Baolin Wang 2020-08-28 1013 blk_account_io_merge_bio(req);
7d7ca7c5269bec Baolin Wang 2020-08-28 1014 return BIO_MERGE_OK;
8e756373d7c8eb Baolin Wang 2020-08-28 1015 }
8e756373d7c8eb Baolin Wang 2020-08-28 1016
eda5cc997abd21 Christoph Hellwig 2020-10-06 1017 static enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
eda5cc997abd21 Christoph Hellwig 2020-10-06 1018 struct request *req, struct bio *bio)
8e756373d7c8eb Baolin Wang 2020-08-28 1019 {
8e756373d7c8eb Baolin Wang 2020-08-28 1020 unsigned short segments = blk_rq_nr_discard_segments(req);
8e756373d7c8eb Baolin Wang 2020-08-28 1021
8e756373d7c8eb Baolin Wang 2020-08-28 1022 if (segments >= queue_max_discard_segments(q))
8e756373d7c8eb Baolin Wang 2020-08-28 1023 goto no_merge;
8e756373d7c8eb Baolin Wang 2020-08-28 1024 if (blk_rq_sectors(req) + bio_sectors(bio) >
8e756373d7c8eb Baolin Wang 2020-08-28 1025 blk_rq_get_max_sectors(req, blk_rq_pos(req)))
8e756373d7c8eb Baolin Wang 2020-08-28 1026 goto no_merge;
8e756373d7c8eb Baolin Wang 2020-08-28 1027
8e756373d7c8eb Baolin Wang 2020-08-28 1028 rq_qos_merge(q, req, bio);
8e756373d7c8eb Baolin Wang 2020-08-28 1029
8e756373d7c8eb Baolin Wang 2020-08-28 1030 req->biotail->bi_next = bio;
8e756373d7c8eb Baolin Wang 2020-08-28 1031 req->biotail = bio;
8e756373d7c8eb Baolin Wang 2020-08-28 1032 req->__data_len += bio->bi_iter.bi_size;
8e756373d7c8eb Baolin Wang 2020-08-28 1033 req->nr_phys_segments = segments + 1;
8e756373d7c8eb Baolin Wang 2020-08-28 1034
8e756373d7c8eb Baolin Wang 2020-08-28 1035 blk_account_io_merge_bio(req);
7d7ca7c5269bec Baolin Wang 2020-08-28 1036 return BIO_MERGE_OK;
8e756373d7c8eb Baolin Wang 2020-08-28 1037 no_merge:
8e756373d7c8eb Baolin Wang 2020-08-28 1038 req_set_nomerge(q, req);
7d7ca7c5269bec Baolin Wang 2020-08-28 1039 return BIO_MERGE_FAILED;
7d7ca7c5269bec Baolin Wang 2020-08-28 1040 }
7d7ca7c5269bec Baolin Wang 2020-08-28 1041
7d7ca7c5269bec Baolin Wang 2020-08-28 1042 static enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q,
7d7ca7c5269bec Baolin Wang 2020-08-28 1043 struct request *rq,
7d7ca7c5269bec Baolin Wang 2020-08-28 1044 struct bio *bio,
7d7ca7c5269bec Baolin Wang 2020-08-28 1045 unsigned int nr_segs,
7d7ca7c5269bec Baolin Wang 2020-08-28 1046 bool sched_allow_merge)
7d7ca7c5269bec Baolin Wang 2020-08-28 1047 {
7d7ca7c5269bec Baolin Wang 2020-08-28 1048 if (!blk_rq_merge_ok(rq, bio))
7d7ca7c5269bec Baolin Wang 2020-08-28 1049 return BIO_MERGE_NONE;
7d7ca7c5269bec Baolin Wang 2020-08-28 1050
7d7ca7c5269bec Baolin Wang 2020-08-28 1051 switch (blk_try_merge(rq, bio)) {
7d7ca7c5269bec Baolin Wang 2020-08-28 1052 case ELEVATOR_BACK_MERGE:
265600b7b6e88f Baolin Wang 2020-09-02 1053 if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))
7d7ca7c5269bec Baolin Wang 2020-08-28 1054 return bio_attempt_back_merge(rq, bio, nr_segs);
7d7ca7c5269bec Baolin Wang 2020-08-28 1055 break;
7d7ca7c5269bec Baolin Wang 2020-08-28 1056 case ELEVATOR_FRONT_MERGE:
265600b7b6e88f Baolin Wang 2020-09-02 1057 if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))
7d7ca7c5269bec Baolin Wang 2020-08-28 1058 return bio_attempt_front_merge(rq, bio, nr_segs);
7d7ca7c5269bec Baolin Wang 2020-08-28 1059 break;
7d7ca7c5269bec Baolin Wang 2020-08-28 1060 case ELEVATOR_DISCARD_MERGE:
7d7ca7c5269bec Baolin Wang 2020-08-28 1061 return bio_attempt_discard_merge(q, rq, bio);
7d7ca7c5269bec Baolin Wang 2020-08-28 1062 default:
7d7ca7c5269bec Baolin Wang 2020-08-28 1063 return BIO_MERGE_NONE;
7d7ca7c5269bec Baolin Wang 2020-08-28 1064 }
7d7ca7c5269bec Baolin Wang 2020-08-28 1065
7d7ca7c5269bec Baolin Wang 2020-08-28 1066 return BIO_MERGE_FAILED;
8e756373d7c8eb Baolin Wang 2020-08-28 1067 }
8e756373d7c8eb Baolin Wang 2020-08-28 1068
8e756373d7c8eb Baolin Wang 2020-08-28 1069 /**
8e756373d7c8eb Baolin Wang 2020-08-28 1070 * blk_attempt_plug_merge - try to merge with %current's plugged list
8e756373d7c8eb Baolin Wang 2020-08-28 1071 * @q: request_queue new bio is being queued at
8e756373d7c8eb Baolin Wang 2020-08-28 1072 * @bio: new bio being queued
8e756373d7c8eb Baolin Wang 2020-08-28 1073 * @nr_segs: number of segments in @bio
87c037d11b83b9 Jens Axboe 2021-10-18 1074 * from the passed in @q already in the plug list
8e756373d7c8eb Baolin Wang 2020-08-28 1075 *
d38a9c04c0d563 Jens Axboe 2021-10-14 1076 * Determine whether @bio being queued on @q can be merged with the previous
d38a9c04c0d563 Jens Axboe 2021-10-14 1077 * request on %current's plugged list. Returns %true if merge was successful,
8e756373d7c8eb Baolin Wang 2020-08-28 1078 * otherwise %false.
8e756373d7c8eb Baolin Wang 2020-08-28 1079 *
8e756373d7c8eb Baolin Wang 2020-08-28 1080 * Plugging coalesces IOs from the same issuer for the same purpose without
8e756373d7c8eb Baolin Wang 2020-08-28 1081 * going through @q->queue_lock. As such it's more of an issuing mechanism
8e756373d7c8eb Baolin Wang 2020-08-28 1082 * than scheduling, and the request, while may have elvpriv data, is not
8e756373d7c8eb Baolin Wang 2020-08-28 1083 * added on the elevator at this point. In addition, we don't have
8e756373d7c8eb Baolin Wang 2020-08-28 1084 * reliable access to the elevator outside queue lock. Only check basic
8e756373d7c8eb Baolin Wang 2020-08-28 1085 * merging parameters without querying the elevator.
8e756373d7c8eb Baolin Wang 2020-08-28 1086 *
8e756373d7c8eb Baolin Wang 2020-08-28 1087 * Caller must ensure !blk_queue_nomerges(q) beforehand.
8e756373d7c8eb Baolin Wang 2020-08-28 1088 */
8e756373d7c8eb Baolin Wang 2020-08-28 @1089 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
0c5bcc92d94a8f Christoph Hellwig 2021-11-23 1090 unsigned int nr_segs)
8e756373d7c8eb Baolin Wang 2020-08-28 1091 {
99a9476b27e895 Damien Le Moal 2024-04-08 1092 struct blk_plug *plug = current->plug;
8e756373d7c8eb Baolin Wang 2020-08-28 1093 struct request *rq;
8e756373d7c8eb Baolin Wang 2020-08-28 1094
a3396b99990d8b Christoph Hellwig 2024-11-13 1095 if (!plug || rq_list_empty(&plug->mq_list))
8e756373d7c8eb Baolin Wang 2020-08-28 1096 return false;
8e756373d7c8eb Baolin Wang 2020-08-28 1097
961296e89dc380 Jens Axboe 2025-06-11 1098 rq = plug->mq_list.tail;
961296e89dc380 Jens Axboe 2025-06-11 1099 if (rq->q == q)
961296e89dc380 Jens Axboe 2025-06-11 1100 return blk_attempt_bio_merge(q, rq, bio, nr_segs, false) ==
961296e89dc380 Jens Axboe 2025-06-11 1101 BIO_MERGE_OK;
961296e89dc380 Jens Axboe 2025-06-11 1102 else if (!plug->multiple_queues)
961296e89dc380 Jens Axboe 2025-06-11 1103 return false;
961296e89dc380 Jens Axboe 2025-06-11 1104
5b2050718d095c Jens Axboe 2022-03-11 1105 rq_list_for_each(&plug->mq_list, rq) {
961296e89dc380 Jens Axboe 2025-06-11 1106 if (rq->q != q)
961296e89dc380 Jens Axboe 2025-06-11 1107 continue;
a1cb65377e7075 Ming Lei 2021-11-02 1108 if (blk_attempt_bio_merge(q, rq, bio, nr_segs, false) ==
a1cb65377e7075 Ming Lei 2021-11-02 1109 BIO_MERGE_OK)
8e756373d7c8eb Baolin Wang 2020-08-28 1110 return true;
5b2050718d095c Jens Axboe 2022-03-11 1111 break;
5b2050718d095c Jens Axboe 2022-03-11 1112 }
8e756373d7c8eb Baolin Wang 2020-08-28 1113 return false;
8e756373d7c8eb Baolin Wang 2020-08-28 1114 }
bdc6a287bc98e8 Baolin Wang 2020-08-28 1115
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply
* Re: [Lsf-pc] [LSF/MM/BPF TOPIC] A block level, active-active replication solution
From: Matthew Wilcox @ 2026-05-01 19:47 UTC (permalink / raw)
To: Haris Iqbal; +Cc: lsf-pc, linux-block, Jia Li
In-Reply-To: <CAJpMwyiVqqbR3ni+URv0uPnE0Bgt_+d+-_+_t6E3wzfiUOT-cg@mail.gmail.com>
On Tue, Feb 03, 2026 at 04:09:59PM +0100, Haris Iqbal via Lsf-pc wrote:
> We are working on a pair of kernel modules which would offer a new
> replication solution in the Linux kernel. It would be a block level,
> active-active replication solution for RDMA transport.
Why is active-active a good idea?
With an active-passive solution, network traffic is directed to the
active node. Over time at some point we get close to saturating the
link and performance drops. At that point, human intervention will
occur and the network link will be upgraded.
With an active-active solution, traffic goes to each node. At smoe point
each link will be about 75% utilised and we won't see any performance
problems. But then a node goes down and all of a sudden the remaining
node is being hit with 150% of the link capacity. There's no gradual
degradation here; the whole solution just goes down.
Of course it doesn't have to be network capacity either; it could be CPU,
RAM or any other resource needed to service the requests.
Active-active is fragile and I would never recommend such a solution.
I won't be in your session, but I thought it worth raising this point.
^ permalink raw reply
* Re: [GIT PULL] Block fixes for 7.1-rc2
From: pr-tracker-bot @ 2026-05-01 18:42 UTC (permalink / raw)
To: Jens Axboe; +Cc: Linus Torvalds, linux-block@vger.kernel.org
In-Reply-To: <8549b9c4-487f-4d3e-8b23-8e83297d11be@kernel.dk>
The pull request you sent on Thu, 30 Apr 2026 19:49:46 -0600:
> https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git tags/block-7.1-20260430
has been merged into torvalds/linux.git:
https://git.kernel.org/torvalds/c/6fe0be6dc7faf984599b1e356ead1c49b64ed3ca
Thank you!
--
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/prtracker.html
^ permalink raw reply
* [PATCHv2] blk-mq: check for stale cached request in blk_mq_submit_bio
From: Keith Busch @ 2026-05-01 17:41 UTC (permalink / raw)
To: axboe, hch, linux-block; +Cc: Keith Busch
From: Keith Busch <kbusch@kernel.org>
When submitting a bio to blk-mq, if the task should sleep after peeking
a cached request, but before it pops it, the plug flushes and calls
blk_mq_free_plug_rqs, freeing the cached_rqs. This creates a
use-after-free bug. Fix this by ensuring the cached_rqs still contains
our peeked request, and retry the bio submission without it if the
request had been freed.
The code had already warned of this possibility, and specifically popped
the request before other known blocking calls, but it didn't handle a
blocking GFP_NOIO alloc. Under memory pressure, allocating the split bio
or the integrity payload are two such cases that can block. The blk-mq
submit_bio function continues using the peeked request that was already
freed and re-initialized, so the driver receives that request with a
NULL'ed mq_hctx, and inevitably panics.
Relevant kernel messages if you should encounter this condition, where
the "WARNING" is the harbinger of the panic about to happen:
------------[ cut here ]------------
WARNING: CPU: 4 PID: 80820 at block/blk-mq.c:3071 blk_mq_submit_bio+0x2cf/0x5b0
...
BUG: kernel NULL pointer dereference, address: 0000000000000100
#PF: supervisor read access in kernel mode
#PF: error_code(0x0000) - not-present page
PGD 6b367b067 P4D 6b367b067 PUD 6bb5eb067 PMD 0
Oops: Oops: 0000 [#1] SMP
...
Call Trace:
<TASK>
blk_mq_dispatch_queue_requests+0x46/0x120
blk_mq_flush_plug_list+0x38/0x130
blk_add_rq_to_plug+0xa2/0x160
blk_mq_submit_bio+0x3ab/0x5b0
__submit_bio+0x3a/0x260
submit_bio_noacct_nocheck+0xc6/0x2b0
btrfs_submit_bbio+0x14d/0x520
? btrfs_get_extent+0x43f/0x640
submit_extent_folio+0x31f/0x340
btrfs_do_readpage+0x2d7/0xac0
btrfs_readahead+0x142/0x200
? clear_state_bit+0x520/0x520
read_pages+0x57/0x200
? folio_alloc_noprof+0x10c/0x310
page_cache_ra_unbounded+0x28c/0x480
? asm_sysvec_call_function+0x16/0x20
? blk_cgroup_congested+0xa/0x50
? page_cache_sync_ra+0x41/0x2d0
filemap_get_pages+0x347/0xd50
filemap_read+0xd3/0x500
? 0xffffffff81000000
__io_read+0x111/0x440
io_read+0x23/0x90
__io_issue_sqe+0x40/0x120
io_issue_sqe+0x3f/0x3a0
io_submit_sqes+0x2bd/0x790
__se_sys_io_uring_enter+0x100/0xc10
? eventfd_read+0x100/0x1f0
? futex_wake+0x1b9/0x260
? syscall_trace_enter+0x34/0x1d0
do_syscall_64+0x6a/0x250
entry_SYSCALL_64_after_hwframe+0x4b/0x53
Fixes: b0077e269f6c1 ("blk-mq: make sure active queue usage is held for bio_integrity_prep()")
Fixes: 7b4f36cd22a65 ("block: ensure we hold a queue reference when using queue limits")
Signed-off-by: Keith Busch <kbusch@kernel.org>
---
v1->v2:
Warn if the cached_rqs is not NULL when the peeked request isn't the
top. This should never happen, but such a bug would be difficult to
figure out was happening without the warning. The previous warn was
essential to figure out the bug this patch is addressing.
If the peeked request was freed, rerun the entire bio setup. The first
version potentially performed operations outside the queue usage
counter protection, so may have resulted in an invalid bio if it was
racing against a driver updating queue limites. This retry also
required clearing the integrity allocation if it was done since
updated limits may make any previous setup invalid.
block/blk-mq.c | 29 +++++++++++++++++++++--------
1 file changed, 21 insertions(+), 8 deletions(-)
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 4c5c16cce4f8f..73ef3e4be5123 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -3096,22 +3096,27 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug,
return rq;
}
-static void blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug,
+static bool blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug,
struct bio *bio)
{
- if (rq_list_pop(&plug->cached_rqs) != rq)
- WARN_ON_ONCE(1);
-
/*
- * If any qos ->throttle() end up blocking, we will have flushed the
- * plug and hence killed the cached_rq list as well. Pop this entry
- * before we throttle.
+ * We will have flushed the plug and hence killed the cached_rq list as
+ * well if anything had scheduled. Pop this entry before we throttle if
+ * the entry is still valid.
*/
+ struct request *popped = rq_list_pop(&plug->cached_rqs);
+
+ if (popped != rq) {
+ WARN_ON_ONCE(popped);
+ return false;
+ }
+
rq_qos_throttle(rq->q, bio);
blk_mq_rq_time_init(rq, blk_time_get_ns());
rq->cmd_flags = bio->bi_opf;
INIT_LIST_HEAD(&rq->queuelist);
+ return true;
}
static bool bio_unaligned(const struct bio *bio, struct request_queue *q)
@@ -3154,6 +3159,7 @@ void blk_mq_submit_bio(struct bio *bio)
*/
rq = blk_mq_peek_cached_request(plug, q, bio->bi_opf);
+retry:
/*
* A BIO that was released from a zone write plug has already been
* through the preparation in this function, already holds a reference
@@ -3211,7 +3217,14 @@ void blk_mq_submit_bio(struct bio *bio)
new_request:
if (rq) {
- blk_mq_use_cached_rq(rq, plug, bio);
+ if (!blk_mq_use_cached_rq(rq, plug, bio)) {
+ struct bio_integrity_payload *bip = bio_integrity(bio);
+
+ if (bip && (bip->bip_flags & BIP_BLOCK_INTEGRITY))
+ bio_integrity_free(bio);
+ rq = NULL;
+ goto retry;
+ }
} else {
rq = blk_mq_get_new_requests(q, plug, bio);
if (unlikely(!rq)) {
--
2.52.0
^ permalink raw reply related
* Re: [LSF/MM/BPF TOPIC] Memory fragmentation with large block sizes
From: Matthew Wilcox @ 2026-05-01 14:33 UTC (permalink / raw)
To: Hannes Reinecke
Cc: lsf-pc, linux-nvme@lists.infradead.org,
linux-block@vger.kernel.org, linux-mm
In-Reply-To: <f22caf98-1375-493a-a275-0500ffac3e81@suse.de>
On Thu, Feb 19, 2026 at 10:54:48AM +0100, Hannes Reinecke wrote:
> I (together with the Czech Technical University) did some experiments trying
> to measure memory fragmentation with large block sizes.
>
> Doing so raised some challenges:
>
> - How do you _generate_ memory fragmentation? The MM subsystem is
> precisely geared up to avoid it, so you would need to come up
> with some idea how to defeat it. With the help from Willy I managed
> to come up with something, but I really would like to discuss
> what would be the best option here.
> - What is acceptable memory fragmentation? Are we good enough if the
> measured fragmentation does not grow during the test runs?
> - Do we have better visibility into memory fragmentation other than
> just reading /proc/buddyinfo?
>
> And, of course, I would like to present (and discuss) the results
> of the testruns done on 4k, 8k, and 16k blocksizes.
I think that Rik's recent work is going to affect discussion of this
topic (summary: with a "small amount" of work, reliable allocation of
1GB folios is possible):
https://lore.kernel.org/linux-mm/20260430202233.111010-1-riel@surriel.com/
but another aspect to it is the recent performance problem reported by
Amazon (summary: compaction takes too long):
https://lore.kernel.org/linux-mm/20260428150240.3009-1-dipiets@amazon.it/
Anyway, I'm putting you on notice that I may hijack this session to talk
about how GFP flags suck. I may even have a proposal for a replacement,
depending how inspired I am over the next few days.
I still think this discussion is useful because we wouldn't want an
attacker to be able to make Linux unreliable. So it's useful to think
about how userspace can make memory unreclaimable and if large folios
make the problem worse in any meaningful way.
^ permalink raw reply
* Re: [PATCH 1/2] block: pass a minsize argument to bio_iov_iter_bounce
From: Pankaj Raghav (Samsung) @ 2026-05-01 12:46 UTC (permalink / raw)
To: Christoph Hellwig
Cc: Jens Axboe, Christian Brauner, Darrick J. Wong, linux-block,
linux-xfs, linux-fsdevel
In-Reply-To: <20260430132019.312405-2-hch@lst.de>
On Thu, Apr 30, 2026 at 03:20:04PM +0200, Christoph Hellwig wrote:
> When bouncing for block size > PAGE_SIZE file systems, the bio needs to
> be big enough to fit an entire block.
We set the alignment to fs_block_size when IOMAP_DIO_FSBLOCK_ALIGNED is
set. And we always set IOMAP_DIO_FSBLOCK_ALIGNED in btrfs for dio
reads/writes. IIUC, it was added to support bs > PS block sizes in btrfs.
But in XFS, we set it only for CoW inodes (xfs_is_always_cow_inode is true).
The commit message seems to indicate this is needed for all filesystems
that support bs > PS configurations. Am I missing something here?
Otherwise the patch looks good to me.
>
> Signed-off-by: Christoph Hellwig <hch@lst.de>
> ---
> block/bio.c | 23 +++++++++++++----------
> fs/iomap/direct-io.c | 2 +-
> include/linux/bio.h | 3 ++-
> 3 files changed, 16 insertions(+), 12 deletions(-)
>
--
Pankaj
^ permalink raw reply
* Re: [PATCH] ublk: don't issue uring_cmd from fallback task work
From: Jens Axboe @ 2026-05-01 11:31 UTC (permalink / raw)
To: linux-block, Ming Lei; +Cc: Caleb Sander Mateos, Uday Shankar
In-Reply-To: <20260501112312.947327-1-tom.leiming@gmail.com>
On Fri, 01 May 2026 19:23:12 +0800, Ming Lei wrote:
> When ublk_ch_uring_cmd_cb() runs as fallback task work (e.g., because
> the submitting task is exiting), the command should not be issued as
> current is a kworker, not the daemon task. This can cause io->task
> to capture the wrong task in __ublk_fetch(), leading to a task
> mismatch warning in ublk_uring_cmd_cancel_fn().
>
> Check tw.cancel and return -ECANCELED instead of issuing the command
> from fallback context.
>
> [...]
Applied, thanks!
[1/1] ublk: don't issue uring_cmd from fallback task work
commit: 845db023a8aeba8b14315a846dcfba31ee727fb1
Best regards,
--
Jens Axboe
^ permalink raw reply
* [PATCH] ublk: don't issue uring_cmd from fallback task work
From: Ming Lei @ 2026-05-01 11:23 UTC (permalink / raw)
To: Jens Axboe, linux-block; +Cc: Caleb Sander Mateos, Uday Shankar, Ming Lei
From: Jens Axboe <axboe@kernel.dk>
When ublk_ch_uring_cmd_cb() runs as fallback task work (e.g., because
the submitting task is exiting), the command should not be issued as
current is a kworker, not the daemon task. This can cause io->task
to capture the wrong task in __ublk_fetch(), leading to a task
mismatch warning in ublk_uring_cmd_cancel_fn().
Check tw.cancel and return -ECANCELED instead of issuing the command
from fallback context.
Fixes: 3421c7f68bba ("ublk: make sure io cmd handled in submitter task context")
Signed-off-by: Ming Lei <tom.leiming@gmail.com>
---
drivers/block/ublk_drv.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 8e5f3738c203..d10460d29e4a 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -3496,8 +3496,10 @@ static void ublk_ch_uring_cmd_cb(struct io_tw_req tw_req, io_tw_token_t tw)
{
unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
- int ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
+ int ret = -ECANCELED;
+ if (!tw.cancel)
+ ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
if (ret != -EIOCBQUEUED)
io_uring_cmd_done(cmd, ret, issue_flags);
}
--
2.53.0
^ permalink raw reply related
* Re: [PATCH] ublk: reject FETCH from non-userspace context
From: Jens Axboe @ 2026-05-01 11:05 UTC (permalink / raw)
To: Ming Lei; +Cc: linux-block, Caleb Sander Mateos, Uday Shankar
In-Reply-To: <afSGfdcmtKHpvR07@fedora>
On 5/1/26 4:54 AM, Ming Lei wrote:
> On Fri, May 01, 2026 at 04:38:22AM -0600, Jens Axboe wrote:
>> On 5/1/26 4:36 AM, Ming Lei wrote:
>>> On Fri, May 01, 2026 at 04:34:11AM -0600, Jens Axboe wrote:
>>>> On 5/1/26 2:52 AM, Ming Lei wrote:
>>>>> __ublk_fetch() sets io->task to current, which is later checked
>>>>> against io_uring_cmd_get_task() in ublk_uring_cmd_cancel_fn().
>>>>> With REQ_F_FORCE_ASYNC, the FETCH uring_cmd can be issued from
>>>>> task work, which can be run from io_uring's fallback workqueue,
>>>>> causing a task mismatch and triggering the WARN in cancel_fn.
>>>>>
>>>>> Reject FETCH if current is not a real userspace task, and it is
>>>>> reasonable for failing it in case of io_uring fallback.
>>>>
>>>> I think this should be caught in ublk_ch_uring_cmd_cb(), which
>>>> should not hit ublk_ch_uring_cmd_local() if tw.cancel is true.
>>>> fallback work should just be terminated, not be issued.
>>>
>>> Yeah, that is definitely better, will take it in V2.
>>
>> Something like this, totally untested. Caveat: probably worth checking
>> the other tw paths in ublk as well!
>
> tw.cancel can replace the check in ublk_dispatch_req() for ublk_cmd_tw_cb() and
> ublk_cmd_list_tw_cb(), which is one cleanup.
>
> ublk_batch_tw_cb() doesn't track io task, so it is just fine.
Sounds good to me.
>> diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
>> index 8e5f3738c203..d10460d29e4a 100644
>> --- a/drivers/block/ublk_drv.c
>> +++ b/drivers/block/ublk_drv.c
>> @@ -3496,8 +3496,10 @@ static void ublk_ch_uring_cmd_cb(struct io_tw_req tw_req, io_tw_token_t tw)
>> {
>> unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
>> struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
>> - int ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
>> + int ret = -ECANCELED;
>>
>> + if (!tw.cancel)
>> + ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
>> if (ret != -EIOCBQUEUED)
>> io_uring_cmd_done(cmd, ret, issue_flags);
>> }
>
> This fix looks good.
Feel free to just run with it or make it part of a fixup series.
--
Jens Axboe
^ permalink raw reply
* Re: [PATCH] ublk: reject FETCH from non-userspace context
From: Ming Lei @ 2026-05-01 10:54 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-block, Caleb Sander Mateos, Uday Shankar
In-Reply-To: <c97fe4d3-9b48-4793-a008-4fbf7e72f5b0@kernel.dk>
On Fri, May 01, 2026 at 04:38:22AM -0600, Jens Axboe wrote:
> On 5/1/26 4:36 AM, Ming Lei wrote:
> > On Fri, May 01, 2026 at 04:34:11AM -0600, Jens Axboe wrote:
> >> On 5/1/26 2:52 AM, Ming Lei wrote:
> >>> __ublk_fetch() sets io->task to current, which is later checked
> >>> against io_uring_cmd_get_task() in ublk_uring_cmd_cancel_fn().
> >>> With REQ_F_FORCE_ASYNC, the FETCH uring_cmd can be issued from
> >>> task work, which can be run from io_uring's fallback workqueue,
> >>> causing a task mismatch and triggering the WARN in cancel_fn.
> >>>
> >>> Reject FETCH if current is not a real userspace task, and it is
> >>> reasonable for failing it in case of io_uring fallback.
> >>
> >> I think this should be caught in ublk_ch_uring_cmd_cb(), which
> >> should not hit ublk_ch_uring_cmd_local() if tw.cancel is true.
> >> fallback work should just be terminated, not be issued.
> >
> > Yeah, that is definitely better, will take it in V2.
>
> Something like this, totally untested. Caveat: probably worth checking
> the other tw paths in ublk as well!
tw.cancel can replace the check in ublk_dispatch_req() for ublk_cmd_tw_cb() and
ublk_cmd_list_tw_cb(), which is one cleanup.
ublk_batch_tw_cb() doesn't track io task, so it is just fine.
>
> diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
> index 8e5f3738c203..d10460d29e4a 100644
> --- a/drivers/block/ublk_drv.c
> +++ b/drivers/block/ublk_drv.c
> @@ -3496,8 +3496,10 @@ static void ublk_ch_uring_cmd_cb(struct io_tw_req tw_req, io_tw_token_t tw)
> {
> unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
> struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
> - int ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
> + int ret = -ECANCELED;
>
> + if (!tw.cancel)
> + ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
> if (ret != -EIOCBQUEUED)
> io_uring_cmd_done(cmd, ret, issue_flags);
> }
This fix looks good.
Thanks,
Ming
^ permalink raw reply
* Re: [PATCH] ublk: reject FETCH from non-userspace context
From: Jens Axboe @ 2026-05-01 10:38 UTC (permalink / raw)
To: Ming Lei; +Cc: linux-block, Caleb Sander Mateos, Uday Shankar
In-Reply-To: <afSCOu40SoLXQpwK@fedora>
On 5/1/26 4:36 AM, Ming Lei wrote:
> On Fri, May 01, 2026 at 04:34:11AM -0600, Jens Axboe wrote:
>> On 5/1/26 2:52 AM, Ming Lei wrote:
>>> __ublk_fetch() sets io->task to current, which is later checked
>>> against io_uring_cmd_get_task() in ublk_uring_cmd_cancel_fn().
>>> With REQ_F_FORCE_ASYNC, the FETCH uring_cmd can be issued from
>>> task work, which can be run from io_uring's fallback workqueue,
>>> causing a task mismatch and triggering the WARN in cancel_fn.
>>>
>>> Reject FETCH if current is not a real userspace task, and it is
>>> reasonable for failing it in case of io_uring fallback.
>>
>> I think this should be caught in ublk_ch_uring_cmd_cb(), which
>> should not hit ublk_ch_uring_cmd_local() if tw.cancel is true.
>> fallback work should just be terminated, not be issued.
>
> Yeah, that is definitely better, will take it in V2.
Something like this, totally untested. Caveat: probably worth checking
the other tw paths in ublk as well!
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 8e5f3738c203..d10460d29e4a 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -3496,8 +3496,10 @@ static void ublk_ch_uring_cmd_cb(struct io_tw_req tw_req, io_tw_token_t tw)
{
unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
- int ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
+ int ret = -ECANCELED;
+ if (!tw.cancel)
+ ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
if (ret != -EIOCBQUEUED)
io_uring_cmd_done(cmd, ret, issue_flags);
}
--
Jens Axboe
^ permalink raw reply related
* Re: [PATCH] ublk: reject FETCH from non-userspace context
From: Ming Lei @ 2026-05-01 10:36 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-block, Caleb Sander Mateos, Uday Shankar
In-Reply-To: <9efbd035-b40e-4155-8382-b42acff75a9e@kernel.dk>
On Fri, May 01, 2026 at 04:34:11AM -0600, Jens Axboe wrote:
> On 5/1/26 2:52 AM, Ming Lei wrote:
> > __ublk_fetch() sets io->task to current, which is later checked
> > against io_uring_cmd_get_task() in ublk_uring_cmd_cancel_fn().
> > With REQ_F_FORCE_ASYNC, the FETCH uring_cmd can be issued from
> > task work, which can be run from io_uring's fallback workqueue,
> > causing a task mismatch and triggering the WARN in cancel_fn.
> >
> > Reject FETCH if current is not a real userspace task, and it is
> > reasonable for failing it in case of io_uring fallback.
>
> I think this should be caught in ublk_ch_uring_cmd_cb(), which
> should not hit ublk_ch_uring_cmd_local() if tw.cancel is true.
> fallback work should just be terminated, not be issued.
Yeah, that is definitely better, will take it in V2.
Thanks,
Ming
^ permalink raw reply
* Re: [PATCH] ublk: reject FETCH from non-userspace context
From: Jens Axboe @ 2026-05-01 10:34 UTC (permalink / raw)
To: Ming Lei, linux-block; +Cc: Caleb Sander Mateos, Uday Shankar
In-Reply-To: <20260501085216.905401-1-tom.leiming@gmail.com>
On 5/1/26 2:52 AM, Ming Lei wrote:
> __ublk_fetch() sets io->task to current, which is later checked
> against io_uring_cmd_get_task() in ublk_uring_cmd_cancel_fn().
> With REQ_F_FORCE_ASYNC, the FETCH uring_cmd can be issued from
> task work, which can be run from io_uring's fallback workqueue,
> causing a task mismatch and triggering the WARN in cancel_fn.
>
> Reject FETCH if current is not a real userspace task, and it is
> reasonable for failing it in case of io_uring fallback.
I think this should be caught in ublk_ch_uring_cmd_cb(), which
should not hit ublk_ch_uring_cmd_local() if tw.cancel is true.
fallback work should just be terminated, not be issued.
--
Jens Axboe
^ permalink raw reply
* [PATCH] ublk: reject FETCH from non-userspace context
From: Ming Lei @ 2026-05-01 8:52 UTC (permalink / raw)
To: Jens Axboe, linux-block; +Cc: Caleb Sander Mateos, Uday Shankar, Ming Lei
__ublk_fetch() sets io->task to current, which is later checked
against io_uring_cmd_get_task() in ublk_uring_cmd_cancel_fn().
With REQ_F_FORCE_ASYNC, the FETCH uring_cmd can be issued from
task work, which can be run from io_uring's fallback workqueue,
causing a task mismatch and triggering the WARN in cancel_fn.
Reject FETCH if current is not a real userspace task, and it is
reasonable for failing it in case of io_uring fallback.
Fixes: 3421c7f68bba ("ublk: make sure io cmd handled in submitter task context")
Signed-off-by: Ming Lei <tom.leiming@gmail.com>
---
drivers/block/ublk_drv.c | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 8e5f3738c203..57abc0e9681f 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -3251,12 +3251,19 @@ static int __ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub,
WARN_ON_ONCE(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV);
- ublk_fill_io_cmd(io, cmd);
-
- if (ublk_dev_support_batch_io(ub))
+ if (ublk_dev_support_batch_io(ub)) {
WRITE_ONCE(io->task, NULL);
- else
+ } else {
+ /*
+ * FETCH must come from a real userspace task, not a
+ * kworker is actually io_uring fallback workqueue.
+ */
+ if (current->flags & (PF_KTHREAD | PF_WQ_WORKER))
+ return -EINVAL;
WRITE_ONCE(io->task, get_task_struct(current));
+ }
+
+ ublk_fill_io_cmd(io, cmd);
return 0;
}
--
2.53.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox