[PATCH 51/79] block: rust: add polled completion support

Andreas Hindborg
2026-02-15T23:42:53Z

Add support for polled I/O completion to the Rust block layer bindings.
This includes the `poll` callback in `Operations` and the
`IoCompletionBatch` type for batched completions.

The `poll` callback is invoked by the block layer when polling for
completed requests on poll queues. Drivers can use `IoCompletionBatch`
to batch multiple completions efficiently.

Signed-off-by: Andreas Hindborg <a.hindborg@kernel.org>
---
drivers/block/rnull/rnull.rs | 1 +
rust/helpers/blk.c | 7 +++
rust/kernel/block/mq.rs | 4 +-
rust/kernel/block/mq/operations.rs | 104 +++++++++++++++++++++++++++++++++++--
rust/kernel/block/mq/request.rs | 5 ++
5 files changed, 117 insertions(+), 4 deletions(-)

diff --git a/drivers/block/rnull/rnull.rs b/drivers/block/rnull/rnull.rs
index 48b2bd598304c..765bbc8101d10 100644
--- a/drivers/block/rnull/rnull.rs
+++ b/drivers/block/rnull/rnull.rs
@@ -733,6 +733,7 @@ fn queue_rq(
this: ArcBorrow<'_, Self>,
rq: Owned<mq::IdleRequest<Self>>,
_is_last: bool,
+ _is_poll: bool,
) -> BlkResult {
if this.bandwidth_limit != 0 {
if !this.bandwidth_timer.active() {
diff --git a/rust/helpers/blk.c b/rust/helpers/blk.c
index 53beba8c7782d..91b72131a04cd 100644
--- a/rust/helpers/blk.c
+++ b/rust/helpers/blk.c
@@ -20,3 +20,10 @@ void rust_helper_bio_advance_iter_single(const struct bio *bio,
{
bio_advance_iter_single(bio, iter, bytes);
}
+
+bool rust_helper_blk_mq_add_to_batch(struct request *req,
+ struct io_comp_batch *iob, bool is_error,
+ void (*complete)(struct io_comp_batch *))
+{
+ return blk_mq_add_to_batch(req, iob, is_error, complete);
+}
diff --git a/rust/kernel/block/mq.rs b/rust/kernel/block/mq.rs
index cd0bfbcbf317a..1d58eea971bd7 100644
--- a/rust/kernel/block/mq.rs
+++ b/rust/kernel/block/mq.rs
@@ -89,7 +89,8 @@
//! _hw_data: (),
//! _queue_data: (),
//! rq: Owned<IdleRequest<Self>>,
-//! _is_last: bool
+//! _is_last: bool,
+//! is_poll: bool
//! ) -> BlkResult {
//! rq.start().end_ok();
//! Ok(())
@@ -130,6 +131,7 @@
mod request_queue;
pub mod tag_set;

+pub use operations::IoCompletionBatch;
pub use operations::Operations;
pub use request::Command;
pub use request::IdleRequest;
diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/operations.rs
index 28dd4b28d203f..bccc1903a0d10 100644
--- a/rust/kernel/block/mq/operations.rs
+++ b/rust/kernel/block/mq/operations.rs
@@ -65,6 +65,7 @@ fn queue_rq(
queue_data: ForeignBorrowed<'_, Self::QueueData>,
rq: Owned<IdleRequest<Self>>,
is_last: bool,
+ is_poll: bool,
) -> BlkResult;

/// Called by the kernel to indicate that queued requests should be submitted.
@@ -84,7 +85,15 @@ fn init_hctx(

/// Called by the kernel to poll the device for completed requests. Only
/// used for poll queues.
- fn poll(_hw_data: ForeignBorrowed<'_, Self::HwData>) -> bool {
+ ///
+ /// Should return `Ok(true)` if any requests were completed during the call,
+ /// `Ok(false)` if no requests were completed, and `Err(e)` to signal an
+ /// error condition.
+ fn poll(
+ _hw_data: ForeignBorrowed<'_, Self::HwData>,
+ _queue_data: ForeignBorrowed<'_, Self::QueueData>,
+ _batch: &mut IoCompletionBatch<Self>,
+ ) -> Result<bool> {
build_error!(crate::error::VTABLE_DEFAULT_ERROR)
}

@@ -168,6 +177,11 @@ impl<T: Operations> OperationsVTable<T> {
// `into_foreign` in `Self::init_hctx_callback`.
let hw_data = unsafe { T::HwData::borrow((*hctx).driver_data) };

+ let is_poll = u32::from(
+ // SAFETY: `hctx` is valid as required by this function.
+ unsafe { (*hctx).type_ },
+ ) == bindings::hctx_type_HCTX_TYPE_POLL;
+
// SAFETY: `hctx` is valid as required by this function.
let queue_data = unsafe { (*(*hctx).queue).queuedata };

@@ -184,6 +198,7 @@ impl<T: Operations> OperationsVTable<T> {
// SAFETY: `bd` is valid as required by the safety requirement for
// this function.
unsafe { (*bd).last },
+ is_poll,
);

if let Err(e) = ret {
@@ -242,13 +257,32 @@ impl<T: Operations> OperationsVTable<T> {
/// previously initialized by a call to `init_hctx_callback`.
unsafe extern "C" fn poll_callback(
hctx: *mut bindings::blk_mq_hw_ctx,
- _iob: *mut bindings::io_comp_batch,
+ iob: *mut bindings::io_comp_batch,
) -> crate::ffi::c_int {
// SAFETY: By function safety requirement, `hctx` was initialized by
// `init_hctx_callback` and thus `driver_data` came from a call to
// `into_foreign`.
let hw_data = unsafe { T::HwData::borrow((*hctx).driver_data) };
- T::poll(hw_data).into()
+
+ // SAFETY: `hctx` is valid as required by this function.
+ let queue_data = unsafe { (*(*hctx).queue).queuedata };
+
+ // SAFETY: `queue.queuedata` was created by `GenDiskBuilder::build` with
+ // a call to `ForeignOwnable::into_foreign` to create `queuedata`.
+ // `ForeignOwnable::from_foreign` is only called when the tagset is
+ // dropped, which happens after we are dropped.
+ let queue_data = unsafe { T::QueueData::borrow(queue_data) };
+
+ let mut batch = IoCompletionBatch {
+ inner: iob,
+ _p: PhantomData,
+ };
+
+ let ret = T::poll(hw_data, queue_data, &mut batch);
+ match ret {
+ Ok(val) => val.into(),
+ Err(e) => e.to_errno(),
+ }
}

/// This function is called by the C kernel. A pointer to this function is
@@ -448,3 +482,67 @@ pub(crate) const fn build() -> &'static bindings::blk_mq_ops {
&Self::VTABLE
}
}
+
+/// A batch of I/O completions for polled I/O.
+///
+/// This struct wraps the C `struct io_comp_batch` and is used to batch
+/// multiple request completions together for improved efficiency during polled
+/// I/O operations.
+///
+/// When the kernel polls for completed requests via [`Operations::poll`], it
+/// passes an `IoCompletionBatch` to collect completed requests. The driver can
+/// add completed requests to the batch using [`add_request`], allowing the
+/// kernel to process multiple completions together rather than one at a time.
+///
+/// # Invariants
+///
+/// - `inner` must point to a valid `io_comp_batch`.
+///
+/// [`add_request`]: IoCompletionBatch::add_request
+#[repr(transparent)]
+pub struct IoCompletionBatch<T> {
+ inner: *mut bindings::io_comp_batch,
+ _p: PhantomData<T>,
+}
+
+impl<T: Operations> IoCompletionBatch<T> {
+ /// Attempt to add a completed request to this batch.
+ ///
+ /// This method tries to add `rq` to the batch for deferred completion. If
+ /// the request is successfully added, ownership is transferred to the batch
+ /// and the request will be completed later when the batch is processed.
+ ///
+ /// # Arguments
+ ///
+ /// - `rq`: The completed request to add to the batch.
+ /// - `error`: Set to `true` if the request completed with an error.
+ ///
+ /// # Return
+ ///
+ /// When this method returns `Err`, the caller is responsible for completing
+ /// the request through other means, such as calling
+ /// [`Request::complete`](super::Request::complete).
+ pub fn add_request(
+ &mut self,
+ rq: Owned<Request<T>>,
+ error: bool,
+ ) -> Result<(), Owned<Request<T>>> {
+ // SAFETY: By type invariant, `self.inner` is a valid `io_comp_batch`.
+ let ret = unsafe {
+ bindings::blk_mq_add_to_batch(
+ rq.as_raw(),
+ self.inner,
+ error,
+ Some(bindings::blk_mq_end_request_batch),
+ )
+ };
+
+ match ret {
+ true => {
+ core::mem::forget(rq);
+ Ok(())
+ }
+ false => Err(rq),
+ }
+ }
+}
diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs
index 4e7adb10929f0..b4f5a98a97b16 100644
--- a/rust/kernel/block/mq/request.rs
+++ b/rust/kernel/block/mq/request.rs
@@ -135,6 +135,11 @@ pub fn queue(&self) -> &RequestQueue<T> {
// SAFETY: By type invariant, self.0 is guaranteed to be valid.
unsafe { RequestQueue::from_raw((*self.0.get()).q) }
}
+
+ /// Return a raw pointer to the underlying C structure.
+ pub fn as_raw(&self) -> *mut bindings::request {
+ self.0.get()
+ }
}

/// A wrapper around a blk-mq [`struct request`]. This represents an IO request.

--
2.51.2