* [PATCH v3 1/7] rust: percpu: introduce a rust API for per-CPU variables
2025-08-28 19:00 [PATCH v3 0/7] rust: Add Per-CPU Variable API Mitchell Levy
@ 2025-08-28 19:00 ` Mitchell Levy
2025-09-03 21:42 ` Yury Norov
2025-08-28 19:00 ` [PATCH v3 2/7] rust: percpu: add a rust per-CPU variable sample Mitchell Levy
` (5 subsequent siblings)
6 siblings, 1 reply; 26+ messages in thread
From: Mitchell Levy @ 2025-08-28 19:00 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Yury Norov, Viresh Kumar
Cc: Tyler Hicks, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
Per-CPU variables are an important tool for reducing lock contention,
especially in systems with many processors. They also provide a
convenient way to handle data that are logically associated with a
particular CPU (e.g., the currently running task). Therefore, add a Rust
API to make use of per-CPU variables.
Add a `CpuGuard` type that disables preemption for its lifetime. Add a
`PerCpuAllocation` type used to track dynamic allocations. Add a
`define_per_cpu!` macro to create static per-CPU allocations. Add
`DynamicPerCpu` and `StaticPerCpu` to provide a high-level API. Add a
`PerCpu` trait to unify the dynamic and static cases.
Co-developed-by: Boqun Feng <boqun.feng@gmail.com>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/helpers/helpers.c | 2 +
rust/helpers/percpu.c | 9 ++
rust/helpers/preempt.c | 14 +++
rust/kernel/lib.rs | 3 +
rust/kernel/percpu.rs | 223 ++++++++++++++++++++++++++++++++++++++++
rust/kernel/percpu/cpu_guard.rs | 35 +++++++
rust/kernel/percpu/dynamic.rs | 83 +++++++++++++++
rust/kernel/percpu/static_.rs | 132 ++++++++++++++++++++++++
8 files changed, 501 insertions(+)
diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c
index 7cf7fe95e41d..2fc8d26cfe66 100644
--- a/rust/helpers/helpers.c
+++ b/rust/helpers/helpers.c
@@ -31,9 +31,11 @@
#include "of.c"
#include "page.c"
#include "pci.c"
+#include "percpu.c"
#include "pid_namespace.c"
#include "platform.c"
#include "poll.c"
+#include "preempt.c"
#include "property.c"
#include "rbtree.c"
#include "rcu.c"
diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c
new file mode 100644
index 000000000000..a091389f730f
--- /dev/null
+++ b/rust/helpers/percpu.c
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/percpu.h>
+
+void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align)
+{
+ return __alloc_percpu(sz, align);
+}
+
diff --git a/rust/helpers/preempt.c b/rust/helpers/preempt.c
new file mode 100644
index 000000000000..2c7529528ddd
--- /dev/null
+++ b/rust/helpers/preempt.c
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/preempt.h>
+
+void rust_helper_preempt_disable(void)
+{
+ preempt_disable();
+}
+
+void rust_helper_preempt_enable(void)
+{
+ preempt_enable();
+}
+
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
index ed53169e795c..ed0d5756dc55 100644
--- a/rust/kernel/lib.rs
+++ b/rust/kernel/lib.rs
@@ -106,6 +106,9 @@
pub mod page;
#[cfg(CONFIG_PCI)]
pub mod pci;
+// Only x86_64 is supported by percpu for now
+#[cfg(CONFIG_X86_64)]
+pub mod percpu;
pub mod pid_namespace;
pub mod platform;
pub mod prelude;
diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs
new file mode 100644
index 000000000000..35afcdba3ccd
--- /dev/null
+++ b/rust/kernel/percpu.rs
@@ -0,0 +1,223 @@
+// SPDX-License-Identifier: GPL-2.0
+//! This module contains abstractions for creating and using per-CPU variables from Rust.
+//! See the define_per_cpu! macro and the DynamicPerCpu<T> type, as well as the PerCpu<T> trait.
+
+pub mod cpu_guard;
+mod dynamic;
+mod static_;
+
+#[doc(inline)]
+pub use dynamic::*;
+#[doc(inline)]
+pub use static_::*;
+
+use bindings::{alloc_percpu, free_percpu};
+
+use crate::alloc::Flags;
+use crate::percpu::cpu_guard::CpuGuard;
+use crate::prelude::*;
+use crate::sync::Arc;
+use crate::types::Opaque;
+use crate::{declare_extern_per_cpu, get_static_per_cpu};
+
+use core::arch::asm;
+use core::cell::{Cell, RefCell, UnsafeCell};
+use core::mem::{align_of, size_of, MaybeUninit};
+
+use ffi::c_void;
+
+/// A per-CPU pointer; that is, an offset into the per-CPU area. Note that this type is NOT a smart
+/// pointer, it does not manage the allocation.
+pub struct PerCpuPtr<T>(*mut MaybeUninit<T>);
+
+/// Represents exclusive access to the memory location pointed at by a particular PerCpu<T>.
+pub struct PerCpuToken<'a, T> {
+ // INVARIANT: the current CPU's memory location associated with the per-CPU variable pointed at
+ // by `ptr` (i.e., the entry in the per-CPU area on the current CPU) has been initialized.
+ _guard: CpuGuard,
+ ptr: &'a PerCpuPtr<T>,
+}
+
+/// Represents access to the memory location pointed at by a particular PerCpu<T> where the type
+/// `T` manages access to the underlying memory to avoid aliaising troubles. (For example, `T`
+/// might be a `Cell` or `RefCell`.)
+pub struct CheckedPerCpuToken<'a, T> {
+ // INVARIANT: the current CPU's memory location associated with the per-CPU variable pointed at
+ // by `ptr` (i.e., the entry in the per-CPU area on the current CPU) has been initialized.
+ _guard: CpuGuard,
+ ptr: &'a PerCpuPtr<T>,
+}
+
+impl<T> PerCpuPtr<T> {
+ /// Makes a new PerCpuPtr from a raw per-CPU pointer.
+ ///
+ /// # Safety
+ /// `ptr` must be a valid per-CPU pointer.
+ pub unsafe fn new(ptr: *mut MaybeUninit<T>) -> Self {
+ Self(ptr)
+ }
+
+ /// Get a `&mut MaybeUninit<T>` to the per-CPU variable on the current CPU represented by `&self`
+ ///
+ /// # Safety
+ /// The returned `&mut T` must follow Rust's aliasing rules. That is, no other `&(mut) T` may
+ /// exist that points to the same location in memory. In practice, this means that `get_(mut_)ref`
+ /// must not be called on another `PerCpuPtr<T>` that is a copy/clone of `&self` for as long as
+ /// the returned reference lives.
+ ///
+ /// CPU preemption must be disabled before calling this function and for the lifetime of the
+ /// returned reference. Otherwise, the returned reference might end up being a reference to a
+ /// different CPU's per-CPU area, causing the potential for a data race.
+ #[allow(clippy::mut_from_ref)] // Safety requirements prevent aliasing issues
+ pub unsafe fn get_mut_ref(&self) -> &mut MaybeUninit<T> {
+ // SAFETY: `self.get_ptr()` returns a valid pointer to a `MaybeUninit<T>` by its contract,
+ // and the safety requirements of this function ensure that the returned reference is
+ // exclusive.
+ unsafe { &mut *(self.get_ptr()) }
+ }
+
+ /// Get a `&MaybeUninit<T>` to the per-CPU variable on the current CPU represented by `&self`
+ ///
+ /// # Safety
+ /// The returned `&T` must follow Rust's aliasing rules. That is, no `&mut T` may exist that
+ /// points to the same location in memory. In practice, this means that `get_mut_ref` must not
+ /// be called on another `PerCpuPtr<T>` that is a copy/clone of `&self` for as long as the
+ /// returned reference lives.
+ ///
+ /// CPU preemption must be disabled before calling this function and for the lifetime of the
+ /// returned reference. Otherwise, the returned reference might end up being a reference to a
+ /// different CPU's per-CPU area, causing the potential for a data race.
+ pub unsafe fn get_ref(&self) -> &MaybeUninit<T> {
+ // SAFETY: `self.get_ptr()` returns a valid pointer to a `MaybeUninit<T>` by its contract.
+ // The safety requirements of this function ensure that the returned reference isn't
+ // aliased by a `&mut MaybeUninit<T>`.
+ unsafe { &*self.get_ptr() }
+ }
+
+ /// Get a `*mut MaybeUninit<T>` to the per-CPU variable on the current CPU represented by
+ /// `&self`. Note that if CPU preemption is not disabled before calling this function, use of
+ /// the returned pointer may cause a data race without some other synchronization mechanism.
+ /// Buyer beware!
+ pub fn get_ptr(&self) -> *mut MaybeUninit<T> {
+ let this_cpu_off_pcpu: PerCpuPtr<*mut c_void> = get_static_per_cpu!(this_cpu_off).0;
+ let mut this_cpu_area: *mut c_void;
+ // SAFETY: gs + this_cpu_off_pcpu is guaranteed to be a valid pointer because `gs` points
+ // to the per-CPU area and this_cpu_off_pcpu is a valid per-CPU allocation.
+ unsafe {
+ asm!(
+ "mov {out}, gs:[{off_val}]",
+ off_val = in(reg) this_cpu_off_pcpu.0,
+ out = out(reg) this_cpu_area,
+ )
+ };
+
+ // This_cpu_area + self.0 is guaranteed to be a valid pointer by the per-CPU subsystem and
+ // the invariant that self.0 is a valid offset into the per-CPU area.
+ (this_cpu_area).wrapping_add(self.0 as usize).cast()
+ }
+}
+
+// SAFETY: Sending a `PerCpuPtr<T>` to another thread is safe because as soon as it's sent, the
+// pointer is logically referring to a different place in memory in the other CPU's per-CPU area.
+// In particular, this means that there are no restrictions on the type `T`.
+unsafe impl<T> Send for PerCpuPtr<T> {}
+
+// SAFETY: Two threads concurrently making use of a `PerCpuPtr<T>` will each see the `T` in their
+// own per-CPU area, so there's no potential for a data race (regardless of whether `T` is itself
+// `Sync`).
+unsafe impl<T> Sync for PerCpuPtr<T> {}
+
+impl<T> Clone for PerCpuPtr<T> {
+ fn clone(&self) -> Self {
+ *self
+ }
+}
+
+/// `PerCpuPtr` is just a wrapper around a pointer.
+impl<T> Copy for PerCpuPtr<T> {}
+
+/// A trait representing a per-CPU variable. This is implemented for both `StaticPerCpu<T>` and
+/// `DynamicPerCpu<T>`. The main usage of this trait is to call `get_mut` to get a `PerCpuToken`
+/// that can be used to access the underlying per-CPU variable. See `PerCpuToken::with`.
+pub trait PerCpu<T> {
+ /// Produces a token, asserting that the holder has exclusive access to the underlying memory
+ /// pointed to by `self`
+ ///
+ /// # Safety
+ /// No other `PerCpuToken` or `CheckedPerCpuToken` may exist on the current CPU (which is a
+ /// sensible notion, since we keep a `CpuGuard` around) that is derived from the same
+ /// `PerCpu<T>` or a clone thereof.
+ unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T>;
+}
+
+/// A marker trait for types that are interior mutable. Types that implement this trait can be used
+/// to create "checked" per-CPU variables. See `CheckedPerCpu<T>`.
+pub trait InteriorMutable {}
+
+impl<T> InteriorMutable for Cell<T> {}
+impl<T> InteriorMutable for RefCell<T> {}
+impl<T> InteriorMutable for UnsafeCell<T> {}
+impl<T> InteriorMutable for Opaque<T> {}
+
+/// A trait representing a per-CPU variable that is usable via a `&T`. The unsafety of `PerCpu<T>`
+/// stems from the fact that the holder of a `PerCpuToken` can use it to get a `&mut T` to the
+/// underlying per-CPU variable. This is problematic because the existence of aliaising `&mut T` is
+/// undefined behavior in Rust. This type avoids that issue by only allowing access via a `&T`,
+/// with the tradeoff that then `T` must be interior mutable or the underlying per-CPU variable
+/// must be a constant for the lifetime of the corresponding `CheckedPerCpu<T>`.
+///
+/// Currently, only the case where `T` is interior mutable has first-class support, though a custom
+/// implementation of `PerCpu<T>`/`CheckedPerCpu<T>` could be created for the const case.
+pub trait CheckedPerCpu<T>: PerCpu<T> {
+ /// Produces a token via which the holder can access the underlying per-CPU variable.
+ fn get(&mut self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T>;
+}
+
+impl<'a, T> PerCpuToken<'a, T> {
+ /// # Safety
+ /// No other `PerCpuToken` or `CheckedPerCpuToken` may exist on the current CPU (which is a
+ /// sensible notion, since we keep a `CpuGuard` around) that uses the same `PerCpuPtr<T>`.
+ ///
+ /// The current CPU's memory location associated with the per-CPU variable pointed at by `ptr`
+ /// (i.e., the entry in the per-CPU area on this CPU) must be initialized.
+ pub unsafe fn new(guard: CpuGuard, ptr: &'a PerCpuPtr<T>) -> PerCpuToken<'a, T> {
+ Self { _guard: guard, ptr }
+ }
+
+ /// Immediately invokes `func` with a `&mut T` that points at the underlying per-CPU variable
+ /// that `&mut self` represents.
+ pub fn with<U>(&mut self, func: U)
+ where
+ U: FnOnce(&mut T),
+ {
+ // SAFETY: The existence of a PerCpuToken means that the requirements for get_mut_ref are
+ // satisfied. Likewise, the requirements for assume_init_mut are satisfied because the
+ // invariants of this type ensure that on the current CPU (which is a sensible notion
+ // because we have a CpuGuard), the memory location pointed to by `ptr` is initialized.
+ func(unsafe { self.ptr.get_mut_ref().assume_init_mut() });
+ }
+}
+
+impl<'a, T> CheckedPerCpuToken<'a, T> {
+ /// # Safety
+ /// The current CPU's memory location associated with the per-CPU variable pointed at by `ptr`
+ /// (i.e., the entry in the per-CPU area on this CPU) must be initialized.
+ pub unsafe fn new(guard: CpuGuard, ptr: &'a PerCpuPtr<T>) -> CheckedPerCpuToken<'a, T> {
+ Self { _guard: guard, ptr }
+ }
+
+ /// Immediately invokes `func` with a `&T` that points at the underlying per-CPU variable that
+ /// `&mut self` represents.
+ pub fn with<U>(&mut self, func: U)
+ where
+ U: FnOnce(&T),
+ {
+ // SAFETY: The existence of a CheckedPerCpuToken means that the requirements for get_ref
+ // are satisfied. Likewise, the requirements for assume_init_ref are satisfied because the
+ // invariants of this type ensure that on the current CPU (which is a sensible notion
+ // because we have a CpuGuard), the memory location pointed to by `ptr` is initialized.
+ func(unsafe { self.ptr.get_ref().assume_init_ref() });
+ }
+}
+
+declare_extern_per_cpu!(this_cpu_off: *mut c_void);
diff --git a/rust/kernel/percpu/cpu_guard.rs b/rust/kernel/percpu/cpu_guard.rs
new file mode 100644
index 000000000000..14c04b12e7f0
--- /dev/null
+++ b/rust/kernel/percpu/cpu_guard.rs
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-2.0
+//! Contains abstractions for disabling CPU preemption. See `CpuGuard`.
+
+/// A RAII guard for bindings::preempt_disable and bindings::preempt_enable. Guarantees preemption
+/// is disabled for as long as this object exists.
+pub struct CpuGuard {
+ // Don't make one without using new()
+ _phantom: (),
+}
+
+impl CpuGuard {
+ /// Create a new CpuGuard. Disables preemption for its lifetime.
+ pub fn new() -> Self {
+ // SAFETY: There are no preconditions required to call preempt_disable
+ unsafe {
+ bindings::preempt_disable();
+ }
+ CpuGuard { _phantom: () }
+ }
+}
+
+impl Default for CpuGuard {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl Drop for CpuGuard {
+ fn drop(&mut self) {
+ // SAFETY: There are no preconditions required to call preempt_enable
+ unsafe {
+ bindings::preempt_enable();
+ }
+ }
+}
diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs
new file mode 100644
index 000000000000..ce95e420f943
--- /dev/null
+++ b/rust/kernel/percpu/dynamic.rs
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0
+//! Dynamically allocated per-CPU variables.
+
+use super::*;
+
+/// Represents a dynamic allocation of a per-CPU variable via alloc_percpu. Calls free_percpu when
+/// dropped.
+pub struct PerCpuAllocation<T>(PerCpuPtr<T>);
+
+impl<T: Zeroable> PerCpuAllocation<T> {
+ /// Dynamically allocates a space in the per-CPU area suitably sized and aligned to hold a `T`,
+ /// initially filled with the zero value for `T`.
+ ///
+ /// Returns `None` under the same circumstances the C function `alloc_percpu` returns `NULL`.
+ pub fn new_zero() -> Option<PerCpuAllocation<T>> {
+ let ptr: *mut MaybeUninit<T> =
+ // SAFETY: No preconditions to call alloc_percpu; MaybeUninit<T> is
+ // `#[repr(transparent)]`, so we can cast a `*mut T` to it.
+ unsafe { alloc_percpu(size_of::<T>(), align_of::<T>()) }.cast();
+ if ptr.is_null() {
+ return None;
+ }
+
+ // alloc_percpu returns zero'ed memory
+ Some(Self(PerCpuPtr(ptr)))
+ }
+}
+
+impl<T> PerCpuAllocation<T> {
+ /// Makes a per-CPU allocation sized and aligned to hold a `T`.
+ ///
+ /// Returns `None` under the same circumstances the C function `alloc_percpu` returns `NULL`.
+ pub fn new_uninit() -> Option<PerCpuAllocation<T>> {
+ let ptr: *mut MaybeUninit<T> =
+ // SAFETY: No preconditions to call alloc_percpu; MaybeUninit<T> is
+ // `#[repr(transparent)]`, so we can cast a `*mut T` to it.
+ unsafe { alloc_percpu(size_of::<T>(), align_of::<T>()) }.cast();
+ if ptr.is_null() {
+ return None;
+ }
+
+ Some(Self(PerCpuPtr(ptr)))
+ }
+}
+
+impl<T> Drop for PerCpuAllocation<T> {
+ fn drop(&mut self) {
+ // SAFETY: self.0.0 was returned by alloc_percpu, and so was a valid pointer into
+ // the percpu area, and has remained valid by the invariants of PerCpuAllocation<T>.
+ unsafe { free_percpu(self.0 .0.cast()) }
+ }
+}
+
+/// Holds a dynamically-allocated per-CPU variable.
+#[derive(Clone)]
+pub struct DynamicPerCpu<T> {
+ // INVARIANT: The memory location in each CPU's per-CPU area pointed at by `alloc.0` has been
+ // initialized.
+ pub(super) alloc: Arc<PerCpuAllocation<T>>,
+}
+
+impl<T: Zeroable> DynamicPerCpu<T> {
+ /// Allocates a new per-CPU variable
+ ///
+ /// # Arguments
+ /// * `flags` - Flags used to allocate an `Arc` that keeps track of the underlying
+ /// `PerCpuAllocation`.
+ pub fn new_zero(flags: Flags) -> Option<Self> {
+ let alloc: PerCpuAllocation<T> = PerCpuAllocation::new_zero()?;
+
+ let arc = Arc::new(alloc, flags).ok()?;
+
+ Some(Self { alloc: arc })
+ }
+}
+
+impl<T> PerCpu<T> for DynamicPerCpu<T> {
+ unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> {
+ // SAFETY: The requirements of `PerCpu::get_mut` and this type's invariant ensure that the
+ // requirements of `PerCpuToken::new` are met.
+ unsafe { PerCpuToken::new(guard, &self.alloc.0) }
+ }
+}
diff --git a/rust/kernel/percpu/static_.rs b/rust/kernel/percpu/static_.rs
new file mode 100644
index 000000000000..be226dd2c3aa
--- /dev/null
+++ b/rust/kernel/percpu/static_.rs
@@ -0,0 +1,132 @@
+// SPDX-License-Identifier: GPL-2.0
+//! Statically allocated per-CPU variables.
+
+use super::*;
+
+/// A wrapper used for declaring static per-CPU variables. These symbols are "virtual" in that the
+/// linker uses them to generate offsets into each CPU's per-CPU area, but shouldn't be read
+/// from/written to directly. The fact that the statics are immutable prevents them being written
+/// to (generally), this struct having _val be non-public prevents reading from them.
+///
+/// The end-user of the per-CPU API should make use of the define_per_cpu! macro instead of
+/// declaring variables of this type directly. All instances of this type must be `static` and
+/// `#[link_section = ".data..percpu"]` (which the macro handles).
+#[repr(transparent)]
+pub struct StaticPerCpuSymbol<T> {
+ _val: T, // generate a correctly sized type
+}
+
+/// Holds a statically-allocated per-CPU variable.
+#[derive(Clone)]
+pub struct StaticPerCpu<T>(pub(super) PerCpuPtr<T>);
+
+impl<T> StaticPerCpu<T> {
+ /// Creates a `StaticPerCpu<T>` from a `StaticPerCpuSymbol<T>`. You should probably be using
+ /// `get_static_per_cpu!` instead.
+ pub fn new(ptr: *const StaticPerCpuSymbol<T>) -> StaticPerCpu<T> {
+ // SAFETY: `StaticPerCpuSymbol<T>` is `#[repr(transparent)]`, so we can safely cast a
+ // pointer to it into a pointer to `MaybeUninit<T>`. The validity of it as a per-CPU
+ // pointer is guaranteed by the per-CPU subsystem and invariants of the StaticPerCpuSymbol
+ // type.
+ let pcpu_ptr = unsafe { PerCpuPtr::new(ptr.cast_mut().cast()) };
+ Self(pcpu_ptr)
+ }
+}
+
+impl<T> PerCpu<T> for StaticPerCpu<T> {
+ unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> {
+ // SAFETY: The requirements of `PerCpu::get_mut` and the fact that statically-allocated
+ // per-CPU variables are initialized by the per-CPU subsystem ensure that the requirements
+ // of `PerCpuToken::new` are met.
+ unsafe { PerCpuToken::new(guard, &self.0) }
+ }
+}
+
+impl<T: InteriorMutable> CheckedPerCpu<T> for StaticPerCpu<T> {
+ fn get(&mut self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T> {
+ // SAFETY: The per-CPU subsystem guarantees that each CPU's instance of a
+ // statically allocated variable begins with a copy of the contents of the
+ // corresponding symbol in `.data..percpu`. Thus, the requirements of
+ // `CheckedPerCpuToken::new` are met.
+ unsafe { CheckedPerCpuToken::new(guard, &self.0) }
+ }
+}
+
+/// Gets a `StaticPerCpu<T>` from a symbol declared with `define_per_cpu!` or
+/// `declare_extern_per_cpu!`.
+///
+/// # Arguments
+/// * `ident` - The identifier declared
+#[macro_export]
+macro_rules! get_static_per_cpu {
+ ($id:ident) => {
+ $crate::percpu::StaticPerCpu::new((&raw const $id).cast())
+ };
+}
+
+/// Declares a StaticPerCpuSymbol corresponding to a per-CPU variable defined in C. Be sure to read
+/// the safety requirements of `PerCpu::get`.
+#[macro_export]
+macro_rules! declare_extern_per_cpu {
+ ($id:ident: $ty:ty) => {
+ extern "C" {
+ static $id: StaticPerCpuSymbol<$ty>;
+ }
+ };
+}
+
+/// define_per_cpu! is analogous to the C DEFINE_PER_CPU macro in that it lets you create a
+/// statically allocated per-CPU variable.
+///
+/// # Example
+/// ```
+/// use kernel::define_per_cpu;
+/// use kernel::percpu::StaticPerCpuSymbol;
+///
+/// define_per_cpu!(pub MY_PERCPU: u64 = 0);
+/// ```
+#[macro_export]
+macro_rules! define_per_cpu {
+ ($vis:vis $id:ident: $ty:ty = $expr:expr) => {
+ $crate::macros::paste! {
+ // We might want to have a per-CPU variable that doesn't implement `Sync` (not paying
+ // sync overhead costs is part of the point), but Rust won't let us declare a static of
+ // a `!Sync` type. Of course, we don't actually have any synchronization issues, since
+ // each CPU will see its own copy of the variable, so we cheat a little bit and tell
+ // Rust it's fine.
+ #[doc(hidden)]
+ #[allow(non_camel_case_types)]
+ #[repr(transparent)] // It needs to be the same size as $ty
+ struct [<__PRIVATE_TYPE_ $id>]($ty);
+
+ impl [<__PRIVATE_TYPE_ $id>] {
+ #[doc(hidden)]
+ const fn new(val: $ty) -> Self {
+ Self(val)
+ }
+ }
+
+ // Expand $expr outside of the unsafe block to avoid silently allowing unsafe code to be
+ // used without a user-facing unsafe block
+ #[doc(hidden)]
+ static [<__INIT_ $id>]: [<__PRIVATE_TYPE_ $id>] = [<__PRIVATE_TYPE_ $id>]::new($expr);
+
+ // SAFETY: This type will ONLY ever be used to declare a `StaticPerCpuSymbol`
+ // (which we then only ever use as input to `&raw`). Reading from the symbol is
+ // already UB, so we won't ever actually have any variables of this type where
+ // synchronization is a concern.
+ #[doc(hidden)]
+ unsafe impl Sync for [<__PRIVATE_TYPE_ $id>] {}
+
+ // SAFETY: StaticPerCpuSymbol<T> is #[repr(transparent)], so we can freely convert from
+ // [<__PRIVATE_TYPE_ $id>], which is also `#[repr(transparent)]` (i.e., everything is
+ // just a `$ty` from a memory layout perspective).
+ #[link_section = ".data..percpu"]
+ $vis static $id: StaticPerCpuSymbol<[<__PRIVATE_TYPE_ $id>]> = unsafe {
+ core::mem::transmute_copy::<
+ [<__PRIVATE_TYPE_ $id>], StaticPerCpuSymbol<[<__PRIVATE_TYPE_ $id>]>
+ >(&[<__INIT_ $id>])
+ };
+ }
+ };
+}
--
2.34.1
^ permalink raw reply related [flat|nested] 26+ messages in thread* Re: [PATCH v3 1/7] rust: percpu: introduce a rust API for per-CPU variables
2025-08-28 19:00 ` [PATCH v3 1/7] rust: percpu: introduce a rust API for per-CPU variables Mitchell Levy
@ 2025-09-03 21:42 ` Yury Norov
2025-09-04 19:53 ` Mitchell Levy
0 siblings, 1 reply; 26+ messages in thread
From: Yury Norov @ 2025-09-03 21:42 UTC (permalink / raw)
To: Mitchell Levy
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Viresh Kumar, Tyler Hicks,
linux-kernel, rust-for-linux, linux-mm
On Thu, Aug 28, 2025 at 12:00:08PM -0700, Mitchell Levy wrote:
> Per-CPU variables are an important tool for reducing lock contention,
> especially in systems with many processors. They also provide a
> convenient way to handle data that are logically associated with a
> particular CPU (e.g., the currently running task). Therefore, add a Rust
> API to make use of per-CPU variables.
>
> Add a `CpuGuard` type that disables preemption for its lifetime. Add a
> `PerCpuAllocation` type used to track dynamic allocations. Add a
> `define_per_cpu!` macro to create static per-CPU allocations. Add
> `DynamicPerCpu` and `StaticPerCpu` to provide a high-level API. Add a
> `PerCpu` trait to unify the dynamic and static cases.
>
> Co-developed-by: Boqun Feng <boqun.feng@gmail.com>
> Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
> Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
> ---
> rust/helpers/helpers.c | 2 +
> rust/helpers/percpu.c | 9 ++
> rust/helpers/preempt.c | 14 +++
> rust/kernel/lib.rs | 3 +
> rust/kernel/percpu.rs | 223 ++++++++++++++++++++++++++++++++++++++++
> rust/kernel/percpu/cpu_guard.rs | 35 +++++++
> rust/kernel/percpu/dynamic.rs | 83 +++++++++++++++
> rust/kernel/percpu/static_.rs | 132 ++++++++++++++++++++++++
> 8 files changed, 501 insertions(+)
That's quite a massive patch. Can you please split-out C-binders, and
maybe make separate patches for each .rs component?
> diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c
> index 7cf7fe95e41d..2fc8d26cfe66 100644
> --- a/rust/helpers/helpers.c
> +++ b/rust/helpers/helpers.c
> @@ -31,9 +31,11 @@
> #include "of.c"
> #include "page.c"
> #include "pci.c"
> +#include "percpu.c"
> #include "pid_namespace.c"
> #include "platform.c"
> #include "poll.c"
> +#include "preempt.c"
> #include "property.c"
> #include "rbtree.c"
> #include "rcu.c"
> diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c
> new file mode 100644
> index 000000000000..a091389f730f
> --- /dev/null
> +++ b/rust/helpers/percpu.c
> @@ -0,0 +1,9 @@
> +// SPDX-License-Identifier: GPL-2.0
> +
> +#include <linux/percpu.h>
> +
> +void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align)
> +{
> + return __alloc_percpu(sz, align);
> +}
> +
> diff --git a/rust/helpers/preempt.c b/rust/helpers/preempt.c
> new file mode 100644
> index 000000000000..2c7529528ddd
> --- /dev/null
> +++ b/rust/helpers/preempt.c
> @@ -0,0 +1,14 @@
> +// SPDX-License-Identifier: GPL-2.0
> +
> +#include <linux/preempt.h>
> +
> +void rust_helper_preempt_disable(void)
> +{
> + preempt_disable();
> +}
> +
> +void rust_helper_preempt_enable(void)
> +{
> + preempt_enable();
> +}
> +
> diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
> index ed53169e795c..ed0d5756dc55 100644
> --- a/rust/kernel/lib.rs
> +++ b/rust/kernel/lib.rs
> @@ -106,6 +106,9 @@
> pub mod page;
> #[cfg(CONFIG_PCI)]
> pub mod pci;
> +// Only x86_64 is supported by percpu for now
> +#[cfg(CONFIG_X86_64)]
> +pub mod percpu;
> pub mod pid_namespace;
> pub mod platform;
> pub mod prelude;
> diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs
> new file mode 100644
> index 000000000000..35afcdba3ccd
> --- /dev/null
> +++ b/rust/kernel/percpu.rs
> @@ -0,0 +1,223 @@
> +// SPDX-License-Identifier: GPL-2.0
> +//! This module contains abstractions for creating and using per-CPU variables from Rust.
> +//! See the define_per_cpu! macro and the DynamicPerCpu<T> type, as well as the PerCpu<T> trait.
> +
> +pub mod cpu_guard;
> +mod dynamic;
> +mod static_;
> +
> +#[doc(inline)]
> +pub use dynamic::*;
> +#[doc(inline)]
> +pub use static_::*;
> +
> +use bindings::{alloc_percpu, free_percpu};
> +
> +use crate::alloc::Flags;
> +use crate::percpu::cpu_guard::CpuGuard;
> +use crate::prelude::*;
> +use crate::sync::Arc;
> +use crate::types::Opaque;
> +use crate::{declare_extern_per_cpu, get_static_per_cpu};
> +
> +use core::arch::asm;
> +use core::cell::{Cell, RefCell, UnsafeCell};
> +use core::mem::{align_of, size_of, MaybeUninit};
> +
> +use ffi::c_void;
> +
> +/// A per-CPU pointer; that is, an offset into the per-CPU area. Note that this type is NOT a smart
> +/// pointer, it does not manage the allocation.
> +pub struct PerCpuPtr<T>(*mut MaybeUninit<T>);
> +
> +/// Represents exclusive access to the memory location pointed at by a particular PerCpu<T>.
> +pub struct PerCpuToken<'a, T> {
> + // INVARIANT: the current CPU's memory location associated with the per-CPU variable pointed at
> + // by `ptr` (i.e., the entry in the per-CPU area on the current CPU) has been initialized.
> + _guard: CpuGuard,
> + ptr: &'a PerCpuPtr<T>,
> +}
> +
> +/// Represents access to the memory location pointed at by a particular PerCpu<T> where the type
> +/// `T` manages access to the underlying memory to avoid aliaising troubles. (For example, `T`
> +/// might be a `Cell` or `RefCell`.)
> +pub struct CheckedPerCpuToken<'a, T> {
> + // INVARIANT: the current CPU's memory location associated with the per-CPU variable pointed at
> + // by `ptr` (i.e., the entry in the per-CPU area on the current CPU) has been initialized.
> + _guard: CpuGuard,
> + ptr: &'a PerCpuPtr<T>,
> +}
> +
> +impl<T> PerCpuPtr<T> {
> + /// Makes a new PerCpuPtr from a raw per-CPU pointer.
> + ///
> + /// # Safety
> + /// `ptr` must be a valid per-CPU pointer.
> + pub unsafe fn new(ptr: *mut MaybeUninit<T>) -> Self {
> + Self(ptr)
> + }
> +
> + /// Get a `&mut MaybeUninit<T>` to the per-CPU variable on the current CPU represented by `&self`
> + ///
> + /// # Safety
> + /// The returned `&mut T` must follow Rust's aliasing rules. That is, no other `&(mut) T` may
> + /// exist that points to the same location in memory. In practice, this means that `get_(mut_)ref`
How long is this line?
> + /// must not be called on another `PerCpuPtr<T>` that is a copy/clone of `&self` for as long as
> + /// the returned reference lives.
> + ///
> + /// CPU preemption must be disabled before calling this function and for the lifetime of the
> + /// returned reference. Otherwise, the returned reference might end up being a reference to a
> + /// different CPU's per-CPU area, causing the potential for a data race.
> + #[allow(clippy::mut_from_ref)] // Safety requirements prevent aliasing issues
> + pub unsafe fn get_mut_ref(&self) -> &mut MaybeUninit<T> {
> + // SAFETY: `self.get_ptr()` returns a valid pointer to a `MaybeUninit<T>` by its contract,
> + // and the safety requirements of this function ensure that the returned reference is
> + // exclusive.
> + unsafe { &mut *(self.get_ptr()) }
> + }
Here and everywhere: would it make sense to enforce it by testing
the CPU with preemptible() before returning a reference?
Thanks,
Yury
> +
> + /// Get a `&MaybeUninit<T>` to the per-CPU variable on the current CPU represented by `&self`
> + ///
> + /// # Safety
> + /// The returned `&T` must follow Rust's aliasing rules. That is, no `&mut T` may exist that
> + /// points to the same location in memory. In practice, this means that `get_mut_ref` must not
> + /// be called on another `PerCpuPtr<T>` that is a copy/clone of `&self` for as long as the
> + /// returned reference lives.
> + ///
> + /// CPU preemption must be disabled before calling this function and for the lifetime of the
> + /// returned reference. Otherwise, the returned reference might end up being a reference to a
> + /// different CPU's per-CPU area, causing the potential for a data race.
> + pub unsafe fn get_ref(&self) -> &MaybeUninit<T> {
> + // SAFETY: `self.get_ptr()` returns a valid pointer to a `MaybeUninit<T>` by its contract.
> + // The safety requirements of this function ensure that the returned reference isn't
> + // aliased by a `&mut MaybeUninit<T>`.
> + unsafe { &*self.get_ptr() }
> + }
> +
> + /// Get a `*mut MaybeUninit<T>` to the per-CPU variable on the current CPU represented by
> + /// `&self`. Note that if CPU preemption is not disabled before calling this function, use of
> + /// the returned pointer may cause a data race without some other synchronization mechanism.
> + /// Buyer beware!
> + pub fn get_ptr(&self) -> *mut MaybeUninit<T> {
> + let this_cpu_off_pcpu: PerCpuPtr<*mut c_void> = get_static_per_cpu!(this_cpu_off).0;
> + let mut this_cpu_area: *mut c_void;
> + // SAFETY: gs + this_cpu_off_pcpu is guaranteed to be a valid pointer because `gs` points
> + // to the per-CPU area and this_cpu_off_pcpu is a valid per-CPU allocation.
> + unsafe {
> + asm!(
> + "mov {out}, gs:[{off_val}]",
> + off_val = in(reg) this_cpu_off_pcpu.0,
> + out = out(reg) this_cpu_area,
> + )
> + };
> +
> + // This_cpu_area + self.0 is guaranteed to be a valid pointer by the per-CPU subsystem and
> + // the invariant that self.0 is a valid offset into the per-CPU area.
> + (this_cpu_area).wrapping_add(self.0 as usize).cast()
> + }
> +}
> +
> +// SAFETY: Sending a `PerCpuPtr<T>` to another thread is safe because as soon as it's sent, the
> +// pointer is logically referring to a different place in memory in the other CPU's per-CPU area.
> +// In particular, this means that there are no restrictions on the type `T`.
> +unsafe impl<T> Send for PerCpuPtr<T> {}
> +
> +// SAFETY: Two threads concurrently making use of a `PerCpuPtr<T>` will each see the `T` in their
> +// own per-CPU area, so there's no potential for a data race (regardless of whether `T` is itself
> +// `Sync`).
> +unsafe impl<T> Sync for PerCpuPtr<T> {}
> +
> +impl<T> Clone for PerCpuPtr<T> {
> + fn clone(&self) -> Self {
> + *self
> + }
> +}
> +
> +/// `PerCpuPtr` is just a wrapper around a pointer.
> +impl<T> Copy for PerCpuPtr<T> {}
> +
> +/// A trait representing a per-CPU variable. This is implemented for both `StaticPerCpu<T>` and
> +/// `DynamicPerCpu<T>`. The main usage of this trait is to call `get_mut` to get a `PerCpuToken`
> +/// that can be used to access the underlying per-CPU variable. See `PerCpuToken::with`.
> +pub trait PerCpu<T> {
> + /// Produces a token, asserting that the holder has exclusive access to the underlying memory
> + /// pointed to by `self`
> + ///
> + /// # Safety
> + /// No other `PerCpuToken` or `CheckedPerCpuToken` may exist on the current CPU (which is a
> + /// sensible notion, since we keep a `CpuGuard` around) that is derived from the same
> + /// `PerCpu<T>` or a clone thereof.
> + unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T>;
> +}
> +
> +/// A marker trait for types that are interior mutable. Types that implement this trait can be used
> +/// to create "checked" per-CPU variables. See `CheckedPerCpu<T>`.
> +pub trait InteriorMutable {}
> +
> +impl<T> InteriorMutable for Cell<T> {}
> +impl<T> InteriorMutable for RefCell<T> {}
> +impl<T> InteriorMutable for UnsafeCell<T> {}
> +impl<T> InteriorMutable for Opaque<T> {}
> +
> +/// A trait representing a per-CPU variable that is usable via a `&T`. The unsafety of `PerCpu<T>`
> +/// stems from the fact that the holder of a `PerCpuToken` can use it to get a `&mut T` to the
> +/// underlying per-CPU variable. This is problematic because the existence of aliaising `&mut T` is
> +/// undefined behavior in Rust. This type avoids that issue by only allowing access via a `&T`,
> +/// with the tradeoff that then `T` must be interior mutable or the underlying per-CPU variable
> +/// must be a constant for the lifetime of the corresponding `CheckedPerCpu<T>`.
> +///
> +/// Currently, only the case where `T` is interior mutable has first-class support, though a custom
> +/// implementation of `PerCpu<T>`/`CheckedPerCpu<T>` could be created for the const case.
> +pub trait CheckedPerCpu<T>: PerCpu<T> {
> + /// Produces a token via which the holder can access the underlying per-CPU variable.
> + fn get(&mut self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T>;
> +}
> +
> +impl<'a, T> PerCpuToken<'a, T> {
> + /// # Safety
> + /// No other `PerCpuToken` or `CheckedPerCpuToken` may exist on the current CPU (which is a
> + /// sensible notion, since we keep a `CpuGuard` around) that uses the same `PerCpuPtr<T>`.
> + ///
> + /// The current CPU's memory location associated with the per-CPU variable pointed at by `ptr`
> + /// (i.e., the entry in the per-CPU area on this CPU) must be initialized.
> + pub unsafe fn new(guard: CpuGuard, ptr: &'a PerCpuPtr<T>) -> PerCpuToken<'a, T> {
> + Self { _guard: guard, ptr }
> + }
> +
> + /// Immediately invokes `func` with a `&mut T` that points at the underlying per-CPU variable
> + /// that `&mut self` represents.
> + pub fn with<U>(&mut self, func: U)
> + where
> + U: FnOnce(&mut T),
> + {
> + // SAFETY: The existence of a PerCpuToken means that the requirements for get_mut_ref are
> + // satisfied. Likewise, the requirements for assume_init_mut are satisfied because the
> + // invariants of this type ensure that on the current CPU (which is a sensible notion
> + // because we have a CpuGuard), the memory location pointed to by `ptr` is initialized.
> + func(unsafe { self.ptr.get_mut_ref().assume_init_mut() });
> + }
> +}
> +
> +impl<'a, T> CheckedPerCpuToken<'a, T> {
> + /// # Safety
> + /// The current CPU's memory location associated with the per-CPU variable pointed at by `ptr`
> + /// (i.e., the entry in the per-CPU area on this CPU) must be initialized.
> + pub unsafe fn new(guard: CpuGuard, ptr: &'a PerCpuPtr<T>) -> CheckedPerCpuToken<'a, T> {
> + Self { _guard: guard, ptr }
> + }
> +
> + /// Immediately invokes `func` with a `&T` that points at the underlying per-CPU variable that
> + /// `&mut self` represents.
> + pub fn with<U>(&mut self, func: U)
> + where
> + U: FnOnce(&T),
> + {
> + // SAFETY: The existence of a CheckedPerCpuToken means that the requirements for get_ref
> + // are satisfied. Likewise, the requirements for assume_init_ref are satisfied because the
> + // invariants of this type ensure that on the current CPU (which is a sensible notion
> + // because we have a CpuGuard), the memory location pointed to by `ptr` is initialized.
> + func(unsafe { self.ptr.get_ref().assume_init_ref() });
> + }
> +}
> +
> +declare_extern_per_cpu!(this_cpu_off: *mut c_void);
> diff --git a/rust/kernel/percpu/cpu_guard.rs b/rust/kernel/percpu/cpu_guard.rs
> new file mode 100644
> index 000000000000..14c04b12e7f0
> --- /dev/null
> +++ b/rust/kernel/percpu/cpu_guard.rs
> @@ -0,0 +1,35 @@
> +// SPDX-License-Identifier: GPL-2.0
> +//! Contains abstractions for disabling CPU preemption. See `CpuGuard`.
> +
> +/// A RAII guard for bindings::preempt_disable and bindings::preempt_enable. Guarantees preemption
> +/// is disabled for as long as this object exists.
> +pub struct CpuGuard {
> + // Don't make one without using new()
> + _phantom: (),
> +}
> +
> +impl CpuGuard {
> + /// Create a new CpuGuard. Disables preemption for its lifetime.
> + pub fn new() -> Self {
> + // SAFETY: There are no preconditions required to call preempt_disable
> + unsafe {
> + bindings::preempt_disable();
> + }
> + CpuGuard { _phantom: () }
> + }
> +}
> +
> +impl Default for CpuGuard {
> + fn default() -> Self {
> + Self::new()
> + }
> +}
> +
> +impl Drop for CpuGuard {
> + fn drop(&mut self) {
> + // SAFETY: There are no preconditions required to call preempt_enable
> + unsafe {
> + bindings::preempt_enable();
> + }
> + }
> +}
> diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs
> new file mode 100644
> index 000000000000..ce95e420f943
> --- /dev/null
> +++ b/rust/kernel/percpu/dynamic.rs
> @@ -0,0 +1,83 @@
> +// SPDX-License-Identifier: GPL-2.0
> +//! Dynamically allocated per-CPU variables.
> +
> +use super::*;
> +
> +/// Represents a dynamic allocation of a per-CPU variable via alloc_percpu. Calls free_percpu when
> +/// dropped.
> +pub struct PerCpuAllocation<T>(PerCpuPtr<T>);
> +
> +impl<T: Zeroable> PerCpuAllocation<T> {
> + /// Dynamically allocates a space in the per-CPU area suitably sized and aligned to hold a `T`,
> + /// initially filled with the zero value for `T`.
> + ///
> + /// Returns `None` under the same circumstances the C function `alloc_percpu` returns `NULL`.
> + pub fn new_zero() -> Option<PerCpuAllocation<T>> {
> + let ptr: *mut MaybeUninit<T> =
> + // SAFETY: No preconditions to call alloc_percpu; MaybeUninit<T> is
> + // `#[repr(transparent)]`, so we can cast a `*mut T` to it.
> + unsafe { alloc_percpu(size_of::<T>(), align_of::<T>()) }.cast();
> + if ptr.is_null() {
> + return None;
> + }
> +
> + // alloc_percpu returns zero'ed memory
> + Some(Self(PerCpuPtr(ptr)))
> + }
> +}
> +
> +impl<T> PerCpuAllocation<T> {
> + /// Makes a per-CPU allocation sized and aligned to hold a `T`.
> + ///
> + /// Returns `None` under the same circumstances the C function `alloc_percpu` returns `NULL`.
> + pub fn new_uninit() -> Option<PerCpuAllocation<T>> {
> + let ptr: *mut MaybeUninit<T> =
> + // SAFETY: No preconditions to call alloc_percpu; MaybeUninit<T> is
> + // `#[repr(transparent)]`, so we can cast a `*mut T` to it.
> + unsafe { alloc_percpu(size_of::<T>(), align_of::<T>()) }.cast();
> + if ptr.is_null() {
> + return None;
> + }
> +
> + Some(Self(PerCpuPtr(ptr)))
> + }
> +}
> +
> +impl<T> Drop for PerCpuAllocation<T> {
> + fn drop(&mut self) {
> + // SAFETY: self.0.0 was returned by alloc_percpu, and so was a valid pointer into
> + // the percpu area, and has remained valid by the invariants of PerCpuAllocation<T>.
> + unsafe { free_percpu(self.0 .0.cast()) }
> + }
> +}
> +
> +/// Holds a dynamically-allocated per-CPU variable.
> +#[derive(Clone)]
> +pub struct DynamicPerCpu<T> {
> + // INVARIANT: The memory location in each CPU's per-CPU area pointed at by `alloc.0` has been
> + // initialized.
> + pub(super) alloc: Arc<PerCpuAllocation<T>>,
> +}
> +
> +impl<T: Zeroable> DynamicPerCpu<T> {
> + /// Allocates a new per-CPU variable
> + ///
> + /// # Arguments
> + /// * `flags` - Flags used to allocate an `Arc` that keeps track of the underlying
> + /// `PerCpuAllocation`.
> + pub fn new_zero(flags: Flags) -> Option<Self> {
> + let alloc: PerCpuAllocation<T> = PerCpuAllocation::new_zero()?;
> +
> + let arc = Arc::new(alloc, flags).ok()?;
> +
> + Some(Self { alloc: arc })
> + }
> +}
> +
> +impl<T> PerCpu<T> for DynamicPerCpu<T> {
> + unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> {
> + // SAFETY: The requirements of `PerCpu::get_mut` and this type's invariant ensure that the
> + // requirements of `PerCpuToken::new` are met.
> + unsafe { PerCpuToken::new(guard, &self.alloc.0) }
> + }
> +}
> diff --git a/rust/kernel/percpu/static_.rs b/rust/kernel/percpu/static_.rs
> new file mode 100644
> index 000000000000..be226dd2c3aa
> --- /dev/null
> +++ b/rust/kernel/percpu/static_.rs
> @@ -0,0 +1,132 @@
> +// SPDX-License-Identifier: GPL-2.0
> +//! Statically allocated per-CPU variables.
> +
> +use super::*;
> +
> +/// A wrapper used for declaring static per-CPU variables. These symbols are "virtual" in that the
> +/// linker uses them to generate offsets into each CPU's per-CPU area, but shouldn't be read
> +/// from/written to directly. The fact that the statics are immutable prevents them being written
> +/// to (generally), this struct having _val be non-public prevents reading from them.
> +///
> +/// The end-user of the per-CPU API should make use of the define_per_cpu! macro instead of
> +/// declaring variables of this type directly. All instances of this type must be `static` and
> +/// `#[link_section = ".data..percpu"]` (which the macro handles).
> +#[repr(transparent)]
> +pub struct StaticPerCpuSymbol<T> {
> + _val: T, // generate a correctly sized type
> +}
> +
> +/// Holds a statically-allocated per-CPU variable.
> +#[derive(Clone)]
> +pub struct StaticPerCpu<T>(pub(super) PerCpuPtr<T>);
> +
> +impl<T> StaticPerCpu<T> {
> + /// Creates a `StaticPerCpu<T>` from a `StaticPerCpuSymbol<T>`. You should probably be using
> + /// `get_static_per_cpu!` instead.
> + pub fn new(ptr: *const StaticPerCpuSymbol<T>) -> StaticPerCpu<T> {
> + // SAFETY: `StaticPerCpuSymbol<T>` is `#[repr(transparent)]`, so we can safely cast a
> + // pointer to it into a pointer to `MaybeUninit<T>`. The validity of it as a per-CPU
> + // pointer is guaranteed by the per-CPU subsystem and invariants of the StaticPerCpuSymbol
> + // type.
> + let pcpu_ptr = unsafe { PerCpuPtr::new(ptr.cast_mut().cast()) };
> + Self(pcpu_ptr)
> + }
> +}
> +
> +impl<T> PerCpu<T> for StaticPerCpu<T> {
> + unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> {
> + // SAFETY: The requirements of `PerCpu::get_mut` and the fact that statically-allocated
> + // per-CPU variables are initialized by the per-CPU subsystem ensure that the requirements
> + // of `PerCpuToken::new` are met.
> + unsafe { PerCpuToken::new(guard, &self.0) }
> + }
> +}
> +
> +impl<T: InteriorMutable> CheckedPerCpu<T> for StaticPerCpu<T> {
> + fn get(&mut self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T> {
> + // SAFETY: The per-CPU subsystem guarantees that each CPU's instance of a
> + // statically allocated variable begins with a copy of the contents of the
> + // corresponding symbol in `.data..percpu`. Thus, the requirements of
> + // `CheckedPerCpuToken::new` are met.
> + unsafe { CheckedPerCpuToken::new(guard, &self.0) }
> + }
> +}
> +
> +/// Gets a `StaticPerCpu<T>` from a symbol declared with `define_per_cpu!` or
> +/// `declare_extern_per_cpu!`.
> +///
> +/// # Arguments
> +/// * `ident` - The identifier declared
> +#[macro_export]
> +macro_rules! get_static_per_cpu {
> + ($id:ident) => {
> + $crate::percpu::StaticPerCpu::new((&raw const $id).cast())
> + };
> +}
> +
> +/// Declares a StaticPerCpuSymbol corresponding to a per-CPU variable defined in C. Be sure to read
> +/// the safety requirements of `PerCpu::get`.
> +#[macro_export]
> +macro_rules! declare_extern_per_cpu {
> + ($id:ident: $ty:ty) => {
> + extern "C" {
> + static $id: StaticPerCpuSymbol<$ty>;
> + }
> + };
> +}
> +
> +/// define_per_cpu! is analogous to the C DEFINE_PER_CPU macro in that it lets you create a
> +/// statically allocated per-CPU variable.
> +///
> +/// # Example
> +/// ```
> +/// use kernel::define_per_cpu;
> +/// use kernel::percpu::StaticPerCpuSymbol;
> +///
> +/// define_per_cpu!(pub MY_PERCPU: u64 = 0);
> +/// ```
> +#[macro_export]
> +macro_rules! define_per_cpu {
> + ($vis:vis $id:ident: $ty:ty = $expr:expr) => {
> + $crate::macros::paste! {
> + // We might want to have a per-CPU variable that doesn't implement `Sync` (not paying
> + // sync overhead costs is part of the point), but Rust won't let us declare a static of
> + // a `!Sync` type. Of course, we don't actually have any synchronization issues, since
> + // each CPU will see its own copy of the variable, so we cheat a little bit and tell
> + // Rust it's fine.
> + #[doc(hidden)]
> + #[allow(non_camel_case_types)]
> + #[repr(transparent)] // It needs to be the same size as $ty
> + struct [<__PRIVATE_TYPE_ $id>]($ty);
> +
> + impl [<__PRIVATE_TYPE_ $id>] {
> + #[doc(hidden)]
> + const fn new(val: $ty) -> Self {
> + Self(val)
> + }
> + }
> +
> + // Expand $expr outside of the unsafe block to avoid silently allowing unsafe code to be
> + // used without a user-facing unsafe block
> + #[doc(hidden)]
> + static [<__INIT_ $id>]: [<__PRIVATE_TYPE_ $id>] = [<__PRIVATE_TYPE_ $id>]::new($expr);
> +
> + // SAFETY: This type will ONLY ever be used to declare a `StaticPerCpuSymbol`
> + // (which we then only ever use as input to `&raw`). Reading from the symbol is
> + // already UB, so we won't ever actually have any variables of this type where
> + // synchronization is a concern.
> + #[doc(hidden)]
> + unsafe impl Sync for [<__PRIVATE_TYPE_ $id>] {}
> +
> + // SAFETY: StaticPerCpuSymbol<T> is #[repr(transparent)], so we can freely convert from
> + // [<__PRIVATE_TYPE_ $id>], which is also `#[repr(transparent)]` (i.e., everything is
> + // just a `$ty` from a memory layout perspective).
> + #[link_section = ".data..percpu"]
> + $vis static $id: StaticPerCpuSymbol<[<__PRIVATE_TYPE_ $id>]> = unsafe {
> + core::mem::transmute_copy::<
> + [<__PRIVATE_TYPE_ $id>], StaticPerCpuSymbol<[<__PRIVATE_TYPE_ $id>]>
> + >(&[<__INIT_ $id>])
> + };
> + }
> + };
> +}
>
> --
> 2.34.1
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH v3 1/7] rust: percpu: introduce a rust API for per-CPU variables
2025-09-03 21:42 ` Yury Norov
@ 2025-09-04 19:53 ` Mitchell Levy
2025-09-04 20:27 ` Yury Norov
0 siblings, 1 reply; 26+ messages in thread
From: Mitchell Levy @ 2025-09-04 19:53 UTC (permalink / raw)
To: Yury Norov
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Viresh Kumar, Tyler Hicks,
linux-kernel, rust-for-linux, linux-mm
On Wed, Sep 03, 2025 at 05:42:08PM -0400, Yury Norov wrote:
> On Thu, Aug 28, 2025 at 12:00:08PM -0700, Mitchell Levy wrote:
> > Per-CPU variables are an important tool for reducing lock contention,
> > especially in systems with many processors. They also provide a
> > convenient way to handle data that are logically associated with a
> > particular CPU (e.g., the currently running task). Therefore, add a Rust
> > API to make use of per-CPU variables.
> >
> > Add a `CpuGuard` type that disables preemption for its lifetime. Add a
> > `PerCpuAllocation` type used to track dynamic allocations. Add a
> > `define_per_cpu!` macro to create static per-CPU allocations. Add
> > `DynamicPerCpu` and `StaticPerCpu` to provide a high-level API. Add a
> > `PerCpu` trait to unify the dynamic and static cases.
> >
> > Co-developed-by: Boqun Feng <boqun.feng@gmail.com>
> > Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
> > Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
> > ---
> > rust/helpers/helpers.c | 2 +
> > rust/helpers/percpu.c | 9 ++
> > rust/helpers/preempt.c | 14 +++
> > rust/kernel/lib.rs | 3 +
> > rust/kernel/percpu.rs | 223 ++++++++++++++++++++++++++++++++++++++++
> > rust/kernel/percpu/cpu_guard.rs | 35 +++++++
> > rust/kernel/percpu/dynamic.rs | 83 +++++++++++++++
> > rust/kernel/percpu/static_.rs | 132 ++++++++++++++++++++++++
> > 8 files changed, 501 insertions(+)
>
> That's quite a massive patch. Can you please split-out C-binders, and
> maybe make separate patches for each .rs component?
Sure, will do so for v4.
> > diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c
> > index 7cf7fe95e41d..2fc8d26cfe66 100644
[...]
> > --- /dev/null
> > +++ b/rust/kernel/percpu.rs
> > @@ -0,0 +1,223 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +//! This module contains abstractions for creating and using per-CPU variables from Rust.
> > +//! See the define_per_cpu! macro and the DynamicPerCpu<T> type, as well as the PerCpu<T> trait.
> > +
> > +pub mod cpu_guard;
> > +mod dynamic;
> > +mod static_;
> > +
> > +#[doc(inline)]
> > +pub use dynamic::*;
> > +#[doc(inline)]
> > +pub use static_::*;
> > +
> > +use bindings::{alloc_percpu, free_percpu};
> > +
> > +use crate::alloc::Flags;
> > +use crate::percpu::cpu_guard::CpuGuard;
> > +use crate::prelude::*;
> > +use crate::sync::Arc;
> > +use crate::types::Opaque;
> > +use crate::{declare_extern_per_cpu, get_static_per_cpu};
> > +
> > +use core::arch::asm;
> > +use core::cell::{Cell, RefCell, UnsafeCell};
> > +use core::mem::{align_of, size_of, MaybeUninit};
> > +
> > +use ffi::c_void;
> > +
> > +/// A per-CPU pointer; that is, an offset into the per-CPU area. Note that this type is NOT a smart
> > +/// pointer, it does not manage the allocation.
> > +pub struct PerCpuPtr<T>(*mut MaybeUninit<T>);
> > +
> > +/// Represents exclusive access to the memory location pointed at by a particular PerCpu<T>.
> > +pub struct PerCpuToken<'a, T> {
> > + // INVARIANT: the current CPU's memory location associated with the per-CPU variable pointed at
> > + // by `ptr` (i.e., the entry in the per-CPU area on the current CPU) has been initialized.
> > + _guard: CpuGuard,
> > + ptr: &'a PerCpuPtr<T>,
> > +}
> > +
> > +/// Represents access to the memory location pointed at by a particular PerCpu<T> where the type
> > +/// `T` manages access to the underlying memory to avoid aliaising troubles. (For example, `T`
> > +/// might be a `Cell` or `RefCell`.)
> > +pub struct CheckedPerCpuToken<'a, T> {
> > + // INVARIANT: the current CPU's memory location associated with the per-CPU variable pointed at
> > + // by `ptr` (i.e., the entry in the per-CPU area on the current CPU) has been initialized.
> > + _guard: CpuGuard,
> > + ptr: &'a PerCpuPtr<T>,
> > +}
> > +
> > +impl<T> PerCpuPtr<T> {
> > + /// Makes a new PerCpuPtr from a raw per-CPU pointer.
> > + ///
> > + /// # Safety
> > + /// `ptr` must be a valid per-CPU pointer.
> > + pub unsafe fn new(ptr: *mut MaybeUninit<T>) -> Self {
> > + Self(ptr)
> > + }
> > +
> > + /// Get a `&mut MaybeUninit<T>` to the per-CPU variable on the current CPU represented by `&self`
> > + ///
> > + /// # Safety
> > + /// The returned `&mut T` must follow Rust's aliasing rules. That is, no other `&(mut) T` may
> > + /// exist that points to the same location in memory. In practice, this means that `get_(mut_)ref`
>
> How long is this line?
102 chars, or 103 if you include the newline. `rustfmt` doesn't break
the line, so I left it as-is for this patch. Happy to change it if it
poses a problem, though.
> > + /// must not be called on another `PerCpuPtr<T>` that is a copy/clone of `&self` for as long as
> > + /// the returned reference lives.
> > + ///
> > + /// CPU preemption must be disabled before calling this function and for the lifetime of the
> > + /// returned reference. Otherwise, the returned reference might end up being a reference to a
> > + /// different CPU's per-CPU area, causing the potential for a data race.
> > + #[allow(clippy::mut_from_ref)] // Safety requirements prevent aliasing issues
> > + pub unsafe fn get_mut_ref(&self) -> &mut MaybeUninit<T> {
> > + // SAFETY: `self.get_ptr()` returns a valid pointer to a `MaybeUninit<T>` by its contract,
> > + // and the safety requirements of this function ensure that the returned reference is
> > + // exclusive.
> > + unsafe { &mut *(self.get_ptr()) }
> > + }
>
> Here and everywhere: would it make sense to enforce it by testing
> the CPU with preemptible() before returning a reference?
The only thing we could do would be to panic, which I don't 100% love.
Another alternative would be to take a &'a CpuGuard and bound the
lifetime of the returned reference to 'a, and then we don't need to do
any run-time checking at all.
Originally, I had left this to the caller because it might make sense
down the line for some complex behavior based on per-CPU (e.g., per-CPU
refcount) to do all its own management of per-CPU variables using
`PerCpuPtr` as a core primitive. In these cases, I believe there are
some times where being non-preemptible wouldn't actually be required
(that said, my thoughts on this aren't well reflected in the safety
comment, since I said it must be disabled... gah). But, the more I think
about it, the more I think these use cases would be better served by
just using `get_ptr` --- conjuring `&mut` references seems like it would
be a big footgun. And the safety comment already actually reflects these
thoughts somewhat :)
For v4 I will probably have this function take a &'a CpuGuard and use
that to bound the liftetime of the returned reference, unless there are
other thoughts on this point.
Thanks,
Mitchell
> Thanks,
> Yury
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH v3 1/7] rust: percpu: introduce a rust API for per-CPU variables
2025-09-04 19:53 ` Mitchell Levy
@ 2025-09-04 20:27 ` Yury Norov
2025-09-04 21:17 ` Mitchell Levy
0 siblings, 1 reply; 26+ messages in thread
From: Yury Norov @ 2025-09-04 20:27 UTC (permalink / raw)
To: Mitchell Levy
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Viresh Kumar, Tyler Hicks,
linux-kernel, rust-for-linux, linux-mm
On Thu, Sep 04, 2025 at 12:53:59PM -0700, Mitchell Levy wrote:
> On Wed, Sep 03, 2025 at 05:42:08PM -0400, Yury Norov wrote:
> > On Thu, Aug 28, 2025 at 12:00:08PM -0700, Mitchell Levy wrote:
...
> > > + /// Get a `&mut MaybeUninit<T>` to the per-CPU variable on the current CPU represented by `&self`
> > > + ///
> > > + /// # Safety
> > > + /// The returned `&mut T` must follow Rust's aliasing rules. That is, no other `&(mut) T` may
> > > + /// exist that points to the same location in memory. In practice, this means that `get_(mut_)ref`
> >
> > How long is this line?
>
> 102 chars, or 103 if you include the newline. `rustfmt` doesn't break
> the line, so I left it as-is for this patch. Happy to change it if it
> poses a problem, though.
Then don't use that tool - it's broken. In kernel we used to have 80-chars
limit for the lines, recently relaxed to 100.
> > > + /// must not be called on another `PerCpuPtr<T>` that is a copy/clone of `&self` for as long as
> > > + /// the returned reference lives.
> > > + ///
> > > + /// CPU preemption must be disabled before calling this function and for the lifetime of the
> > > + /// returned reference. Otherwise, the returned reference might end up being a reference to a
> > > + /// different CPU's per-CPU area, causing the potential for a data race.
> > > + #[allow(clippy::mut_from_ref)] // Safety requirements prevent aliasing issues
> > > + pub unsafe fn get_mut_ref(&self) -> &mut MaybeUninit<T> {
> > > + // SAFETY: `self.get_ptr()` returns a valid pointer to a `MaybeUninit<T>` by its contract,
> > > + // and the safety requirements of this function ensure that the returned reference is
> > > + // exclusive.
> > > + unsafe { &mut *(self.get_ptr()) }
> > > + }
> >
> > Here and everywhere: would it make sense to enforce it by testing
> > the CPU with preemptible() before returning a reference?
>
> The only thing we could do would be to panic, which I don't 100% love.
> Another alternative would be to take a &'a CpuGuard and bound the
> lifetime of the returned reference to 'a, and then we don't need to do
> any run-time checking at all.
>
> Originally, I had left this to the caller because it might make sense
> down the line for some complex behavior based on per-CPU (e.g., per-CPU
> refcount) to do all its own management of per-CPU variables using
> `PerCpuPtr` as a core primitive. In these cases, I believe there are
> some times where being non-preemptible wouldn't actually be required
> (that said, my thoughts on this aren't well reflected in the safety
> comment, since I said it must be disabled... gah). But, the more I think
> about it, the more I think these use cases would be better served by
> just using `get_ptr` --- conjuring `&mut` references seems like it would
> be a big footgun. And the safety comment already actually reflects these
> thoughts somewhat :)
If you think that in future there will be a user who will not need to
disable preemption before dereferencing a percpu pointer, then you can
add another less restricted flavor of the helper.
> For v4 I will probably have this function take a &'a CpuGuard and use
> that to bound the liftetime of the returned reference, unless there are
> other thoughts on this point.
I don't want you to panic just because of invlid user call, but
whatever you call in comment must be enforced in code, right?
You can use the guard, if it guarantees the preemption disabled; or
you can return None; you can create CONFIG_RUST_PERCPU_HARDENED for
panics.
Please refer the recent bitmap API wrapper, and how erroneous request
is handled there.
https://lore.kernel.org/all/20250904165015.3791895-4-bqe@google.com/
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH v3 1/7] rust: percpu: introduce a rust API for per-CPU variables
2025-09-04 20:27 ` Yury Norov
@ 2025-09-04 21:17 ` Mitchell Levy
0 siblings, 0 replies; 26+ messages in thread
From: Mitchell Levy @ 2025-09-04 21:17 UTC (permalink / raw)
To: Yury Norov
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Viresh Kumar, Tyler Hicks,
linux-kernel, rust-for-linux, linux-mm
On Thu, Sep 04, 2025 at 04:27:43PM -0400, Yury Norov wrote:
> On Thu, Sep 04, 2025 at 12:53:59PM -0700, Mitchell Levy wrote:
> > On Wed, Sep 03, 2025 at 05:42:08PM -0400, Yury Norov wrote:
> > > On Thu, Aug 28, 2025 at 12:00:08PM -0700, Mitchell Levy wrote:
>
> ...
>
> > > > + /// Get a `&mut MaybeUninit<T>` to the per-CPU variable on the current CPU represented by `&self`
> > > > + ///
> > > > + /// # Safety
> > > > + /// The returned `&mut T` must follow Rust's aliasing rules. That is, no other `&(mut) T` may
> > > > + /// exist that points to the same location in memory. In practice, this means that `get_(mut_)ref`
> > >
> > > How long is this line?
> >
> > 102 chars, or 103 if you include the newline. `rustfmt` doesn't break
> > the line, so I left it as-is for this patch. Happy to change it if it
> > poses a problem, though.
>
> Then don't use that tool - it's broken. In kernel we used to have 80-chars
> limit for the lines, recently relaxed to 100.
Sure, will fix.
> > > > + /// must not be called on another `PerCpuPtr<T>` that is a copy/clone of `&self` for as long as
> > > > + /// the returned reference lives.
> > > > + ///
> > > > + /// CPU preemption must be disabled before calling this function and for the lifetime of the
> > > > + /// returned reference. Otherwise, the returned reference might end up being a reference to a
> > > > + /// different CPU's per-CPU area, causing the potential for a data race.
> > > > + #[allow(clippy::mut_from_ref)] // Safety requirements prevent aliasing issues
> > > > + pub unsafe fn get_mut_ref(&self) -> &mut MaybeUninit<T> {
> > > > + // SAFETY: `self.get_ptr()` returns a valid pointer to a `MaybeUninit<T>` by its contract,
> > > > + // and the safety requirements of this function ensure that the returned reference is
> > > > + // exclusive.
> > > > + unsafe { &mut *(self.get_ptr()) }
> > > > + }
> > >
> > > Here and everywhere: would it make sense to enforce it by testing
> > > the CPU with preemptible() before returning a reference?
> >
> > The only thing we could do would be to panic, which I don't 100% love.
> > Another alternative would be to take a &'a CpuGuard and bound the
> > lifetime of the returned reference to 'a, and then we don't need to do
> > any run-time checking at all.
> >
> > Originally, I had left this to the caller because it might make sense
> > down the line for some complex behavior based on per-CPU (e.g., per-CPU
> > refcount) to do all its own management of per-CPU variables using
> > `PerCpuPtr` as a core primitive. In these cases, I believe there are
> > some times where being non-preemptible wouldn't actually be required
> > (that said, my thoughts on this aren't well reflected in the safety
> > comment, since I said it must be disabled... gah). But, the more I think
> > about it, the more I think these use cases would be better served by
> > just using `get_ptr` --- conjuring `&mut` references seems like it would
> > be a big footgun. And the safety comment already actually reflects these
> > thoughts somewhat :)
>
> If you think that in future there will be a user who will not need to
> disable preemption before dereferencing a percpu pointer, then you can
> add another less restricted flavor of the helper.
Yeah, that's fair.
> > For v4 I will probably have this function take a &'a CpuGuard and use
> > that to bound the liftetime of the returned reference, unless there are
> > other thoughts on this point.
>
> I don't want you to panic just because of invlid user call, but
> whatever you call in comment must be enforced in code, right?
>
>
> You can use the guard, if it guarantees the preemption disabled; or
> you can return None; you can create CONFIG_RUST_PERCPU_HARDENED for
> panics.
Yes, the existence of a `CpuGuard` guarantees that preemption is
disabled.
> Please refer the recent bitmap API wrapper, and how erroneous request
> is handled there.
>
> https://lore.kernel.org/all/20250904165015.3791895-4-bqe@google.com/
Thanks,
Mitchell
^ permalink raw reply [flat|nested] 26+ messages in thread
* [PATCH v3 2/7] rust: percpu: add a rust per-CPU variable sample
2025-08-28 19:00 [PATCH v3 0/7] rust: Add Per-CPU Variable API Mitchell Levy
2025-08-28 19:00 ` [PATCH v3 1/7] rust: percpu: introduce a rust API for per-CPU variables Mitchell Levy
@ 2025-08-28 19:00 ` Mitchell Levy
2025-08-28 19:00 ` [PATCH v3 3/7] rust: cpumask: Add a `Cpumask` iterator Mitchell Levy
` (4 subsequent siblings)
6 siblings, 0 replies; 26+ messages in thread
From: Mitchell Levy @ 2025-08-28 19:00 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Yury Norov, Viresh Kumar
Cc: Tyler Hicks, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
Add a short exercise for Rust's per-CPU variable API, modelled after
lib/percpu_test.c
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/helpers/percpu.c | 6 ++
samples/rust/Kconfig | 9 +++
samples/rust/Makefile | 1 +
samples/rust/rust_percpu.rs | 163 ++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 179 insertions(+)
diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c
index a091389f730f..8cc01d094752 100644
--- a/rust/helpers/percpu.c
+++ b/rust/helpers/percpu.c
@@ -1,9 +1,15 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/percpu.h>
+#include <linux/smp.h>
void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align)
{
return __alloc_percpu(sz, align);
}
+void rust_helper_on_each_cpu(smp_call_func_t func, void *info, int wait)
+{
+ on_each_cpu(func, info, wait);
+}
+
diff --git a/samples/rust/Kconfig b/samples/rust/Kconfig
index 7f7371a004ee..23e35d64ac78 100644
--- a/samples/rust/Kconfig
+++ b/samples/rust/Kconfig
@@ -105,6 +105,15 @@ config SAMPLE_RUST_DRIVER_AUXILIARY
If unsure, say N.
+config SAMPLE_RUST_PERCPU
+ tristate "Per-CPU support"
+ depends on m
+ help
+ Enable this option to build a module which demonstrates Rust per-CPU
+ operations.
+
+ If unsure, say N.
+
config SAMPLE_RUST_HOSTPROGS
bool "Host programs"
help
diff --git a/samples/rust/Makefile b/samples/rust/Makefile
index bd2faad63b4f..8a34d9d74754 100644
--- a/samples/rust/Makefile
+++ b/samples/rust/Makefile
@@ -10,6 +10,7 @@ obj-$(CONFIG_SAMPLE_RUST_DRIVER_PLATFORM) += rust_driver_platform.o
obj-$(CONFIG_SAMPLE_RUST_DRIVER_FAUX) += rust_driver_faux.o
obj-$(CONFIG_SAMPLE_RUST_DRIVER_AUXILIARY) += rust_driver_auxiliary.o
obj-$(CONFIG_SAMPLE_RUST_CONFIGFS) += rust_configfs.o
+obj-$(CONFIG_SAMPLE_RUST_PERCPU) += rust_percpu.o
rust_print-y := rust_print_main.o rust_print_events.o
diff --git a/samples/rust/rust_percpu.rs b/samples/rust/rust_percpu.rs
new file mode 100644
index 000000000000..98ca1c781b6b
--- /dev/null
+++ b/samples/rust/rust_percpu.rs
@@ -0,0 +1,163 @@
+// SPDX-License-Identifier: GPL-2.0
+//! A simple demonstration of the rust per-CPU API.
+
+use core::cell::RefCell;
+use core::ffi::c_void;
+
+use kernel::{
+ bindings::on_each_cpu,
+ cpu::CpuId,
+ define_per_cpu, get_static_per_cpu,
+ percpu::{cpu_guard::*, *},
+ pr_info,
+ prelude::*,
+};
+
+module! {
+ type: PerCpuMod,
+ name: "rust_percpu",
+ authors: ["Mitchell Levy"],
+ description: "Sample to demonstrate the Rust per-CPU API",
+ license: "GPL v2",
+}
+
+struct PerCpuMod;
+
+define_per_cpu!(PERCPU: i64 = 0);
+define_per_cpu!(UPERCPU: u64 = 0);
+define_per_cpu!(CHECKED: RefCell<u64> = RefCell::new(0));
+
+impl kernel::Module for PerCpuMod {
+ fn init(_module: &'static ThisModule) -> Result<Self, Error> {
+ pr_info!("rust percpu test start\n");
+
+ let mut native: i64 = 0;
+ let mut pcpu: StaticPerCpu<i64> = get_static_per_cpu!(PERCPU);
+
+ // SAFETY: We only have one PerCpu that points at PERCPU
+ unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut i64| {
+ pr_info!("The contents of pcpu are {}\n", val);
+
+ native += -1;
+ *val += -1;
+ pr_info!("Native: {}, *pcpu: {}\n", native, val);
+ assert!(native == *val && native == -1);
+
+ native += 1;
+ *val += 1;
+ pr_info!("Native: {}, *pcpu: {}\n", native, val);
+ assert!(native == *val && native == 0);
+ });
+
+ let mut unative: u64 = 0;
+ let mut upcpu: StaticPerCpu<u64> = get_static_per_cpu!(UPERCPU);
+
+ // SAFETY: We only have one PerCpu pointing at UPERCPU
+ unsafe { upcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| {
+ unative += 1;
+ *val += 1;
+ pr_info!("Unative: {}, *upcpu: {}\n", unative, val);
+ assert!(unative == *val && unative == 1);
+
+ unative = unative.wrapping_add((-1i64) as u64);
+ *val = val.wrapping_add((-1i64) as u64);
+ pr_info!("Unative: {}, *upcpu: {}\n", unative, val);
+ assert!(unative == *val && unative == 0);
+
+ unative = unative.wrapping_add((-1i64) as u64);
+ *val = val.wrapping_add((-1i64) as u64);
+ pr_info!("Unative: {}, *upcpu: {}\n", unative, val);
+ assert!(unative == *val && unative == (-1i64) as u64);
+
+ unative = 0;
+ *val = 0;
+
+ unative = unative.wrapping_sub(1);
+ *val = val.wrapping_sub(1);
+ pr_info!("Unative: {}, *upcpu: {}\n", unative, val);
+ assert!(unative == *val && unative == (-1i64) as u64);
+ assert!(unative == *val && unative == u64::MAX);
+ });
+
+ let mut checked_native: u64 = 0;
+ let mut checked: StaticPerCpu<RefCell<u64>> = get_static_per_cpu!(CHECKED);
+ checked.get(CpuGuard::new()).with(|val: &RefCell<u64>| {
+ checked_native += 1;
+ *val.borrow_mut() += 1;
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == 1);
+
+ checked_native = checked_native.wrapping_add((-1i64) as u64);
+ val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64));
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == 0);
+
+ checked_native = checked_native.wrapping_add((-1i64) as u64);
+ val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64));
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64);
+
+ checked_native = 0;
+ *val.borrow_mut() = 0;
+
+ checked_native = checked_native.wrapping_sub(1);
+ val.replace_with(|old: &mut u64| old.wrapping_sub(1));
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64);
+ assert!(checked_native == *val.borrow() && checked_native == u64::MAX);
+ });
+
+ pr_info!("rust static percpu test done\n");
+
+ pr_info!("rust dynamic percpu test start\n");
+ let mut test: DynamicPerCpu<u64> = DynamicPerCpu::new_zero(GFP_KERNEL).unwrap();
+
+ // SAFETY: No prerequisites for on_each_cpu.
+ unsafe {
+ on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0);
+ on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0);
+ on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0);
+ on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 1);
+ on_each_cpu(Some(check_percpu), (&raw mut test).cast(), 1);
+ }
+
+ pr_info!("rust dynamic percpu test done\n");
+
+ // Return Err to unload the module
+ Result::Err(EINVAL)
+ }
+}
+
+extern "C" fn inc_percpu(info: *mut c_void) {
+ // SAFETY: We know that info is a void *const DynamicPerCpu<u64> and DynamicPerCpu<u64> is Send.
+ let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<u64>)).clone() };
+ pr_info!("Incrementing on {}\n", CpuId::current().as_u32());
+
+ // SAFETY: We don't have multiple clones of pcpu in scope
+ unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| *val += 1);
+}
+
+extern "C" fn check_percpu(info: *mut c_void) {
+ // SAFETY: We know that info is a void *const DynamicPerCpu<u64> and DynamicPerCpu<u64> is Send.
+ let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<u64>)).clone() };
+ pr_info!("Asserting on {}\n", CpuId::current().as_u32());
+
+ // SAFETY: We don't have multiple clones of pcpu in scope
+ unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| assert!(*val == 4));
+}
--
2.34.1
^ permalink raw reply related [flat|nested] 26+ messages in thread* [PATCH v3 3/7] rust: cpumask: Add a `Cpumask` iterator
2025-08-28 19:00 [PATCH v3 0/7] rust: Add Per-CPU Variable API Mitchell Levy
2025-08-28 19:00 ` [PATCH v3 1/7] rust: percpu: introduce a rust API for per-CPU variables Mitchell Levy
2025-08-28 19:00 ` [PATCH v3 2/7] rust: percpu: add a rust per-CPU variable sample Mitchell Levy
@ 2025-08-28 19:00 ` Mitchell Levy
2025-08-29 5:19 ` Viresh Kumar
2025-08-28 19:00 ` [PATCH v3 4/7] rust: cpumask: Add getters for globally defined cpumasks Mitchell Levy
` (3 subsequent siblings)
6 siblings, 1 reply; 26+ messages in thread
From: Mitchell Levy @ 2025-08-28 19:00 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Yury Norov, Viresh Kumar
Cc: Tyler Hicks, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
Add an iterator for `Cpumask` making use of C's `cpumask_next`.
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/helpers/cpumask.c | 5 +++++
rust/kernel/cpumask.rs | 48 +++++++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 52 insertions(+), 1 deletion(-)
diff --git a/rust/helpers/cpumask.c b/rust/helpers/cpumask.c
index eb10598a0242..d95bfa111191 100644
--- a/rust/helpers/cpumask.c
+++ b/rust/helpers/cpumask.c
@@ -42,6 +42,11 @@ bool rust_helper_cpumask_full(struct cpumask *srcp)
return cpumask_full(srcp);
}
+unsigned int rust_helper_cpumask_next(int n, struct cpumask *srcp)
+{
+ return cpumask_next(n, srcp);
+}
+
unsigned int rust_helper_cpumask_weight(struct cpumask *srcp)
{
return cpumask_weight(srcp);
diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs
index 3fcbff438670..b7401848f59e 100644
--- a/rust/kernel/cpumask.rs
+++ b/rust/kernel/cpumask.rs
@@ -6,7 +6,7 @@
use crate::{
alloc::{AllocError, Flags},
- cpu::CpuId,
+ cpu::{self, CpuId},
prelude::*,
types::Opaque,
};
@@ -161,6 +161,52 @@ pub fn copy(&self, dstp: &mut Self) {
}
}
+/// Iterator for a `Cpumask`.
+pub struct CpumaskIter<'a> {
+ mask: &'a Cpumask,
+ last: Option<u32>,
+}
+
+impl<'a> CpumaskIter<'a> {
+ /// Creates a new `CpumaskIter` for the given `Cpumask`.
+ fn new(mask: &'a Cpumask) -> CpumaskIter<'a> {
+ Self { mask, last: None }
+ }
+}
+
+impl<'a> Iterator for CpumaskIter<'a> {
+ type Item = CpuId;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ // SAFETY: By the type invariant, `self.mask.as_raw` is a `struct cpumask *`.
+ let next = unsafe {
+ bindings::cpumask_next(
+ if let Some(last) = self.last {
+ last.try_into().unwrap()
+ } else {
+ -1
+ },
+ self.mask.as_raw(),
+ )
+ };
+
+ if next == cpu::nr_cpu_ids() {
+ None
+ } else {
+ self.last = Some(next);
+ // SAFETY: `cpumask_next` returns either `nr_cpu_ids` or a valid CPU ID.
+ unsafe { Some(CpuId::from_u32_unchecked(next)) }
+ }
+ }
+}
+
+impl Cpumask {
+ /// Returns an iterator over the set bits in the cpumask.
+ pub fn iter(&self) -> CpumaskIter<'_> {
+ CpumaskIter::new(self)
+ }
+}
+
/// A CPU Mask pointer.
///
/// Rust abstraction for the C `struct cpumask_var_t`.
--
2.34.1
^ permalink raw reply related [flat|nested] 26+ messages in thread* Re: [PATCH v3 3/7] rust: cpumask: Add a `Cpumask` iterator
2025-08-28 19:00 ` [PATCH v3 3/7] rust: cpumask: Add a `Cpumask` iterator Mitchell Levy
@ 2025-08-29 5:19 ` Viresh Kumar
0 siblings, 0 replies; 26+ messages in thread
From: Viresh Kumar @ 2025-08-29 5:19 UTC (permalink / raw)
To: Mitchell Levy
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Yury Norov, Tyler Hicks,
linux-kernel, rust-for-linux, linux-mm
On 28-08-25, 12:00, Mitchell Levy wrote:
> Add an iterator for `Cpumask` making use of C's `cpumask_next`.
>
> Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
> ---
> rust/helpers/cpumask.c | 5 +++++
> rust/kernel/cpumask.rs | 48 +++++++++++++++++++++++++++++++++++++++++++++++-
> 2 files changed, 52 insertions(+), 1 deletion(-)
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
--
viresh
^ permalink raw reply [flat|nested] 26+ messages in thread
* [PATCH v3 4/7] rust: cpumask: Add getters for globally defined cpumasks
2025-08-28 19:00 [PATCH v3 0/7] rust: Add Per-CPU Variable API Mitchell Levy
` (2 preceding siblings ...)
2025-08-28 19:00 ` [PATCH v3 3/7] rust: cpumask: Add a `Cpumask` iterator Mitchell Levy
@ 2025-08-28 19:00 ` Mitchell Levy
2025-08-29 5:20 ` Viresh Kumar
2025-09-03 22:03 ` Yury Norov
2025-08-28 19:00 ` [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu Mitchell Levy
` (2 subsequent siblings)
6 siblings, 2 replies; 26+ messages in thread
From: Mitchell Levy @ 2025-08-28 19:00 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Yury Norov, Viresh Kumar
Cc: Tyler Hicks, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
Add getters for the global cpumasks documented in
`include/linux/cpumask.h`, specifically:
- cpu_possible_mask
- cpu_online_mask
- cpu_enabled_mask
- cpu_present_mask
- cpu_active_mask
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/kernel/cpumask.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 46 insertions(+)
diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs
index b7401848f59e..ca9aa6875b4a 100644
--- a/rust/kernel/cpumask.rs
+++ b/rust/kernel/cpumask.rs
@@ -77,6 +77,52 @@ pub unsafe fn as_ref<'a>(ptr: *const bindings::cpumask) -> &'a Self {
unsafe { &*ptr.cast() }
}
+ /// Get a CPU mask representing possible CPUs; has bit `cpu` set iff cpu is populatable
+ #[inline]
+ pub fn possible() -> &'static Self {
+ // SAFETY: `__cpu_possible_mask` is a valid global provided by the kernel that lives
+ // forever.
+ unsafe { Cpumask::as_ref(&raw const bindings::__cpu_possible_mask) }
+ }
+
+ /// Get a CPU mask representing online CPUs; has bit `cpu` set iff cpu available to the
+ /// scheduler
+ #[inline]
+ pub fn online() -> &'static Self {
+ // SAFETY: `__cpu_online_mask` is a valid global provided by the kernel that lives forever.
+ // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_online_mask`
+ // may change its value.
+ unsafe { Cpumask::as_ref(&raw const bindings::__cpu_online_mask) }
+ }
+
+ /// Get a CPU mask representing enabled CPUs; has bit `cpu` set iff cpu can be brought online
+ #[inline]
+ pub fn enabled() -> &'static Self {
+ // SAFETY: `__cpu_enabled_mask` is a valid global provided by the kernel that lives forever.
+ // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_enabled_mask`
+ // may change its value.
+ unsafe { Cpumask::as_ref(&raw const bindings::__cpu_enabled_mask) }
+ }
+
+ /// Get a CPU mask representing present CPUs; has bit `cpu` set iff cpu is populated
+ #[inline]
+ pub fn present() -> &'static Self {
+ // SAFETY: `__cpu_present_mask` is a valid global provided by the kernel that lives
+ // forever. Since we wrap the returned pointer in an `Opaque`, it's ok that
+ // `__cpu_present_mask` may change its value.
+ unsafe { Cpumask::as_ref(&raw const bindings::__cpu_present_mask) }
+ }
+
+ /// Get a CPU mask representing active CPUs; has bit `cpu` set iff cpu is available to
+ /// migration.
+ #[inline]
+ pub fn active() -> &'static Self {
+ // SAFETY: `__cpu_active_mask` is a valid global provided by the kernel that lives forever.
+ // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_active_mask`
+ // may change its value.
+ unsafe { Cpumask::as_ref(&raw const bindings::__cpu_active_mask) }
+ }
+
/// Obtain the raw `struct cpumask` pointer.
pub fn as_raw(&self) -> *mut bindings::cpumask {
let this: *const Self = self;
--
2.34.1
^ permalink raw reply related [flat|nested] 26+ messages in thread* Re: [PATCH v3 4/7] rust: cpumask: Add getters for globally defined cpumasks
2025-08-28 19:00 ` [PATCH v3 4/7] rust: cpumask: Add getters for globally defined cpumasks Mitchell Levy
@ 2025-08-29 5:20 ` Viresh Kumar
2025-09-03 22:03 ` Yury Norov
1 sibling, 0 replies; 26+ messages in thread
From: Viresh Kumar @ 2025-08-29 5:20 UTC (permalink / raw)
To: Mitchell Levy
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Yury Norov, Tyler Hicks,
linux-kernel, rust-for-linux, linux-mm
On 28-08-25, 12:00, Mitchell Levy wrote:
> Add getters for the global cpumasks documented in
> `include/linux/cpumask.h`, specifically:
> - cpu_possible_mask
> - cpu_online_mask
> - cpu_enabled_mask
> - cpu_present_mask
> - cpu_active_mask
>
> Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
> ---
> rust/kernel/cpumask.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 46 insertions(+)
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
--
viresh
^ permalink raw reply [flat|nested] 26+ messages in thread
* Re: [PATCH v3 4/7] rust: cpumask: Add getters for globally defined cpumasks
2025-08-28 19:00 ` [PATCH v3 4/7] rust: cpumask: Add getters for globally defined cpumasks Mitchell Levy
2025-08-29 5:20 ` Viresh Kumar
@ 2025-09-03 22:03 ` Yury Norov
2025-09-04 19:55 ` Mitchell Levy
1 sibling, 1 reply; 26+ messages in thread
From: Yury Norov @ 2025-09-03 22:03 UTC (permalink / raw)
To: Mitchell Levy
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Viresh Kumar, Tyler Hicks,
linux-kernel, rust-for-linux, linux-mm
On Thu, Aug 28, 2025 at 12:00:11PM -0700, Mitchell Levy wrote:
> Add getters for the global cpumasks documented in
> `include/linux/cpumask.h`, specifically:
> - cpu_possible_mask
> - cpu_online_mask
> - cpu_enabled_mask
> - cpu_present_mask
> - cpu_active_mask
>
> Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
> ---
> rust/kernel/cpumask.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 46 insertions(+)
>
> diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs
> index b7401848f59e..ca9aa6875b4a 100644
> --- a/rust/kernel/cpumask.rs
> +++ b/rust/kernel/cpumask.rs
> @@ -77,6 +77,52 @@ pub unsafe fn as_ref<'a>(ptr: *const bindings::cpumask) -> &'a Self {
> unsafe { &*ptr.cast() }
> }
>
> + /// Get a CPU mask representing possible CPUs; has bit `cpu` set iff cpu is populatable
> + #[inline]
> + pub fn possible() -> &'static Self {
> + // SAFETY: `__cpu_possible_mask` is a valid global provided by the kernel that lives
> + // forever.
> + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_possible_mask) }
> + }
This function and the following look like returning a boolean.
Maybe possible_cpus()?
The general rule is to keep the rust naming as close to C as
possible, but in this case Cpumask::cpu_possible_mask is, OK,
somewhat excessive.
Thanks,
Yury
> +
> + /// Get a CPU mask representing online CPUs; has bit `cpu` set iff cpu available to the
> + /// scheduler
> + #[inline]
> + pub fn online() -> &'static Self {
> + // SAFETY: `__cpu_online_mask` is a valid global provided by the kernel that lives forever.
> + // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_online_mask`
> + // may change its value.
> + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_online_mask) }
> + }
> +
> + /// Get a CPU mask representing enabled CPUs; has bit `cpu` set iff cpu can be brought online
> + #[inline]
> + pub fn enabled() -> &'static Self {
> + // SAFETY: `__cpu_enabled_mask` is a valid global provided by the kernel that lives forever.
> + // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_enabled_mask`
> + // may change its value.
> + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_enabled_mask) }
> + }
> +
> + /// Get a CPU mask representing present CPUs; has bit `cpu` set iff cpu is populated
> + #[inline]
> + pub fn present() -> &'static Self {
> + // SAFETY: `__cpu_present_mask` is a valid global provided by the kernel that lives
> + // forever. Since we wrap the returned pointer in an `Opaque`, it's ok that
> + // `__cpu_present_mask` may change its value.
> + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_present_mask) }
> + }
> +
> + /// Get a CPU mask representing active CPUs; has bit `cpu` set iff cpu is available to
> + /// migration.
> + #[inline]
> + pub fn active() -> &'static Self {
> + // SAFETY: `__cpu_active_mask` is a valid global provided by the kernel that lives forever.
> + // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_active_mask`
> + // may change its value.
> + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_active_mask) }
> + }
> +
> /// Obtain the raw `struct cpumask` pointer.
> pub fn as_raw(&self) -> *mut bindings::cpumask {
> let this: *const Self = self;
>
> --
> 2.34.1
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH v3 4/7] rust: cpumask: Add getters for globally defined cpumasks
2025-09-03 22:03 ` Yury Norov
@ 2025-09-04 19:55 ` Mitchell Levy
0 siblings, 0 replies; 26+ messages in thread
From: Mitchell Levy @ 2025-09-04 19:55 UTC (permalink / raw)
To: Yury Norov
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Viresh Kumar, Tyler Hicks,
linux-kernel, rust-for-linux, linux-mm
On Wed, Sep 03, 2025 at 06:03:51PM -0400, Yury Norov wrote:
> On Thu, Aug 28, 2025 at 12:00:11PM -0700, Mitchell Levy wrote:
> > Add getters for the global cpumasks documented in
> > `include/linux/cpumask.h`, specifically:
> > - cpu_possible_mask
> > - cpu_online_mask
> > - cpu_enabled_mask
> > - cpu_present_mask
> > - cpu_active_mask
> >
> > Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
> > ---
> > rust/kernel/cpumask.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++++
> > 1 file changed, 46 insertions(+)
> >
> > diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs
> > index b7401848f59e..ca9aa6875b4a 100644
> > --- a/rust/kernel/cpumask.rs
> > +++ b/rust/kernel/cpumask.rs
> > @@ -77,6 +77,52 @@ pub unsafe fn as_ref<'a>(ptr: *const bindings::cpumask) -> &'a Self {
> > unsafe { &*ptr.cast() }
> > }
> >
> > + /// Get a CPU mask representing possible CPUs; has bit `cpu` set iff cpu is populatable
> > + #[inline]
> > + pub fn possible() -> &'static Self {
> > + // SAFETY: `__cpu_possible_mask` is a valid global provided by the kernel that lives
> > + // forever.
> > + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_possible_mask) }
> > + }
>
> This function and the following look like returning a boolean.
> Maybe possible_cpus()?
Agreed, I think that's definitely better. Will do for v4.
> The general rule is to keep the rust naming as close to C as
> possible, but in this case Cpumask::cpu_possible_mask is, OK,
> somewhat excessive.
Agreed :)
Thanks,
Mitchell
> Thanks,
> Yury
>
> > +
> > + /// Get a CPU mask representing online CPUs; has bit `cpu` set iff cpu available to the
> > + /// scheduler
> > + #[inline]
> > + pub fn online() -> &'static Self {
> > + // SAFETY: `__cpu_online_mask` is a valid global provided by the kernel that lives forever.
> > + // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_online_mask`
> > + // may change its value.
> > + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_online_mask) }
> > + }
> > +
> > + /// Get a CPU mask representing enabled CPUs; has bit `cpu` set iff cpu can be brought online
> > + #[inline]
> > + pub fn enabled() -> &'static Self {
> > + // SAFETY: `__cpu_enabled_mask` is a valid global provided by the kernel that lives forever.
> > + // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_enabled_mask`
> > + // may change its value.
> > + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_enabled_mask) }
> > + }
> > +
> > + /// Get a CPU mask representing present CPUs; has bit `cpu` set iff cpu is populated
> > + #[inline]
> > + pub fn present() -> &'static Self {
> > + // SAFETY: `__cpu_present_mask` is a valid global provided by the kernel that lives
> > + // forever. Since we wrap the returned pointer in an `Opaque`, it's ok that
> > + // `__cpu_present_mask` may change its value.
> > + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_present_mask) }
> > + }
> > +
> > + /// Get a CPU mask representing active CPUs; has bit `cpu` set iff cpu is available to
> > + /// migration.
> > + #[inline]
> > + pub fn active() -> &'static Self {
> > + // SAFETY: `__cpu_active_mask` is a valid global provided by the kernel that lives forever.
> > + // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_active_mask`
> > + // may change its value.
> > + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_active_mask) }
> > + }
> > +
> > /// Obtain the raw `struct cpumask` pointer.
> > pub fn as_raw(&self) -> *mut bindings::cpumask {
> > let this: *const Self = self;
> >
> > --
> > 2.34.1
^ permalink raw reply [flat|nested] 26+ messages in thread
* [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu
2025-08-28 19:00 [PATCH v3 0/7] rust: Add Per-CPU Variable API Mitchell Levy
` (3 preceding siblings ...)
2025-08-28 19:00 ` [PATCH v3 4/7] rust: cpumask: Add getters for globally defined cpumasks Mitchell Levy
@ 2025-08-28 19:00 ` Mitchell Levy
2025-09-03 22:19 ` Yury Norov
2025-09-03 23:05 ` Miguel Ojeda
2025-08-28 19:00 ` [PATCH v3 6/7] rust: percpu: Add pin-hole optimizations for numerics Mitchell Levy
2025-08-28 19:00 ` [PATCH v3 7/7] rust: percpu: cache per-CPU pointers in the dynamic case Mitchell Levy
6 siblings, 2 replies; 26+ messages in thread
From: Mitchell Levy @ 2025-08-28 19:00 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Yury Norov, Viresh Kumar
Cc: Tyler Hicks, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
Add functionality to `PerCpuPtr` to compute pointers to per-CPU variable
slots on other CPUs. Use this facility to initialize per-CPU variables
on all possible CPUs when a dynamic per-CPU variable is created with a
non-zeroable type. Since `RefCell` and other `Cell`-like types fall into
this category, `impl CheckedPerCpu` on `DynamicPerCpu` for these
`InteriorMutable` types since they can now be used. Add examples of
these usages to `samples/rust/rust_percpu.rs`.
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/helpers/percpu.c | 5 +++
rust/kernel/percpu.rs | 15 +++++++
rust/kernel/percpu/dynamic.rs | 40 +++++++++++++++++
samples/rust/rust_percpu.rs | 99 ++++++++++++++++++++++++++++++++++++++++---
4 files changed, 152 insertions(+), 7 deletions(-)
diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c
index 8cc01d094752..8d83b6b86106 100644
--- a/rust/helpers/percpu.c
+++ b/rust/helpers/percpu.c
@@ -8,6 +8,11 @@ void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align)
return __alloc_percpu(sz, align);
}
+void *rust_helper_per_cpu_ptr(void __percpu *ptr, unsigned int cpu)
+{
+ return per_cpu_ptr(ptr, cpu);
+}
+
void rust_helper_on_each_cpu(smp_call_func_t func, void *info, int wait)
{
on_each_cpu(func, info, wait);
diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs
index 35afcdba3ccd..c68c7520b67f 100644
--- a/rust/kernel/percpu.rs
+++ b/rust/kernel/percpu.rs
@@ -14,6 +14,7 @@
use bindings::{alloc_percpu, free_percpu};
use crate::alloc::Flags;
+use crate::cpu::CpuId;
use crate::percpu::cpu_guard::CpuGuard;
use crate::prelude::*;
use crate::sync::Arc;
@@ -115,6 +116,20 @@ pub fn get_ptr(&self) -> *mut MaybeUninit<T> {
// the invariant that self.0 is a valid offset into the per-CPU area.
(this_cpu_area).wrapping_add(self.0 as usize).cast()
}
+
+ /// Get a `*mut MaybeUninit<T>` to the per-CPU variable on the CPU represented by `cpu`. Note
+ /// that without some kind of synchronization, use of the returned pointer may cause a data
+ /// race. It is the caller's responsibility to use the returned pointer in a reasonable way.
+ ///
+ /// # Safety
+ /// - The returned pointer is valid only if `self` is (that is, it points to a live allocation
+ /// correctly sized and aligned to hold a `T`)
+ /// - The returned pointer is valid only if the bit corresponding to `cpu` is set in
+ /// `Cpumask::possible()`.
+ pub unsafe fn get_remote_ptr(&self, cpu: CpuId) -> *mut MaybeUninit<T> {
+ // SAFETY: The requirements of this function ensure this call is safe.
+ unsafe { bindings::per_cpu_ptr(self.0.cast(), cpu.as_u32()) }.cast()
+ }
}
// SAFETY: Sending a `PerCpuPtr<T>` to another thread is safe because as soon as it's sent, the
diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs
index ce95e420f943..64f04cef3705 100644
--- a/rust/kernel/percpu/dynamic.rs
+++ b/rust/kernel/percpu/dynamic.rs
@@ -3,6 +3,8 @@
use super::*;
+use crate::cpumask::Cpumask;
+
/// Represents a dynamic allocation of a per-CPU variable via alloc_percpu. Calls free_percpu when
/// dropped.
pub struct PerCpuAllocation<T>(PerCpuPtr<T>);
@@ -74,6 +76,36 @@ pub fn new_zero(flags: Flags) -> Option<Self> {
}
}
+impl<T: Clone> DynamicPerCpu<T> {
+ /// Allocates a new per-CPU variable
+ ///
+ /// # Arguments
+ /// * `val` - The initial value of the per-CPU variable on all CPUs.
+ /// * `flags` - Flags used to allocate an `Arc` that keeps track of the underlying
+ /// `PerCpuAllocation`.
+ pub fn new_with(val: T, flags: Flags) -> Option<Self> {
+ let alloc: PerCpuAllocation<T> = PerCpuAllocation::new_uninit()?;
+ let ptr = alloc.0;
+
+ for cpu in Cpumask::possible().iter() {
+ // SAFETY: `ptr` is a valid allocation, and `cpu` appears in `Cpumask::possible()`
+ let remote_ptr = unsafe { ptr.get_remote_ptr(cpu) };
+ // SAFETY: Each CPU's slot corresponding to `ptr` is currently uninitialized, and no
+ // one else has a reference to it. Therefore, we can freely write to it without
+ // worrying about the need to drop what was there or whether we're racing with someone
+ // else. `PerCpuPtr::get_remote_ptr` guarantees that the pointer is valid since we
+ // derived it from a valid allocation and `cpu`.
+ unsafe {
+ (*remote_ptr).write(val.clone());
+ }
+ }
+
+ let arc = Arc::new(alloc, flags).ok()?;
+
+ Some(Self { alloc: arc })
+ }
+}
+
impl<T> PerCpu<T> for DynamicPerCpu<T> {
unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> {
// SAFETY: The requirements of `PerCpu::get_mut` and this type's invariant ensure that the
@@ -81,3 +113,11 @@ unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> {
unsafe { PerCpuToken::new(guard, &self.alloc.0) }
}
}
+
+impl<T: InteriorMutable> CheckedPerCpu<T> for DynamicPerCpu<T> {
+ fn get(&mut self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T> {
+ // SAFETY: By the invariant of `DynamicPerCpu`, the memory location in each CPU's
+ // per-CPU area corresponding to this variable has been initialized.
+ unsafe { CheckedPerCpuToken::new(guard, &self.alloc.0) }
+ }
+}
diff --git a/samples/rust/rust_percpu.rs b/samples/rust/rust_percpu.rs
index 98ca1c781b6b..06b322019134 100644
--- a/samples/rust/rust_percpu.rs
+++ b/samples/rust/rust_percpu.rs
@@ -130,13 +130,72 @@ fn init(_module: &'static ThisModule) -> Result<Self, Error> {
// SAFETY: No prerequisites for on_each_cpu.
unsafe {
- on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0);
- on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0);
- on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0);
- on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 1);
- on_each_cpu(Some(check_percpu), (&raw mut test).cast(), 1);
+ on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0);
+ on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0);
+ on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0);
+ on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 1);
+ on_each_cpu(Some(check_percpu_u64), (&raw mut test).cast(), 1);
}
+ let mut checked: DynamicPerCpu<RefCell<u64>> =
+ DynamicPerCpu::new_with(RefCell::new(100), GFP_KERNEL).unwrap();
+
+ // SAFETY: No prerequisites for on_each_cpu.
+ unsafe {
+ on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0);
+ on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0);
+ on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0);
+ on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 1);
+ on_each_cpu(Some(check_percpu_refcell_u64), (&raw mut checked).cast(), 1);
+ }
+
+ checked.get(CpuGuard::new()).with(|val: &RefCell<u64>| {
+ assert!(*val.borrow() == 104);
+
+ let mut checked_native = 0;
+ *val.borrow_mut() = 0;
+
+ checked_native += 1;
+ *val.borrow_mut() += 1;
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == 1);
+
+ checked_native = checked_native.wrapping_add((-1i64) as u64);
+ val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64));
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == 0);
+
+ checked_native = checked_native.wrapping_add((-1i64) as u64);
+ val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64));
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64);
+
+ checked_native = 0;
+ *val.borrow_mut() = 0;
+
+ checked_native = checked_native.wrapping_sub(1);
+ val.replace_with(|old: &mut u64| old.wrapping_sub(1));
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64);
+ assert!(checked_native == *val.borrow() && checked_native == u64::MAX);
+ });
+
pr_info!("rust dynamic percpu test done\n");
// Return Err to unload the module
@@ -144,7 +203,7 @@ fn init(_module: &'static ThisModule) -> Result<Self, Error> {
}
}
-extern "C" fn inc_percpu(info: *mut c_void) {
+extern "C" fn inc_percpu_u64(info: *mut c_void) {
// SAFETY: We know that info is a void *const DynamicPerCpu<u64> and DynamicPerCpu<u64> is Send.
let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<u64>)).clone() };
pr_info!("Incrementing on {}\n", CpuId::current().as_u32());
@@ -153,7 +212,7 @@ extern "C" fn inc_percpu(info: *mut c_void) {
unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| *val += 1);
}
-extern "C" fn check_percpu(info: *mut c_void) {
+extern "C" fn check_percpu_u64(info: *mut c_void) {
// SAFETY: We know that info is a void *const DynamicPerCpu<u64> and DynamicPerCpu<u64> is Send.
let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<u64>)).clone() };
pr_info!("Asserting on {}\n", CpuId::current().as_u32());
@@ -161,3 +220,29 @@ extern "C" fn check_percpu(info: *mut c_void) {
// SAFETY: We don't have multiple clones of pcpu in scope
unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| assert!(*val == 4));
}
+
+extern "C" fn inc_percpu_refcell_u64(info: *mut c_void) {
+ // SAFETY: We know that info is a void *const DynamicPerCpu<RefCell<u64>> and
+ // DynamicPerCpu<RefCell<u64>> is Send.
+ let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<RefCell<u64>>)).clone() };
+ // SAFETY: smp_processor_id has no preconditions
+ pr_info!("Incrementing on {}\n", CpuId::current().as_u32());
+
+ pcpu.get(CpuGuard::new()).with(|val: &RefCell<u64>| {
+ let mut val = val.borrow_mut();
+ *val += 1;
+ });
+}
+
+extern "C" fn check_percpu_refcell_u64(info: *mut c_void) {
+ // SAFETY: We know that info is a void *const DynamicPerCpu<RefCell<u64>> and
+ // DynamicPerCpu<RefCell<u64>> is Send.
+ let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<RefCell<u64>>)).clone() };
+ // SAFETY: smp_processor_id has no preconditions
+ pr_info!("Asserting on {}\n", CpuId::current().as_u32());
+
+ pcpu.get(CpuGuard::new()).with(|val: &RefCell<u64>| {
+ let val = val.borrow();
+ assert!(*val == 104);
+ });
+}
--
2.34.1
^ permalink raw reply related [flat|nested] 26+ messages in thread* Re: [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu
2025-08-28 19:00 ` [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu Mitchell Levy
@ 2025-09-03 22:19 ` Yury Norov
2025-09-04 20:26 ` Mitchell Levy
2025-09-03 23:05 ` Miguel Ojeda
1 sibling, 1 reply; 26+ messages in thread
From: Yury Norov @ 2025-09-03 22:19 UTC (permalink / raw)
To: Mitchell Levy
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Viresh Kumar, Tyler Hicks,
linux-kernel, rust-for-linux, linux-mm
On Thu, Aug 28, 2025 at 12:00:12PM -0700, Mitchell Levy wrote:
> Add functionality to `PerCpuPtr` to compute pointers to per-CPU variable
> slots on other CPUs. Use this facility to initialize per-CPU variables
> on all possible CPUs when a dynamic per-CPU variable is created with a
> non-zeroable type. Since `RefCell` and other `Cell`-like types fall into
> this category, `impl CheckedPerCpu` on `DynamicPerCpu` for these
> `InteriorMutable` types since they can now be used. Add examples of
> these usages to `samples/rust/rust_percpu.rs`.
>
> Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
> ---
> rust/helpers/percpu.c | 5 +++
> rust/kernel/percpu.rs | 15 +++++++
> rust/kernel/percpu/dynamic.rs | 40 +++++++++++++++++
> samples/rust/rust_percpu.rs | 99 ++++++++++++++++++++++++++++++++++++++++---
> 4 files changed, 152 insertions(+), 7 deletions(-)
>
> diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c
> index 8cc01d094752..8d83b6b86106 100644
> --- a/rust/helpers/percpu.c
> +++ b/rust/helpers/percpu.c
> @@ -8,6 +8,11 @@ void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align)
> return __alloc_percpu(sz, align);
> }
>
> +void *rust_helper_per_cpu_ptr(void __percpu *ptr, unsigned int cpu)
> +{
> + return per_cpu_ptr(ptr, cpu);
> +}
> +
> void rust_helper_on_each_cpu(smp_call_func_t func, void *info, int wait)
> {
> on_each_cpu(func, info, wait);
> diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs
> index 35afcdba3ccd..c68c7520b67f 100644
> --- a/rust/kernel/percpu.rs
> +++ b/rust/kernel/percpu.rs
> @@ -14,6 +14,7 @@
> use bindings::{alloc_percpu, free_percpu};
>
> use crate::alloc::Flags;
> +use crate::cpu::CpuId;
> use crate::percpu::cpu_guard::CpuGuard;
> use crate::prelude::*;
> use crate::sync::Arc;
> @@ -115,6 +116,20 @@ pub fn get_ptr(&self) -> *mut MaybeUninit<T> {
> // the invariant that self.0 is a valid offset into the per-CPU area.
> (this_cpu_area).wrapping_add(self.0 as usize).cast()
> }
> +
> + /// Get a `*mut MaybeUninit<T>` to the per-CPU variable on the CPU represented by `cpu`. Note
> + /// that without some kind of synchronization, use of the returned pointer may cause a data
> + /// race. It is the caller's responsibility to use the returned pointer in a reasonable way.
> + ///
> + /// # Safety
> + /// - The returned pointer is valid only if `self` is (that is, it points to a live allocation
> + /// correctly sized and aligned to hold a `T`)
> + /// - The returned pointer is valid only if the bit corresponding to `cpu` is set in
> + /// `Cpumask::possible()`.
Instead of explaining those rules in comments, can you just enforce
them in code? Not sure about the 1st rule, but the 2nd one looks like
a trivial check.
> + pub unsafe fn get_remote_ptr(&self, cpu: CpuId) -> *mut MaybeUninit<T> {
> + // SAFETY: The requirements of this function ensure this call is safe.
> + unsafe { bindings::per_cpu_ptr(self.0.cast(), cpu.as_u32()) }.cast()
> + }
> }
>
> // SAFETY: Sending a `PerCpuPtr<T>` to another thread is safe because as soon as it's sent, the
> diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs
> index ce95e420f943..64f04cef3705 100644
> --- a/rust/kernel/percpu/dynamic.rs
> +++ b/rust/kernel/percpu/dynamic.rs
> @@ -3,6 +3,8 @@
>
> use super::*;
>
> +use crate::cpumask::Cpumask;
> +
> /// Represents a dynamic allocation of a per-CPU variable via alloc_percpu. Calls free_percpu when
> /// dropped.
> pub struct PerCpuAllocation<T>(PerCpuPtr<T>);
> @@ -74,6 +76,36 @@ pub fn new_zero(flags: Flags) -> Option<Self> {
> }
> }
>
> +impl<T: Clone> DynamicPerCpu<T> {
> + /// Allocates a new per-CPU variable
> + ///
> + /// # Arguments
> + /// * `val` - The initial value of the per-CPU variable on all CPUs.
> + /// * `flags` - Flags used to allocate an `Arc` that keeps track of the underlying
> + /// `PerCpuAllocation`.
> + pub fn new_with(val: T, flags: Flags) -> Option<Self> {
> + let alloc: PerCpuAllocation<T> = PerCpuAllocation::new_uninit()?;
> + let ptr = alloc.0;
> +
> + for cpu in Cpumask::possible().iter() {
In C we've got the 'for_each_possible_cpu()'. Is there any way to
preserve that semantics in rust? I really believe that similar
semantics on higher level on both sides will help _a_lot_ for those
transitioning into the rust world (like me).
Thanks,
Yury
> + // SAFETY: `ptr` is a valid allocation, and `cpu` appears in `Cpumask::possible()`
> + let remote_ptr = unsafe { ptr.get_remote_ptr(cpu) };
> + // SAFETY: Each CPU's slot corresponding to `ptr` is currently uninitialized, and no
> + // one else has a reference to it. Therefore, we can freely write to it without
> + // worrying about the need to drop what was there or whether we're racing with someone
> + // else. `PerCpuPtr::get_remote_ptr` guarantees that the pointer is valid since we
> + // derived it from a valid allocation and `cpu`.
> + unsafe {
> + (*remote_ptr).write(val.clone());
> + }
> + }
> +
> + let arc = Arc::new(alloc, flags).ok()?;
> +
> + Some(Self { alloc: arc })
> + }
> +}
> +
> impl<T> PerCpu<T> for DynamicPerCpu<T> {
> unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> {
> // SAFETY: The requirements of `PerCpu::get_mut` and this type's invariant ensure that the
> @@ -81,3 +113,11 @@ unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> {
> unsafe { PerCpuToken::new(guard, &self.alloc.0) }
> }
> }
> +
> +impl<T: InteriorMutable> CheckedPerCpu<T> for DynamicPerCpu<T> {
> + fn get(&mut self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T> {
> + // SAFETY: By the invariant of `DynamicPerCpu`, the memory location in each CPU's
> + // per-CPU area corresponding to this variable has been initialized.
> + unsafe { CheckedPerCpuToken::new(guard, &self.alloc.0) }
> + }
> +}
> diff --git a/samples/rust/rust_percpu.rs b/samples/rust/rust_percpu.rs
> index 98ca1c781b6b..06b322019134 100644
> --- a/samples/rust/rust_percpu.rs
> +++ b/samples/rust/rust_percpu.rs
> @@ -130,13 +130,72 @@ fn init(_module: &'static ThisModule) -> Result<Self, Error> {
>
> // SAFETY: No prerequisites for on_each_cpu.
> unsafe {
> - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0);
> - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0);
> - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0);
> - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 1);
> - on_each_cpu(Some(check_percpu), (&raw mut test).cast(), 1);
> + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0);
> + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0);
> + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0);
> + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 1);
> + on_each_cpu(Some(check_percpu_u64), (&raw mut test).cast(), 1);
> }
>
> + let mut checked: DynamicPerCpu<RefCell<u64>> =
> + DynamicPerCpu::new_with(RefCell::new(100), GFP_KERNEL).unwrap();
> +
> + // SAFETY: No prerequisites for on_each_cpu.
> + unsafe {
> + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0);
> + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0);
> + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0);
> + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 1);
> + on_each_cpu(Some(check_percpu_refcell_u64), (&raw mut checked).cast(), 1);
> + }
> +
> + checked.get(CpuGuard::new()).with(|val: &RefCell<u64>| {
> + assert!(*val.borrow() == 104);
> +
> + let mut checked_native = 0;
> + *val.borrow_mut() = 0;
> +
> + checked_native += 1;
> + *val.borrow_mut() += 1;
> + pr_info!(
> + "Checked native: {}, *checked: {}\n",
> + checked_native,
> + val.borrow()
> + );
> + assert!(checked_native == *val.borrow() && checked_native == 1);
> +
> + checked_native = checked_native.wrapping_add((-1i64) as u64);
> + val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64));
> + pr_info!(
> + "Checked native: {}, *checked: {}\n",
> + checked_native,
> + val.borrow()
> + );
> + assert!(checked_native == *val.borrow() && checked_native == 0);
> +
> + checked_native = checked_native.wrapping_add((-1i64) as u64);
> + val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64));
> + pr_info!(
> + "Checked native: {}, *checked: {}\n",
> + checked_native,
> + val.borrow()
> + );
> + assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64);
> +
> + checked_native = 0;
> + *val.borrow_mut() = 0;
> +
> + checked_native = checked_native.wrapping_sub(1);
> + val.replace_with(|old: &mut u64| old.wrapping_sub(1));
> + pr_info!(
> + "Checked native: {}, *checked: {}\n",
> + checked_native,
> + val.borrow()
> + );
> + assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64);
> + assert!(checked_native == *val.borrow() && checked_native == u64::MAX);
> + });
> +
> pr_info!("rust dynamic percpu test done\n");
>
> // Return Err to unload the module
> @@ -144,7 +203,7 @@ fn init(_module: &'static ThisModule) -> Result<Self, Error> {
> }
> }
>
> -extern "C" fn inc_percpu(info: *mut c_void) {
> +extern "C" fn inc_percpu_u64(info: *mut c_void) {
> // SAFETY: We know that info is a void *const DynamicPerCpu<u64> and DynamicPerCpu<u64> is Send.
> let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<u64>)).clone() };
> pr_info!("Incrementing on {}\n", CpuId::current().as_u32());
> @@ -153,7 +212,7 @@ extern "C" fn inc_percpu(info: *mut c_void) {
> unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| *val += 1);
> }
>
> -extern "C" fn check_percpu(info: *mut c_void) {
> +extern "C" fn check_percpu_u64(info: *mut c_void) {
> // SAFETY: We know that info is a void *const DynamicPerCpu<u64> and DynamicPerCpu<u64> is Send.
> let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<u64>)).clone() };
> pr_info!("Asserting on {}\n", CpuId::current().as_u32());
> @@ -161,3 +220,29 @@ extern "C" fn check_percpu(info: *mut c_void) {
> // SAFETY: We don't have multiple clones of pcpu in scope
> unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| assert!(*val == 4));
> }
> +
> +extern "C" fn inc_percpu_refcell_u64(info: *mut c_void) {
> + // SAFETY: We know that info is a void *const DynamicPerCpu<RefCell<u64>> and
> + // DynamicPerCpu<RefCell<u64>> is Send.
> + let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<RefCell<u64>>)).clone() };
> + // SAFETY: smp_processor_id has no preconditions
> + pr_info!("Incrementing on {}\n", CpuId::current().as_u32());
> +
> + pcpu.get(CpuGuard::new()).with(|val: &RefCell<u64>| {
> + let mut val = val.borrow_mut();
> + *val += 1;
> + });
> +}
> +
> +extern "C" fn check_percpu_refcell_u64(info: *mut c_void) {
> + // SAFETY: We know that info is a void *const DynamicPerCpu<RefCell<u64>> and
> + // DynamicPerCpu<RefCell<u64>> is Send.
> + let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<RefCell<u64>>)).clone() };
> + // SAFETY: smp_processor_id has no preconditions
> + pr_info!("Asserting on {}\n", CpuId::current().as_u32());
> +
> + pcpu.get(CpuGuard::new()).with(|val: &RefCell<u64>| {
> + let val = val.borrow();
> + assert!(*val == 104);
> + });
> +}
>
> --
> 2.34.1
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu
2025-09-03 22:19 ` Yury Norov
@ 2025-09-04 20:26 ` Mitchell Levy
2025-09-04 20:37 ` Yury Norov
0 siblings, 1 reply; 26+ messages in thread
From: Mitchell Levy @ 2025-09-04 20:26 UTC (permalink / raw)
To: Yury Norov
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Viresh Kumar, Tyler Hicks,
linux-kernel, rust-for-linux, linux-mm
On Wed, Sep 03, 2025 at 06:19:25PM -0400, Yury Norov wrote:
> On Thu, Aug 28, 2025 at 12:00:12PM -0700, Mitchell Levy wrote:
> > Add functionality to `PerCpuPtr` to compute pointers to per-CPU variable
> > slots on other CPUs. Use this facility to initialize per-CPU variables
> > on all possible CPUs when a dynamic per-CPU variable is created with a
> > non-zeroable type. Since `RefCell` and other `Cell`-like types fall into
> > this category, `impl CheckedPerCpu` on `DynamicPerCpu` for these
> > `InteriorMutable` types since they can now be used. Add examples of
> > these usages to `samples/rust/rust_percpu.rs`.
> >
> > Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
> > ---
> > rust/helpers/percpu.c | 5 +++
> > rust/kernel/percpu.rs | 15 +++++++
> > rust/kernel/percpu/dynamic.rs | 40 +++++++++++++++++
> > samples/rust/rust_percpu.rs | 99 ++++++++++++++++++++++++++++++++++++++++---
> > 4 files changed, 152 insertions(+), 7 deletions(-)
> >
> > diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c
> > index 8cc01d094752..8d83b6b86106 100644
> > --- a/rust/helpers/percpu.c
> > +++ b/rust/helpers/percpu.c
> > @@ -8,6 +8,11 @@ void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align)
> > return __alloc_percpu(sz, align);
> > }
> >
> > +void *rust_helper_per_cpu_ptr(void __percpu *ptr, unsigned int cpu)
> > +{
> > + return per_cpu_ptr(ptr, cpu);
> > +}
> > +
> > void rust_helper_on_each_cpu(smp_call_func_t func, void *info, int wait)
> > {
> > on_each_cpu(func, info, wait);
> > diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs
> > index 35afcdba3ccd..c68c7520b67f 100644
> > --- a/rust/kernel/percpu.rs
> > +++ b/rust/kernel/percpu.rs
> > @@ -14,6 +14,7 @@
> > use bindings::{alloc_percpu, free_percpu};
> >
> > use crate::alloc::Flags;
> > +use crate::cpu::CpuId;
> > use crate::percpu::cpu_guard::CpuGuard;
> > use crate::prelude::*;
> > use crate::sync::Arc;
> > @@ -115,6 +116,20 @@ pub fn get_ptr(&self) -> *mut MaybeUninit<T> {
> > // the invariant that self.0 is a valid offset into the per-CPU area.
> > (this_cpu_area).wrapping_add(self.0 as usize).cast()
> > }
> > +
> > + /// Get a `*mut MaybeUninit<T>` to the per-CPU variable on the CPU represented by `cpu`. Note
> > + /// that without some kind of synchronization, use of the returned pointer may cause a data
> > + /// race. It is the caller's responsibility to use the returned pointer in a reasonable way.
> > + ///
> > + /// # Safety
> > + /// - The returned pointer is valid only if `self` is (that is, it points to a live allocation
> > + /// correctly sized and aligned to hold a `T`)
> > + /// - The returned pointer is valid only if the bit corresponding to `cpu` is set in
> > + /// `Cpumask::possible()`.
>
> Instead of explaining those rules in comments, can you just enforce
> them in code? Not sure about the 1st rule, but the 2nd one looks like
> a trivial check.
I don't think the first one is possible to check (at least, certainly
not without peeking deeply into the per-CPU allocation management
system).
For the second, I will probably be converting this function to a safe
function (see my reply to Miguel), with these being requirements for the
returned pointer to be valid, so panicking when the requirements aren't
met would probably not be expected behavior.
> > + pub unsafe fn get_remote_ptr(&self, cpu: CpuId) -> *mut MaybeUninit<T> {
> > + // SAFETY: The requirements of this function ensure this call is safe.
> > + unsafe { bindings::per_cpu_ptr(self.0.cast(), cpu.as_u32()) }.cast()
> > + }
> > }
> >
> > // SAFETY: Sending a `PerCpuPtr<T>` to another thread is safe because as soon as it's sent, the
> > diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs
> > index ce95e420f943..64f04cef3705 100644
> > --- a/rust/kernel/percpu/dynamic.rs
> > +++ b/rust/kernel/percpu/dynamic.rs
> > @@ -3,6 +3,8 @@
> >
> > use super::*;
> >
> > +use crate::cpumask::Cpumask;
> > +
> > /// Represents a dynamic allocation of a per-CPU variable via alloc_percpu. Calls free_percpu when
> > /// dropped.
> > pub struct PerCpuAllocation<T>(PerCpuPtr<T>);
> > @@ -74,6 +76,36 @@ pub fn new_zero(flags: Flags) -> Option<Self> {
> > }
> > }
> >
> > +impl<T: Clone> DynamicPerCpu<T> {
> > + /// Allocates a new per-CPU variable
> > + ///
> > + /// # Arguments
> > + /// * `val` - The initial value of the per-CPU variable on all CPUs.
> > + /// * `flags` - Flags used to allocate an `Arc` that keeps track of the underlying
> > + /// `PerCpuAllocation`.
> > + pub fn new_with(val: T, flags: Flags) -> Option<Self> {
> > + let alloc: PerCpuAllocation<T> = PerCpuAllocation::new_uninit()?;
> > + let ptr = alloc.0;
> > +
> > + for cpu in Cpumask::possible().iter() {
>
> In C we've got the 'for_each_possible_cpu()'. Is there any way to
> preserve that semantics in rust? I really believe that similar
> semantics on higher level on both sides will help _a_lot_ for those
> transitioning into the rust world (like me).
I'm not sure I understand what you mean --- I believe the semantics
should be the same here (`cpu` takes on each value in
`cpu_possible_mask`). Could you please clarify?
Thanks,
Mitchell
> Thanks,
> Yury
>
> > + // SAFETY: `ptr` is a valid allocation, and `cpu` appears in `Cpumask::possible()`
> > + let remote_ptr = unsafe { ptr.get_remote_ptr(cpu) };
> > + // SAFETY: Each CPU's slot corresponding to `ptr` is currently uninitialized, and no
> > + // one else has a reference to it. Therefore, we can freely write to it without
> > + // worrying about the need to drop what was there or whether we're racing with someone
> > + // else. `PerCpuPtr::get_remote_ptr` guarantees that the pointer is valid since we
> > + // derived it from a valid allocation and `cpu`.
> > + unsafe {
> > + (*remote_ptr).write(val.clone());
> > + }
> > + }
> > +
> > + let arc = Arc::new(alloc, flags).ok()?;
> > +
> > + Some(Self { alloc: arc })
> > + }
> > +}
> > +
> > impl<T> PerCpu<T> for DynamicPerCpu<T> {
> > unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> {
> > // SAFETY: The requirements of `PerCpu::get_mut` and this type's invariant ensure that the
> > @@ -81,3 +113,11 @@ unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> {
> > unsafe { PerCpuToken::new(guard, &self.alloc.0) }
> > }
> > }
> > +
> > +impl<T: InteriorMutable> CheckedPerCpu<T> for DynamicPerCpu<T> {
> > + fn get(&mut self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T> {
> > + // SAFETY: By the invariant of `DynamicPerCpu`, the memory location in each CPU's
> > + // per-CPU area corresponding to this variable has been initialized.
> > + unsafe { CheckedPerCpuToken::new(guard, &self.alloc.0) }
> > + }
> > +}
> > diff --git a/samples/rust/rust_percpu.rs b/samples/rust/rust_percpu.rs
> > index 98ca1c781b6b..06b322019134 100644
> > --- a/samples/rust/rust_percpu.rs
> > +++ b/samples/rust/rust_percpu.rs
> > @@ -130,13 +130,72 @@ fn init(_module: &'static ThisModule) -> Result<Self, Error> {
> >
> > // SAFETY: No prerequisites for on_each_cpu.
> > unsafe {
> > - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0);
> > - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0);
> > - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0);
> > - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 1);
> > - on_each_cpu(Some(check_percpu), (&raw mut test).cast(), 1);
> > + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0);
> > + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0);
> > + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0);
> > + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 1);
> > + on_each_cpu(Some(check_percpu_u64), (&raw mut test).cast(), 1);
> > }
> >
> > + let mut checked: DynamicPerCpu<RefCell<u64>> =
> > + DynamicPerCpu::new_with(RefCell::new(100), GFP_KERNEL).unwrap();
> > +
> > + // SAFETY: No prerequisites for on_each_cpu.
> > + unsafe {
> > + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0);
> > + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0);
> > + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0);
> > + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 1);
> > + on_each_cpu(Some(check_percpu_refcell_u64), (&raw mut checked).cast(), 1);
> > + }
> > +
> > + checked.get(CpuGuard::new()).with(|val: &RefCell<u64>| {
> > + assert!(*val.borrow() == 104);
> > +
> > + let mut checked_native = 0;
> > + *val.borrow_mut() = 0;
> > +
> > + checked_native += 1;
> > + *val.borrow_mut() += 1;
> > + pr_info!(
> > + "Checked native: {}, *checked: {}\n",
> > + checked_native,
> > + val.borrow()
> > + );
> > + assert!(checked_native == *val.borrow() && checked_native == 1);
> > +
> > + checked_native = checked_native.wrapping_add((-1i64) as u64);
> > + val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64));
> > + pr_info!(
> > + "Checked native: {}, *checked: {}\n",
> > + checked_native,
> > + val.borrow()
> > + );
> > + assert!(checked_native == *val.borrow() && checked_native == 0);
> > +
> > + checked_native = checked_native.wrapping_add((-1i64) as u64);
> > + val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64));
> > + pr_info!(
> > + "Checked native: {}, *checked: {}\n",
> > + checked_native,
> > + val.borrow()
> > + );
> > + assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64);
> > +
> > + checked_native = 0;
> > + *val.borrow_mut() = 0;
> > +
> > + checked_native = checked_native.wrapping_sub(1);
> > + val.replace_with(|old: &mut u64| old.wrapping_sub(1));
> > + pr_info!(
> > + "Checked native: {}, *checked: {}\n",
> > + checked_native,
> > + val.borrow()
> > + );
> > + assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64);
> > + assert!(checked_native == *val.borrow() && checked_native == u64::MAX);
> > + });
> > +
> > pr_info!("rust dynamic percpu test done\n");
> >
> > // Return Err to unload the module
> > @@ -144,7 +203,7 @@ fn init(_module: &'static ThisModule) -> Result<Self, Error> {
> > }
> > }
> >
> > -extern "C" fn inc_percpu(info: *mut c_void) {
> > +extern "C" fn inc_percpu_u64(info: *mut c_void) {
> > // SAFETY: We know that info is a void *const DynamicPerCpu<u64> and DynamicPerCpu<u64> is Send.
> > let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<u64>)).clone() };
> > pr_info!("Incrementing on {}\n", CpuId::current().as_u32());
> > @@ -153,7 +212,7 @@ extern "C" fn inc_percpu(info: *mut c_void) {
> > unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| *val += 1);
> > }
> >
> > -extern "C" fn check_percpu(info: *mut c_void) {
> > +extern "C" fn check_percpu_u64(info: *mut c_void) {
> > // SAFETY: We know that info is a void *const DynamicPerCpu<u64> and DynamicPerCpu<u64> is Send.
> > let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<u64>)).clone() };
> > pr_info!("Asserting on {}\n", CpuId::current().as_u32());
> > @@ -161,3 +220,29 @@ extern "C" fn check_percpu(info: *mut c_void) {
> > // SAFETY: We don't have multiple clones of pcpu in scope
> > unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| assert!(*val == 4));
> > }
> > +
> > +extern "C" fn inc_percpu_refcell_u64(info: *mut c_void) {
> > + // SAFETY: We know that info is a void *const DynamicPerCpu<RefCell<u64>> and
> > + // DynamicPerCpu<RefCell<u64>> is Send.
> > + let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<RefCell<u64>>)).clone() };
> > + // SAFETY: smp_processor_id has no preconditions
> > + pr_info!("Incrementing on {}\n", CpuId::current().as_u32());
> > +
> > + pcpu.get(CpuGuard::new()).with(|val: &RefCell<u64>| {
> > + let mut val = val.borrow_mut();
> > + *val += 1;
> > + });
> > +}
> > +
> > +extern "C" fn check_percpu_refcell_u64(info: *mut c_void) {
> > + // SAFETY: We know that info is a void *const DynamicPerCpu<RefCell<u64>> and
> > + // DynamicPerCpu<RefCell<u64>> is Send.
> > + let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<RefCell<u64>>)).clone() };
> > + // SAFETY: smp_processor_id has no preconditions
> > + pr_info!("Asserting on {}\n", CpuId::current().as_u32());
> > +
> > + pcpu.get(CpuGuard::new()).with(|val: &RefCell<u64>| {
> > + let val = val.borrow();
> > + assert!(*val == 104);
> > + });
> > +}
> >
> > --
> > 2.34.1
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu
2025-09-04 20:26 ` Mitchell Levy
@ 2025-09-04 20:37 ` Yury Norov
2025-09-04 21:05 ` Mitchell Levy
0 siblings, 1 reply; 26+ messages in thread
From: Yury Norov @ 2025-09-04 20:37 UTC (permalink / raw)
To: Mitchell Levy
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Viresh Kumar, Tyler Hicks,
linux-kernel, rust-for-linux, linux-mm
On Thu, Sep 04, 2025 at 01:26:07PM -0700, Mitchell Levy wrote:
> On Wed, Sep 03, 2025 at 06:19:25PM -0400, Yury Norov wrote:
> > On Thu, Aug 28, 2025 at 12:00:12PM -0700, Mitchell Levy wrote:
...
> > > +impl<T: Clone> DynamicPerCpu<T> {
> > > + /// Allocates a new per-CPU variable
> > > + ///
> > > + /// # Arguments
> > > + /// * `val` - The initial value of the per-CPU variable on all CPUs.
> > > + /// * `flags` - Flags used to allocate an `Arc` that keeps track of the underlying
> > > + /// `PerCpuAllocation`.
> > > + pub fn new_with(val: T, flags: Flags) -> Option<Self> {
> > > + let alloc: PerCpuAllocation<T> = PerCpuAllocation::new_uninit()?;
> > > + let ptr = alloc.0;
> > > +
> > > + for cpu in Cpumask::possible().iter() {
> >
> > In C we've got the 'for_each_possible_cpu()'. Is there any way to
> > preserve that semantics in rust? I really believe that similar
> > semantics on higher level on both sides will help _a_lot_ for those
> > transitioning into the rust world (like me).
>
> I'm not sure I understand what you mean --- I believe the semantics
> should be the same here (`cpu` takes on each value in
> `cpu_possible_mask`). Could you please clarify?
>
I mean:
for_each_possible_cpu(cpu) {
let remote_ptr = unsafe { ptr.get_remote_ptr(cpu) };
unsafe { (*remote_ptr).write(val.clone()); }
let arc = Arc::new(alloc, flags).ok()?;
Some(Self { alloc: arc })
}
Is it possible to do the above in rust?
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu
2025-09-04 20:37 ` Yury Norov
@ 2025-09-04 21:05 ` Mitchell Levy
2025-09-04 21:46 ` Yury Norov
2025-09-04 21:57 ` Miguel Ojeda
0 siblings, 2 replies; 26+ messages in thread
From: Mitchell Levy @ 2025-09-04 21:05 UTC (permalink / raw)
To: Yury Norov
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Viresh Kumar, Tyler Hicks,
linux-kernel, rust-for-linux, linux-mm
On Thu, Sep 04, 2025 at 04:37:02PM -0400, Yury Norov wrote:
> On Thu, Sep 04, 2025 at 01:26:07PM -0700, Mitchell Levy wrote:
> > On Wed, Sep 03, 2025 at 06:19:25PM -0400, Yury Norov wrote:
> > > On Thu, Aug 28, 2025 at 12:00:12PM -0700, Mitchell Levy wrote:
>
> ...
>
> > > > +impl<T: Clone> DynamicPerCpu<T> {
> > > > + /// Allocates a new per-CPU variable
> > > > + ///
> > > > + /// # Arguments
> > > > + /// * `val` - The initial value of the per-CPU variable on all CPUs.
> > > > + /// * `flags` - Flags used to allocate an `Arc` that keeps track of the underlying
> > > > + /// `PerCpuAllocation`.
> > > > + pub fn new_with(val: T, flags: Flags) -> Option<Self> {
> > > > + let alloc: PerCpuAllocation<T> = PerCpuAllocation::new_uninit()?;
> > > > + let ptr = alloc.0;
> > > > +
> > > > + for cpu in Cpumask::possible().iter() {
> > >
> > > In C we've got the 'for_each_possible_cpu()'. Is there any way to
> > > preserve that semantics in rust? I really believe that similar
> > > semantics on higher level on both sides will help _a_lot_ for those
> > > transitioning into the rust world (like me).
> >
> > I'm not sure I understand what you mean --- I believe the semantics
> > should be the same here (`cpu` takes on each value in
> > `cpu_possible_mask`). Could you please clarify?
> >
>
> I mean:
>
> for_each_possible_cpu(cpu) {
> let remote_ptr = unsafe { ptr.get_remote_ptr(cpu) };
> unsafe { (*remote_ptr).write(val.clone()); }
> let arc = Arc::new(alloc, flags).ok()?;
> Some(Self { alloc: arc })
> }
>
> Is it possible to do the above in rust?
Ah, I see.
The syntax would be slightly different, probably something like
use cpu::for_each_possible_cpu;
for_each_possible_cpu(|&cpu| {
let remote_ptr = unsafe { ptr.get_remote_ptr(cpu) };
// ...
})
it *might* also be possible to use a macro and dispense with the need for
a closure, though I'm not familiar enough with proc macros to say for
sure. That would probably look like
for_each_possible_cpu!(cpu) {
let remote_ptr = unsafe { ptr.get_remote_ptr(cpu) };
// ...
}
though personally I think the first one is better (simpler
implementation without too much syntactic overhead, especially since
closures are already used some within R4L).
Thanks,
Mitchell
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu
2025-09-04 21:05 ` Mitchell Levy
@ 2025-09-04 21:46 ` Yury Norov
2025-09-04 21:57 ` Miguel Ojeda
1 sibling, 0 replies; 26+ messages in thread
From: Yury Norov @ 2025-09-04 21:46 UTC (permalink / raw)
To: Mitchell Levy
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Viresh Kumar, Tyler Hicks,
linux-kernel, rust-for-linux, linux-mm
> >
> > for_each_possible_cpu(cpu) {
> > let remote_ptr = unsafe { ptr.get_remote_ptr(cpu) };
> > unsafe { (*remote_ptr).write(val.clone()); }
> > let arc = Arc::new(alloc, flags).ok()?;
> > Some(Self { alloc: arc })
> > }
> >
> > Is it possible to do the above in rust?
>
> Ah, I see.
>
> The syntax would be slightly different, probably something like
>
> use cpu::for_each_possible_cpu;
>
> for_each_possible_cpu(|&cpu| {
> let remote_ptr = unsafe { ptr.get_remote_ptr(cpu) };
> // ...
> })
>
> it *might* also be possible to use a macro and dispense with the need for
> a closure, though I'm not familiar enough with proc macros to say for
> sure. That would probably look like
>
> for_each_possible_cpu!(cpu) {
> let remote_ptr = unsafe { ptr.get_remote_ptr(cpu) };
> // ...
> }
>
> though personally I think the first one is better (simpler
> implementation without too much syntactic overhead, especially since
> closures are already used some within R4L).
Sure, #1 is OK if you prefer it.
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu
2025-09-04 21:05 ` Mitchell Levy
2025-09-04 21:46 ` Yury Norov
@ 2025-09-04 21:57 ` Miguel Ojeda
1 sibling, 0 replies; 26+ messages in thread
From: Miguel Ojeda @ 2025-09-04 21:57 UTC (permalink / raw)
To: Mitchell Levy
Cc: Yury Norov, Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Viresh Kumar, Tyler Hicks,
linux-kernel, rust-for-linux, linux-mm
On Thu, Sep 4, 2025 at 11:05 PM Mitchell Levy <levymitchell0@gmail.com> wrote:
>
> it *might* also be possible to use a macro and dispense with the need for
> a closure, though I'm not familiar enough with proc macros to say for
> sure. That would probably look like
>
> for_each_possible_cpu!(cpu) {
> let remote_ptr = unsafe { ptr.get_remote_ptr(cpu) };
> // ...
> }
>
> though personally I think the first one is better (simpler
> implementation without too much syntactic overhead, especially since
> closures are already used some within R4L).
Yeah, please avoid macros as much as possible. Sometimes macros do
have advantages, but if it is just to avoid a closure, then no, please
avoid it.
Similarly, unless there is a concrete advantage needed with the
function, please avoid it too -- the original `for` with the iterator
is the normal way of doing it and already used plenty in the kernel.
Thanks!
Cheers,
Miguel
^ permalink raw reply [flat|nested] 26+ messages in thread
* Re: [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu
2025-08-28 19:00 ` [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu Mitchell Levy
2025-09-03 22:19 ` Yury Norov
@ 2025-09-03 23:05 ` Miguel Ojeda
2025-09-04 20:17 ` Mitchell Levy
1 sibling, 1 reply; 26+ messages in thread
From: Miguel Ojeda @ 2025-09-03 23:05 UTC (permalink / raw)
To: Mitchell Levy
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Yury Norov, Viresh Kumar,
Tyler Hicks, linux-kernel, rust-for-linux, linux-mm
On Thu, Aug 28, 2025 at 9:01 PM Mitchell Levy <levymitchell0@gmail.com> wrote:
>
> + /// Get a `*mut MaybeUninit<T>` to the per-CPU variable on the CPU represented by `cpu`. Note
> + /// that without some kind of synchronization, use of the returned pointer may cause a data
> + /// race. It is the caller's responsibility to use the returned pointer in a reasonable way.
Please try to make the first paragraph ("short description" / title) smaller.
Does "reasonable" mean anything different than any other raw pointer?
> + /// # Safety
Newline after section headers (also elsewhere).
> + /// - The returned pointer is valid only if `self` is (that is, it points to a live allocation
> + /// correctly sized and aligned to hold a `T`)
> + /// - The returned pointer is valid only if the bit corresponding to `cpu` is set in
> + /// `Cpumask::possible()`.
It sounds like the returned pointer can be invalid without triggering
UB -- could you please clarify why the method is `unsafe`?
In addition, please use intra-doc links wherever possible (e.g. there
a was also an `Arc` elsewhere).
> + // SAFETY: The requirements of this function ensure this call is safe.
> + unsafe { bindings::per_cpu_ptr(self.0.cast(), cpu.as_u32()) }.cast()
Please try to explain why, not just that it is. It isn't clear how the
safety preconditions, which only seem to talk about the returned
pointer, make this OK.
Thanks!
Cheers,
Miguel
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu
2025-09-03 23:05 ` Miguel Ojeda
@ 2025-09-04 20:17 ` Mitchell Levy
2025-09-04 20:37 ` Miguel Ojeda
0 siblings, 1 reply; 26+ messages in thread
From: Mitchell Levy @ 2025-09-04 20:17 UTC (permalink / raw)
To: Miguel Ojeda
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Yury Norov, Viresh Kumar,
Tyler Hicks, linux-kernel, rust-for-linux, linux-mm
On Thu, Sep 04, 2025 at 01:05:36AM +0200, Miguel Ojeda wrote:
> On Thu, Aug 28, 2025 at 9:01 PM Mitchell Levy <levymitchell0@gmail.com> wrote:
> >
> > + /// Get a `*mut MaybeUninit<T>` to the per-CPU variable on the CPU represented by `cpu`. Note
> > + /// that without some kind of synchronization, use of the returned pointer may cause a data
> > + /// race. It is the caller's responsibility to use the returned pointer in a reasonable way.
>
> Please try to make the first paragraph ("short description" / title) smaller.
Will do.
> Does "reasonable" mean anything different than any other raw pointer?
This will depend very heavily on `T`, there's a detailed discussion
here: https://docs.kernel.org/core-api/this_cpu_ops.html#remote-access-to-per-cpu-data
In general, remote accesses are strongly discouraged, and my intention
was mostly just wanting to reflect that in this documentation.
> > + /// # Safety
>
> Newline after section headers (also elsewhere).
Will do.
> > + /// - The returned pointer is valid only if `self` is (that is, it points to a live allocation
> > + /// correctly sized and aligned to hold a `T`)
> > + /// - The returned pointer is valid only if the bit corresponding to `cpu` is set in
> > + /// `Cpumask::possible()`.
>
> It sounds like the returned pointer can be invalid without triggering
> UB -- could you please clarify why the method is `unsafe`?
Yes, this is true; strictly speaking, calling this function without
dereferencing the returned pointer is always safe. I suppose I find it
clearer that, when a function has preconditions that are necessary for
it to return a valid pointer, the "safe-ness" has more to do with the
functions preconditions than the act of dereferencing the pointer.
In this case, the pointer shouldn't be going very far, but I think this
logic applies especially well in cases where pointers might be getting
stored away for later (and so the validity of the dereference later on
might rely on non-local conditions).
All that said, I'm alright with having this be a safe function, with the
documentation noting these requirements for the returned pointer to be
valid.
> In addition, please use intra-doc links wherever possible (e.g. there
> a was also an `Arc` elsewhere).
Will do.
> > + // SAFETY: The requirements of this function ensure this call is safe.
> > + unsafe { bindings::per_cpu_ptr(self.0.cast(), cpu.as_u32()) }.cast()
>
> Please try to explain why, not just that it is. It isn't clear how the
> safety preconditions, which only seem to talk about the returned
> pointer, make this OK.
This flows from the first requirement (that `self` is a live allocation,
which is necessary for `per_cpu_ptr` to return a valid pointer). Though,
as above, simply possessing this pointer isn't UB, so it's arguable that
any call to `per_cpu_ptr` (on x86 at least, I'm not sure how it's
implemented on other arches) is always safe. Regardless, I agree this
comment should be more clear (even if the function is safe, it's
probably worth noting why the pointer returned is valid when the
function preconditions are met); will fix.
Thanks,
Mitchell
> Thanks!
>
> Cheers,
> Miguel
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu
2025-09-04 20:17 ` Mitchell Levy
@ 2025-09-04 20:37 ` Miguel Ojeda
2025-09-04 21:50 ` Mitchell Levy
0 siblings, 1 reply; 26+ messages in thread
From: Miguel Ojeda @ 2025-09-04 20:37 UTC (permalink / raw)
To: Mitchell Levy
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Yury Norov, Viresh Kumar,
Tyler Hicks, linux-kernel, rust-for-linux, linux-mm
On Thu, Sep 4, 2025 at 10:17 PM Mitchell Levy <levymitchell0@gmail.com> wrote:
>
> Will do.
By the way, sorry -- just to clarify, I didn't mean to remove
anything, but rather split it (likely at the first full stop).
> Yes, this is true; strictly speaking, calling this function without
> dereferencing the returned pointer is always safe. I suppose I find i
t> clearer that, when a function has preconditions that are necessary for
> it to return a valid pointer, the "safe-ness" has more to do with the
> functions preconditions than the act of dereferencing the pointer.
In that case, I would make it safe -- we don't use safety
preconditions to mark "dangerous" operations, so to speak.
> In this case, the pointer shouldn't be going very far, but I think this
> logic applies especially well in cases where pointers might be getting
> stored away for later (and so the validity of the dereference later on
> might rely on non-local conditions).
But that applies to any returned raw pointer, no?
> This flows from the first requirement (that `self` is a live allocation,
> which is necessary for `per_cpu_ptr` to return a valid pointer). Though,
> as above, simply possessing this pointer isn't UB, so it's arguable that
> any call to `per_cpu_ptr` (on x86 at least, I'm not sure how it's
> implemented on other arches) is always safe. Regardless, I agree this
> comment should be more clear (even if the function is safe, it's
> probably worth noting why the pointer returned is valid when the
> function preconditions are met); will fix.
Sounds good, thanks!
If you have cases you think other architectures may have different
requirements, and you think it is likely one could enable the support
for other arches and break it, then I would suggest adding a comment
about it.
Or, ideally, try to see if other architectures are fine already, even
if you still don't enable the code in other arches.
Cheers,
Miguel
^ permalink raw reply [flat|nested] 26+ messages in thread
* Re: [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu
2025-09-04 20:37 ` Miguel Ojeda
@ 2025-09-04 21:50 ` Mitchell Levy
0 siblings, 0 replies; 26+ messages in thread
From: Mitchell Levy @ 2025-09-04 21:50 UTC (permalink / raw)
To: Miguel Ojeda
Cc: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Yury Norov, Viresh Kumar,
Tyler Hicks, linux-kernel, rust-for-linux, linux-mm
On Thu, Sep 04, 2025 at 10:37:48PM +0200, Miguel Ojeda wrote:
> On Thu, Sep 4, 2025 at 10:17 PM Mitchell Levy <levymitchell0@gmail.com> wrote:
> >
> > Will do.
>
> By the way, sorry -- just to clarify, I didn't mean to remove
> anything, but rather split it (likely at the first full stop).
Oh no worries, I gotcha
> > Yes, this is true; strictly speaking, calling this function without
> > dereferencing the returned pointer is always safe. I suppose I find i
> t> clearer that, when a function has preconditions that are necessary for
> > it to return a valid pointer, the "safe-ness" has more to do with the
> > functions preconditions than the act of dereferencing the pointer.
>
> In that case, I would make it safe -- we don't use safety
> preconditions to mark "dangerous" operations, so to speak.
Understood.
> > In this case, the pointer shouldn't be going very far, but I think this
> > logic applies especially well in cases where pointers might be getting
> > stored away for later (and so the validity of the dereference later on
> > might rely on non-local conditions).
>
> But that applies to any returned raw pointer, no?
This is a fair point :)
> > This flows from the first requirement (that `self` is a live allocation,
> > which is necessary for `per_cpu_ptr` to return a valid pointer). Though,
> > as above, simply possessing this pointer isn't UB, so it's arguable that
> > any call to `per_cpu_ptr` (on x86 at least, I'm not sure how it's
> > implemented on other arches) is always safe. Regardless, I agree this
> > comment should be more clear (even if the function is safe, it's
> > probably worth noting why the pointer returned is valid when the
> > function preconditions are met); will fix.
>
> Sounds good, thanks!
>
> If you have cases you think other architectures may have different
> requirements, and you think it is likely one could enable the support
> for other arches and break it, then I would suggest adding a comment
> about it.
>
> Or, ideally, try to see if other architectures are fine already, even
> if you still don't enable the code in other arches.
Yeah agreed, I'd like to get ARM64 support going soon-ish at the very
least. My hope is that I should only need to mess with `percpu::numeric`
and `PerCpuPtr::get_ptr`... usage of `per_cpu_ptr` *shouldn't* have any
prerequisites, but there's just this note:
https://elixir.bootlin.com/linux/v6.17-rc4/source/include/asm-generic/percpu.h#L29
so I don't want to make strong claims :)
(note, I think the comment about x86_64 might be out-of-date, see
https://elixir.bootlin.com/linux/v6.17-rc4/source/arch/x86/kernel/setup_percpu.c#L32
and
https://elixir.bootlin.com/linux/v6.17-rc4/source/arch/x86/kernel/setup_percpu.c#L165
)
Thanks,
Mitchell
> Cheers,
> Miguel
^ permalink raw reply [flat|nested] 26+ messages in thread
* [PATCH v3 6/7] rust: percpu: Add pin-hole optimizations for numerics
2025-08-28 19:00 [PATCH v3 0/7] rust: Add Per-CPU Variable API Mitchell Levy
` (4 preceding siblings ...)
2025-08-28 19:00 ` [PATCH v3 5/7] rust: percpu: Support non-zeroable types for DynamicPerCpu Mitchell Levy
@ 2025-08-28 19:00 ` Mitchell Levy
2025-08-28 19:00 ` [PATCH v3 7/7] rust: percpu: cache per-CPU pointers in the dynamic case Mitchell Levy
6 siblings, 0 replies; 26+ messages in thread
From: Mitchell Levy @ 2025-08-28 19:00 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Yury Norov, Viresh Kumar
Cc: Tyler Hicks, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
The C implementations of `this_cpu_add`, `this_cpu_sub`, etc., are
optimized to save an instruction by avoiding having to compute
`this_cpu_ptr(&x)` for some per-CPU variable `x`. For example, rather
than
u64 *x_ptr = this_cpu_ptr(&x);
*x_ptr += 5;
the implementation of `this_cpu_add` is clever enough to make use of the
fact that per-CPU variables are implemented on x86 via segment
registers, and so we can use only a single instruction (where we assume
`&x` is already in `rax`)
add gs:[rax], 5
Add this optimization via a `PerCpuNumeric` type to enable code-reuse
between `DynamicPerCpu` and `StaticPerCpu`.
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/kernel/percpu.rs | 1 +
rust/kernel/percpu/dynamic.rs | 2 +-
rust/kernel/percpu/numeric.rs | 128 ++++++++++++++++++++++++++++++++++++++++++
samples/rust/rust_percpu.rs | 36 ++++++++++++
4 files changed, 166 insertions(+), 1 deletion(-)
diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs
index c68c7520b67f..c693d16518d2 100644
--- a/rust/kernel/percpu.rs
+++ b/rust/kernel/percpu.rs
@@ -4,6 +4,7 @@
pub mod cpu_guard;
mod dynamic;
+pub mod numeric;
mod static_;
#[doc(inline)]
diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs
index 64f04cef3705..aad08e4b4251 100644
--- a/rust/kernel/percpu/dynamic.rs
+++ b/rust/kernel/percpu/dynamic.rs
@@ -7,7 +7,7 @@
/// Represents a dynamic allocation of a per-CPU variable via alloc_percpu. Calls free_percpu when
/// dropped.
-pub struct PerCpuAllocation<T>(PerCpuPtr<T>);
+pub struct PerCpuAllocation<T>(pub(super) PerCpuPtr<T>);
impl<T: Zeroable> PerCpuAllocation<T> {
/// Dynamically allocates a space in the per-CPU area suitably sized and aligned to hold a `T`,
diff --git a/rust/kernel/percpu/numeric.rs b/rust/kernel/percpu/numeric.rs
new file mode 100644
index 000000000000..4de93f653f0e
--- /dev/null
+++ b/rust/kernel/percpu/numeric.rs
@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: GPL-2.0
+//! Pin-hole optimizations for PerCpu<T> where T is a numeric type.
+
+use super::*;
+use core::arch::asm;
+
+/// Represents a per-CPU variable that can be manipulated with machine-intrinsic numeric
+/// operations.
+pub struct PerCpuNumeric<'a, T> {
+ // INVARIANT: `ptr.0` is a valid offset into the per-CPU area and is initialized on all CPUs
+ // (since we don't have a CPU guard, we have to be pessimistic and assume we could be on any
+ // CPU).
+ ptr: &'a PerCpuPtr<T>,
+}
+
+macro_rules! impl_ops {
+ ($ty:ty, $reg:tt) => {
+ impl DynamicPerCpu<$ty> {
+ /// Returns a `PerCpuNumeric` that can be used to manipulate the underlying per-CPU variable.
+ pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> {
+ // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that
+ // this pointer is valid and initialized on all CPUs.
+ PerCpuNumeric { ptr: &self.alloc.0 }
+ }
+ }
+ impl StaticPerCpu<$ty> {
+ /// Returns a `PerCpuNumeric` that can be used to manipulate the underlying per-CPU variable.
+ pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> {
+ // The invariant is satisfied because `StaticPerCpu`'s invariant guarantees that
+ // this pointer is valid and initialized on all CPUs.
+ PerCpuNumeric { ptr: &self.0 }
+ }
+ }
+
+ impl PerCpuNumeric<'_, $ty> {
+ /// Adds `rhs` to the per-CPU variable.
+ pub fn add(&mut self, rhs: $ty) {
+ // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a
+ // pointer relative to the `gs` segment register) by the invariants of this type.
+ unsafe {
+ asm!(
+ concat!("add gs:[{off}], {val:", $reg, "}"),
+ off = in(reg) self.ptr.0.cast::<*mut $ty>(),
+ val = in(reg) rhs,
+ );
+ }
+ }
+ }
+ impl PerCpuNumeric<'_, $ty> {
+ /// Subtracts `rhs` from the per-CPU variable.
+ pub fn sub(&mut self, rhs: $ty) {
+ // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a
+ // pointer relative to the `gs` segment register) by the invariants of this type.
+ unsafe {
+ asm!(
+ concat!("sub gs:[{off}], {val:", $reg, "}"),
+ off = in(reg) self.ptr.0.cast::<*mut $ty>(),
+ val = in(reg) rhs,
+ );
+ }
+ }
+ }
+ };
+}
+
+macro_rules! impl_ops_byte {
+ ($ty:ty) => {
+ impl DynamicPerCpu<$ty> {
+ /// Returns a `PerCpuNumeric` that can be used to manipulate the underlying per-CPU
+ /// variable.
+ pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> {
+ // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that
+ // this pointer is valid and initialized on all CPUs.
+ PerCpuNumeric { ptr: &self.alloc.0 }
+ }
+ }
+ impl StaticPerCpu<$ty> {
+ /// Returns a `PerCpuNumeric` that can be used to manipulate the underlying per-CPU
+ /// variable.
+ pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> {
+ // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that
+ // this pointer is valid and initialized on all CPUs.
+ PerCpuNumeric { ptr: &self.0 }
+ }
+ }
+
+ impl PerCpuNumeric<'_, $ty> {
+ /// Adds `rhs` to the per-CPU variable.
+ pub fn add(&mut self, rhs: $ty) {
+ // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a
+ // pointer relative to the `gs` segment register) by the invariants of this type.
+ unsafe {
+ asm!(
+ concat!("add gs:[{off}], {val}"),
+ off = in(reg) self.ptr.0.cast::<*mut $ty>(),
+ val = in(reg_byte) rhs,
+ );
+ }
+ }
+ }
+ impl PerCpuNumeric<'_, $ty> {
+ /// Subtracts `rhs` from the per-CPU variable.
+ pub fn sub(&mut self, rhs: $ty) {
+ // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a
+ // pointer relative to the `gs` segment register) by the invariants of this type.
+ unsafe {
+ asm!(
+ concat!("sub gs:[{off}], {val}"),
+ off = in(reg) self.ptr.0.cast::<*mut $ty>(),
+ val = in(reg_byte) rhs,
+ );
+ }
+ }
+ }
+ };
+}
+
+impl_ops_byte!(i8);
+impl_ops!(i16, "x");
+impl_ops!(i32, "e");
+impl_ops!(i64, "r");
+impl_ops!(isize, "r");
+
+impl_ops_byte!(u8);
+impl_ops!(u16, "x");
+impl_ops!(u32, "e");
+impl_ops!(u64, "r");
+impl_ops!(usize, "r");
diff --git a/samples/rust/rust_percpu.rs b/samples/rust/rust_percpu.rs
index 06b322019134..e3a46a053b8d 100644
--- a/samples/rust/rust_percpu.rs
+++ b/samples/rust/rust_percpu.rs
@@ -27,6 +27,26 @@
define_per_cpu!(UPERCPU: u64 = 0);
define_per_cpu!(CHECKED: RefCell<u64> = RefCell::new(0));
+macro_rules! make_optimization_test {
+ ($ty:ty) => {
+ let mut test: DynamicPerCpu<$ty> = DynamicPerCpu::new_zero(GFP_KERNEL).unwrap();
+ {
+ let _ = CpuGuard::new();
+ // SAFETY: No other usage of `test`
+ unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| *val = 10);
+ test.num().add(1);
+ // SAFETY: No other usage of `test`
+ unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 11));
+ test.num().add(10);
+ // SAFETY: No other usage of `test`
+ unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 21));
+ test.num().sub(5);
+ // SAFETY: No other usage of `test`
+ unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 16));
+ }
+ };
+}
+
impl kernel::Module for PerCpuMod {
fn init(_module: &'static ThisModule) -> Result<Self, Error> {
pr_info!("rust percpu test start\n");
@@ -198,6 +218,22 @@ fn init(_module: &'static ThisModule) -> Result<Self, Error> {
pr_info!("rust dynamic percpu test done\n");
+ pr_info!("rust numeric optimizations test start\n");
+
+ make_optimization_test!(u8);
+ make_optimization_test!(u16);
+ make_optimization_test!(u32);
+ make_optimization_test!(u64);
+ make_optimization_test!(usize);
+
+ make_optimization_test!(i8);
+ make_optimization_test!(i16);
+ make_optimization_test!(i32);
+ make_optimization_test!(i64);
+ make_optimization_test!(isize);
+
+ pr_info!("rust numeric optimizations test done\n");
+
// Return Err to unload the module
Result::Err(EINVAL)
}
--
2.34.1
^ permalink raw reply related [flat|nested] 26+ messages in thread* [PATCH v3 7/7] rust: percpu: cache per-CPU pointers in the dynamic case
2025-08-28 19:00 [PATCH v3 0/7] rust: Add Per-CPU Variable API Mitchell Levy
` (5 preceding siblings ...)
2025-08-28 19:00 ` [PATCH v3 6/7] rust: percpu: Add pin-hole optimizations for numerics Mitchell Levy
@ 2025-08-28 19:00 ` Mitchell Levy
6 siblings, 0 replies; 26+ messages in thread
From: Mitchell Levy @ 2025-08-28 19:00 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Boqun Feng, Gary Guo,
Björn Roy Baron, Andreas Hindborg, Alice Ryhl, Trevor Gross,
Andrew Morton, Dennis Zhou, Tejun Heo, Christoph Lameter,
Danilo Krummrich, Benno Lossin, Yury Norov, Viresh Kumar
Cc: Tyler Hicks, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
Currently, the creation of a `PerCpuNumeric` requires a memory read via
the `Arc` managing the dynamic allocation. While the compiler might be
clever enough to consolidate these reads in some cases, the read must
happen *somewhere*, which, when we're concerning ourselves with
individual instructions, is a very high burden.
Instead, cache the `PerCpuPointer` inside the `DynamicPerCpu` structure;
then, the `Arc` is used solely to manage the allocation.
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/kernel/percpu/dynamic.rs | 8 ++++++--
rust/kernel/percpu/numeric.rs | 4 ++--
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs
index aad08e4b4251..8683e94bcd9f 100644
--- a/rust/kernel/percpu/dynamic.rs
+++ b/rust/kernel/percpu/dynamic.rs
@@ -58,7 +58,10 @@ fn drop(&mut self) {
pub struct DynamicPerCpu<T> {
// INVARIANT: The memory location in each CPU's per-CPU area pointed at by `alloc.0` has been
// initialized.
+ // INVARIANT: `ptr` is the per-CPU pointer managed by `alloc`, which does not change for the
+ // lifetime of `self`.
pub(super) alloc: Arc<PerCpuAllocation<T>>,
+ pub(super) ptr: PerCpuPtr<T>,
}
impl<T: Zeroable> DynamicPerCpu<T> {
@@ -70,9 +73,10 @@ impl<T: Zeroable> DynamicPerCpu<T> {
pub fn new_zero(flags: Flags) -> Option<Self> {
let alloc: PerCpuAllocation<T> = PerCpuAllocation::new_zero()?;
+ let ptr = alloc.0;
let arc = Arc::new(alloc, flags).ok()?;
- Some(Self { alloc: arc })
+ Some(Self { alloc: arc, ptr })
}
}
@@ -102,7 +106,7 @@ pub fn new_with(val: T, flags: Flags) -> Option<Self> {
let arc = Arc::new(alloc, flags).ok()?;
- Some(Self { alloc: arc })
+ Some(Self { alloc: arc, ptr })
}
}
diff --git a/rust/kernel/percpu/numeric.rs b/rust/kernel/percpu/numeric.rs
index 4de93f653f0e..3412c2de85fd 100644
--- a/rust/kernel/percpu/numeric.rs
+++ b/rust/kernel/percpu/numeric.rs
@@ -20,7 +20,7 @@ impl DynamicPerCpu<$ty> {
pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> {
// The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that
// this pointer is valid and initialized on all CPUs.
- PerCpuNumeric { ptr: &self.alloc.0 }
+ PerCpuNumeric { ptr: &self.ptr }
}
}
impl StaticPerCpu<$ty> {
@@ -71,7 +71,7 @@ impl DynamicPerCpu<$ty> {
pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> {
// The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that
// this pointer is valid and initialized on all CPUs.
- PerCpuNumeric { ptr: &self.alloc.0 }
+ PerCpuNumeric { ptr: &self.ptr }
}
}
impl StaticPerCpu<$ty> {
--
2.34.1
^ permalink raw reply related [flat|nested] 26+ messages in thread