From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from kanga.kvack.org (kanga.kvack.org [205.233.56.17]) by smtp.lore.kernel.org (Postfix) with ESMTP id D7E09C7EE32 for ; Tue, 24 Jun 2025 22:11:41 +0000 (UTC) Received: by kanga.kvack.org (Postfix) id 820BF6B00A6; Tue, 24 Jun 2025 18:11:40 -0400 (EDT) Received: by kanga.kvack.org (Postfix, from userid 40) id 7D0EE6B00A7; Tue, 24 Jun 2025 18:11:40 -0400 (EDT) X-Delivered-To: int-list-linux-mm@kvack.org Received: by kanga.kvack.org (Postfix, from userid 63042) id 6E6346B00AC; Tue, 24 Jun 2025 18:11:40 -0400 (EDT) X-Delivered-To: linux-mm@kvack.org Received: from relay.hostedemail.com (smtprelay0013.hostedemail.com [216.40.44.13]) by kanga.kvack.org (Postfix) with ESMTP id 5B91F6B00A6 for ; Tue, 24 Jun 2025 18:11:40 -0400 (EDT) Received: from smtpin22.hostedemail.com (a10.router.float.18 [10.200.18.1]) by unirelay07.hostedemail.com (Postfix) with ESMTP id 1A9801605CA for ; Tue, 24 Jun 2025 22:11:40 +0000 (UTC) X-FDA: 83591691960.22.3618352 Received: from mail-pl1-f181.google.com (mail-pl1-f181.google.com [209.85.214.181]) by imf27.hostedemail.com (Postfix) with ESMTP id 1124340006 for ; Tue, 24 Jun 2025 22:11:37 +0000 (UTC) Authentication-Results: imf27.hostedemail.com; dkim=pass header.d=gmail.com header.s=20230601 header.b=TPk5ATWp; spf=pass (imf27.hostedemail.com: domain of levymitchell0@gmail.com designates 209.85.214.181 as permitted sender) smtp.mailfrom=levymitchell0@gmail.com; dmarc=pass (policy=none) header.from=gmail.com ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=hostedemail.com; s=arc-20220608; t=1750803098; h=from:from:sender:reply-to:subject:subject:date:date: message-id:message-id:to:to:cc:cc:mime-version:mime-version: content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references:dkim-signature; bh=k1qfz7npuSV263RU/XyMTPXjzl3/t+g8dNO+Vafq5Vc=; b=0JEQZG5SgmLNr8Jmc68rDalY+q6dyluLF0kDkr8Oq7GOEWPDTZeA6CG6+2XC4+dU6/OcGx CC9itdoRU4NV6C0Bpap0YDHjTrelXPRL+pDxtOp60JjD/qCcTkMQvUm15aM7xBW2Mr7C4Q tpf9H9v0S7qP5aYi6rVkeaeUqYTj0cI= ARC-Authentication-Results: i=1; imf27.hostedemail.com; dkim=pass header.d=gmail.com header.s=20230601 header.b=TPk5ATWp; spf=pass (imf27.hostedemail.com: domain of levymitchell0@gmail.com designates 209.85.214.181 as permitted sender) smtp.mailfrom=levymitchell0@gmail.com; dmarc=pass (policy=none) header.from=gmail.com ARC-Seal: i=1; s=arc-20220608; d=hostedemail.com; t=1750803098; a=rsa-sha256; cv=none; b=sf+pkL7hCX4pbUZwJzzFEwsWuvFWZekH48bF1MEig7qLgTOnNdd8N86+lK4DzmOvSkrot+ aWJLhuLwZ0ctCA/+oej0gRh7lDvgtIHP+V/hnEBVKAZVYL+lv4vqyEvNkdu1+AbIzC1I5P +BbPAl4mzFjqDcHqJKK4fBif3npwJLY= Received: by mail-pl1-f181.google.com with SMTP id d9443c01a7336-2363e973db1so3467335ad.0 for ; Tue, 24 Jun 2025 15:11:37 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1750803097; x=1751407897; darn=kvack.org; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :mime-version:subject:date:from:from:to:cc:subject:date:message-id :reply-to; bh=k1qfz7npuSV263RU/XyMTPXjzl3/t+g8dNO+Vafq5Vc=; b=TPk5ATWpygj9rsjOZ7u3HntAUQil6nJxeif80imFkK1n2pS9PFNVxSbAs/SdtDwFoi SbO2/cIXouNu+pzkKnAc70GpZbwmDNISBmJv/+hQQDMbm4YMVNfAnvvpQ4tmpqdAMvkP xj9ADS/2mkH4X/Dd3DIWiX4wBE/1yutFA9fWD8f7J3GBx58XncwihWu0vVbr4GO886OH HqAtdmvZowmevyC55cE31xudYyawRjRZNyPLQuMTmVq2F6OGQQNEnImGWsRdCkfKFO1m CyURsONppft11uSwIMhTIL6k1Ix2iUC1BzB/iNQswVe1PoVHlyIMGnXqpiGY/x9U3k2A /cYQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1750803097; x=1751407897; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :mime-version:subject:date:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=k1qfz7npuSV263RU/XyMTPXjzl3/t+g8dNO+Vafq5Vc=; b=HwuB8itUS7V0yzgtGw4wWzzXkpr5zcuFN+eHiRmvSqJaDq8pFnfBaEwASjSZuH/4Yq d+8JmrXk6Jdm5BniKQUWSWrRgtSp70fnoS1L7wxE2y4W6dQB4ya2FoLl5WbMbdN7ggDU 9AnSanSQBZcMWyLdiF+qNIHdy7JCrbL7Z+B6Okw7NpctQvw5nanqa3Li+P3b5XgScbbg ZXh/6124aQqsDadDR0Qwjgs7HsPWfKNQyTG+mjJvePol3Korqen4NFBqNR/lNrIjMiS/ HxYrihxUQ6DmqYlC2cBaSkeMqGugCEZH/aipfWCWZSuauw5a9/NrEwVbsg6KQeP31GqR SRXA== X-Forwarded-Encrypted: i=1; AJvYcCU63AAzLyJvTRscJZk0u1b041Eeqy8YQFbmjBh/q8aC8ao+4nh0mYWN3DrzNG2plX4dp2icASmYnQ==@kvack.org X-Gm-Message-State: AOJu0Yyl3I5J3TuGk3uL7fxCzMfzuCjH82naUG8/LXH7yN9ot3EZRqAi Hv6RSwQiFz2Mt1xEX/pVIxT4sbX8HLJQN5zrzMJLl+5emg6TMS9d1RJWgUijVCNn X-Gm-Gg: ASbGncu0Z7aIpmsuwXJxZFV0U7dReWxd9ZwbDO01DNE8217i7682C3TZHD9UkIf6ZCx HgCAfniJJSpBA6Jkoi2hIbEZcI32DZORme4LUT98r2qtoZppAdtVZAqkStvfvM3M3iZLMfDoQPf naytGPyrllFv/rbxRuSlo30ERkq6ul59TyfM9ruNQY1pjkpG5B3i6vXFFQddi3aYyFgN/HkCF5x kbmv86s4ecE7+NLpXcAiybKHnISz61twQpJ1EhW7mcynBn8N9MddfuM4je4SKTW/U6B+sWMrUW2 h+c7CBp52SPqq21BlFaNPc7DyVyVaYixfwz00CBxab2Qne841dGKYXSgXBCWJqOmxwkHvfz43wE 6i2x3hmRCn7v5Se4BU9iR2m18X/DxdqO1ZCbx X-Google-Smtp-Source: AGHT+IHoR9fiqp6ip71cYAnZJhavz9a3WeQJL/OnTgAV6bvnKoELPcHgQjb4d+FplsKuexEMI1F26A== X-Received: by 2002:a17:902:f68b:b0:232:59b:5923 with SMTP id d9443c01a7336-238253ec8b3mr9686685ad.23.1750803096739; Tue, 24 Jun 2025 15:11:36 -0700 (PDT) Received: from mitchelllevy.localdomain (82.sub-174-224-205.myvzw.com. [174.224.205.82]) by smtp.gmail.com with ESMTPSA id d9443c01a7336-237d8393741sm114580555ad.15.2025.06.24.15.11.35 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 24 Jun 2025 15:11:36 -0700 (PDT) From: Mitchell Levy Date: Tue, 24 Jun 2025 15:10:39 -0700 Subject: [PATCH 1/5] rust: percpu: introduce a rust API for per-CPU variables MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit Message-Id: <20250624-rust-percpu-v1-1-9c59b07d2a9c@gmail.com> References: <20250624-rust-percpu-v1-0-9c59b07d2a9c@gmail.com> In-Reply-To: <20250624-rust-percpu-v1-0-9c59b07d2a9c@gmail.com> To: Miguel Ojeda , Alex Gaynor , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Andreas Hindborg , Alice Ryhl , Trevor Gross , Andrew Morton , Dennis Zhou , Tejun Heo , Christoph Lameter , Danilo Krummrich , Benno Lossin Cc: linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, linux-mm@kvack.org, Mitchell Levy X-Mailer: b4 0.14.0 X-Developer-Signature: v=1; a=ed25519-sha256; t=1750803093; l=15606; i=levymitchell0@gmail.com; s=20240719; h=from:subject:message-id; bh=2fYVqdvVZOyhVdWES/18qc3/KomIzWwxDfp2+bGaXfw=; b=owgvvji1ROx81yigs8sJ/SU6nZxYmDVydO9bTpnfzG0fF3LIPscm98TeBFAnOSJ4u8J4DRJ98 R/lO168ofM9CpMzoMIIy0UV12Gsckj1oHyEm9phovR2C2EiTNuyUzc5 X-Developer-Key: i=levymitchell0@gmail.com; a=ed25519; pk=n6kBmUnb+UNmjVkTnDwrLwTJAEKUfs2e8E+MFPZI93E= X-Rspamd-Queue-Id: 1124340006 X-Stat-Signature: 6q37auagwxnouw5khazf1sg8c191ktmj X-Rspamd-Server: rspam09 X-Rspam-User: X-HE-Tag: 1750803097-469712 X-HE-Meta: U2FsdGVkX1/n/FqSCdriV8E5jUrRnYwOVJi2mIrsX+YNSv1g4RAysHf3al157T6ZkdFZwWr6CuqOJnGL8D1IZv9+4roqPdhcxzgki0IYpUym4Q2KSVPLMcB6BMZdJqPX37K2TQcq2DWZgQ4kra7hu2FXIsNVupnn9fYSP5sK1TyBkqwOPQdFzYgeMut9z+4PSawtN/prDIX97Jg7gAXjDB5YpQgLMuGK2dB0cfl2EeUZBW8p7ZHDjDJUorVaiqDPv5wYHDHueJl8e2p0CZfjqef1LdcP2MYGye105x8BW70Kef7p3aqajcu/w+N9d2UTrs8b5RE39Gtc/VP5PQqlSNEGWP4lZn2GRwWnJw8HtQLHr7U41q7rvycvhxL11loKHSussTI/PMoNsgIFGWNcYxI6o5+eRNyVHFX5+9dhlGNIWXe9E7KKFjsVPeHzZ3Uo28Sz4+OE50yXcaMNGSTkiKW5JF1GLBbeVyyCyvcVpbjfCoD8kUXOwLgS8f2PocDgdBNtNEWrbMd7/7fnBUZqxF7KmQxwf2co9TImM1lunzbFptRBKm3/7aS3Bv53EDEKIsaRcP1MR0fgQOMl6RcKIUM4DKrwmX7Yz4ywfD3icMF44WagnoQ7oJdaJWI1615qohGL/vDFW2eD+VA3782FMTkcMP7U1hCiPvpH7PL5gliO1+bF+hVG0E1d30CxRP6VSUKIwxTVSF74MVKWAARH0ZCqLHJ5wSPcoFRj/Czl385H9wJSA8gIgvZALpx1UMqdZe3bWiXcKj64sMZK3gIC3y90JLfdPTMArEBfm6FKfGzGTYH5NLQzYvaNmD2JmPudxay2we1vJeEx3jURU1EzdsTAkYyky9Kk0uPaboGTXbM0N002S/d3AvP1SZ8weXtBf98oVvAnWU5eoRc9rJ7uXb3Gk0NXVYHc6peRcLxc5SPMvhUi4it0Ed9ym+Fw6HybBsWih2f1bIcIEdRd4fV dijRE2OI niZ8yb3DMepG5MaZXFDUK/YPO5CCgdmMbfIrabN9HowOQg6oRDNIAdhKo/hZS1fDCF697rZhlndHTmnBGg3j3g1kFlI5iBsRg/xnqlgzP2uiwVq0s4oY8c/KoqhgHibOp+rGoxg9nL3IBQQAI1A4X40bogstTxkQCOmv358dfmmZVRdszju64rlcRCU/qzP3JkNpgxpWPRVIQGx3hsKfUZJLlrZfYmhUsyvHR7qMd4pcCLpft7k/SowwHYl+3kV6BN4QHeH2ghjOe369VzQenbHmdI6uBSuspD20Ln2KmDCaEw093UjeKK2vG0h9K/m8hz57TS/Kag0iNhhjIf5YJVUhM+yxtwET8Wq3/JzKAsKyVvuoMoueLIXnvuoL95Ec5FGSV5+y3QxR4zyIYvgIwIZKRtR5Ytg0M/zycRGz4cVh7xK8WdAMMcYPz5Z6g5+UC508MfZq7ngh3EUU= X-Bogosity: Ham, tests=bogofilter, spamicity=0.000000, version=1.2.4 Sender: owner-linux-mm@kvack.org Precedence: bulk X-Loop: owner-majordomo@kvack.org List-ID: List-Subscribe: List-Unsubscribe: Add a `CpuGuard` type that disables preemption for its lifetime. Add a `PerCpuAllocation` type used to track dynamic allocations. Add a `define_per_cpu!` macro to create static per-CPU allocations. Add `DynamicPerCpu` and `StaticPerCpu` to provide a high-level API. Add a `PerCpu` trait to unify the dynamic and static cases. Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Signed-off-by: Mitchell Levy --- rust/helpers/helpers.c | 2 + rust/helpers/percpu.c | 9 ++ rust/helpers/preempt.c | 14 ++ rust/kernel/lib.rs | 3 + rust/kernel/percpu.rs | 299 ++++++++++++++++++++++++++++++++++++++++ rust/kernel/percpu/cpu_guard.rs | 35 +++++ 6 files changed, 362 insertions(+) diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 0f1b5d115985..d56bbe6334d3 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -29,7 +29,9 @@ #include "page.c" #include "platform.c" #include "pci.c" +#include "percpu.c" #include "pid_namespace.c" +#include "preempt.c" #include "rbtree.c" #include "rcu.c" #include "refcount.c" diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c new file mode 100644 index 000000000000..a091389f730f --- /dev/null +++ b/rust/helpers/percpu.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align) +{ + return __alloc_percpu(sz, align); +} + diff --git a/rust/helpers/preempt.c b/rust/helpers/preempt.c new file mode 100644 index 000000000000..2c7529528ddd --- /dev/null +++ b/rust/helpers/preempt.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void rust_helper_preempt_disable(void) +{ + preempt_disable(); +} + +void rust_helper_preempt_enable(void) +{ + preempt_enable(); +} + diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 6b4774b2b1c3..733f9ff8b888 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -95,6 +95,9 @@ pub mod page; #[cfg(CONFIG_PCI)] pub mod pci; +// Only x86_64 is supported by percpu for now +#[cfg(CONFIG_X86_64)] +pub mod percpu; pub mod pid_namespace; pub mod platform; pub mod prelude; diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs new file mode 100644 index 000000000000..a912f74349e0 --- /dev/null +++ b/rust/kernel/percpu.rs @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0 +//! This module contains abstractions for creating and using per-CPU variables from Rust. +//! See the define_per_cpu! macro and the PerCpu type. +pub mod cpu_guard; + +use bindings::{alloc_percpu, free_percpu}; + +use crate::alloc::Flags; +use crate::percpu::cpu_guard::CpuGuard; +use crate::sync::Arc; + +use core::arch::asm; + +use ffi::c_void; + +/// A per-CPU pointer; that is, an offset into the per-CPU area. +pub struct PerCpuPtr(*mut T); + +/// Represents a dynamic allocation of a per-CPU variable via alloc_percpu. Calls free_percpu when +/// dropped. +pub struct PerCpuAllocation(PerCpuPtr); + +/// Holds a dynamically-allocated per-CPU variable. +pub struct DynamicPerCpu { + alloc: Arc>, +} + +/// Holds a statically-allocated per-CPU variable. +pub struct StaticPerCpu(PerCpuPtr); + +/// Represents exclusive access to the memory location pointed at by a particular PerCpu. +pub struct PerCpuToken<'a, T> { + _guard: CpuGuard, + ptr: &'a PerCpuPtr, +} + +/// A wrapper used for declaring static per-CPU variables. These symbols are "virtual" in that the +/// linker uses them to generate offsets into each CPU's per-CPU area, but shouldn't be read +/// from/written to directly. The fact that the statics are immutable prevents them being written +/// to (generally), this struct having _val be non-public prevents reading from them. +/// +/// The end-user of the per-CPU API should make use of the define_per_cpu! macro instead of +/// declaring variables of this type directly. +#[repr(transparent)] +pub struct StaticPerCpuSymbol { + _val: T, // generate a correctly sized type +} + +impl PerCpuPtr { + /// Makes a new PerCpuPtr from a raw per-CPU pointer. + /// + /// # Safety + /// `ptr` must be a valid per-CPU pointer. + pub unsafe fn new(ptr: *mut T) -> Self { + Self(ptr) + } + + /// Get a `&mut T` to the per-CPU variable represented by `&self` + /// + /// # Safety + /// The returned `&mut T` must follow Rust's aliasing rules. That is, no other `&(mut) T` may + /// exist that points to the same location in memory. In practice, this means that any PerCpu + /// on the same CPU holding a reference to the same PerCpuAllocation as `self` mut not call + /// `get_ref` for as long as the returned reference lives. + /// + /// CPU preemption must be disabled before calling this function and for the lifetime of the + /// returned reference. Otherwise, the returned &mut T might end up being a reference to a + /// different CPU's per-CPU area, causing the potential for a data race. + #[allow(clippy::mut_from_ref)] // Safety requirements prevent aliasing issues + pub unsafe fn get_ref(&self) -> &mut T { + let this_cpu_off_pcpu = core::ptr::addr_of!(this_cpu_off); + let mut this_cpu_area: *mut c_void; + // SAFETY: gs + this_cpu_off_pcpu is guaranteed to be a valid pointer because `gs` points + // to the per-CPU area and this_cpu_off_pcpu is a valid per-CPU allocation. + unsafe { + asm!( + "mov {out}, gs:[{off_val}]", + off_val = in(reg) this_cpu_off_pcpu, + out = out(reg) this_cpu_area, + ) + }; + // SAFETY: this_cpu_area + self.0 is guaranteed to be a valid pointer by the per-CPU + // subsystem and the invariant that self.0 is a valid offset into the per-CPU area. + // + // We know no-one else has a reference to the underlying pcpu variable because of the + // safety requirements of this function. + unsafe { &mut *((this_cpu_area).wrapping_add(self.0 as usize) as *mut T) } + } +} + +impl Clone for PerCpuPtr { + fn clone(&self) -> Self { + *self + } +} + +/// PerCpuPtr is just a pointer, so it's safe to copy. +impl Copy for PerCpuPtr {} + +impl PerCpuAllocation { + /// Dynamically allocates a space in the per-CPU area suitably sized and aligned to hold a `T`. + /// + /// Returns `None` under the same circumstances the C function `alloc_percpu` returns `NULL`. + pub fn new() -> Option> { + // SAFETY: No preconditions to call alloc_percpu + let ptr: *mut T = unsafe { alloc_percpu(size_of::(), align_of::()) } as *mut T; + if ptr.is_null() { + return None; + } + + Some(Self(PerCpuPtr(ptr))) + } +} + +impl Drop for PerCpuAllocation { + fn drop(&mut self) { + // SAFETY: self.0.0 was returned by alloc_percpu, and so was a valid pointer into + // the percpu area, and has remained valid by the invariants of PerCpuAllocation. + unsafe { free_percpu(self.0 .0 as *mut c_void) } + } +} + +/// A trait representing a per-CPU variable. This is implemented for both `StaticPerCpu` and +/// `DynamicPerCpu`. The main usage of this trait is to call `get` to get a `PerCpuToken` that +/// can be used to access the underlying per-CPU variable. See `PerCpuToken::with`. +/// +/// # Safety +/// The returned value from `ptr` must be valid for the lifetime of `&mut self`. +pub unsafe trait PerCpu { + /// Gets a `PerCpuPtr` to the per-CPU variable represented by `&mut self` + /// + /// # Safety + /// `self` may be doing all sorts of things to track when the underlying per-CPU variable can + /// be deallocated. You almost certainly shouldn't be calling this function directly (it's + /// essentially an implementation detail of the trait), and you certainly shouldn't be making + /// copies of the returned `PerCpuPtr` that may outlive `&mut self`. + /// + /// Implementers of this trait should ensure that the returned `PerCpuPtr` is valid for + /// the lifetime of `&mut self`. + unsafe fn ptr(&mut self) -> &PerCpuPtr; + + /// Produces a token, asserting that the holder has exclusive access to the underlying memory + /// pointed to by `self` + /// + /// # Safety + /// `func` (or its callees that execute on the same CPU) may not call `get_ref` on another + /// `PerCpu` that represents the same per-CPU variable as `&mut self` (that is, they must + /// not be `clone()`s of each other or, in the case of statically allocated variables, + /// additionally can't both have come from the same `define_per_cpu!`) for the lifetime of the + /// returned token. + /// + /// In particular, this requires that the underlying per-CPU variable cannot ever be mutated + /// from an interrupt context, unless irqs are disabled for the lifetime of the returned + /// `PerCpuToken`. + unsafe fn get(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> { + PerCpuToken { + _guard: guard, + // SAFETY: The lifetime of the returned `PerCpuToken<'_, T>` is bounded by the lifetime + // of `&mut self`. + ptr: unsafe { self.ptr() }, + } + } +} + +impl StaticPerCpu { + /// Creates a new PerCpu pointing to the statically allocated variable at `ptr`. End-users + /// should probably be using the `unsafe_get_per_cpu!` macro instead of calling this function. + /// + /// # Safety + /// `ptr` must be a valid pointer to a per-CPU variable. This means that it must be a valid + /// offset into the per-CPU area, and that the per-CPU area must be suitably sized and aligned + /// to hold a `T`. + pub unsafe fn new(ptr: *mut T) -> Self { + Self(PerCpuPtr(ptr)) + } +} + +// SAFETY: The `PerCpuPtr` returned by `ptr` is valid for the lifetime of `self` (and in fact, +// forever). +unsafe impl PerCpu for StaticPerCpu { + unsafe fn ptr(&mut self) -> &PerCpuPtr { + &self.0 + } +} + +impl Clone for StaticPerCpu { + fn clone(&self) -> Self { + Self(self.0) + } +} + +impl DynamicPerCpu { + /// Allocates a new per-CPU variable + /// + /// # Arguments + /// * `flags` - Flags used to allocate an `Arc` that keeps track of the underlying + /// `PerCpuAllocation`. + pub fn new(flags: Flags) -> Option { + let alloc: PerCpuAllocation = PerCpuAllocation::new()?; + + let arc = Arc::new(alloc, flags).ok()?; + + Some(Self { alloc: arc }) + } + + /// Wraps a `PerCpuAllocation` in a `PerCpu` + /// + /// # Arguments + /// * `alloc` - The allocation to use + /// * `flags` - The flags used to allocate an `Arc` that keeps track of the `PerCpuAllocation`. + pub fn new_from_allocation(alloc: PerCpuAllocation, flags: Flags) -> Option { + let arc = Arc::new(alloc, flags).ok()?; + Some(Self { alloc: arc }) + } +} + +// SAFETY: The `PerCpuPtr` returned by `ptr` is valid for the lifetime of `self` because we +// don't deallocate the underlying `PerCpuAllocation` until `self` is dropped. +unsafe impl PerCpu for DynamicPerCpu { + unsafe fn ptr(&mut self) -> &PerCpuPtr { + &self.alloc.0 + } +} + +impl Clone for DynamicPerCpu { + fn clone(&self) -> Self { + Self { + alloc: self.alloc.clone(), + } + } +} + +impl PerCpuToken<'_, T> { + /// Immediately invokes `func` with a `&mut T` that points at the underlying per-CPU variable + /// that `&mut self` represents. + pub fn with(&mut self, func: U) + where + U: FnOnce(&mut T), + { + // SAFETY: The existence of a PerCpuToken means that the requirements for get_ref are + // satisfied. + func(unsafe { self.ptr.get_ref() }); + } +} + +/// define_per_cpu! is analogous to the C DEFINE_PER_CPU macro in that it lets you create a +/// statically allocated per-CPU variable. +/// +/// # Example +/// ``` +/// use kernel::define_per_cpu; +/// use kernel::percpu::StaticPerCpuSymbol; +/// +/// define_per_cpu!(pub MY_PERCPU: u64 = 0); +/// ``` +#[macro_export] +macro_rules! define_per_cpu { + ($vis:vis $id:ident: $ty:ty = $expr:expr) => { + $crate::macros::paste! { + // Expand $expr outside of the unsafe block to avoid silently allowing unsafe code to be + // used without a user-facing unsafe block + static [<__INIT_ $id>]: $ty = $expr; + + // SAFETY: StaticPerCpuSymbol is #[repr(transparent)], so we can freely convert from T + #[link_section = ".data..percpu"] + $vis static $id: StaticPerCpuSymbol<$ty> = unsafe { + core::mem::transmute::<$ty, StaticPerCpuSymbol<$ty>>([<__INIT_ $id>]) + }; + } + }; +} + +/// Gets a `PerCpu` from a symbol declared with `define_per_cpu!` or `declare_extern_per_cpu!`. +/// +/// # Arguments +/// * `ident` - The identifier declared +/// +/// # Safety +/// `$id` must be declared with either `define_per_cpu!` or `declare_extern_per_cpu!`, and the +/// returned value must be stored in a `PerCpu` where `T` matches the declared type of `$id`. +#[macro_export] +macro_rules! unsafe_get_per_cpu { + ($id:ident) => {{ + $crate::percpu::StaticPerCpu::new((&$id) as *const _ as *mut _) + }}; +} + +/// Declares a StaticPerCpuSymbol corresponding to a per-CPU variable defined in C. Be sure to read +/// the safety requirements of `PerCpu::get`. +#[macro_export] +macro_rules! declare_extern_per_cpu { + ($id:ident: $ty:ty) => { + extern "C" { + static $id: StaticPerCpuSymbol<$ty>; + } + }; +} + +declare_extern_per_cpu!(this_cpu_off: u64); diff --git a/rust/kernel/percpu/cpu_guard.rs b/rust/kernel/percpu/cpu_guard.rs new file mode 100644 index 000000000000..14c04b12e7f0 --- /dev/null +++ b/rust/kernel/percpu/cpu_guard.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +//! Contains abstractions for disabling CPU preemption. See `CpuGuard`. + +/// A RAII guard for bindings::preempt_disable and bindings::preempt_enable. Guarantees preemption +/// is disabled for as long as this object exists. +pub struct CpuGuard { + // Don't make one without using new() + _phantom: (), +} + +impl CpuGuard { + /// Create a new CpuGuard. Disables preemption for its lifetime. + pub fn new() -> Self { + // SAFETY: There are no preconditions required to call preempt_disable + unsafe { + bindings::preempt_disable(); + } + CpuGuard { _phantom: () } + } +} + +impl Default for CpuGuard { + fn default() -> Self { + Self::new() + } +} + +impl Drop for CpuGuard { + fn drop(&mut self) { + // SAFETY: There are no preconditions required to call preempt_enable + unsafe { + bindings::preempt_enable(); + } + } +} -- 2.34.1