From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from kanga.kvack.org (kanga.kvack.org [205.233.56.17]) by smtp.lore.kernel.org (Postfix) with ESMTP id 46C76C83F10 for ; Sat, 12 Jul 2025 21:32:10 +0000 (UTC) Received: by kanga.kvack.org (Postfix) id 98B406B00B8; Sat, 12 Jul 2025 17:32:09 -0400 (EDT) Received: by kanga.kvack.org (Postfix, from userid 40) id 89EC66B00BA; Sat, 12 Jul 2025 17:32:09 -0400 (EDT) X-Delivered-To: int-list-linux-mm@kvack.org Received: by kanga.kvack.org (Postfix, from userid 63042) id 73F066B00BC; Sat, 12 Jul 2025 17:32:09 -0400 (EDT) X-Delivered-To: linux-mm@kvack.org Received: from relay.hostedemail.com (smtprelay0015.hostedemail.com [216.40.44.15]) by kanga.kvack.org (Postfix) with ESMTP id 575816B00B8 for ; Sat, 12 Jul 2025 17:32:09 -0400 (EDT) Received: from smtpin14.hostedemail.com (a10.router.float.18 [10.200.18.1]) by unirelay04.hostedemail.com (Postfix) with ESMTP id 1EEA51A02B2 for ; Sat, 12 Jul 2025 21:32:09 +0000 (UTC) X-FDA: 83656910778.14.092F23C Received: from mail-pg1-f180.google.com (mail-pg1-f180.google.com [209.85.215.180]) by imf13.hostedemail.com (Postfix) with ESMTP id 0E9B220016 for ; Sat, 12 Jul 2025 21:32:06 +0000 (UTC) Authentication-Results: imf13.hostedemail.com; dkim=pass header.d=gmail.com header.s=20230601 header.b="mY/N8tUG"; spf=pass (imf13.hostedemail.com: domain of levymitchell0@gmail.com designates 209.85.215.180 as permitted sender) smtp.mailfrom=levymitchell0@gmail.com; dmarc=pass (policy=none) header.from=gmail.com ARC-Seal: i=1; s=arc-20220608; d=hostedemail.com; t=1752355927; a=rsa-sha256; cv=none; b=iZKjpc/GbFu0oZUbPZCm4ncaOV2xYitkr4NDqTWZyf+FKQV+bCQ7X8vz1Tiyo35c/UNBMU aDN8jRwnNDAhYdhcsYejJ7z2t8K4NkjwoyqmqzKcewxWpMcnPgcYn26S11H3kIcrdNRMEf fi0OGTJgfc79ak0HKs821kp85ALPr+M= ARC-Authentication-Results: i=1; imf13.hostedemail.com; dkim=pass header.d=gmail.com header.s=20230601 header.b="mY/N8tUG"; spf=pass (imf13.hostedemail.com: domain of levymitchell0@gmail.com designates 209.85.215.180 as permitted sender) smtp.mailfrom=levymitchell0@gmail.com; dmarc=pass (policy=none) header.from=gmail.com ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=hostedemail.com; s=arc-20220608; t=1752355927; h=from:from:sender:reply-to:subject:subject:date:date: message-id:message-id:to:to:cc:cc:mime-version:mime-version: content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references:dkim-signature; bh=CTtoMD2jUnmzV6fdblunt8Pya7UDWPcBKIFiwvtZyXU=; b=dKH4c6cb67Sebo4Sbeyxw3mND82JiZRAOmj/SdXB7FnDnFx5f7EAX4cGTDxFY+/hxnvC7c o10xUc4lVnxVx++8BfWrUcXuGlGBzftI8bvJaeR1hBQ7P7o3ghrxdxzYedS7eiQXSQlZ3r zWdSBDBlpPV5xdaUxsAqGwmElMHQj/M= Received: by mail-pg1-f180.google.com with SMTP id 41be03b00d2f7-b390136ed88so2377675a12.2 for ; Sat, 12 Jul 2025 14:32:06 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1752355926; x=1752960726; darn=kvack.org; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :mime-version:subject:date:from:from:to:cc:subject:date:message-id :reply-to; bh=CTtoMD2jUnmzV6fdblunt8Pya7UDWPcBKIFiwvtZyXU=; b=mY/N8tUG3SM0qOpiwxNxQrdHMPFod4Ob24YLo+2b7xShHxlVsXTuoK1STHyMBg8WPt vxE3BQBhmUW7r2pHIPtBtqEydXIcgy3F417Xqg+kD9JFG6lH6OVZma287jveZCIIEsQy n6ZMY01rTnNzlNmsMVnQmgquwjE+QvqDxpWo7/Iv554jUOV5laMnMkC842cZ5Lh5Wd28 rQU5L+SKlmqIhB0U5wskw5wSHzH8oOTMRPuc7+BRz7p8El1K21WAl9HURhvAMmfBMAxH hF3Gy9b0xZIvcGSMzJUqpgaP05HHXsKsLCdJghFj93HORvGMQzlpX7AVmtMcT44VLr9I epqQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1752355926; x=1752960726; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :mime-version:subject:date:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=CTtoMD2jUnmzV6fdblunt8Pya7UDWPcBKIFiwvtZyXU=; b=gKQWOGwlKPb8p28eqwubp4iXC1bJPVaaxY9HvKaJ9lbvGtoz1pKL/1QHn+P1h94xvS HUeAZgdz59an99HSfV1/3YTKs07226Ro9u/s1q2XSIHhDYQv5is58dh1pr8ADmXCn6P/ bhInnNtlxgDqkUB0CNYSWw1GeQGOAn/bu2ArGRKZ9aa4EZnY2B24LqDJzD46+ZXSDLQ6 3U75igeydLqmVHyOj2JD8qvrvb38hg0WC9DkfkmZO7pKdU5iOn5R0ea0MwWSbDOZqwgp NAh2aaNO8c31rWZqPxmu2utGMi3XxOyuaK63jZmScw6x62z3LyQZixppAZPynGs1ryaS NV8g== X-Forwarded-Encrypted: i=1; AJvYcCWfAHzQopVtm1XCni9Y+x4vFi9cT+FrkT25aHm4EfkulKUP7MWFQtCbD8/98zHkZpMpunVmx6N4eA==@kvack.org X-Gm-Message-State: AOJu0YyOZmcEoznYVSs+gZz1W0gJAe8tbWTuhJwkvO9TKVsGHMP38gZ3 +2ab1xO/LUDh1zXgB4g5KhiH4+j9bffBFkQpHk3mqNptPpqBhzW/qfPw X-Gm-Gg: ASbGncuKOse9DltG/ycrxUCdqF/nk2LWem8J4RGEsDi1qubJwCoEFaP/c59/qHagtpX gFan7S5Fd5Vgs2uI6srcRONsfRW2V2XpAHvPiUPpas0jNo/RhY58n8OLxdRU3z7hO+T55B41pYv BYOuIEcGhn8xvwBvDZ6fTdVMWWt32mkLfsjf/+YDlgNUDP2QpDoNFeuiOwhcX8el8yqHhq0UWQO 8fuwRC1l9GjdiGqP4svPlFE7A9WXduW98BiJK1tTj7Wavm/+8riDfZn8/h747rpM7JCIk2Cq0ta +pni/TVrJhnFqNPftDslr0gV8HpV38a6PAHEIU3LXzdN77UcQH05pn98JF7Xg4EntNbCnTmul3L nEXCpexnojIjfTbg6Iv6HH5mR4jWIl3mSwLes1FrE/v/7pYIMEfIF9N1E X-Google-Smtp-Source: AGHT+IHm5qpR1gjPtQZ/rA3SjPJLCrptEAwSAPqdZIC/RIhV1B17wk9gihcdL3O+6GYe4cBg0KSDew== X-Received: by 2002:a17:90a:e18e:b0:312:1ae9:1529 with SMTP id 98e67ed59e1d1-31c4cd8b792mr13311488a91.27.1752355925629; Sat, 12 Jul 2025 14:32:05 -0700 (PDT) Received: from mitchelllevy.localdomain ([174.127.224.194]) by smtp.gmail.com with ESMTPSA id 98e67ed59e1d1-31c3eb65aa2sm8003935a91.40.2025.07.12.14.32.04 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sat, 12 Jul 2025 14:32:05 -0700 (PDT) From: Mitchell Levy Date: Sat, 12 Jul 2025 14:31:12 -0700 Subject: [PATCH v2 1/5] rust: percpu: introduce a rust API for per-CPU variables MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit Message-Id: <20250712-rust-percpu-v2-1-826f2567521b@gmail.com> References: <20250712-rust-percpu-v2-0-826f2567521b@gmail.com> In-Reply-To: <20250712-rust-percpu-v2-0-826f2567521b@gmail.com> To: Miguel Ojeda , Alex Gaynor , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Andreas Hindborg , Alice Ryhl , Trevor Gross , Andrew Morton , Dennis Zhou , Tejun Heo , Christoph Lameter , Danilo Krummrich , Benno Lossin Cc: linux-kernel@vger.kernel.org, rust-for-linux@vger.kernel.org, linux-mm@kvack.org, Mitchell Levy X-Mailer: b4 0.14.0 X-Developer-Signature: v=1; a=ed25519-sha256; t=1752355923; l=15933; i=levymitchell0@gmail.com; s=20240719; h=from:subject:message-id; bh=/wdff0O+TBccIF+YWwxyFnAcyvOePCDxb+KptptoB68=; b=fGprdJqyWMecWZn0BZns5u9xCisKn6z932FGaZjQybSNKSx6RtIY/nAITGYWL2SRjzCB7MQIi XRcE08T71eXAgh4d8Zxj22bRAI+gzPsvVUhbxuCfGiO09XNkEGjbYkw X-Developer-Key: i=levymitchell0@gmail.com; a=ed25519; pk=n6kBmUnb+UNmjVkTnDwrLwTJAEKUfs2e8E+MFPZI93E= X-Rspam-User: X-Rspamd-Server: rspam05 X-Rspamd-Queue-Id: 0E9B220016 X-Stat-Signature: dh7r3szfqt71g5fah575zttspgp7oc1o X-HE-Tag: 1752355926-546704 X-HE-Meta: U2FsdGVkX1/1ZpbI/3awKNfXFfO6YvNIEUBjiL5VYwFKWkGz9bWw+4YsL/Y7j37Yj4PB/b89Xba+AHIwipvBjoKFcrrVFowIYP717IjS/19nattZtsut186kv64kURr8ntSzYMyLcWd7QS9hma3a4LAfNTVwdjw5c/qbbm1mOaR3IGAekcppeMQVpG7r9a4g2ELZganoaJSpfxdtMrChbrFn7+nq3O2rESmJIIP/aNF9eZ/TWcJlGqrRHLMzrB7dxHA90P/MNArjvZtk9ZTAX2ZROf3fWk6LjMJyojO13TNWbwtePZxaEHQuPcXqciY6yVOrUbLmEBT0/YAjAjWFeBrvxmKd4Ncxui/7Ypxs7Ied9VYkPa5G1rJJAw9fhXIJ1HICJMO48pXV8OLR5wpNcWi6A1L3dqPoMHGI+XuVdqqHUYh9DnHYt5Aqzbz4QZuXtA9cxOCo7lNd1f73zO6pk/HD0TaypVpxm2x1DzgkQqcMQVHbYyZgjP6QC7WD60BBpvNHDRb3jQj2E6+aMmmE15j9ggZYctv0WWKlOqugkA253e3Cy7olSebpuxa3sYTRfWRjfW56tBfcvDcUciejelNOElnKKpKZfmHWV5YNzy+OnETEDRWl8ND7n1HA5Mm83K/wdxD9DYy5VarSD3ipGlrTT1Bh6E4xeZTTUrpZZnQRx0eeHI4eA+lJHWzEgzxuglWCxjHyFbSTofMSBM45tW2Hc4cg4cAKnz2GJERYGLsyEIeZvjBwXfe6vk467IBokzlJnt8/AN/yQrbrs/PzxcYPP28k9dkdLZhkHKoYGNb517R9V3LHsUvFIuTCQ9GHP4e+MLBS9nERC3/PMdfsqTtnS2SGJ9nvajpdCdIYoRgrqdM3/d8iUjLW4jv4dOZp5nVLmSNrB24I+0WvO1kanE+dh/SeMmJCgrNiI6qdsZqxJqCUCOqDuXD8FPDq6WeppjAb6bWu1FJ30K5b/vi fw794uxY 9LKVLFgJEQJbNVESSp+r/ACTO1Guo0Z2VuQ3x7kNmSKgKRyEpIC4cmHp2UNYzOh9Ab2iwF7pOXN5CjPgPrv3OOprTy3uif7L41cj4yIjJdI1E+FOdN37hjqkXKCt/e70Y4Zq+nyHK+b8m6M4xPiKOC375LVsqp/2Npg3DQvfp1NMadmIToAYNsS2tSA7KWwhsHcBzZiSUYXwShOelQHHyx2jQWHIeJTgXfD1VNSApPcQfgzMZ7zP9vzfKMC75XFAHR+6Zq9yByFVblSkrkG+01dhIWWWDu1+n9GmtqHi0l1uDW2AmR/RM947zTFdYAsRgugXAUbU0uJR4pvsgjzDWTyq+gn8uH6diULNzWmzEKg+OwjEhqIyK2HGO//gfS3Rxyd3uBh/k8ZiYYYK6CuPQaDUUA1iqvlbVELBk52i7svNIBFbu/kH7UMoh4sJPo/VYBK+LjkAM7M9STYo= X-Bogosity: Ham, tests=bogofilter, spamicity=0.000000, version=1.2.4 Sender: owner-linux-mm@kvack.org Precedence: bulk X-Loop: owner-majordomo@kvack.org List-ID: List-Subscribe: List-Unsubscribe: Add a `CpuGuard` type that disables preemption for its lifetime. Add a `PerCpuAllocation` type used to track dynamic allocations. Add a `define_per_cpu!` macro to create static per-CPU allocations. Add `DynamicPerCpu` and `StaticPerCpu` to provide a high-level API. Add a `PerCpu` trait to unify the dynamic and static cases. Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Signed-off-by: Mitchell Levy --- rust/helpers/helpers.c | 2 + rust/helpers/percpu.c | 9 ++ rust/helpers/preempt.c | 14 ++ rust/kernel/lib.rs | 3 + rust/kernel/percpu.rs | 308 ++++++++++++++++++++++++++++++++++++++++ rust/kernel/percpu/cpu_guard.rs | 35 +++++ 6 files changed, 371 insertions(+) diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 0f1b5d115985..d56bbe6334d3 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -29,7 +29,9 @@ #include "page.c" #include "platform.c" #include "pci.c" +#include "percpu.c" #include "pid_namespace.c" +#include "preempt.c" #include "rbtree.c" #include "rcu.c" #include "refcount.c" diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c new file mode 100644 index 000000000000..a091389f730f --- /dev/null +++ b/rust/helpers/percpu.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align) +{ + return __alloc_percpu(sz, align); +} + diff --git a/rust/helpers/preempt.c b/rust/helpers/preempt.c new file mode 100644 index 000000000000..2c7529528ddd --- /dev/null +++ b/rust/helpers/preempt.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void rust_helper_preempt_disable(void) +{ + preempt_disable(); +} + +void rust_helper_preempt_enable(void) +{ + preempt_enable(); +} + diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 6b4774b2b1c3..733f9ff8b888 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -95,6 +95,9 @@ pub mod page; #[cfg(CONFIG_PCI)] pub mod pci; +// Only x86_64 is supported by percpu for now +#[cfg(CONFIG_X86_64)] +pub mod percpu; pub mod pid_namespace; pub mod platform; pub mod prelude; diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs new file mode 100644 index 000000000000..7dfceb6aefd7 --- /dev/null +++ b/rust/kernel/percpu.rs @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0 +//! This module contains abstractions for creating and using per-CPU variables from Rust. +//! See the define_per_cpu! macro and the DynamicPerCpu type, as well as the PerCpu trait. +pub mod cpu_guard; + +use bindings::{alloc_percpu, free_percpu}; + +use crate::alloc::Flags; +use crate::percpu::cpu_guard::CpuGuard; +use crate::prelude::*; +use crate::sync::Arc; + +use core::arch::asm; +use core::mem::{align_of, size_of}; + +use ffi::c_void; + +/// A per-CPU pointer; that is, an offset into the per-CPU area. Note that this type is NOT a smart +/// pointer, it does not manage the allocation. +pub struct PerCpuPtr(*mut T); + +/// Represents a dynamic allocation of a per-CPU variable via alloc_percpu. Calls free_percpu when +/// dropped. +pub struct PerCpuAllocation(PerCpuPtr); + +/// Holds a dynamically-allocated per-CPU variable. +pub struct DynamicPerCpu { + alloc: Arc>, +} + +/// Holds a statically-allocated per-CPU variable. +pub struct StaticPerCpu(PerCpuPtr); + +/// Represents exclusive access to the memory location pointed at by a particular PerCpu. +pub struct PerCpuToken<'a, T> { + _guard: CpuGuard, + ptr: &'a PerCpuPtr, +} + +/// A wrapper used for declaring static per-CPU variables. These symbols are "virtual" in that the +/// linker uses them to generate offsets into each CPU's per-CPU area, but shouldn't be read +/// from/written to directly. The fact that the statics are immutable prevents them being written +/// to (generally), this struct having _val be non-public prevents reading from them. +/// +/// The end-user of the per-CPU API should make use of the define_per_cpu! macro instead of +/// declaring variables of this type directly. +#[repr(transparent)] +pub struct StaticPerCpuSymbol { + _val: T, // generate a correctly sized type +} + +impl PerCpuPtr { + /// Makes a new PerCpuPtr from a raw per-CPU pointer. + /// + /// # Safety + /// `ptr` must be a valid per-CPU pointer. + pub unsafe fn new(ptr: *mut T) -> Self { + Self(ptr) + } + + /// Get a `&mut T` to the per-CPU variable represented by `&self` + /// + /// # Safety + /// The returned `&mut T` must follow Rust's aliasing rules. That is, no other `&(mut) T` may + /// exist that points to the same location in memory. In practice, this means that `get_ref` + /// must not be called on another `PerCpuPtr` that is a copy/clone of `&self` for as long as + /// the returned reference lives. + /// + /// CPU preemption must be disabled before calling this function and for the lifetime of the + /// returned reference. Otherwise, the returned &mut T might end up being a reference to a + /// different CPU's per-CPU area, causing the potential for a data race. + #[allow(clippy::mut_from_ref)] // Safety requirements prevent aliasing issues + pub unsafe fn get_ref(&self) -> &mut T { + let this_cpu_off_pcpu = core::ptr::addr_of!(this_cpu_off); + let mut this_cpu_area: *mut c_void; + // SAFETY: gs + this_cpu_off_pcpu is guaranteed to be a valid pointer because `gs` points + // to the per-CPU area and this_cpu_off_pcpu is a valid per-CPU allocation. + unsafe { + asm!( + "mov {out}, gs:[{off_val}]", + off_val = in(reg) this_cpu_off_pcpu, + out = out(reg) this_cpu_area, + ) + }; + // SAFETY: this_cpu_area + self.0 is guaranteed to be a valid pointer by the per-CPU + // subsystem and the invariant that self.0 is a valid offset into the per-CPU area. + // + // We know no-one else has a reference to the underlying pcpu variable because of the + // safety requirements of this function. + unsafe { &mut *((this_cpu_area).wrapping_add(self.0 as usize) as *mut T) } + } +} + +impl Clone for PerCpuPtr { + fn clone(&self) -> Self { + *self + } +} + +/// PerCpuPtr is just a pointer, so it's safe to copy. +impl Copy for PerCpuPtr {} + +impl PerCpuAllocation { + /// Dynamically allocates a space in the per-CPU area suitably sized and aligned to hold a `T`. + /// + /// Returns `None` under the same circumstances the C function `alloc_percpu` returns `NULL`. + pub fn new() -> Option> { + // SAFETY: No preconditions to call alloc_percpu + let ptr: *mut T = unsafe { alloc_percpu(size_of::(), align_of::()) } as *mut T; + if ptr.is_null() { + return None; + } + + Some(Self(PerCpuPtr(ptr))) + } +} + +impl Drop for PerCpuAllocation { + fn drop(&mut self) { + // SAFETY: self.0.0 was returned by alloc_percpu, and so was a valid pointer into + // the percpu area, and has remained valid by the invariants of PerCpuAllocation. + unsafe { free_percpu(self.0 .0 as *mut c_void) } + } +} + +/// A trait representing a per-CPU variable. This is implemented for both `StaticPerCpu` and +/// `DynamicPerCpu`. The main usage of this trait is to call `get` to get a `PerCpuToken` that +/// can be used to access the underlying per-CPU variable. See `PerCpuToken::with`. +/// +/// # Safety +/// The returned value from `ptr` must be valid for the lifetime of `&mut self`. +pub unsafe trait PerCpu { + /// Gets a `PerCpuPtr` to the per-CPU variable represented by `&mut self` + /// + /// # Safety + /// `self` may be doing all sorts of things to track when the underlying per-CPU variable can + /// be deallocated. You almost certainly shouldn't be calling this function directly (it's + /// essentially an implementation detail of the trait), and you certainly shouldn't be making + /// copies of the returned `PerCpuPtr` that may outlive `&mut self`. + /// + /// Implementers of this trait must ensure that the returned `PerCpuPtr` is valid for the + /// lifetime of `&mut self`. + unsafe fn ptr(&mut self) -> &PerCpuPtr; + + /// Produces a token, asserting that the holder has exclusive access to the underlying memory + /// pointed to by `self` + /// + /// # Safety + /// `func` (or its callees that execute on the same CPU) may not, for any `x: PerCpu` that + /// is a `clone` of `&mut self` (or, for a statically allocated variable, a `StaticPerCpu` + /// that came from the same `define_per_cpu!`): + /// - call `x.get()` + /// - make use of the value returned by `x.ptr()` + /// + /// `func` and its callees must not access or modify the memory associated with `&mut self`'s + /// allocation in the per-CPU area, except via (reborrows of) the reference passed to `func`. + /// + /// The underlying per-CPU variable cannot ever be mutated from an interrupt context, unless + /// irqs are disabled for the lifetime of the returned `PerCpuToken`. + unsafe fn get(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> { + PerCpuToken { + _guard: guard, + // SAFETY: The lifetime of the returned `PerCpuToken<'_, T>` is bounded by the lifetime + // of `&mut self`. + ptr: unsafe { self.ptr() }, + } + } +} + +impl StaticPerCpu { + /// Creates a new PerCpu pointing to the statically allocated variable at `ptr`. End-users + /// should probably be using the `unsafe_get_per_cpu!` macro instead of calling this function. + /// + /// # Safety + /// `ptr` must be a valid pointer to a per-CPU variable. This means that it must be a valid + /// offset into the per-CPU area, and that the per-CPU area must be suitably sized and aligned + /// to hold a `T`. + pub unsafe fn new(ptr: *mut T) -> Self { + Self(PerCpuPtr(ptr)) + } +} + +// SAFETY: The `PerCpuPtr` returned by `ptr` is valid for the lifetime of `self` (and in fact, +// forever). +unsafe impl PerCpu for StaticPerCpu { + unsafe fn ptr(&mut self) -> &PerCpuPtr { + &self.0 + } +} + +impl Clone for StaticPerCpu { + fn clone(&self) -> Self { + Self(self.0) + } +} + +impl DynamicPerCpu { + /// Allocates a new per-CPU variable + /// + /// # Arguments + /// * `flags` - Flags used to allocate an `Arc` that keeps track of the underlying + /// `PerCpuAllocation`. + pub fn new(flags: Flags) -> Option { + let alloc: PerCpuAllocation = PerCpuAllocation::new()?; + + let arc = Arc::new(alloc, flags).ok()?; + + Some(Self { alloc: arc }) + } +} + +impl DynamicPerCpu { + /// Wraps a `PerCpuAllocation` in a `PerCpu` + /// + /// # Arguments + /// * `alloc` - The allocation to use + /// * `flags` - The flags used to allocate an `Arc` that keeps track of the `PerCpuAllocation`. + pub fn new_from_allocation(alloc: PerCpuAllocation, flags: Flags) -> Option { + let arc = Arc::new(alloc, flags).ok()?; + Some(Self { alloc: arc }) + } +} + +// SAFETY: The `PerCpuPtr` returned by `ptr` is valid for the lifetime of `self` because we +// don't deallocate the underlying `PerCpuAllocation` until `self` is dropped. +unsafe impl PerCpu for DynamicPerCpu { + unsafe fn ptr(&mut self) -> &PerCpuPtr { + &self.alloc.0 + } +} + +impl Clone for DynamicPerCpu { + fn clone(&self) -> Self { + Self { + alloc: self.alloc.clone(), + } + } +} + +impl PerCpuToken<'_, T> { + /// Immediately invokes `func` with a `&mut T` that points at the underlying per-CPU variable + /// that `&mut self` represents. + pub fn with(&mut self, func: U) + where + U: FnOnce(&mut T), + { + // SAFETY: The existence of a PerCpuToken means that the requirements for get_ref are + // satisfied. + func(unsafe { self.ptr.get_ref() }); + } +} + +/// define_per_cpu! is analogous to the C DEFINE_PER_CPU macro in that it lets you create a +/// statically allocated per-CPU variable. +/// +/// # Example +/// ``` +/// use kernel::define_per_cpu; +/// use kernel::percpu::StaticPerCpuSymbol; +/// +/// define_per_cpu!(pub MY_PERCPU: u64 = 0); +/// ``` +#[macro_export] +macro_rules! define_per_cpu { + ($vis:vis $id:ident: $ty:ty = $expr:expr) => { + $crate::macros::paste! { + // Expand $expr outside of the unsafe block to avoid silently allowing unsafe code to be + // used without a user-facing unsafe block + static [<__INIT_ $id>]: $ty = $expr; + + // SAFETY: StaticPerCpuSymbol is #[repr(transparent)], so we can freely convert from T + #[link_section = ".data..percpu"] + $vis static $id: StaticPerCpuSymbol<$ty> = unsafe { + core::mem::transmute::<$ty, StaticPerCpuSymbol<$ty>>([<__INIT_ $id>]) + }; + } + }; +} + +/// Gets a `StaticPerCpu` from a symbol declared with `define_per_cpu!` or +/// `declare_extern_per_cpu!`. +/// +/// # Arguments +/// * `ident` - The identifier declared +/// +/// # Safety +/// `$id` must be declared with either `define_per_cpu!` or `declare_extern_per_cpu!`, and the +/// returned value must be stored in a `StaticPerCpu` where `T` matches the declared type of +/// `$id`. +#[macro_export] +macro_rules! unsafe_get_per_cpu { + ($id:ident) => {{ + $crate::percpu::StaticPerCpu::new((&$id) as *const _ as *mut _) + }}; +} + +/// Declares a StaticPerCpuSymbol corresponding to a per-CPU variable defined in C. Be sure to read +/// the safety requirements of `PerCpu::get`. +#[macro_export] +macro_rules! declare_extern_per_cpu { + ($id:ident: $ty:ty) => { + extern "C" { + static $id: StaticPerCpuSymbol<$ty>; + } + }; +} + +declare_extern_per_cpu!(this_cpu_off: u64); diff --git a/rust/kernel/percpu/cpu_guard.rs b/rust/kernel/percpu/cpu_guard.rs new file mode 100644 index 000000000000..14c04b12e7f0 --- /dev/null +++ b/rust/kernel/percpu/cpu_guard.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +//! Contains abstractions for disabling CPU preemption. See `CpuGuard`. + +/// A RAII guard for bindings::preempt_disable and bindings::preempt_enable. Guarantees preemption +/// is disabled for as long as this object exists. +pub struct CpuGuard { + // Don't make one without using new() + _phantom: (), +} + +impl CpuGuard { + /// Create a new CpuGuard. Disables preemption for its lifetime. + pub fn new() -> Self { + // SAFETY: There are no preconditions required to call preempt_disable + unsafe { + bindings::preempt_disable(); + } + CpuGuard { _phantom: () } + } +} + +impl Default for CpuGuard { + fn default() -> Self { + Self::new() + } +} + +impl Drop for CpuGuard { + fn drop(&mut self) { + // SAFETY: There are no preconditions required to call preempt_enable + unsafe { + bindings::preempt_enable(); + } + } +} -- 2.34.1