From mboxrd@z Thu Jan 1 00:00:00 1970 From: Andrew Jones Date: Fri, 19 Apr 2024 15:53:25 +0200 Subject: [PATCH v2 3/6] riscv: Add Zawrs support for spinlocks In-Reply-To: <20240419135321.70781-8-ajones@ventanamicro.com> References: <20240419135321.70781-8-ajones@ventanamicro.com> Message-ID: <20240419135321.70781-11-ajones@ventanamicro.com> List-Id: To: kvm-riscv@lists.infradead.org MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit From: Christoph M?llner RISC-V code uses the generic ticket lock implementation, which calls the macros smp_cond_load_relaxed() and smp_cond_load_acquire(). Introduce a RISC-V specific implementation of smp_cond_load_relaxed() which applies WRS.NTO of the Zawrs extension in order to reduce power consumption while waiting and allows hypervisors to enable guests to trap while waiting. smp_cond_load_acquire() doesn't need a RISC-V specific implementation as the generic implementation is based on smp_cond_load_relaxed() and smp_acquire__after_ctrl_dep() sufficiently provides the acquire semantics. This implementation is heavily based on Arm's approach which is the approach Andrea Parri also suggested. The Zawrs specification can be found here: https://github.com/riscv/riscv-zawrs/blob/main/zawrs.adoc Signed-off-by: Christoph M?llner Co-developed-by: Andrew Jones Signed-off-by: Andrew Jones --- arch/riscv/Kconfig | 13 ++++++++ arch/riscv/include/asm/barrier.h | 45 ++++++++++++++++++--------- arch/riscv/include/asm/cmpxchg.h | 51 +++++++++++++++++++++++++++++++ arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/include/asm/insn-def.h | 2 ++ arch/riscv/kernel/cpufeature.c | 1 + 6 files changed, 98 insertions(+), 15 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 7427d8088337..34bbe6b70546 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -578,6 +578,19 @@ config RISCV_ISA_V_PREEMPTIVE preemption. Enabling this config will result in higher memory consumption due to the allocation of per-task's kernel Vector context. +config RISCV_ISA_ZAWRS + bool "Zawrs extension support for more efficient busy waiting" + depends on RISCV_ALTERNATIVE + default y + help + The Zawrs extension defines instructions to be used in polling loops + which allow a hart to enter a low-power state or to trap to the + hypervisor while waiting on a store to a memory location. Enable the + use of these instructions in the kernel when the Zawrs extension is + detected at boot. + + If you don't know what to do here, say Y. + config TOOLCHAIN_HAS_ZBB bool default y diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h index 880b56d8480d..e1d9bf1deca6 100644 --- a/arch/riscv/include/asm/barrier.h +++ b/arch/riscv/include/asm/barrier.h @@ -11,6 +11,7 @@ #define _ASM_RISCV_BARRIER_H #ifndef __ASSEMBLY__ +#include #include #define nop() __asm__ __volatile__ ("nop") @@ -28,21 +29,6 @@ #define __smp_rmb() RISCV_FENCE(r, r) #define __smp_wmb() RISCV_FENCE(w, w) -#define __smp_store_release(p, v) \ -do { \ - compiletime_assert_atomic_type(*p); \ - RISCV_FENCE(rw, w); \ - WRITE_ONCE(*p, v); \ -} while (0) - -#define __smp_load_acquire(p) \ -({ \ - typeof(*p) ___p1 = READ_ONCE(*p); \ - compiletime_assert_atomic_type(*p); \ - RISCV_FENCE(r, rw); \ - ___p1; \ -}) - /* * This is a very specific barrier: it's currently only used in two places in * the kernel, both in the scheduler. See include/linux/spinlock.h for the two @@ -70,6 +56,35 @@ do { \ */ #define smp_mb__after_spinlock() RISCV_FENCE(iorw, iorw) +#define __smp_store_release(p, v) \ +do { \ + compiletime_assert_atomic_type(*p); \ + RISCV_FENCE(rw, w); \ + WRITE_ONCE(*p, v); \ +} while (0) + +#define __smp_load_acquire(p) \ +({ \ + typeof(*p) ___p1 = READ_ONCE(*p); \ + compiletime_assert_atomic_type(*p); \ + RISCV_FENCE(r, rw); \ + ___p1; \ +}) + +#ifdef CONFIG_RISCV_ISA_ZAWRS +#define smp_cond_load_relaxed(ptr, cond_expr) ({ \ + typeof(ptr) __PTR = (ptr); \ + __unqual_scalar_typeof(*ptr) VAL; \ + for (;;) { \ + VAL = READ_ONCE(*__PTR); \ + if (cond_expr) \ + break; \ + __cmpwait_relaxed(ptr, VAL); \ + } \ + (typeof(*ptr))VAL; \ +}) +#endif + #include #endif /* __ASSEMBLY__ */ diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h index 2fee65cc8443..0926ac7f4ca6 100644 --- a/arch/riscv/include/asm/cmpxchg.h +++ b/arch/riscv/include/asm/cmpxchg.h @@ -8,7 +8,10 @@ #include +#include #include +#include +#include #define __xchg_relaxed(ptr, new, size) \ ({ \ @@ -359,4 +362,52 @@ arch_cmpxchg_relaxed((ptr), (o), (n)); \ }) +#ifdef CONFIG_RISCV_ISA_ZAWRS +static __always_inline void __cmpwait(volatile void *ptr, + unsigned long val, + int size) +{ + unsigned long tmp; + + asm goto(ALTERNATIVE("j %l[no_zawrs]", "nop", + 0, RISCV_ISA_EXT_ZAWRS, 1) + : : : : no_zawrs); + + switch (size) { + case 4: + asm volatile( + " lr.w %0, %1\n" + " xor %0, %0, %2\n" + " bnez %0, 1f\n" + ZAWRS_WRS_NTO "\n" + "1:" + : "=&r" (tmp), "+A" (*(u32 *)ptr) + : "r" (val)); + break; +#if __riscv_xlen == 64 + case 8: + asm volatile( + " lr.d %0, %1\n" + " xor %0, %0, %2\n" + " bnez %0, 1f\n" + ZAWRS_WRS_NTO "\n" + "1:" + : "=&r" (tmp), "+A" (*(u64 *)ptr) + : "r" (val)); + break; +#endif + default: + BUILD_BUG(); + } + + return; + +no_zawrs: + asm volatile(RISCV_PAUSE : : : "memory"); +} + +#define __cmpwait_relaxed(ptr, val) \ + __cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr))) +#endif + #endif /* _ASM_RISCV_CMPXCHG_H */ diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index e17d0078a651..5b358c3cf212 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -81,6 +81,7 @@ #define RISCV_ISA_EXT_ZTSO 72 #define RISCV_ISA_EXT_ZACAS 73 #define RISCV_ISA_EXT_XANDESPMU 74 +#define RISCV_ISA_EXT_ZAWRS 75 #define RISCV_ISA_EXT_XLINUXENVCFG 127 diff --git a/arch/riscv/include/asm/insn-def.h b/arch/riscv/include/asm/insn-def.h index 64dffaa21bfa..9a913010cdd9 100644 --- a/arch/riscv/include/asm/insn-def.h +++ b/arch/riscv/include/asm/insn-def.h @@ -197,5 +197,7 @@ RS1(base), SIMM12(4)) #define RISCV_PAUSE ".4byte 0x100000f" +#define ZAWRS_WRS_NTO ".4byte 0x00d00073" +#define ZAWRS_WRS_STO ".4byte 0x01d00073" #endif /* __ASM_INSN_DEF_H */ diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 3ed2359eae35..02de9eaa3f42 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -257,6 +257,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), + __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS), __RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA), __RISCV_ISA_EXT_DATA(zfh, RISCV_ISA_EXT_ZFH), __RISCV_ISA_EXT_DATA(zfhmin, RISCV_ISA_EXT_ZFHMIN), -- 2.44.0 From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from bombadil.infradead.org (bombadil.infradead.org [198.137.202.133]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.lore.kernel.org (Postfix) with ESMTPS id 36DB4C4345F for ; Fri, 19 Apr 2024 13:53:43 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=lists.infradead.org; s=bombadil.20210309; h=Sender: Content-Transfer-Encoding:Content-Type:List-Subscribe:List-Help:List-Post: List-Archive:List-Unsubscribe:List-Id:MIME-Version:References:In-Reply-To: Message-ID:Date:Subject:Cc:To:From:Reply-To:Content-ID:Content-Description: Resent-Date:Resent-From:Resent-Sender:Resent-To:Resent-Cc:Resent-Message-ID: List-Owner; bh=sGNnvc2gbFB1+ZfJaHJGXk3n8i29HMaO4e0cmg92SYA=; b=ggXojA7GXfESW0 rQIu95c/lMkTrCKtSVJfXhaabiVstHwblt3ORelIbvp4WYwf7puYjk6o9RcfC+nlwhVaZApaqCiOa vtTc6VQ2uWcaLV72qVsgFElvFMxxbmE0OU/g1vAFcPcWG2cKYMhrymJVqBkPrBmMr/atq5OEg/1tD ey0MVcU26tj0KvaXXrSRYvdqfyvq3JIpDgkeOEcdppMOepgyCBnB6q7fbc0Phv+K2SZ9DmtUmfurC BOQmSbGhxqhCVLyBSEXi4No3mJx5OD7p2+kkyuo8gOU3G3OwWGWrB4eEcTINQfpiNWoEAV1PBi8nJ YIIyzM01odqRKNpSCsyw==; Received: from localhost ([::1] helo=bombadil.infradead.org) by bombadil.infradead.org with esmtp (Exim 4.97.1 #2 (Red Hat Linux)) id 1rxogM-00000005pJA-126J; Fri, 19 Apr 2024 13:53:38 +0000 Received: from mail-ed1-x52c.google.com ([2a00:1450:4864:20::52c]) by bombadil.infradead.org with esmtps (Exim 4.97.1 #2 (Red Hat Linux)) id 1rxogE-00000005pCR-28LN for linux-riscv@lists.infradead.org; Fri, 19 Apr 2024 13:53:32 +0000 Received: by mail-ed1-x52c.google.com with SMTP id 4fb4d7f45d1cf-571c22d9de4so2213632a12.3 for ; Fri, 19 Apr 2024 06:53:29 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=ventanamicro.com; s=google; t=1713534808; x=1714139608; darn=lists.infradead.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=nEizurJcSR1aY1vBLFtX+JNkUfZx2QKjVF6QEsuG1UY=; b=XR636xQoQ1f/dyBx1t2HsNEXwDFWXqZfCZJLETFuKaaDZwfx5Dtm2ec2YuSCcEjJZv 1z7q8x1ZEs0kc0Agh7cWuEeeM5LQhvFw8YIFTbPsCOtFLEP9Dc+S2AuxG7xQwZNzW6xq guWmIybbyvI/PVj1ZOuSer7P0M+ew9Qd1atJn6FQLBAJ0nji0u2b1jeA/y6d4PYbbwpM NTyCtkNL/DaQLT7LXuZG9OJNG4j/xMMs2RE12arMltHsEQFiQBYV6zW8Qx0WOJfzzMvj 9ybHQtVeI+0ivxW5jOceEu4+XhPaCNeV6DOyKJqj3x+9fb1KggjHQq4YfoeCXTweqUOu MTGg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1713534808; x=1714139608; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=nEizurJcSR1aY1vBLFtX+JNkUfZx2QKjVF6QEsuG1UY=; b=q3OcuKfGx0nd43Fwqm3tBAJIUiOkYBksdYoEHe+rj9k/rznvnJy6NPBSBGuquJIjDe UwLPcYYq1q3QC6+N7iyltsYGTsgfwT4FpH+G7Mq16gk6HmqhCxAC5gmZVD3WMRZWuyg9 Dj4mO5+Sb++FDrbgclOdxEfi8Ly3JYUy2pn0vCtRgTySEpm7sdtOXh9H067QcLwbieB/ wrX2egtF683VKtUSheXpL4NdlF85Or8TNfM2Y79vZhh4nFhRAmK0svPQY5IFza+PP6V8 JqYMQhU7Z1oWTPKEYt8tp/SX+J4DCsTAZpiYrMc6KE+6L+uAgeFODQaCIocssN0b9bB6 N8Xw== X-Gm-Message-State: AOJu0Yxs/uk6l1/XQr5IRxi54dntYu3VxHQSP7LQkgJKrIJoY9vOxfEG oadAvg0fQZi61FsGoqt6wda1JyJ1mBxu6evAUu7u9oPzQIOOQ3f/CQwxTTliMRbOhAQbTAHLfq9 O+qs= X-Google-Smtp-Source: AGHT+IGkQt2hO18a+SNSAYfEOPucXpHAMV1WR0RrkuDvLuCw0MpZGKjuj++zRhwHCrnJC9FEvEAlqg== X-Received: by 2002:a50:baad:0:b0:56e:323b:d7e7 with SMTP id x42-20020a50baad000000b0056e323bd7e7mr1751219ede.34.1713534807639; Fri, 19 Apr 2024 06:53:27 -0700 (PDT) Received: from localhost (2001-1ae9-1c2-4c00-20f-c6b4-1e57-7965.ip6.tmcz.cz. [2001:1ae9:1c2:4c00:20f:c6b4:1e57:7965]) by smtp.gmail.com with ESMTPSA id d6-20020a05640208c600b0056e72c4a330sm2148891edz.41.2024.04.19.06.53.27 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 19 Apr 2024 06:53:27 -0700 (PDT) From: Andrew Jones To: linux-riscv@lists.infradead.org, kvm-riscv@lists.infradead.org, devicetree@vger.kernel.org Cc: paul.walmsley@sifive.com, palmer@dabbelt.com, aou@eecs.berkeley.edu, conor.dooley@microchip.com, anup@brainfault.org, atishp@atishpatra.org, robh@kernel.org, krzysztof.kozlowski+dt@linaro.org, conor+dt@kernel.org, christoph.muellner@vrull.eu, heiko@sntech.de, charlie@rivosinc.com, David.Laight@ACULAB.COM, parri.andrea@gmail.com, luxu.kernel@bytedance.com Subject: [PATCH v2 3/6] riscv: Add Zawrs support for spinlocks Date: Fri, 19 Apr 2024 15:53:25 +0200 Message-ID: <20240419135321.70781-11-ajones@ventanamicro.com> X-Mailer: git-send-email 2.44.0 In-Reply-To: <20240419135321.70781-8-ajones@ventanamicro.com> References: <20240419135321.70781-8-ajones@ventanamicro.com> MIME-Version: 1.0 X-CRM114-Version: 20100106-BlameMichelson ( TRE 0.8.0 (BSD) ) MR-646709E3 X-CRM114-CacheID: sfid-20240419_065330_689051_DF2D410C X-CRM114-Status: GOOD ( 20.06 ) X-BeenThere: linux-riscv@lists.infradead.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: base64 Sender: "linux-riscv" Errors-To: linux-riscv-bounces+linux-riscv=archiver.kernel.org@lists.infradead.org RnJvbTogQ2hyaXN0b3BoIE3DvGxsbmVyIDxjaHJpc3RvcGgubXVlbGxuZXJAdnJ1bGwuZXU+CgpS SVNDLVYgY29kZSB1c2VzIHRoZSBnZW5lcmljIHRpY2tldCBsb2NrIGltcGxlbWVudGF0aW9uLCB3 aGljaCBjYWxscwp0aGUgbWFjcm9zIHNtcF9jb25kX2xvYWRfcmVsYXhlZCgpIGFuZCBzbXBfY29u ZF9sb2FkX2FjcXVpcmUoKS4KSW50cm9kdWNlIGEgUklTQy1WIHNwZWNpZmljIGltcGxlbWVudGF0 aW9uIG9mIHNtcF9jb25kX2xvYWRfcmVsYXhlZCgpCndoaWNoIGFwcGxpZXMgV1JTLk5UTyBvZiB0 aGUgWmF3cnMgZXh0ZW5zaW9uIGluIG9yZGVyIHRvIHJlZHVjZSBwb3dlcgpjb25zdW1wdGlvbiB3 aGlsZSB3YWl0aW5nIGFuZCBhbGxvd3MgaHlwZXJ2aXNvcnMgdG8gZW5hYmxlIGd1ZXN0cyB0bwp0 cmFwIHdoaWxlIHdhaXRpbmcuIHNtcF9jb25kX2xvYWRfYWNxdWlyZSgpIGRvZXNuJ3QgbmVlZCBh IFJJU0MtVgpzcGVjaWZpYyBpbXBsZW1lbnRhdGlvbiBhcyB0aGUgZ2VuZXJpYyBpbXBsZW1lbnRh dGlvbiBpcyBiYXNlZCBvbgpzbXBfY29uZF9sb2FkX3JlbGF4ZWQoKSBhbmQgc21wX2FjcXVpcmVf X2FmdGVyX2N0cmxfZGVwKCkgc3VmZmljaWVudGx5CnByb3ZpZGVzIHRoZSBhY3F1aXJlIHNlbWFu dGljcy4KClRoaXMgaW1wbGVtZW50YXRpb24gaXMgaGVhdmlseSBiYXNlZCBvbiBBcm0ncyBhcHBy b2FjaCB3aGljaCBpcyB0aGUKYXBwcm9hY2ggQW5kcmVhIFBhcnJpIGFsc28gc3VnZ2VzdGVkLgoK VGhlIFphd3JzIHNwZWNpZmljYXRpb24gY2FuIGJlIGZvdW5kIGhlcmU6Cmh0dHBzOi8vZ2l0aHVi LmNvbS9yaXNjdi9yaXNjdi16YXdycy9ibG9iL21haW4vemF3cnMuYWRvYwoKU2lnbmVkLW9mZi1i eTogQ2hyaXN0b3BoIE3DvGxsbmVyIDxjaHJpc3RvcGgubXVlbGxuZXJAdnJ1bGwuZXU+CkNvLWRl dmVsb3BlZC1ieTogQW5kcmV3IEpvbmVzIDxham9uZXNAdmVudGFuYW1pY3JvLmNvbT4KU2lnbmVk LW9mZi1ieTogQW5kcmV3IEpvbmVzIDxham9uZXNAdmVudGFuYW1pY3JvLmNvbT4KLS0tCiBhcmNo L3Jpc2N2L0tjb25maWcgICAgICAgICAgICAgICAgfCAxMyArKysrKysrKwogYXJjaC9yaXNjdi9p bmNsdWRlL2FzbS9iYXJyaWVyLmggIHwgNDUgKysrKysrKysrKysrKysrKysrLS0tLS0tLS0tCiBh cmNoL3Jpc2N2L2luY2x1ZGUvYXNtL2NtcHhjaGcuaCAgfCA1MSArKysrKysrKysrKysrKysrKysr KysrKysrKysrKysrCiBhcmNoL3Jpc2N2L2luY2x1ZGUvYXNtL2h3Y2FwLmggICAgfCAgMSArCiBh cmNoL3Jpc2N2L2luY2x1ZGUvYXNtL2luc24tZGVmLmggfCAgMiArKwogYXJjaC9yaXNjdi9rZXJu ZWwvY3B1ZmVhdHVyZS5jICAgIHwgIDEgKwogNiBmaWxlcyBjaGFuZ2VkLCA5OCBpbnNlcnRpb25z KCspLCAxNSBkZWxldGlvbnMoLSkKCmRpZmYgLS1naXQgYS9hcmNoL3Jpc2N2L0tjb25maWcgYi9h cmNoL3Jpc2N2L0tjb25maWcKaW5kZXggNzQyN2Q4MDg4MzM3Li4zNGJiZTZiNzA1NDYgMTAwNjQ0 Ci0tLSBhL2FyY2gvcmlzY3YvS2NvbmZpZworKysgYi9hcmNoL3Jpc2N2L0tjb25maWcKQEAgLTU3 OCw2ICs1NzgsMTkgQEAgY29uZmlnIFJJU0NWX0lTQV9WX1BSRUVNUFRJVkUKIAkgIHByZWVtcHRp b24uIEVuYWJsaW5nIHRoaXMgY29uZmlnIHdpbGwgcmVzdWx0IGluIGhpZ2hlciBtZW1vcnkKIAkg IGNvbnN1bXB0aW9uIGR1ZSB0byB0aGUgYWxsb2NhdGlvbiBvZiBwZXItdGFzaydzIGtlcm5lbCBW ZWN0b3IgY29udGV4dC4KIAorY29uZmlnIFJJU0NWX0lTQV9aQVdSUworCWJvb2wgIlphd3JzIGV4 dGVuc2lvbiBzdXBwb3J0IGZvciBtb3JlIGVmZmljaWVudCBidXN5IHdhaXRpbmciCisJZGVwZW5k cyBvbiBSSVNDVl9BTFRFUk5BVElWRQorCWRlZmF1bHQgeQorCWhlbHAKKwkgIFRoZSBaYXdycyBl eHRlbnNpb24gZGVmaW5lcyBpbnN0cnVjdGlvbnMgdG8gYmUgdXNlZCBpbiBwb2xsaW5nIGxvb3Bz CisJICB3aGljaCBhbGxvdyBhIGhhcnQgdG8gZW50ZXIgYSBsb3ctcG93ZXIgc3RhdGUgb3IgdG8g dHJhcCB0byB0aGUKKwkgIGh5cGVydmlzb3Igd2hpbGUgd2FpdGluZyBvbiBhIHN0b3JlIHRvIGEg bWVtb3J5IGxvY2F0aW9uLiBFbmFibGUgdGhlCisJICB1c2Ugb2YgdGhlc2UgaW5zdHJ1Y3Rpb25z IGluIHRoZSBrZXJuZWwgd2hlbiB0aGUgWmF3cnMgZXh0ZW5zaW9uIGlzCisJICBkZXRlY3RlZCBh dCBib290LgorCisJICBJZiB5b3UgZG9uJ3Qga25vdyB3aGF0IHRvIGRvIGhlcmUsIHNheSBZLgor CiBjb25maWcgVE9PTENIQUlOX0hBU19aQkIKIAlib29sCiAJZGVmYXVsdCB5CmRpZmYgLS1naXQg YS9hcmNoL3Jpc2N2L2luY2x1ZGUvYXNtL2JhcnJpZXIuaCBiL2FyY2gvcmlzY3YvaW5jbHVkZS9h c20vYmFycmllci5oCmluZGV4IDg4MGI1NmQ4NDgwZC4uZTFkOWJmMWRlY2E2IDEwMDY0NAotLS0g YS9hcmNoL3Jpc2N2L2luY2x1ZGUvYXNtL2JhcnJpZXIuaAorKysgYi9hcmNoL3Jpc2N2L2luY2x1 ZGUvYXNtL2JhcnJpZXIuaApAQCAtMTEsNiArMTEsNyBAQAogI2RlZmluZSBfQVNNX1JJU0NWX0JB UlJJRVJfSAogCiAjaWZuZGVmIF9fQVNTRU1CTFlfXworI2luY2x1ZGUgPGFzbS9jbXB4Y2hnLmg+ CiAjaW5jbHVkZSA8YXNtL2ZlbmNlLmg+CiAKICNkZWZpbmUgbm9wKCkJCV9fYXNtX18gX192b2xh dGlsZV9fICgibm9wIikKQEAgLTI4LDIxICsyOSw2IEBACiAjZGVmaW5lIF9fc21wX3JtYigpCVJJ U0NWX0ZFTkNFKHIsIHIpCiAjZGVmaW5lIF9fc21wX3dtYigpCVJJU0NWX0ZFTkNFKHcsIHcpCiAK LSNkZWZpbmUgX19zbXBfc3RvcmVfcmVsZWFzZShwLCB2KQkJCQkJXAotZG8gewkJCQkJCQkJCVwK LQljb21waWxldGltZV9hc3NlcnRfYXRvbWljX3R5cGUoKnApOwkJCQlcCi0JUklTQ1ZfRkVOQ0Uo cncsIHcpOwkJCQkJCVwKLQlXUklURV9PTkNFKCpwLCB2KTsJCQkJCQlcCi19IHdoaWxlICgwKQot Ci0jZGVmaW5lIF9fc21wX2xvYWRfYWNxdWlyZShwKQkJCQkJCVwKLSh7CQkJCQkJCQkJXAotCXR5 cGVvZigqcCkgX19fcDEgPSBSRUFEX09OQ0UoKnApOwkJCQlcCi0JY29tcGlsZXRpbWVfYXNzZXJ0 X2F0b21pY190eXBlKCpwKTsJCQkJXAotCVJJU0NWX0ZFTkNFKHIsIHJ3KTsJCQkJCQlcCi0JX19f cDE7CQkJCQkJCQlcCi19KQotCiAvKgogICogVGhpcyBpcyBhIHZlcnkgc3BlY2lmaWMgYmFycmll cjogaXQncyBjdXJyZW50bHkgb25seSB1c2VkIGluIHR3byBwbGFjZXMgaW4KICAqIHRoZSBrZXJu ZWwsIGJvdGggaW4gdGhlIHNjaGVkdWxlci4gIFNlZSBpbmNsdWRlL2xpbnV4L3NwaW5sb2NrLmgg Zm9yIHRoZSB0d28KQEAgLTcwLDYgKzU2LDM1IEBAIGRvIHsJCQkJCQkJCQlcCiAgKi8KICNkZWZp bmUgc21wX21iX19hZnRlcl9zcGlubG9jaygpCVJJU0NWX0ZFTkNFKGlvcncsIGlvcncpCiAKKyNk ZWZpbmUgX19zbXBfc3RvcmVfcmVsZWFzZShwLCB2KQkJCQkJXAorZG8gewkJCQkJCQkJCVwKKwlj b21waWxldGltZV9hc3NlcnRfYXRvbWljX3R5cGUoKnApOwkJCQlcCisJUklTQ1ZfRkVOQ0Uocncs IHcpOwkJCQkJCVwKKwlXUklURV9PTkNFKCpwLCB2KTsJCQkJCQlcCit9IHdoaWxlICgwKQorCisj ZGVmaW5lIF9fc21wX2xvYWRfYWNxdWlyZShwKQkJCQkJCVwKKyh7CQkJCQkJCQkJXAorCXR5cGVv ZigqcCkgX19fcDEgPSBSRUFEX09OQ0UoKnApOwkJCQlcCisJY29tcGlsZXRpbWVfYXNzZXJ0X2F0 b21pY190eXBlKCpwKTsJCQkJXAorCVJJU0NWX0ZFTkNFKHIsIHJ3KTsJCQkJCQlcCisJX19fcDE7 CQkJCQkJCQlcCit9KQorCisjaWZkZWYgQ09ORklHX1JJU0NWX0lTQV9aQVdSUworI2RlZmluZSBz bXBfY29uZF9sb2FkX3JlbGF4ZWQocHRyLCBjb25kX2V4cHIpICh7CQkJXAorCXR5cGVvZihwdHIp IF9fUFRSID0gKHB0cik7CQkJCQlcCisJX191bnF1YWxfc2NhbGFyX3R5cGVvZigqcHRyKSBWQUw7 CQkJCVwKKwlmb3IgKDs7KSB7CQkJCQkJCVwKKwkJVkFMID0gUkVBRF9PTkNFKCpfX1BUUik7CQkJ CVwKKwkJaWYgKGNvbmRfZXhwcikJCQkJCQlcCisJCQlicmVhazsJCQkJCQlcCisJCV9fY21wd2Fp dF9yZWxheGVkKHB0ciwgVkFMKTsJCQkJXAorCX0JCQkJCQkJCVwKKwkodHlwZW9mKCpwdHIpKVZB TDsJCQkJCQlcCit9KQorI2VuZGlmCisKICNpbmNsdWRlIDxhc20tZ2VuZXJpYy9iYXJyaWVyLmg+ CiAKICNlbmRpZiAvKiBfX0FTU0VNQkxZX18gKi8KZGlmZiAtLWdpdCBhL2FyY2gvcmlzY3YvaW5j bHVkZS9hc20vY21weGNoZy5oIGIvYXJjaC9yaXNjdi9pbmNsdWRlL2FzbS9jbXB4Y2hnLmgKaW5k ZXggMmZlZTY1Y2M4NDQzLi4wOTI2YWM3ZjRjYTYgMTAwNjQ0Ci0tLSBhL2FyY2gvcmlzY3YvaW5j bHVkZS9hc20vY21weGNoZy5oCisrKyBiL2FyY2gvcmlzY3YvaW5jbHVkZS9hc20vY21weGNoZy5o CkBAIC04LDcgKzgsMTAgQEAKIAogI2luY2x1ZGUgPGxpbnV4L2J1Zy5oPgogCisjaW5jbHVkZSA8 YXNtL2FsdGVybmF0aXZlLW1hY3Jvcy5oPgogI2luY2x1ZGUgPGFzbS9mZW5jZS5oPgorI2luY2x1 ZGUgPGFzbS9od2NhcC5oPgorI2luY2x1ZGUgPGFzbS9pbnNuLWRlZi5oPgogCiAjZGVmaW5lIF9f eGNoZ19yZWxheGVkKHB0ciwgbmV3LCBzaXplKQkJCQkJXAogKHsJCQkJCQkJCQlcCkBAIC0zNTks NCArMzYyLDUyIEBACiAJYXJjaF9jbXB4Y2hnX3JlbGF4ZWQoKHB0ciksIChvKSwgKG4pKTsJCQkJ XAogfSkKIAorI2lmZGVmIENPTkZJR19SSVNDVl9JU0FfWkFXUlMKK3N0YXRpYyBfX2Fsd2F5c19p bmxpbmUgdm9pZCBfX2NtcHdhaXQodm9sYXRpbGUgdm9pZCAqcHRyLAorCQkJCSAgICAgIHVuc2ln bmVkIGxvbmcgdmFsLAorCQkJCSAgICAgIGludCBzaXplKQoreworCXVuc2lnbmVkIGxvbmcgdG1w OworCisJYXNtIGdvdG8oQUxURVJOQVRJVkUoImogJWxbbm9femF3cnNdIiwgIm5vcCIsCisJCQkg ICAgIDAsIFJJU0NWX0lTQV9FWFRfWkFXUlMsIDEpCisJCSA6IDogOiA6IG5vX3phd3JzKTsKKwor CXN3aXRjaCAoc2l6ZSkgeworCWNhc2UgNDoKKwkJYXNtIHZvbGF0aWxlKAorCQkiCWxyLncJJTAs ICUxXG4iCisJCSIJeG9yCSUwLCAlMCwgJTJcbiIKKwkJIglibmV6CSUwLCAxZlxuIgorCQkJWkFX UlNfV1JTX05UTyAiXG4iCisJCSIxOiIKKwkJOiAiPSZyIiAodG1wKSwgIitBIiAoKih1MzIgKilw dHIpCisJCTogInIiICh2YWwpKTsKKwkJYnJlYWs7CisjaWYgX19yaXNjdl94bGVuID09IDY0CisJ Y2FzZSA4OgorCQlhc20gdm9sYXRpbGUoCisJCSIJbHIuZAklMCwgJTFcbiIKKwkJIgl4b3IJJTAs ICUwLCAlMlxuIgorCQkiCWJuZXoJJTAsIDFmXG4iCisJCQlaQVdSU19XUlNfTlRPICJcbiIKKwkJ IjE6IgorCQk6ICI9JnIiICh0bXApLCAiK0EiICgqKHU2NCAqKXB0cikKKwkJOiAiciIgKHZhbCkp OworCQlicmVhazsKKyNlbmRpZgorCWRlZmF1bHQ6CisJCUJVSUxEX0JVRygpOworCX0KKworCXJl dHVybjsKKworbm9femF3cnM6CisJYXNtIHZvbGF0aWxlKFJJU0NWX1BBVVNFIDogOiA6ICJtZW1v cnkiKTsKK30KKworI2RlZmluZSBfX2NtcHdhaXRfcmVsYXhlZChwdHIsIHZhbCkgXAorCV9fY21w d2FpdCgocHRyKSwgKHVuc2lnbmVkIGxvbmcpKHZhbCksIHNpemVvZigqKHB0cikpKQorI2VuZGlm CisKICNlbmRpZiAvKiBfQVNNX1JJU0NWX0NNUFhDSEdfSCAqLwpkaWZmIC0tZ2l0IGEvYXJjaC9y aXNjdi9pbmNsdWRlL2FzbS9od2NhcC5oIGIvYXJjaC9yaXNjdi9pbmNsdWRlL2FzbS9od2NhcC5o CmluZGV4IGUxN2QwMDc4YTY1MS4uNWIzNThjM2NmMjEyIDEwMDY0NAotLS0gYS9hcmNoL3Jpc2N2 L2luY2x1ZGUvYXNtL2h3Y2FwLmgKKysrIGIvYXJjaC9yaXNjdi9pbmNsdWRlL2FzbS9od2NhcC5o CkBAIC04MSw2ICs4MSw3IEBACiAjZGVmaW5lIFJJU0NWX0lTQV9FWFRfWlRTTwkJNzIKICNkZWZp bmUgUklTQ1ZfSVNBX0VYVF9aQUNBUwkJNzMKICNkZWZpbmUgUklTQ1ZfSVNBX0VYVF9YQU5ERVNQ TVUJCTc0CisjZGVmaW5lIFJJU0NWX0lTQV9FWFRfWkFXUlMJCTc1CiAKICNkZWZpbmUgUklTQ1Zf SVNBX0VYVF9YTElOVVhFTlZDRkcJMTI3CiAKZGlmZiAtLWdpdCBhL2FyY2gvcmlzY3YvaW5jbHVk ZS9hc20vaW5zbi1kZWYuaCBiL2FyY2gvcmlzY3YvaW5jbHVkZS9hc20vaW5zbi1kZWYuaAppbmRl eCA2NGRmZmFhMjFiZmEuLjlhOTEzMDEwY2RkOSAxMDA2NDQKLS0tIGEvYXJjaC9yaXNjdi9pbmNs dWRlL2FzbS9pbnNuLWRlZi5oCisrKyBiL2FyY2gvcmlzY3YvaW5jbHVkZS9hc20vaW5zbi1kZWYu aApAQCAtMTk3LDUgKzE5Nyw3IEBACiAJICAgICAgIFJTMShiYXNlKSwgU0lNTTEyKDQpKQogCiAj ZGVmaW5lIFJJU0NWX1BBVVNFCSIuNGJ5dGUgMHgxMDAwMDBmIgorI2RlZmluZSBaQVdSU19XUlNf TlRPCSIuNGJ5dGUgMHgwMGQwMDA3MyIKKyNkZWZpbmUgWkFXUlNfV1JTX1NUTwkiLjRieXRlIDB4 MDFkMDAwNzMiCiAKICNlbmRpZiAvKiBfX0FTTV9JTlNOX0RFRl9IICovCmRpZmYgLS1naXQgYS9h cmNoL3Jpc2N2L2tlcm5lbC9jcHVmZWF0dXJlLmMgYi9hcmNoL3Jpc2N2L2tlcm5lbC9jcHVmZWF0 dXJlLmMKaW5kZXggM2VkMjM1OWVhZTM1Li4wMmRlOWVhYTNmNDIgMTAwNjQ0Ci0tLSBhL2FyY2gv cmlzY3Yva2VybmVsL2NwdWZlYXR1cmUuYworKysgYi9hcmNoL3Jpc2N2L2tlcm5lbC9jcHVmZWF0 dXJlLmMKQEAgLTI1Nyw2ICsyNTcsNyBAQCBjb25zdCBzdHJ1Y3QgcmlzY3ZfaXNhX2V4dF9kYXRh IHJpc2N2X2lzYV9leHRbXSA9IHsKIAlfX1JJU0NWX0lTQV9FWFRfREFUQSh6aWhpbnRwYXVzZSwg UklTQ1ZfSVNBX0VYVF9aSUhJTlRQQVVTRSksCiAJX19SSVNDVl9JU0FfRVhUX0RBVEEoemlocG0s IFJJU0NWX0lTQV9FWFRfWklIUE0pLAogCV9fUklTQ1ZfSVNBX0VYVF9EQVRBKHphY2FzLCBSSVND Vl9JU0FfRVhUX1pBQ0FTKSwKKwlfX1JJU0NWX0lTQV9FWFRfREFUQSh6YXdycywgUklTQ1ZfSVNB X0VYVF9aQVdSUyksCiAJX19SSVNDVl9JU0FfRVhUX0RBVEEoemZhLCBSSVNDVl9JU0FfRVhUX1pG QSksCiAJX19SSVNDVl9JU0FfRVhUX0RBVEEoemZoLCBSSVNDVl9JU0FfRVhUX1pGSCksCiAJX19S SVNDVl9JU0FfRVhUX0RBVEEoemZobWluLCBSSVNDVl9JU0FfRVhUX1pGSE1JTiksCi0tIAoyLjQ0 LjAKCgpfX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fXwpsaW51 eC1yaXNjdiBtYWlsaW5nIGxpc3QKbGludXgtcmlzY3ZAbGlzdHMuaW5mcmFkZWFkLm9yZwpodHRw Oi8vbGlzdHMuaW5mcmFkZWFkLm9yZy9tYWlsbWFuL2xpc3RpbmZvL2xpbnV4LXJpc2N2Cg== From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mail-ed1-f53.google.com (mail-ed1-f53.google.com [209.85.208.53]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7A7FA12C49C for ; Fri, 19 Apr 2024 13:53:29 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.208.53 ARC-Seal:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1713534811; cv=none; b=VDGhWdXmP4A7Mn0Wqh23/r6WZD0fmGPTHA8POIWo2id15ah9qpEgtXC2O11sgvvcyC1WX58w/gjiYNohIAkMbaBiUXuqupoXlLxCW4Ir2frGtiHIJkdOvMI3xwHdZeZQotOInRRJUv9+t2TqliEeQ6BoorehSqcnCeC/9Y3FCCA= ARC-Message-Signature:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1713534811; c=relaxed/simple; bh=5X0BUssV9Kt9dqZ168A9hVbOXB9i6otaw8DFvHNaeiw=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type:Content-type; b=YV4+B3OoWs5lF0hZ7ouYpn1iywQBvaLV5YwKxwIIY6e5RO/COY0FrN+pTXryvkOiRQkGI8NV32OMGb9/EGvNaqkfM8GX3xM94UrxHQgNO7Vky/kYz2/s9XzgwyW8DLfm8JtIZ1qbei3a/pW6kJG+SzJph0dk0P0N3S57VxTQuE8= ARC-Authentication-Results:i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=ventanamicro.com; spf=pass smtp.mailfrom=ventanamicro.com; dkim=pass (2048-bit key) header.d=ventanamicro.com header.i=@ventanamicro.com header.b=X1rk6Ukx; arc=none smtp.client-ip=209.85.208.53 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=ventanamicro.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=ventanamicro.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=ventanamicro.com header.i=@ventanamicro.com header.b="X1rk6Ukx" Received: by mail-ed1-f53.google.com with SMTP id 4fb4d7f45d1cf-571c22d9de4so2213631a12.3 for ; Fri, 19 Apr 2024 06:53:29 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=ventanamicro.com; s=google; t=1713534808; x=1714139608; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=nEizurJcSR1aY1vBLFtX+JNkUfZx2QKjVF6QEsuG1UY=; b=X1rk6UkxpnrDNVXYp9yMjKW3Q+s67j2eWi/W7xEu79er2oY4oXWw0zKbssyZHfiEa1 Col3drA1shbP10xpV/tiVSpN6U1uI1KMluj/tUw8wJnYWNRqFGwsJrWMOorZ40N6kG0U WQzh0FEs2Elk+l7qsOi2Qk5JMN+kCP3sITvkA++0TW5Jij0CcRUa8MiMEXbFZelaQ/5T U5ts0lf9uyETt/woCSPoQdoAWDcZTZ8n4F++OGsPQfM9dSICm776WXnBExQaxd96ktXF hY/s8zkutk8gL5S6zatE7Zr9c4OtJ54420yMIgkmFNLYU3k2yVkMh9YhGadH2RjUUJkl hfsg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1713534808; x=1714139608; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=nEizurJcSR1aY1vBLFtX+JNkUfZx2QKjVF6QEsuG1UY=; b=X4p1516Clig2PY8j+rW2kaOs9RSE4iAkP2M6rEzVG5PjrtLBkNeMewFRXNeHgoFfI6 aDviQI/j/sGcXAYL4+9PutDBJQLZzWaluu3qlYAuRtgiElO0/UHqJgZhhU1FI6MWaJVf R1duQA3maFdmFfTj+MQGL9Ps6/KhzVtC5tWoDZctS53FgijxLJE2RTAZ8E+AZk6DE5Hc AOnBk+FWIU4uEt1EoDSicsYMkSxfO4HPfyTsLMRIaRAm7TNlJXxvanSh974/9O2qPLLy sdkfIcyS+r4PJamsYrvFM7PBnlWCzmL5WEjfoGdkAUpcRRHR802Nf5Rm7C4+J46Jao2D tJiQ== X-Forwarded-Encrypted: i=1; AJvYcCX9GRqe77tgUlud/tIO7ydZKwMXh1J5czfO9WmV+p8+Ac73zH4nS5QKCbo4UJ6VbgdwB/fUcMIp8rerpPTN+xtFZcteVM7yPlXlNw== X-Gm-Message-State: AOJu0YyBcKOkyZM06+OG1/c6ufs1LMA1u+1kEMMtwJept5dkLLlN/7c2 t03GVji6TkAtjf0e4WPaUdIEGIbHaBDWS8AOsJpYQbok1EnLWtE0QlxXKMLLyi8= X-Google-Smtp-Source: AGHT+IGkQt2hO18a+SNSAYfEOPucXpHAMV1WR0RrkuDvLuCw0MpZGKjuj++zRhwHCrnJC9FEvEAlqg== X-Received: by 2002:a50:baad:0:b0:56e:323b:d7e7 with SMTP id x42-20020a50baad000000b0056e323bd7e7mr1751219ede.34.1713534807639; Fri, 19 Apr 2024 06:53:27 -0700 (PDT) Received: from localhost (2001-1ae9-1c2-4c00-20f-c6b4-1e57-7965.ip6.tmcz.cz. [2001:1ae9:1c2:4c00:20f:c6b4:1e57:7965]) by smtp.gmail.com with ESMTPSA id d6-20020a05640208c600b0056e72c4a330sm2148891edz.41.2024.04.19.06.53.27 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 19 Apr 2024 06:53:27 -0700 (PDT) From: Andrew Jones To: linux-riscv@lists.infradead.org, kvm-riscv@lists.infradead.org, devicetree@vger.kernel.org Cc: paul.walmsley@sifive.com, palmer@dabbelt.com, aou@eecs.berkeley.edu, conor.dooley@microchip.com, anup@brainfault.org, atishp@atishpatra.org, robh@kernel.org, krzysztof.kozlowski+dt@linaro.org, conor+dt@kernel.org, christoph.muellner@vrull.eu, heiko@sntech.de, charlie@rivosinc.com, David.Laight@ACULAB.COM, parri.andrea@gmail.com, luxu.kernel@bytedance.com Subject: [PATCH v2 3/6] riscv: Add Zawrs support for spinlocks Date: Fri, 19 Apr 2024 15:53:25 +0200 Message-ID: <20240419135321.70781-11-ajones@ventanamicro.com> X-Mailer: git-send-email 2.44.0 In-Reply-To: <20240419135321.70781-8-ajones@ventanamicro.com> References: <20240419135321.70781-8-ajones@ventanamicro.com> Precedence: bulk X-Mailing-List: devicetree@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-type: text/plain Content-Transfer-Encoding: 8bit From: Christoph Müllner RISC-V code uses the generic ticket lock implementation, which calls the macros smp_cond_load_relaxed() and smp_cond_load_acquire(). Introduce a RISC-V specific implementation of smp_cond_load_relaxed() which applies WRS.NTO of the Zawrs extension in order to reduce power consumption while waiting and allows hypervisors to enable guests to trap while waiting. smp_cond_load_acquire() doesn't need a RISC-V specific implementation as the generic implementation is based on smp_cond_load_relaxed() and smp_acquire__after_ctrl_dep() sufficiently provides the acquire semantics. This implementation is heavily based on Arm's approach which is the approach Andrea Parri also suggested. The Zawrs specification can be found here: https://github.com/riscv/riscv-zawrs/blob/main/zawrs.adoc Signed-off-by: Christoph Müllner Co-developed-by: Andrew Jones Signed-off-by: Andrew Jones --- arch/riscv/Kconfig | 13 ++++++++ arch/riscv/include/asm/barrier.h | 45 ++++++++++++++++++--------- arch/riscv/include/asm/cmpxchg.h | 51 +++++++++++++++++++++++++++++++ arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/include/asm/insn-def.h | 2 ++ arch/riscv/kernel/cpufeature.c | 1 + 6 files changed, 98 insertions(+), 15 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 7427d8088337..34bbe6b70546 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -578,6 +578,19 @@ config RISCV_ISA_V_PREEMPTIVE preemption. Enabling this config will result in higher memory consumption due to the allocation of per-task's kernel Vector context. +config RISCV_ISA_ZAWRS + bool "Zawrs extension support for more efficient busy waiting" + depends on RISCV_ALTERNATIVE + default y + help + The Zawrs extension defines instructions to be used in polling loops + which allow a hart to enter a low-power state or to trap to the + hypervisor while waiting on a store to a memory location. Enable the + use of these instructions in the kernel when the Zawrs extension is + detected at boot. + + If you don't know what to do here, say Y. + config TOOLCHAIN_HAS_ZBB bool default y diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h index 880b56d8480d..e1d9bf1deca6 100644 --- a/arch/riscv/include/asm/barrier.h +++ b/arch/riscv/include/asm/barrier.h @@ -11,6 +11,7 @@ #define _ASM_RISCV_BARRIER_H #ifndef __ASSEMBLY__ +#include #include #define nop() __asm__ __volatile__ ("nop") @@ -28,21 +29,6 @@ #define __smp_rmb() RISCV_FENCE(r, r) #define __smp_wmb() RISCV_FENCE(w, w) -#define __smp_store_release(p, v) \ -do { \ - compiletime_assert_atomic_type(*p); \ - RISCV_FENCE(rw, w); \ - WRITE_ONCE(*p, v); \ -} while (0) - -#define __smp_load_acquire(p) \ -({ \ - typeof(*p) ___p1 = READ_ONCE(*p); \ - compiletime_assert_atomic_type(*p); \ - RISCV_FENCE(r, rw); \ - ___p1; \ -}) - /* * This is a very specific barrier: it's currently only used in two places in * the kernel, both in the scheduler. See include/linux/spinlock.h for the two @@ -70,6 +56,35 @@ do { \ */ #define smp_mb__after_spinlock() RISCV_FENCE(iorw, iorw) +#define __smp_store_release(p, v) \ +do { \ + compiletime_assert_atomic_type(*p); \ + RISCV_FENCE(rw, w); \ + WRITE_ONCE(*p, v); \ +} while (0) + +#define __smp_load_acquire(p) \ +({ \ + typeof(*p) ___p1 = READ_ONCE(*p); \ + compiletime_assert_atomic_type(*p); \ + RISCV_FENCE(r, rw); \ + ___p1; \ +}) + +#ifdef CONFIG_RISCV_ISA_ZAWRS +#define smp_cond_load_relaxed(ptr, cond_expr) ({ \ + typeof(ptr) __PTR = (ptr); \ + __unqual_scalar_typeof(*ptr) VAL; \ + for (;;) { \ + VAL = READ_ONCE(*__PTR); \ + if (cond_expr) \ + break; \ + __cmpwait_relaxed(ptr, VAL); \ + } \ + (typeof(*ptr))VAL; \ +}) +#endif + #include #endif /* __ASSEMBLY__ */ diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h index 2fee65cc8443..0926ac7f4ca6 100644 --- a/arch/riscv/include/asm/cmpxchg.h +++ b/arch/riscv/include/asm/cmpxchg.h @@ -8,7 +8,10 @@ #include +#include #include +#include +#include #define __xchg_relaxed(ptr, new, size) \ ({ \ @@ -359,4 +362,52 @@ arch_cmpxchg_relaxed((ptr), (o), (n)); \ }) +#ifdef CONFIG_RISCV_ISA_ZAWRS +static __always_inline void __cmpwait(volatile void *ptr, + unsigned long val, + int size) +{ + unsigned long tmp; + + asm goto(ALTERNATIVE("j %l[no_zawrs]", "nop", + 0, RISCV_ISA_EXT_ZAWRS, 1) + : : : : no_zawrs); + + switch (size) { + case 4: + asm volatile( + " lr.w %0, %1\n" + " xor %0, %0, %2\n" + " bnez %0, 1f\n" + ZAWRS_WRS_NTO "\n" + "1:" + : "=&r" (tmp), "+A" (*(u32 *)ptr) + : "r" (val)); + break; +#if __riscv_xlen == 64 + case 8: + asm volatile( + " lr.d %0, %1\n" + " xor %0, %0, %2\n" + " bnez %0, 1f\n" + ZAWRS_WRS_NTO "\n" + "1:" + : "=&r" (tmp), "+A" (*(u64 *)ptr) + : "r" (val)); + break; +#endif + default: + BUILD_BUG(); + } + + return; + +no_zawrs: + asm volatile(RISCV_PAUSE : : : "memory"); +} + +#define __cmpwait_relaxed(ptr, val) \ + __cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr))) +#endif + #endif /* _ASM_RISCV_CMPXCHG_H */ diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index e17d0078a651..5b358c3cf212 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -81,6 +81,7 @@ #define RISCV_ISA_EXT_ZTSO 72 #define RISCV_ISA_EXT_ZACAS 73 #define RISCV_ISA_EXT_XANDESPMU 74 +#define RISCV_ISA_EXT_ZAWRS 75 #define RISCV_ISA_EXT_XLINUXENVCFG 127 diff --git a/arch/riscv/include/asm/insn-def.h b/arch/riscv/include/asm/insn-def.h index 64dffaa21bfa..9a913010cdd9 100644 --- a/arch/riscv/include/asm/insn-def.h +++ b/arch/riscv/include/asm/insn-def.h @@ -197,5 +197,7 @@ RS1(base), SIMM12(4)) #define RISCV_PAUSE ".4byte 0x100000f" +#define ZAWRS_WRS_NTO ".4byte 0x00d00073" +#define ZAWRS_WRS_STO ".4byte 0x01d00073" #endif /* __ASM_INSN_DEF_H */ diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 3ed2359eae35..02de9eaa3f42 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -257,6 +257,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), + __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS), __RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA), __RISCV_ISA_EXT_DATA(zfh, RISCV_ISA_EXT_ZFH), __RISCV_ISA_EXT_DATA(zfhmin, RISCV_ISA_EXT_ZFHMIN), -- 2.44.0