* [PATCH 0/2] s390/preempt: mark all functions __always_inline
@ 2024-03-20 22:47 Ilya Leoshkevich
2024-03-20 22:47 ` [PATCH 1/2] s390/atomic: " Ilya Leoshkevich
` (2 more replies)
0 siblings, 3 replies; 4+ messages in thread
From: Ilya Leoshkevich @ 2024-03-20 22:47 UTC (permalink / raw)
To: Heiko Carstens, Vasily Gorbik, Alexander Gordeev
Cc: Christian Borntraeger, Sven Schnelle, Mark Rutland, linux-s390,
Ilya Leoshkevich
Hi,
This series marks all functions in asm/atomic_ops.h, asm/atomic.h and
asm/preempt.h __always_inline, and is based on the discussion with Mark
[1]. It's one of the changes required to unbreak the work-in-progress
KMSAN support on s390x after commit 5ec8e8ea8b77 ("mm/sparsemem: fix
race in accessing memory_section->usage"). But it also makes sense on
its own, and may prevent issues with the other sanitizers in the
future.
bloat-o-meter says:
add/remove: 4/5 grow/shrink: 58/186 up/down: 4408/-6368 (-1960)
[...]
Total: Before=29725530, After=29723570, chg -0.01%
Even though there are changes in the code generation, they are
insignificant.
[1] https://lore.kernel.org/lkml/ZfhI0F-vXMMw1GzC@FVFF77S0Q05N/
Best regards,
Ilya
Ilya Leoshkevich (2):
s390/atomic: mark all functions __always_inline
s390/preempt: mark all functions __always_inline
arch/s390/include/asm/atomic.h | 44 +++++++++++++++---------------
arch/s390/include/asm/atomic_ops.h | 22 +++++++--------
arch/s390/include/asm/preempt.h | 36 ++++++++++++------------
3 files changed, 51 insertions(+), 51 deletions(-)
--
2.44.0
^ permalink raw reply [flat|nested] 4+ messages in thread* [PATCH 1/2] s390/atomic: mark all functions __always_inline 2024-03-20 22:47 [PATCH 0/2] s390/preempt: mark all functions __always_inline Ilya Leoshkevich @ 2024-03-20 22:47 ` Ilya Leoshkevich 2024-03-20 22:47 ` [PATCH 2/2] s390/preempt: " Ilya Leoshkevich 2024-03-21 6:45 ` [PATCH 0/2] " Heiko Carstens 2 siblings, 0 replies; 4+ messages in thread From: Ilya Leoshkevich @ 2024-03-20 22:47 UTC (permalink / raw) To: Heiko Carstens, Vasily Gorbik, Alexander Gordeev Cc: Christian Borntraeger, Sven Schnelle, Mark Rutland, linux-s390, Ilya Leoshkevich Atomic functions are quite ubiquitous and may be called by noinstr ones, introducing unwanted instrumentation. They are very small, so there are no significant downsides to force-inlining them. Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> --- arch/s390/include/asm/atomic.h | 44 +++++++++++++++--------------- arch/s390/include/asm/atomic_ops.h | 22 +++++++-------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index 7138d189cc42..0c4cad7d5a5b 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -15,31 +15,31 @@ #include <asm/barrier.h> #include <asm/cmpxchg.h> -static inline int arch_atomic_read(const atomic_t *v) +static __always_inline int arch_atomic_read(const atomic_t *v) { return __atomic_read(v); } #define arch_atomic_read arch_atomic_read -static inline void arch_atomic_set(atomic_t *v, int i) +static __always_inline void arch_atomic_set(atomic_t *v, int i) { __atomic_set(v, i); } #define arch_atomic_set arch_atomic_set -static inline int arch_atomic_add_return(int i, atomic_t *v) +static __always_inline int arch_atomic_add_return(int i, atomic_t *v) { return __atomic_add_barrier(i, &v->counter) + i; } #define arch_atomic_add_return arch_atomic_add_return -static inline int arch_atomic_fetch_add(int i, atomic_t *v) +static __always_inline int arch_atomic_fetch_add(int i, atomic_t *v) { return __atomic_add_barrier(i, &v->counter); } #define arch_atomic_fetch_add arch_atomic_fetch_add -static inline void arch_atomic_add(int i, atomic_t *v) +static __always_inline void arch_atomic_add(int i, atomic_t *v) { __atomic_add(i, &v->counter); } @@ -50,11 +50,11 @@ static inline void arch_atomic_add(int i, atomic_t *v) #define arch_atomic_fetch_sub(_i, _v) arch_atomic_fetch_add(-(int)(_i), _v) #define ATOMIC_OPS(op) \ -static inline void arch_atomic_##op(int i, atomic_t *v) \ +static __always_inline void arch_atomic_##op(int i, atomic_t *v) \ { \ __atomic_##op(i, &v->counter); \ } \ -static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ +static __always_inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ { \ return __atomic_##op##_barrier(i, &v->counter); \ } @@ -74,7 +74,7 @@ ATOMIC_OPS(xor) #define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new)) -static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) +static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) { return __atomic_cmpxchg(&v->counter, old, new); } @@ -82,31 +82,31 @@ static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) #define ATOMIC64_INIT(i) { (i) } -static inline s64 arch_atomic64_read(const atomic64_t *v) +static __always_inline s64 arch_atomic64_read(const atomic64_t *v) { return __atomic64_read(v); } #define arch_atomic64_read arch_atomic64_read -static inline void arch_atomic64_set(atomic64_t *v, s64 i) +static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) { __atomic64_set(v, i); } #define arch_atomic64_set arch_atomic64_set -static inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) +static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) { return __atomic64_add_barrier(i, (long *)&v->counter) + i; } #define arch_atomic64_add_return arch_atomic64_add_return -static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) +static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) { return __atomic64_add_barrier(i, (long *)&v->counter); } #define arch_atomic64_fetch_add arch_atomic64_fetch_add -static inline void arch_atomic64_add(s64 i, atomic64_t *v) +static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v) { __atomic64_add(i, (long *)&v->counter); } @@ -114,20 +114,20 @@ static inline void arch_atomic64_add(s64 i, atomic64_t *v) #define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), new)) -static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) +static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) { return __atomic64_cmpxchg((long *)&v->counter, old, new); } #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg -#define ATOMIC64_OPS(op) \ -static inline void arch_atomic64_##op(s64 i, atomic64_t *v) \ -{ \ - __atomic64_##op(i, (long *)&v->counter); \ -} \ -static inline long arch_atomic64_fetch_##op(s64 i, atomic64_t *v) \ -{ \ - return __atomic64_##op##_barrier(i, (long *)&v->counter); \ +#define ATOMIC64_OPS(op) \ +static __always_inline void arch_atomic64_##op(s64 i, atomic64_t *v) \ +{ \ + __atomic64_##op(i, (long *)&v->counter); \ +} \ +static __always_inline long arch_atomic64_fetch_##op(s64 i, atomic64_t *v) \ +{ \ + return __atomic64_##op##_barrier(i, (long *)&v->counter); \ } ATOMIC64_OPS(and) diff --git a/arch/s390/include/asm/atomic_ops.h b/arch/s390/include/asm/atomic_ops.h index 50510e08b893..7fa5f96a553a 100644 --- a/arch/s390/include/asm/atomic_ops.h +++ b/arch/s390/include/asm/atomic_ops.h @@ -8,7 +8,7 @@ #ifndef __ARCH_S390_ATOMIC_OPS__ #define __ARCH_S390_ATOMIC_OPS__ -static inline int __atomic_read(const atomic_t *v) +static __always_inline int __atomic_read(const atomic_t *v) { int c; @@ -18,14 +18,14 @@ static inline int __atomic_read(const atomic_t *v) return c; } -static inline void __atomic_set(atomic_t *v, int i) +static __always_inline void __atomic_set(atomic_t *v, int i) { asm volatile( " st %1,%0\n" : "=R" (v->counter) : "d" (i)); } -static inline s64 __atomic64_read(const atomic64_t *v) +static __always_inline s64 __atomic64_read(const atomic64_t *v) { s64 c; @@ -35,7 +35,7 @@ static inline s64 __atomic64_read(const atomic64_t *v) return c; } -static inline void __atomic64_set(atomic64_t *v, s64 i) +static __always_inline void __atomic64_set(atomic64_t *v, s64 i) { asm volatile( " stg %1,%0\n" @@ -45,7 +45,7 @@ static inline void __atomic64_set(atomic64_t *v, s64 i) #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES #define __ATOMIC_OP(op_name, op_type, op_string, op_barrier) \ -static inline op_type op_name(op_type val, op_type *ptr) \ +static __always_inline op_type op_name(op_type val, op_type *ptr) \ { \ op_type old; \ \ @@ -96,7 +96,7 @@ __ATOMIC_CONST_OPS(__atomic64_add_const, long, "agsi") #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ #define __ATOMIC_OP(op_name, op_string) \ -static inline int op_name(int val, int *ptr) \ +static __always_inline int op_name(int val, int *ptr) \ { \ int old, new; \ \ @@ -122,7 +122,7 @@ __ATOMIC_OPS(__atomic_xor, "xr") #undef __ATOMIC_OPS #define __ATOMIC64_OP(op_name, op_string) \ -static inline long op_name(long val, long *ptr) \ +static __always_inline long op_name(long val, long *ptr) \ { \ long old, new; \ \ @@ -154,7 +154,7 @@ __ATOMIC64_OPS(__atomic64_xor, "xgr") #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ -static inline int __atomic_cmpxchg(int *ptr, int old, int new) +static __always_inline int __atomic_cmpxchg(int *ptr, int old, int new) { asm volatile( " cs %[old],%[new],%[ptr]" @@ -164,7 +164,7 @@ static inline int __atomic_cmpxchg(int *ptr, int old, int new) return old; } -static inline bool __atomic_cmpxchg_bool(int *ptr, int old, int new) +static __always_inline bool __atomic_cmpxchg_bool(int *ptr, int old, int new) { int old_expected = old; @@ -176,7 +176,7 @@ static inline bool __atomic_cmpxchg_bool(int *ptr, int old, int new) return old == old_expected; } -static inline long __atomic64_cmpxchg(long *ptr, long old, long new) +static __always_inline long __atomic64_cmpxchg(long *ptr, long old, long new) { asm volatile( " csg %[old],%[new],%[ptr]" @@ -186,7 +186,7 @@ static inline long __atomic64_cmpxchg(long *ptr, long old, long new) return old; } -static inline bool __atomic64_cmpxchg_bool(long *ptr, long old, long new) +static __always_inline bool __atomic64_cmpxchg_bool(long *ptr, long old, long new) { long old_expected = old; -- 2.44.0 ^ permalink raw reply related [flat|nested] 4+ messages in thread
* [PATCH 2/2] s390/preempt: mark all functions __always_inline 2024-03-20 22:47 [PATCH 0/2] s390/preempt: mark all functions __always_inline Ilya Leoshkevich 2024-03-20 22:47 ` [PATCH 1/2] s390/atomic: " Ilya Leoshkevich @ 2024-03-20 22:47 ` Ilya Leoshkevich 2024-03-21 6:45 ` [PATCH 0/2] " Heiko Carstens 2 siblings, 0 replies; 4+ messages in thread From: Ilya Leoshkevich @ 2024-03-20 22:47 UTC (permalink / raw) To: Heiko Carstens, Vasily Gorbik, Alexander Gordeev Cc: Christian Borntraeger, Sven Schnelle, Mark Rutland, linux-s390, Ilya Leoshkevich preempt_count-related functions are quite ubiquitous and may be called by noinstr ones, introducing unwanted instrumentation. Here is one example call chain: irqentry_nmi_enter() # noinstr lockdep_hardirqs_enabled() this_cpu_read() __pcpu_size_call_return() this_cpu_read_*() this_cpu_generic_read() __this_cpu_generic_read_nopreempt() preempt_disable_notrace() __preempt_count_inc() __preempt_count_add() They are very small, so there are no significant downsides to force-inlining them. Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> --- arch/s390/include/asm/preempt.h | 36 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h index bf15da0fedbc..0e3da500e98c 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -12,12 +12,12 @@ #define PREEMPT_NEED_RESCHED 0x80000000 #define PREEMPT_ENABLED (0 + PREEMPT_NEED_RESCHED) -static inline int preempt_count(void) +static __always_inline int preempt_count(void) { return READ_ONCE(S390_lowcore.preempt_count) & ~PREEMPT_NEED_RESCHED; } -static inline void preempt_count_set(int pc) +static __always_inline void preempt_count_set(int pc) { int old, new; @@ -29,22 +29,22 @@ static inline void preempt_count_set(int pc) old, new) != old); } -static inline void set_preempt_need_resched(void) +static __always_inline void set_preempt_need_resched(void) { __atomic_and(~PREEMPT_NEED_RESCHED, &S390_lowcore.preempt_count); } -static inline void clear_preempt_need_resched(void) +static __always_inline void clear_preempt_need_resched(void) { __atomic_or(PREEMPT_NEED_RESCHED, &S390_lowcore.preempt_count); } -static inline bool test_preempt_need_resched(void) +static __always_inline bool test_preempt_need_resched(void) { return !(READ_ONCE(S390_lowcore.preempt_count) & PREEMPT_NEED_RESCHED); } -static inline void __preempt_count_add(int val) +static __always_inline void __preempt_count_add(int val) { /* * With some obscure config options and CONFIG_PROFILE_ALL_BRANCHES @@ -59,17 +59,17 @@ static inline void __preempt_count_add(int val) __atomic_add(val, &S390_lowcore.preempt_count); } -static inline void __preempt_count_sub(int val) +static __always_inline void __preempt_count_sub(int val) { __preempt_count_add(-val); } -static inline bool __preempt_count_dec_and_test(void) +static __always_inline bool __preempt_count_dec_and_test(void) { return __atomic_add(-1, &S390_lowcore.preempt_count) == 1; } -static inline bool should_resched(int preempt_offset) +static __always_inline bool should_resched(int preempt_offset) { return unlikely(READ_ONCE(S390_lowcore.preempt_count) == preempt_offset); @@ -79,45 +79,45 @@ static inline bool should_resched(int preempt_offset) #define PREEMPT_ENABLED (0) -static inline int preempt_count(void) +static __always_inline int preempt_count(void) { return READ_ONCE(S390_lowcore.preempt_count); } -static inline void preempt_count_set(int pc) +static __always_inline void preempt_count_set(int pc) { S390_lowcore.preempt_count = pc; } -static inline void set_preempt_need_resched(void) +static __always_inline void set_preempt_need_resched(void) { } -static inline void clear_preempt_need_resched(void) +static __always_inline void clear_preempt_need_resched(void) { } -static inline bool test_preempt_need_resched(void) +static __always_inline bool test_preempt_need_resched(void) { return false; } -static inline void __preempt_count_add(int val) +static __always_inline void __preempt_count_add(int val) { S390_lowcore.preempt_count += val; } -static inline void __preempt_count_sub(int val) +static __always_inline void __preempt_count_sub(int val) { S390_lowcore.preempt_count -= val; } -static inline bool __preempt_count_dec_and_test(void) +static __always_inline bool __preempt_count_dec_and_test(void) { return !--S390_lowcore.preempt_count && tif_need_resched(); } -static inline bool should_resched(int preempt_offset) +static __always_inline bool should_resched(int preempt_offset) { return unlikely(preempt_count() == preempt_offset && tif_need_resched()); -- 2.44.0 ^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [PATCH 0/2] s390/preempt: mark all functions __always_inline 2024-03-20 22:47 [PATCH 0/2] s390/preempt: mark all functions __always_inline Ilya Leoshkevich 2024-03-20 22:47 ` [PATCH 1/2] s390/atomic: " Ilya Leoshkevich 2024-03-20 22:47 ` [PATCH 2/2] s390/preempt: " Ilya Leoshkevich @ 2024-03-21 6:45 ` Heiko Carstens 2 siblings, 0 replies; 4+ messages in thread From: Heiko Carstens @ 2024-03-21 6:45 UTC (permalink / raw) To: Ilya Leoshkevich Cc: Vasily Gorbik, Alexander Gordeev, Christian Borntraeger, Sven Schnelle, Mark Rutland, linux-s390 On Wed, Mar 20, 2024 at 11:47:47PM +0100, Ilya Leoshkevich wrote: > Hi, > > This series marks all functions in asm/atomic_ops.h, asm/atomic.h and > asm/preempt.h __always_inline, and is based on the discussion with Mark > [1]. It's one of the changes required to unbreak the work-in-progress > KMSAN support on s390x after commit 5ec8e8ea8b77 ("mm/sparsemem: fix > race in accessing memory_section->usage"). But it also makes sense on > its own, and may prevent issues with the other sanitizers in the > future. ... > Ilya Leoshkevich (2): > s390/atomic: mark all functions __always_inline > s390/preempt: mark all functions __always_inline > > arch/s390/include/asm/atomic.h | 44 +++++++++++++++--------------- > arch/s390/include/asm/atomic_ops.h | 22 +++++++-------- > arch/s390/include/asm/preempt.h | 36 ++++++++++++------------ > 3 files changed, 51 insertions(+), 51 deletions(-) Applied, thanks! ^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2024-03-21 6:45 UTC | newest] Thread overview: 4+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2024-03-20 22:47 [PATCH 0/2] s390/preempt: mark all functions __always_inline Ilya Leoshkevich 2024-03-20 22:47 ` [PATCH 1/2] s390/atomic: " Ilya Leoshkevich 2024-03-20 22:47 ` [PATCH 2/2] s390/preempt: " Ilya Leoshkevich 2024-03-21 6:45 ` [PATCH 0/2] " Heiko Carstens
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox