Re: [PATCH 09/10] x86/rcu: Add rcu_preempt_count

[Frederic Weisbecker <frederic@kernel.org>]

Le Thu, Mar 28, 2024 at 03:53:17PM +0800, Lai Jiangshan a écrit :
> From: Lai Jiangshan <jiangshan.ljs@antgroup.com>
> 
> Implement PCPU_RCU_PREEMPT_COUNT for x86.
> Mainly copied from asm/preempt.h
> 
> Make rcu_read_[un]lock() inlined for rcu-preempt.
> Make rcu_read_lock() only one instruction.
> Make rcu_read_unlock() only two instructions in the fast path.
> 
> Cc: "Paul E. McKenney" <paulmck@kernel.org>
> Cc: Peter Zijlstra <peterz@infradead.org>
> Cc: Frederic Weisbecker <frederic@kernel.org>
> Signed-off-by: Lai Jiangshan <jiangshan.ljs@antgroup.com>
> ---
>  arch/x86/Kconfig                   |   1 +
>  arch/x86/include/asm/current.h     |   3 +
>  arch/x86/include/asm/rcu_preempt.h | 107 +++++++++++++++++++++++++++++
>  arch/x86/kernel/cpu/common.c       |   7 +-
>  4 files changed, 115 insertions(+), 3 deletions(-)
>  create mode 100644 arch/x86/include/asm/rcu_preempt.h
> 
> diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
> index 78050d5d7fac..7eb17c12f7b7 100644
> --- a/arch/x86/Kconfig
> +++ b/arch/x86/Kconfig
> @@ -257,6 +257,7 @@ config X86
>  	select HAVE_OBJTOOL			if X86_64
>  	select HAVE_OPTPROBES
>  	select HAVE_PAGE_SIZE_4KB
> +	select HAVE_PCPU_RCU_PREEMPT_COUNT
>  	select HAVE_PCSPKR_PLATFORM
>  	select HAVE_PERF_EVENTS
>  	select HAVE_PERF_EVENTS_NMI
> diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
> index bf5953883ec3..dcc2ef784120 100644
> --- a/arch/x86/include/asm/current.h
> +++ b/arch/x86/include/asm/current.h
> @@ -24,6 +24,9 @@ struct pcpu_hot {
>  			unsigned long		top_of_stack;
>  			void			*hardirq_stack_ptr;
>  			u16			softirq_pending;
> +#ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
> +			int			rcu_preempt_count;
> +#endif // #ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
>  #ifdef CONFIG_X86_64
>  			bool			hardirq_stack_inuse;
>  #else
> diff --git a/arch/x86/include/asm/rcu_preempt.h b/arch/x86/include/asm/rcu_preempt.h
> new file mode 100644
> index 000000000000..cb25ebe038a5
> --- /dev/null
> +++ b/arch/x86/include/asm/rcu_preempt.h
> @@ -0,0 +1,107 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef __ASM_RCU_PREEMPT_H
> +#define __ASM_RCU_PREEMPT_H
> +
> +#include <asm/rmwcc.h>
> +#include <asm/percpu.h>
> +#include <asm/current.h>
> +
> +#ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
> +
> +/* We use the MSB mostly because its available */

I think you can safely remove the "We " from all the comments :-)

> +#define RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED	0x80000000

How about RCU_PREEMPT_UNLOCK_FASTPATH ?

> +
> +/*
> + * We use the RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED bit as an inverted
> + * current->rcu_read_unlock_special.s such that a decrement hitting 0
> + * means we can and should call rcu_read_unlock_special().
> + */
> +#define RCU_PREEMPT_INIT	(0 + RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED)

Or simply:

#define RCU_PREEMPT_INIT     RCU_PREEMPT_UNLOCK_FASTPATH

Or you can even remove RCU_PREEMPT_INIT and use RCU_PREEMPT_UNLOCK_FASTPATH directly.

> +/*
> + * Because we keep RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED set when we do
> + * _not_ need to handle unlock-special for a fast-path decrement.
> + */
> +static __always_inline bool pcpu_rcu_preempt_count_dec_and_test(void)
> +{
> +	return GEN_UNARY_RMWcc("decl", __my_cpu_var(pcpu_hot.rcu_preempt_count), e,
> +			       __percpu_arg([var]));
> +}
> +
> +#define pcpu_rcu_read_unlock_special()						\
> +do {										\
> +	rcu_read_unlock_special();						\
> +} while (0)

Why not just call that directly?

> +
> +#endif // #ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
> +
> +#endif /* __ASM_RCU_PREEMPT_H */
> diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
> index ba8cf5e9ce56..0b204a649442 100644
> --- a/arch/x86/kernel/cpu/common.c
> +++ b/arch/x86/kernel/cpu/common.c
> @@ -1992,9 +1992,10 @@ static __init int setup_clearcpuid(char *arg)
>  __setup("clearcpuid=", setup_clearcpuid);
>  
>  DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
> -	.current_task	= &init_task,
> -	.preempt_count	= INIT_PREEMPT_COUNT,
> -	.top_of_stack	= TOP_OF_INIT_STACK,
> +	.current_task		= &init_task,
> +	.preempt_count		= INIT_PREEMPT_COUNT,
> +	.top_of_stack		= TOP_OF_INIT_STACK,
> +	.rcu_preempt_count	= RCU_PREEMPT_INIT,

#ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT ?

Thanks.


>  };
>  EXPORT_PER_CPU_SYMBOL(pcpu_hot);
>  EXPORT_PER_CPU_SYMBOL(const_pcpu_hot);
> -- 
> 2.19.1.6.gb485710b
>