Re: [PATCH] powerpc/64s: Add support for ASB_Notify on POWER9

[Benjamin Herrenschmidt <benh@au1.ibm.com>]

On Fri, 2017-08-04 at 16:56 +0200, Christophe Lombard wrote:
> The POWER9 core supports a new feature: ASB_Notify which requires the
> support of the Special Purpose Register: TIDR.
> 
> The ASB_Notify command, generated by the AFU, will attempt to
> wake-up the host thread identified by the particular LPID:PID:TID.
> 
> The special register TIDR has to be updated to with the same value
> present in the process element.
> 
> If the length of the register TIDR is 64bits, the CAPI Translation
> Service Layer core (XSL9) for Power9 systems limits the size (16bits) of
> the Thread ID when it generates the ASB_Notify message adding
> PID:LPID:TID information from the context.
> 
> The content of the internal kernel Thread ID (32bits) can not therefore
> be used to fulfill the register TIDR.
> 
> This patch allows to avoid this limitation by adding a new interface
> for the user. The instructions mfspr/mtspr SPRN_TIDR are emulated,
> save/restore SPRs (context switch) are updated and a new feature
> (CPU_FTR_TIDR) is added to POWER9 system.

Those CPU_FTR_* are internal to the kernel. You probably also need a
feature in AT_HWCAP2 to indicate to userspace that this is supported.

Also you put the onus of allocating the TIDs onto userspace which is a
bit tricky. What happens if there are duplicate TIDs for example ? (ie,
userspace doesn't allocate it or uses a library that spawns a thread)

Ben.
> 
> Signed-off-by: Christophe Lombard <clombard@linux.vnet.ibm.com>
> ---
>  arch/powerpc/include/asm/cputable.h     |  4 +++-
>  arch/powerpc/include/asm/emulated_ops.h |  2 ++
>  arch/powerpc/include/asm/ppc-opcode.h   |  4 ++++
>  arch/powerpc/include/asm/processor.h    |  1 +
>  arch/powerpc/kernel/process.c           |  8 ++++++++
>  arch/powerpc/kernel/traps.c             | 21 +++++++++++++++++++++
>  6 files changed, 39 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h
> index d02ad93..706f668 100644
> --- a/arch/powerpc/include/asm/cputable.h
> +++ b/arch/powerpc/include/asm/cputable.h
> @@ -215,6 +215,7 @@ enum {
>  #define CPU_FTR_DABRX			LONG_ASM_CONST(0x0800000000000000)
>  #define CPU_FTR_PMAO_BUG		LONG_ASM_CONST(0x1000000000000000)
>  #define CPU_FTR_POWER9_DD1		LONG_ASM_CONST(0x4000000000000000)
> +#define CPU_FTR_TIDR			LONG_ASM_CONST(0x8000000000000000)
>  
>  #ifndef __ASSEMBLY__
>  
> @@ -474,7 +475,8 @@ enum {
>  	    CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \
>  	    CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \
>  	    CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_DAWR | \
> -	    CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_ARCH_300)
> +	    CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | \
> +	    CPU_FTR_TIDR)
>  #define CPU_FTRS_POWER9_DD1 ((CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD1) & \
>  			     (~CPU_FTR_SAO))
>  #define CPU_FTRS_CELL	(CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \
> diff --git a/arch/powerpc/include/asm/emulated_ops.h b/arch/powerpc/include/asm/emulated_ops.h
> index f00e10e..e83ad42 100644
> --- a/arch/powerpc/include/asm/emulated_ops.h
> +++ b/arch/powerpc/include/asm/emulated_ops.h
> @@ -54,6 +54,8 @@ extern struct ppc_emulated {
>  #ifdef CONFIG_PPC64
>  	struct ppc_emulated_entry mfdscr;
>  	struct ppc_emulated_entry mtdscr;
> +	struct ppc_emulated_entry mftidr;
> +	struct ppc_emulated_entry mttidr;
>  	struct ppc_emulated_entry lq_stq;
>  #endif
>  } ppc_emulated;
> diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h
> index fa9ebae..3ebc446 100644
> --- a/arch/powerpc/include/asm/ppc-opcode.h
> +++ b/arch/powerpc/include/asm/ppc-opcode.h
> @@ -241,6 +241,10 @@
>  #define PPC_INST_MFSPR_DSCR_USER_MASK	0xfc1ffffe
>  #define PPC_INST_MTSPR_DSCR_USER	0x7c0303a6
>  #define PPC_INST_MTSPR_DSCR_USER_MASK	0xfc1ffffe
> +#define PPC_INST_MFSPR_TIDR		0x7d2452a6
> +#define PPC_INST_MFSPR_TIDR_MASK	0xfd2ffffe
> +#define PPC_INST_MTSPR_TIDR		0x7d2453a6
> +#define PPC_INST_MTSPR_TIDR_MASK	0xfd2ffffe
>  #define PPC_INST_MFVSRD			0x7c000066
>  #define PPC_INST_MTVSRD			0x7c000166
>  #define PPC_INST_SLBFEE			0x7c0007a7
> diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
> index fab7ff8..58cc212 100644
> --- a/arch/powerpc/include/asm/processor.h
> +++ b/arch/powerpc/include/asm/processor.h
> @@ -329,6 +329,7 @@ struct thread_struct {
>  	 */
>  	int		dscr_inherit;
>  	unsigned long	ppr;	/* used to save/restore SMT priority */
> +	unsigned long	tidr;
>  #endif
>  #ifdef CONFIG_PPC_BOOK3S_64
>  	unsigned long	tar;
> diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
> index 9f3e2c9..f06ea10 100644
> --- a/arch/powerpc/kernel/process.c
> +++ b/arch/powerpc/kernel/process.c
> @@ -1084,6 +1084,9 @@ static inline void save_sprs(struct thread_struct *t)
>  	if (cpu_has_feature(CPU_FTR_DSCR))
>  		t->dscr = mfspr(SPRN_DSCR);
>  
> +	if (cpu_has_feature(CPU_FTR_TIDR))
> +		t->tidr = mfspr(SPRN_TIDR);
> +
>  	if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
>  		t->bescr = mfspr(SPRN_BESCR);
>  		t->ebbhr = mfspr(SPRN_EBBHR);
> @@ -1120,6 +1123,11 @@ static inline void restore_sprs(struct thread_struct *old_thread,
>  			mtspr(SPRN_DSCR, dscr);
>  	}
>  
> +	if (cpu_has_feature(CPU_FTR_TIDR)) {
> +		if (old_thread->tidr != new_thread->tidr)
> +			mtspr(SPRN_TIDR, new_thread->tidr);
> +	}
> +
>  	if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
>  		if (old_thread->bescr != new_thread->bescr)
>  			mtspr(SPRN_BESCR, new_thread->bescr);
> diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
> index bfcfd9e..b829de7 100644
> --- a/arch/powerpc/kernel/traps.c
> +++ b/arch/powerpc/kernel/traps.c
> @@ -1141,6 +1141,25 @@ static int emulate_instruction(struct pt_regs *regs)
>  		mtspr(SPRN_DSCR, current->thread.dscr);
>  		return 0;
>  	}
> +	/* Emulate the mfspr rD, TIDR. */
> +	if (((instword & PPC_INST_MFSPR_TIDR_MASK) ==
> +		PPC_INST_MFSPR_TIDR) &&
> +			cpu_has_feature(CPU_FTR_TIDR)) {
> +		PPC_WARN_EMULATED(mftidr, regs);
> +		rd = (instword >> 21) & 0x1f;
> +		regs->gpr[rd] = mfspr(SPRN_TIDR);
> +		return 0;
> +	}
> +	/* Emulate the mtspr TIDR, rD. */
> +	if (((instword & PPC_INST_MTSPR_TIDR_MASK) ==
> +		PPC_INST_MTSPR_TIDR) &&
> +			cpu_has_feature(CPU_FTR_TIDR)) {
> +		PPC_WARN_EMULATED(mttidr, regs);
> +		rd = (instword >> 21) & 0x1f;
> +		current->thread.tidr = regs->gpr[rd];
> +		mtspr(SPRN_TIDR, regs->gpr[rd]);
> +		return 0;
> +	}
>  #endif
>  
>  	return -EINVAL;
> @@ -2036,6 +2055,8 @@ struct ppc_emulated ppc_emulated = {
>  #ifdef CONFIG_PPC64
>  	WARN_EMULATED_SETUP(mfdscr),
>  	WARN_EMULATED_SETUP(mtdscr),
> +	WARN_EMULATED_SETUP(mftidr),
> +	WARN_EMULATED_SETUP(mttidr),
>  	WARN_EMULATED_SETUP(lq_stq),
>  #endif
>  };