Re: [RFC PATCH 3/7] x86/entry: Implement atomic-IST-entry

Peter Zijlstra
2023-04-06T21:02:31Z

On Mon, Apr 03, 2023 at 10:06:01PM +0800, Lai Jiangshan wrote:

> diff --git a/arch/x86/entry/Makefile b/arch/x86/entry/Makefile
> index ca2fe186994b..7cc1254ca519 100644
> --- a/arch/x86/entry/Makefile
> +++ b/arch/x86/entry/Makefile
> @@ -8,11 +8,14 @@ UBSAN_SANITIZE := n
> KCOV_INSTRUMENT := n
>
> CFLAGS_REMOVE_common.o = $(CC_FLAGS_FTRACE)
> +CFLAGS_REMOVE_ist_entry.o = $(CC_FLAGS_FTRACE) $(RETHUNK_CFLAGS)

This ^^^

> diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
> index 49ddc4dd3117..50a24cc83581 100644
> --- a/arch/x86/entry/entry_64.S
> +++ b/arch/x86/entry/entry_64.S
> @@ -443,6 +443,184 @@ SYM_CODE_END(\asmsym)

> +.macro idtentry_ist vector asmsym cfunc user_cfunc has_error_code:req, stack_offset:req
> +SYM_CODE_START(\asmsym)
> + UNWIND_HINT_IRET_REGS offset=\has_error_code*8
> + ENDBR
> +
> + /*
> + * Clear X86_EFLAGS_AC, X86_EFLAGS_DF and set a default ORIG_RAX.
> + *
> + * The code setting ORIG_RAX will not be replicated if interrupted.
> + */
> + ASM_CLAC
> + cld
> +
> + .if \has_error_code == 0
> + pushq $-1 /* ORIG_RAX: no syscall to restart */
> + .endif
> +
> + /*
> + * No register can be touched except %rsp,%rflags,%rip before
> + * pushing all the registers. It is indispensable for nested
> + * atomic-IST-entry to replicate pushing the registers.
> + */
> + PUSH_REGS
> +
> + /*
> + * Finished pushing register, all registers can be touched by now.
> + *
> + * Clear registers for the C function ist_copy_regs_to_main_stack()
> + * and the handler to avoid any possible exploitation of any
> + * speculation attack.
> + */
> + CLEAR_REGS
> +
> + /*
> + * Copy the pt_regs to the IST main stack including the pt_regs of
> + * the interrupted atomic-IST-entris, if any, by replicating.
> + */
> + movq %rsp, %rdi /* pt_regs pointer on its own IST stack */
> + leaq PTREGS_SIZE-\stack_offset(%rsp), %rsi /* struct cea_exception_stacks pointer */
> + call ist_copy_regs_to_main_stack

IIUC you do a CALL+RET here, before you call paranoid_entry ...

> +
> + /*
> + * Commit stage.
> + */
> +SYM_INNER_LABEL(start_commit_\asmsym, SYM_L_GLOBAL)
> + /*
> + * Switches to the IST main stack. Before the switching is done,
> + * %rax is the copied pt_regs pointer in IST main stack.
> + */
> + movq %rax, %rsp
> +
> + /*
> + * The label should be immediate after the instruction that switches
> + * the stack since there is code assuming there is only one single
> + * instruction in the commit stage and the code assumes "%rsp in the
> + * IST main stack is also the sign of ending a atomic-IST-entry".
> + * (The code will be removed in future when %rip-based identifying
> + * is added.)
> + */
> +SYM_INNER_LABEL(commit_\asmsym, SYM_L_GLOBAL)
> +
> + /*
> + * Now, it is on the IST main stack. For the whole kernel, the entries
> + * of the IST exceptions can be seen from here because the inside
> + * of the atomic-IST-entry can not be seen from the whole kernel
> + * except in the atomic-IST-entry or #DF.
> + */
> + UNWIND_HINT_REGS
> + ENCODE_FRAME_POINTER
> +
> + /*
> + * The code setting ORIG_RAX will not be replicated if interrupted.
> + * So redo it here.
> + */
> + .if \has_error_code == 0
> + movq $-1, ORIG_RAX(%rsp) /* ORIG_RAX: no syscall to restart */
> + .endif
> +
> + /*
> + * If the entry is from userspace, switch stacks and treat it as
> + * a normal entry.
> + */
> + testb $3, CS(%rsp)
> + jnz .Lfrom_usermode_switch_stack_\@
> +
> + /*
> + * paranoid_entry returns GS/CR3/SPEC_CTL information for
> + * paranoid_exit in RBX/R14/R15.
> + */
> + call paranoid_entry

... all the way down here, which will do:

IBRS_ENTER;
UNTRAIN_RET_FROM_CALL;

Which thus breaks the whole RetBleed mess, since that must not do RET
before that happens.