Re: [kvm-unit-tests PATCH v3 17/17] x86/cet: Add testcases to verify KVM rejects emulation of CET instructions

Mathias Krause
2025-11-14T15:19:35Z

[-- Attachment #1: Type: text/plain, Size: 11279 bytes --]

On 14.11.25 01:12, Sean Christopherson wrote:
> Add SHSTK and IBT testcases to verify that KVM rejects (forced) emulation
> of instructions that interact with SHSTK and/or IBT state, as KVM doesn't
> support emulating SHSTK or IBT (rejecting emulation is preferable to
> compromising guest security).
>
> Signed-off-by: Sean Christopherson <seanjc@google.com>
> ---
> x86/cet.c | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
> 1 file changed, 121 insertions(+), 2 deletions(-)
>
> diff --git a/x86/cet.c b/x86/cet.c
> index 26cd1c9b..34c78210 100644
> --- a/x86/cet.c
> +++ b/x86/cet.c
> @@ -75,6 +75,113 @@ static uint64_t cet_ibt_func(void)
> return 0;
> }
>
> +#define __CET_TEST_UNSUPPORTED_INSTRUCTION(insn) \
> +({ \
> + struct far_pointer32 fp = { \
> + .offset = 0, \
> + .selector = USER_CS, \
> + }; \
> + \
> + asm volatile ("push %%rax\n" \
> + ASM_TRY_FEP("1f") insn "\n\t" \
> + "1:" \
> + "pop %%rax\n" \
> + : : "m" (fp), "a" (NONCANONICAL) : "memory"); \
> + \
> + exception_vector(); \
> +})
> +
> +#define SHSTK_TEST_UNSUPPORTED_INSTRUCTION(insn) \
> +do { \
> + uint8_t vector = __CET_TEST_UNSUPPORTED_INSTRUCTION(insn); \
> + \
> + report(vector == UD_VECTOR, "Wanted #UD on %s, got %s", \
> + insn, exception_mnemonic(vector)); \
> +} while (0)
> +
> +/*
> + * Treat IRET as unsupported with IBT even though the minimal interactions with
> + * IBT _could_ be easily emulated by KVM, as KVM doesn't support emulating IRET
> + * outside of Real Mode.
> + */
> +#define CET_TEST_UNSUPPORTED_INSTRUCTIONS(CET) \
> +do { \
> + CET##_TEST_UNSUPPORTED_INSTRUCTION("callq *%%rax"); \
> + CET##_TEST_UNSUPPORTED_INSTRUCTION("lcall *%0"); \

Maybe spell out that this is a 32bit far call ("lcalll *%0"), as only
these are supported on AMD *and* Intel? (Intel can do 64bit ones too.)

> + CET##_TEST_UNSUPPORTED_INSTRUCTION("syscall"); \
> + CET##_TEST_UNSUPPORTED_INSTRUCTION("sysenter"); \
> + CET##_TEST_UNSUPPORTED_INSTRUCTION("iretq"); \

These, if emulated, would be rather disastrous to the test. :D
- SYSCALL messes badly with the register state.
- SYSENTER should #UD in long mode on AMD anyway but Intel allows it.
However, it'll likely #GP early, prior to messing with register state,
as IA32_SYSENTER_CS doesn't get initialized as the CPU expects it to
be.
- The stack frame for IRETQ isn't properly set up (lacks CS, RFLAGS, SS,
RSP) but even if it'll #GP early for the non-canonical address, it'll
have modified RSP, making the recovery "pop %rax" pop an unrelated
stack word, likely making the code crash soon after.

So these instructions really *rely* on KVM to #UD on these early.
Dunno, but it feels like the tests should be made more robust wrt.
possible future emulation by KVM?

> +} while (0)
> +
> +static uint64_t cet_shstk_emulation(void)
> +{
> + CET_TEST_UNSUPPORTED_INSTRUCTIONS(SHSTK);
> +
> + SHSTK_TEST_UNSUPPORTED_INSTRUCTION("call 1f");
> + SHSTK_TEST_UNSUPPORTED_INSTRUCTION("retq");
> + SHSTK_TEST_UNSUPPORTED_INSTRUCTION("retq $10");
> + SHSTK_TEST_UNSUPPORTED_INSTRUCTION("lretq");
> + SHSTK_TEST_UNSUPPORTED_INSTRUCTION("lretq $10");
> +
> + /* Do a handful of JMPs to verify they aren't impacted by SHSTK. */
> + asm volatile(KVM_FEP "jmp 1f\n\t"
> + "1:\n\t"
> + KVM_FEP "lea 2f(%%rip), %%rax\n\t"
> + KVM_FEP "jmp *%%rax\n\t"
^^^^^^^^^^
Fortunately, this indirect branch runs only with shadow stack enabled,
no IBT. If it would, than below misses an ENDBR64.
vvv
> + "2:\n\t"> + KVM_FEP "push $" xstr(USER_CS) "\n\t"
> + KVM_FEP "lea 3f(%%rip), %%rax\n\t"
> + KVM_FEP "push %%rax\n\t"
> + /* Manually encode ljmpq, which gas doesn't recognize :-( */

LJMPQ is actually only supported on Intel systems. AMD doesn't support
it and one has to emulate it via "push $cs; push $rip; lretq". Dunno if
it's a feature or bug of KVM to emulate LJMPQ fine on AMD -- if it does,
that is!

> + KVM_FEP ".byte 0x48\n\t"
> + "ljmpl *(%%rsp)\n\t"
> + "3:\n\t"
> + KVM_FEP "pop %%rax\n\t"
> + KVM_FEP "pop %%rax\n\t"
> + ::: "eax");
> +
> + return 0;
> +}
> +

> +/*
> + * Don't invoke printf() or report() in the IBT testcase, as it will likely
> + * generate an indirect branch without an endbr64 annotation and thus #CP.

Uhm, x86/Makefile.x86_64 has this:

fcf_protection_full := $(call cc-option, -fcf-protection=full,)
COMMON_CFLAGS += -mno-red-zone -mno-sse -mno-sse2 $(fcf_protection_full)
[...]
ifneq ($(fcf_protection_full),)
tests += $(TEST_DIR)/cet.$(exe)
endif

So all code that needs it should have a leading ENDBR64 or the cet test
wouldn't be part of the test suite.

However, I also notice it isn't working...

[digging in, hours later...]

After tweaking the exception handling to be able to tweak the IBT
tracker state (which would be stuck at WAIT_FOR_ENDBRANCH for a missing
ENDBR64), it still wouldn't work for me. Bummer! Further instrumentation
showed, the code triggered within exception_mnemonic() which caused only
more question marks -- it's just a simple switch, right? Though, looking
at the disassembly made it crystal clear:

000000000040707c <exception_mnemonic>:
40707c: endbr64
407080: cmp $0x1e,%edi
407083: ja 407117 <exception_mnemonic+0x9b>
407089: mov %edi,%edi
40708b: notrack jmp *0x4107e0(,%rdi,8)
::
4070b1: mov $0x411c7c,%eax # <-- #CP(3) here

The switch block caused gcc to emit a jump table and an indirect jump
for it (to 40708b). The jump is prefixed with 'notrack' to keep the size
of the jump target stubs small, allowing for omitting the ENDBR64
instruction. However, the IBT test code doesn't enable that in the MSR,
leading the CPU to raising a #CP(3) for this indirect jump. D'oh!

So we can either disable the generation of jump tables in gcc
(-fno-jump-tables) or just allow the 'notrack' handling. I guess the
latter is easier.

Attached patch gets me this (on some early CET series patch set, not
v6.18-rcX):

Running user mode Shadow Stack tests
The return-address in shadow-stack = 0x4035ab, in normal stack = 0x4035ab
Try to temper the return-address, this causes #CP on returning...
PASS: NEAR RET shadow-stack protection test
Try to temper the return-address of far-called function...
PASS: FAR RET shadow-stack protection test
PASS: SHSTK: Wanted #UD on callq *%%rax, got #UD
PASS: SHSTK: Wanted #UD on lcall *%0, got #UD
PASS: SHSTK: Wanted #UD on syscall, got #UD
PASS: SHSTK: Wanted #UD on sysenter, got #UD
PASS: SHSTK: Wanted #UD on iretq, got #UD
PASS: SHSTK: Wanted #UD on call 1f, got #UD
FAIL: SHSTK: Wanted #UD on retq, got #GP
FAIL: SHSTK: Wanted #UD on retq $10, got #GP
PASS: SHSTK: Wanted #UD on lretq, got #UD
PASS: SHSTK: Wanted #UD on lretq $10, got #UD
PASS: MSR_IA32_PL3_SSP alignment test.
No endbr64 instruction at jmp target, this triggers #CP...
PASS: Indirect-branch tracking test
PASS: IBT: Wanted #UD on callq *%%rax, got #UD
PASS: IBT: Wanted #UD on lcall *%0, got #UD
PASS: IBT: Wanted #UD on syscall, got #UD
PASS: IBT: Wanted #UD on sysenter, got #UD
PASS: IBT: Wanted #UD on iretq, got #UD
PASS: IBT: Wanted #UD on jmp *%%rax, got #UD
Unhandled exception 13 #GP(0) at ip 0000000000000000
error_code=0000 rflags=00010006 cs=0000004b
rax=aaaaaaaaaaaaaaaa rcx=00000000000003fd rdx=00000000000003f8
rbx=aaaaaaaaaaaaaaaa
rbp=0000000000713e70 rsi=0000000000411cb7 rdi=000000000000000a
r8=0000000000411cb7 r9=00000000000003f8 r10=000000000000000d
r11=0000000000000000
r12=0000000000000006 r13=0000000000000000 r14=0000000000000000
r15=0000000000000000
cr0=0000000080010011 cr2=0000000000000000 cr3=000000000107f000
cr4=0000000000800020
cr8=0000000000000000
STACK: @0 4035ab 400deb 4001bd
b'0x0000000000000000: ?? ??:0'

Progress, but still something to fix, namely the "ljmp *%0" which is,
apparently, successfully branching to address 0 and then triggering a
#GP(0). But maybe that's just because I'm on an old version of the KVM
CET series, haven't build v6.18-rcX yet.

Anyhow, one less puzzle, so should be good?

> + * Return the line number of the macro invocation to signal failure.
> + */
> +#define IBT_TEST_UNSUPPORTED_INSTRUCTION(insn) \
> +do { \
> + uint8_t vector = __CET_TEST_UNSUPPORTED_INSTRUCTION(insn); \
> + \
> + if (vector != UD_VECTOR) \
> + return __LINE__; \
> +} while (0)
> +
> +static uint64_t cet_ibt_emulation(void)
> +{
> + CET_TEST_UNSUPPORTED_INSTRUCTIONS(IBT);
> +
> + IBT_TEST_UNSUPPORTED_INSTRUCTION("jmp *%%rax");
> + IBT_TEST_UNSUPPORTED_INSTRUCTION("ljmp *%0");
> +
> + /* Verify direct CALLs and JMPs, and all RETs aren't impacted by IBT. */
> + asm volatile(KVM_FEP "jmp 2f\n\t"
> + "1: " KVM_FEP " ret\n\t"
> + "2: " KVM_FEP " call 1b\n\t"
> + KVM_FEP "push $" xstr(USER_CS) "\n\t"
> + KVM_FEP "lea 3f(%%rip), %%rax\n\t"
> + KVM_FEP "push %%rax\n\t"
> + KVM_FEP "lretq\n\t"
> + "3:\n\t"
> + KVM_FEP "push $0x55555555\n\t"
> + KVM_FEP "push $" xstr(USER_CS) "\n\t"
> + KVM_FEP "lea 4f(%%rip), %%rax\n\t"
> + KVM_FEP "push %%rax\n\t"
> + KVM_FEP "lretq $8\n\t"
> + "4:\n\t"
> + ::: "eax");
> + return 0;
> +}
> +
> #define CP_ERR_NEAR_RET 0x0001
> #define CP_ERR_FAR_RET 0x0002
> #define CP_ERR_ENDBR 0x0003
> @@ -117,15 +224,20 @@ static void test_shstk(void)
> /* Store shadow-stack pointer. */
> wrmsr(MSR_IA32_PL3_SSP, (u64)(shstk_virt + 0x1000));
>
> - printf("Unit tests for CET user mode...\n");
> + printf("Running user mode Shadow Stack tests\n");
> run_in_user(cet_shstk_func, CP_VECTOR, 0, 0, 0, 0, &rvc);
> report(rvc && exception_error_code() == CP_ERR_NEAR_RET,
> "NEAR RET shadow-stack protection test");
> -
> run_in_user(cet_shstk_far_ret, CP_VECTOR, 0, 0, 0, 0, &rvc);
> report(rvc && exception_error_code() == CP_ERR_FAR_RET,
> "FAR RET shadow-stack protection test");
>
> + if (is_fep_available &&
> + (run_in_user(cet_shstk_emulation, CP_VECTOR, 0, 0, 0, 0, &rvc) || rvc))
> + report_fail("Forced emulation with SHSTK generated %s(%u)",
> + exception_mnemonic(exception_vector()),
> + exception_error_code());
> +
> /* SSP should be 4-Byte aligned */
> vector = wrmsr_safe(MSR_IA32_PL3_SSP, 0x1);
> report(vector == GP_VECTOR, "MSR_IA32_PL3_SSP alignment test.");
> @@ -133,6 +245,7 @@ static void test_shstk(void)
>
> static void test_ibt(void)
> {
> + uint64_t l;
> bool rvc;
>
> if (!this_cpu_has(X86_FEATURE_IBT)) {
> @@ -146,6 +259,12 @@ static void test_ibt(void)
> run_in_user(cet_ibt_func, CP_VECTOR, 0, 0, 0, 0, &rvc);
> report(rvc && exception_error_code() == CP_ERR_ENDBR,
> "Indirect-branch tracking test");
> +
> + if (is_fep_available &&
> + ((l = run_in_user(cet_ibt_emulation, CP_VECTOR, 0, 0, 0, 0, &rvc)) || rvc))
> + report_fail("Forced emulation with IBT generated %s(%u) at line %lu",
> + exception_mnemonic(exception_vector()),
> + exception_error_code(), l);
> }
>
> int main(int ac, char **av)Reviewed-by: Mathias Krause <minipli@grsecurity.net>

Thanks,
Mathias

[-- Attachment #2: 0001-x86-cet-Enable-NOTRACK-handling-for-IBT-tests.patch --]
[-- Type: text/x-patch, Size: 2610 bytes --]

From 5cba4a9ccdff8e682ae53ee69849879edc326746 Mon Sep 17 00:00:00 2001
From: Mathias Krause <minipli@grsecurity.net>
Date: Fri, 14 Nov 2025 15:53:10 +0100
Subject: [kvm-unit-tests PATCH] x86: cet: Enable NOTRACK handling for IBT
tests

gcc's jump table handling makes use of 'notrack' indirect jumps, causing
spurious #CP(3) exceptions.

Enable 'notrack' handling for the IBT tests instead of disabling jump
tables as we may want to make use of 'notrack' ourselves in future
tests.

With that fixed, we can make the IBT tests report() successfully, just
as the shadow stack tests do.

Signed-off-by: Mathias Krause <minipli@grsecurity.net>
---
x86/cet.c | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/x86/cet.c b/x86/cet.c
index 34c782103a10..801d8da6e929 100644
--- a/x86/cet.c
+++ b/x86/cet.c
@@ -95,7 +95,7 @@ static uint64_t cet_ibt_func(void)
do { \
uint8_t vector = __CET_TEST_UNSUPPORTED_INSTRUCTION(insn); \
\
- report(vector == UD_VECTOR, "Wanted #UD on %s, got %s", \
+ report(vector == UD_VECTOR, "SHSTK: Wanted #UD on %s, got %s", \
insn, exception_mnemonic(vector)); \
} while (0)

@@ -152,8 +152,8 @@ static uint64_t cet_shstk_emulation(void)
do { \
uint8_t vector = __CET_TEST_UNSUPPORTED_INSTRUCTION(insn); \
\
- if (vector != UD_VECTOR) \
- return __LINE__; \
+ report(vector == UD_VECTOR, "IBT: Wanted #UD on %s, got %s", \
+ insn, exception_mnemonic(vector)); \
} while (0)

static uint64_t cet_ibt_emulation(void)
@@ -189,8 +189,9 @@ static uint64_t cet_ibt_emulation(void)
#define CP_ERR_SETSSBSY 0x0005
#define CP_ERR_ENCL BIT(15)

-#define ENABLE_SHSTK_BIT 0x1
-#define ENABLE_IBT_BIT 0x4
+#define CET_ENABLE_SHSTK BIT(0)
+#define CET_ENABLE_IBT BIT(2)
+#define CET_ENABLE_NOTRACK BIT(4)

static void test_shstk(void)
{
@@ -219,7 +220,7 @@ static void test_shstk(void)
install_pte(current_page_table(), 1, shstk_virt, pte, 0);

/* Enable shadow-stack protection */
- wrmsr(MSR_IA32_U_CET, ENABLE_SHSTK_BIT);
+ wrmsr(MSR_IA32_U_CET, CET_ENABLE_SHSTK);

/* Store shadow-stack pointer. */
wrmsr(MSR_IA32_PL3_SSP, (u64)(shstk_virt + 0x1000));
@@ -253,8 +254,8 @@ static void test_ibt(void)
return;
}

- /* Enable indirect-branch tracking */
- wrmsr(MSR_IA32_U_CET, ENABLE_IBT_BIT);
+ /* Enable indirect-branch tracking (notrack handling for jump tables) */
+ wrmsr(MSR_IA32_U_CET, CET_ENABLE_IBT | CET_ENABLE_NOTRACK);

run_in_user(cet_ibt_func, CP_VECTOR, 0, 0, 0, 0, &rvc);
report(rvc && exception_error_code() == CP_ERR_ENDBR,
--
2.51.0