[PATCH 6.6] x86/cpu/kvm: SRSO: Fix possible missing IBPB on VM-Exit

[PATCH 6.6] x86/cpu/kvm: SRSO: Fix possible missing IBPB on VM-Exit

[Patrick Bellasi]

commit 318e8c339c9a0891c389298bb328ed0762a9935e upstream.

In [1] the meaning of the synthetic IBPB flags has been redefined for a
better separation of concerns:
 - ENTRY_IBPB     -- issue IBPB on entry only
 - IBPB_ON_VMEXIT -- issue IBPB on VM-Exit only
and the Retbleed mitigations have been updated to match this new
semantics.

Commit [2] was merged shortly before [1], and their interaction was not
handled properly. This resulted in IBPB not being triggered on VM-Exit
in all SRSO mitigation configs requesting an IBPB there.

Specifically, an IBPB on VM-Exit is triggered only when
X86_FEATURE_IBPB_ON_VMEXIT is set. However:

 - X86_FEATURE_IBPB_ON_VMEXIT is not set for "spec_rstack_overflow=ibpb",
   because before [1] having X86_FEATURE_ENTRY_IBPB was enough. Hence,
   an IBPB is triggered on entry but the expected IBPB on VM-exit is
   not.

 - X86_FEATURE_IBPB_ON_VMEXIT is not set also when
   "spec_rstack_overflow=ibpb-vmexit" if X86_FEATURE_ENTRY_IBPB is
   already set.

   That's because before [1] this was effectively redundant. Hence, e.g.
   a "retbleed=ibpb spec_rstack_overflow=bpb-vmexit" config mistakenly
   reports the machine still vulnerable to SRSO, despite an IBPB being
   triggered both on entry and VM-Exit, because of the Retbleed selected
   mitigation config.

 - UNTRAIN_RET_VM won't still actually do anything unless
   CONFIG_MITIGATION_IBPB_ENTRY is set.

For "spec_rstack_overflow=ibpb", enable IBPB on both entry and VM-Exit
and clear X86_FEATURE_RSB_VMEXIT which is made superfluous by
X86_FEATURE_IBPB_ON_VMEXIT. This effectively makes this mitigation
option similar to the one for 'retbleed=ibpb', thus re-order the code
for the RETBLEED_MITIGATION_IBPB option to be less confusing by having
all features enabling before the disabling of the not needed ones.

For "spec_rstack_overflow=ibpb-vmexit", guard this mitigation setting
with CONFIG_MITIGATION_IBPB_ENTRY to ensure UNTRAIN_RET_VM sequence is
effectively compiled in. Drop instead the CONFIG_MITIGATION_SRSO guard,
since none of the SRSO compile cruft is required in this configuration.
Also, check only that the required microcode is present to effectively
enabled the IBPB on VM-Exit.

Finally, update the KConfig description for CONFIG_MITIGATION_IBPB_ENTRY
to list also all SRSO config settings enabled by this guard.

Fixes: 864bcaa38ee4 ("x86/cpu/kvm: Provide UNTRAIN_RET_VM") [1]
Fixes: d893832d0e1e ("x86/srso: Add IBPB on VMEXIT") [2]
Reported-by: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: Patrick Bellasi <derkling@google.com>
Reviewed-by: Borislav Petkov (AMD) <bp@alien8.de>
Cc: stable@kernel.org
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 arch/x86/Kconfig           |  3 ++-
 arch/x86/kernel/cpu/bugs.c | 21 ++++++++++++++-------
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 05c82fd5d0f60..989d432b58345 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2514,7 +2514,8 @@ config CPU_IBPB_ENTRY
 	depends on CPU_SUP_AMD && X86_64
 	default y
 	help
-	  Compile the kernel with support for the retbleed=ibpb mitigation.
+	  Compile the kernel with support for the retbleed=ibpb and
+	  spec_rstack_overflow={ibpb,ibpb-vmexit} mitigations.
 
 config CPU_IBRS_ENTRY
 	bool "Enable IBRS on kernel entry"
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 7b5ba5b8592a2..7df458a6553eb 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -1113,6 +1113,8 @@ static void __init retbleed_select_mitigation(void)
 
 	case RETBLEED_MITIGATION_IBPB:
 		setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
+		setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
+		mitigate_smt = true;
 
 		/*
 		 * IBPB on entry already obviates the need for
@@ -1122,9 +1124,6 @@ static void __init retbleed_select_mitigation(void)
 		setup_clear_cpu_cap(X86_FEATURE_UNRET);
 		setup_clear_cpu_cap(X86_FEATURE_RETHUNK);
 
-		setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
-		mitigate_smt = true;
-
 		/*
 		 * There is no need for RSB filling: entry_ibpb() ensures
 		 * all predictions, including the RSB, are invalidated,
@@ -2626,6 +2625,7 @@ static void __init srso_select_mitigation(void)
 		if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) {
 			if (has_microcode) {
 				setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
+				setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
 				srso_mitigation = SRSO_MITIGATION_IBPB;
 
 				/*
@@ -2635,6 +2635,13 @@ static void __init srso_select_mitigation(void)
 				 */
 				setup_clear_cpu_cap(X86_FEATURE_UNRET);
 				setup_clear_cpu_cap(X86_FEATURE_RETHUNK);
+
+				/*
+				 * There is no need for RSB filling: entry_ibpb() ensures
+				 * all predictions, including the RSB, are invalidated,
+				 * regardless of IBPB implementation.
+				 */
+				setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT);
 			}
 		} else {
 			pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n");
@@ -2643,8 +2650,8 @@ static void __init srso_select_mitigation(void)
 		break;
 
 	case SRSO_CMD_IBPB_ON_VMEXIT:
-		if (IS_ENABLED(CONFIG_CPU_SRSO)) {
-			if (!boot_cpu_has(X86_FEATURE_ENTRY_IBPB) && has_microcode) {
+		if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) {
+			if (has_microcode) {
 				setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
 				srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT;
 
@@ -2656,9 +2663,9 @@ static void __init srso_select_mitigation(void)
 				setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT);
 			}
 		} else {
-			pr_err("WARNING: kernel not compiled with CPU_SRSO.\n");
+			pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n");
 			goto pred_cmd;
-                }
+		}
 		break;
 
 	default:
-- 
2.48.1.601.g30ceb7b040-goog

Re: [PATCH 6.6] x86/cpu/kvm: SRSO: Fix possible missing IBPB on VM-Exit

[Borislav Petkov]

On Fri, Feb 21, 2025 at 02:20:02PM +0000, Patrick Bellasi wrote:
> commit 318e8c339c9a0891c389298bb328ed0762a9935e upstream.
> 
> In [1] the meaning of the synthetic IBPB flags has been redefined for a
> better separation of concerns:
>  - ENTRY_IBPB     -- issue IBPB on entry only
>  - IBPB_ON_VMEXIT -- issue IBPB on VM-Exit only
> and the Retbleed mitigations have been updated to match this new
> semantics.

All 4 backports by Patrick:

Feb 21           Patrick Bellasi ( : 148|) [PATCH 6.6] x86/cpu/kvm: SRSO: Fix possible missing IBPB on VM-Exit
Feb 21           Patrick Bellasi ( : 147|) [PATCH 6.1] x86/cpu/kvm: SRSO: Fix possible missing IBPB on VM-Exit
Feb 21           Patrick Bellasi ( : 147|) [PATCH 5.15] x86/cpu/kvm: SRSO: Fix possible missing IBPB on VM-Exit
Feb 21           Patrick Bellasi ( : 147|) [PATCH 5.10] x86/cpu/kvm: SRSO: Fix possible missing IBPB on VM-Exit

LGTM.

Thanks!

-- 
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette

Re: [PATCH 6.6] x86/cpu/kvm: SRSO: Fix possible missing IBPB on VM-Exit

[Sasha Levin]

[ Sasha's backport helper bot ]

Hi,

✅ All tests passed successfully. No issues detected.
No action required from the submitter.

The upstream commit SHA1 provided is correct: 318e8c339c9a0891c389298bb328ed0762a9935e

Note: The patch differs from the upstream commit:
---
1:  318e8c339c9a0 ! 1:  7e78323cfe696 x86/cpu/kvm: SRSO: Fix possible missing IBPB on VM-Exit
    @@ Metadata
      ## Commit message ##
         x86/cpu/kvm: SRSO: Fix possible missing IBPB on VM-Exit
     
    +    commit 318e8c339c9a0891c389298bb328ed0762a9935e upstream.
    +
         In [1] the meaning of the synthetic IBPB flags has been redefined for a
         better separation of concerns:
          - ENTRY_IBPB     -- issue IBPB on entry only
    @@ Commit message
         Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
     
      ## arch/x86/Kconfig ##
    -@@ arch/x86/Kconfig: config MITIGATION_IBPB_ENTRY
    +@@ arch/x86/Kconfig: config CPU_IBPB_ENTRY
      	depends on CPU_SUP_AMD && X86_64
      	default y
      	help
    @@ arch/x86/Kconfig: config MITIGATION_IBPB_ENTRY
     +	  Compile the kernel with support for the retbleed=ibpb and
     +	  spec_rstack_overflow={ibpb,ibpb-vmexit} mitigations.
      
    - config MITIGATION_IBRS_ENTRY
    + config CPU_IBRS_ENTRY
      	bool "Enable IBRS on kernel entry"
     
      ## arch/x86/kernel/cpu/bugs.c ##
    @@ arch/x86/kernel/cpu/bugs.c: static void __init retbleed_select_mitigation(void)
      		 * There is no need for RSB filling: entry_ibpb() ensures
      		 * all predictions, including the RSB, are invalidated,
     @@ arch/x86/kernel/cpu/bugs.c: static void __init srso_select_mitigation(void)
    - 		if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {
    + 		if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) {
      			if (has_microcode) {
      				setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
     +				setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
    @@ arch/x86/kernel/cpu/bugs.c: static void __init srso_select_mitigation(void)
     +				setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT);
      			}
      		} else {
    - 			pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n");
    + 			pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n");
     @@ arch/x86/kernel/cpu/bugs.c: static void __init srso_select_mitigation(void)
    + 		break;
      
    - ibpb_on_vmexit:
      	case SRSO_CMD_IBPB_ON_VMEXIT:
    --		if (IS_ENABLED(CONFIG_MITIGATION_SRSO)) {
    +-		if (IS_ENABLED(CONFIG_CPU_SRSO)) {
     -			if (!boot_cpu_has(X86_FEATURE_ENTRY_IBPB) && has_microcode) {
    -+		if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {
    ++		if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) {
     +			if (has_microcode) {
      				setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
      				srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT;
    @@ arch/x86/kernel/cpu/bugs.c: static void __init srso_select_mitigation(void)
      				setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT);
      			}
      		} else {
    --			pr_err("WARNING: kernel not compiled with MITIGATION_SRSO.\n");
    +-			pr_err("WARNING: kernel not compiled with CPU_SRSO.\n");
    ++			pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n");
    + 			goto pred_cmd;
     -                }
    -+			pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n");
     +		}
      		break;
    + 
      	default:
    - 		break;
---

Results of testing on various branches:

| Branch                    | Patch Apply | Build Test |
|---------------------------|-------------|------------|
| stable/linux-6.6.y        |  Success    |  Success   |