[PATCH 1/6] xor: add missing vzeroupper to AVX code

[PATCH 1/6] xor: add missing vzeroupper to AVX code

[Eric Biggers]

Since the AVX optimized XOR code uses YMM registers, execute vzeroupper
before returning from it.  This is needed to avoid degrading the
performance of any later SSE code that may happen to be executed.

Fixes: ea4d26ae24e5 ("raid5: add AVX optimized RAID5 checksumming")
Cc: stable@vger.kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
 lib/raid/xor/x86/xor-avx.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/raid/xor/x86/xor-avx.c b/lib/raid/xor/x86/xor-avx.c
index f7777d7aa269..95b21e7225e8 100644
--- a/lib/raid/xor/x86/xor-avx.c
+++ b/lib/raid/xor/x86/xor-avx.c
@@ -147,6 +147,7 @@ static void xor_gen_avx(void *dest, void **srcs, unsigned int src_cnt,
 {
 	kernel_fpu_begin();
 	xor_gen_avx_inner(dest, srcs, src_cnt, bytes);
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
-- 
2.55.0

[PATCH 2/6] raid6: add missing vzeroupper to AVX2 code

[Eric Biggers]

Since the AVX2 optimized RAID6 code uses YMM registers, execute
vzeroupper before returning from it.  This is needed to avoid degrading
the performance of any later SSE code that may happen to be executed.

Fixes: 2c935842bdb4 ("lib/raid6: Add AVX2 optimized gen_syndrome functions")
Fixes: 7056741fd9fc ("lib/raid6: Add AVX2 optimized recovery functions")
Cc: stable@vger.kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
 lib/raid/raid6/x86/avx2.c       | 6 ++++++
 lib/raid/raid6/x86/recov_avx2.c | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/lib/raid/raid6/x86/avx2.c b/lib/raid/raid6/x86/avx2.c
index 7d829c669ea7..3cc2fe7ac42c 100644
--- a/lib/raid/raid6/x86/avx2.c
+++ b/lib/raid/raid6/x86/avx2.c
@@ -67,6 +67,7 @@ static void raid6_avx21_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -115,6 +116,7 @@ static void raid6_avx21_xor_syndrome(int disks, int start, int stop,
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -175,6 +177,7 @@ static void raid6_avx22_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -243,6 +246,7 @@ static void raid6_avx22_xor_syndrome(int disks, int start, int stop,
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -334,6 +338,7 @@ static void raid6_avx24_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -444,6 +449,7 @@ static void raid6_avx24_xor_syndrome(int disks, int start, int stop,
 		asm volatile("vmovntdq %%ymm14,%0" : "=m" (q[d+96]));
 	}
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
diff --git a/lib/raid/raid6/x86/recov_avx2.c b/lib/raid/raid6/x86/recov_avx2.c
index a714a780a2d8..820871046e30 100644
--- a/lib/raid/raid6/x86/recov_avx2.c
+++ b/lib/raid/raid6/x86/recov_avx2.c
@@ -176,6 +176,7 @@ static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila,
 #endif
 	}
 
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -293,6 +294,7 @@ static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila,
 #endif
 	}
 
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
-- 
2.55.0

[PATCH 3/6] raid6: add missing vzeroupper to AVX-512 code

[Eric Biggers]

Since the AVX-512 optimized RAID6 code uses ZMM registers, execute
vzeroupper before returning from it.  This is needed to avoid degrading
the performance of any later SSE code that may happen to be executed.

Fixes: e0a491c12968 ("lib/raid6: Add AVX512 optimized gen_syndrome functions")
Fixes: 13c520b2993c ("lib/raid6: Add AVX512 optimized recovery functions")
Cc: stable@vger.kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
 lib/raid/raid6/x86/avx512.c       | 6 ++++++
 lib/raid/raid6/x86/recov_avx512.c | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/lib/raid/raid6/x86/avx512.c b/lib/raid/raid6/x86/avx512.c
index e671eb5bde63..772bfc4af6df 100644
--- a/lib/raid/raid6/x86/avx512.c
+++ b/lib/raid/raid6/x86/avx512.c
@@ -78,6 +78,7 @@ static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -137,6 +138,7 @@ static void raid6_avx5121_xor_syndrome(int disks, int start, int stop,
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -208,6 +210,7 @@ static void raid6_avx5122_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -292,6 +295,7 @@ static void raid6_avx5122_xor_syndrome(int disks, int start, int stop,
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -396,6 +400,7 @@ static void raid6_avx5124_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -529,6 +534,7 @@ static void raid6_avx5124_xor_syndrome(int disks, int start, int stop,
 			       "m" (q[d+128]), "m" (q[d+192]));
 	}
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 const struct raid6_calls raid6_avx512x4 = {
diff --git a/lib/raid/raid6/x86/recov_avx512.c b/lib/raid/raid6/x86/recov_avx512.c
index ec72d5a30c01..299a3f044d61 100644
--- a/lib/raid/raid6/x86/recov_avx512.c
+++ b/lib/raid/raid6/x86/recov_avx512.c
@@ -211,6 +211,7 @@ static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila,
 #endif
 	}
 
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -353,6 +354,7 @@ static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila,
 #endif
 	}
 
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
-- 
2.55.0

[PATCH 4/6] crypto: x86/aria - add missing vzeroupper in AVX2 code

[Eric Biggers]

Since the AVX2 optimized ARIA code uses YMM registers, execute
vzeroupper before returning from it.  This is needed to avoid degrading
the performance of any later SSE code that may happen to be executed.

Fixes: 37d8d3ae7a58 ("crypto: x86/aria - implement aria-avx2")
Cc: stable@vger.kernel.org
Cc: Taehee Yoo <ap420073@gmail.com>
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
 arch/x86/crypto/aria-aesni-avx2-asm_64.S | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/arch/x86/crypto/aria-aesni-avx2-asm_64.S b/arch/x86/crypto/aria-aesni-avx2-asm_64.S
index ed53d4f46bd7..fda8cb8a99a8 100644
--- a/arch/x86/crypto/aria-aesni-avx2-asm_64.S
+++ b/arch/x86/crypto/aria-aesni-avx2-asm_64.S
@@ -982,6 +982,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_encrypt_32way)
 		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
 		     %ymm15, %rax);
 
+	vzeroupper;
 	FRAME_END
 	RET;
 SYM_FUNC_END(aria_aesni_avx2_encrypt_32way)
@@ -1007,6 +1008,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_decrypt_32way)
 		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
 		     %ymm15, %rax);
 
+	vzeroupper;
 	FRAME_END
 	RET;
 SYM_FUNC_END(aria_aesni_avx2_decrypt_32way)
@@ -1209,6 +1211,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_ctr_crypt_32way)
 		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
 		     %ymm15, %r10);
 
+	vzeroupper;
 	FRAME_END
 	RET;
 SYM_FUNC_END(aria_aesni_avx2_ctr_crypt_32way)
@@ -1359,6 +1362,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_encrypt_32way)
 		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
 		     %ymm15, %rax);
 
+	vzeroupper;
 	FRAME_END
 	RET;
 SYM_FUNC_END(aria_aesni_avx2_gfni_encrypt_32way)
@@ -1384,6 +1388,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_decrypt_32way)
 		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
 		     %ymm15, %rax);
 
+	vzeroupper;
 	FRAME_END
 	RET;
 SYM_FUNC_END(aria_aesni_avx2_gfni_decrypt_32way)
@@ -1428,6 +1433,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_ctr_crypt_32way)
 		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
 		     %ymm15, %r10);
 
+	vzeroupper;
 	FRAME_END
 	RET;
 SYM_FUNC_END(aria_aesni_avx2_gfni_ctr_crypt_32way)
-- 
2.55.0

[PATCH 5/6] crypto: x86/aria - add missing vzeroupper in AVX-512 code

[Eric Biggers]

Since the AVX-512 optimized ARIA code uses ZMM registers, execute
vzeroupper before returning from it.  This is needed to avoid degrading
the performance of any later SSE code that may happen to be executed.

Fixes: c970d42001f2 ("crypto: x86/aria - implement aria-avx512")
Cc: stable@vger.kernel.org
Cc: Taehee Yoo <ap420073@gmail.com>
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
 arch/x86/crypto/aria-gfni-avx512-asm_64.S | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/arch/x86/crypto/aria-gfni-avx512-asm_64.S b/arch/x86/crypto/aria-gfni-avx512-asm_64.S
index 860887e5d02e..ca83eb126e06 100644
--- a/arch/x86/crypto/aria-gfni-avx512-asm_64.S
+++ b/arch/x86/crypto/aria-gfni-avx512-asm_64.S
@@ -800,6 +800,7 @@ SYM_TYPED_FUNC_START(aria_gfni_avx512_encrypt_64way)
 		     %zmm9, %zmm8, %zmm11, %zmm10, %zmm12, %zmm13, %zmm14,
 		     %zmm15, %rax);
 
+	vzeroupper;
 	FRAME_END
 	RET;
 SYM_FUNC_END(aria_gfni_avx512_encrypt_64way)
@@ -825,6 +826,7 @@ SYM_TYPED_FUNC_START(aria_gfni_avx512_decrypt_64way)
 		     %zmm9, %zmm8, %zmm11, %zmm10, %zmm12, %zmm13, %zmm14,
 		     %zmm15, %rax);
 
+	vzeroupper;
 	FRAME_END
 	RET;
 SYM_FUNC_END(aria_gfni_avx512_decrypt_64way)
@@ -966,6 +968,7 @@ SYM_TYPED_FUNC_START(aria_gfni_avx512_ctr_crypt_64way)
 		     %zmm9, %zmm8, %zmm11, %zmm10, %zmm12, %zmm13, %zmm14,
 		     %zmm15, %r10);
 
+	vzeroupper;
 	FRAME_END
 	RET;
 SYM_FUNC_END(aria_gfni_avx512_ctr_crypt_64way)
-- 
2.55.0

[PATCH 6/6] netfilter: nft_set_pipapo_avx2: add missing vzeroupper

[Eric Biggers]

Since pipapo_get_avx2() uses YMM registers, execute vzeroupper before
returning from it.  This is needed to avoid degrading the performance of
any later SSE code that may happen to be executed.

Fixes: 7400b063969b ("nft_set_pipapo: Introduce AVX2-based lookup implementation")
Cc: stable@vger.kernel.org
Cc: Florian Westphal <fw@strlen.de>
Cc: Stefano Brivio <sbrivio@redhat.com>
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
 net/netfilter/nft_set_pipapo_avx2.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/net/netfilter/nft_set_pipapo_avx2.c b/net/netfilter/nft_set_pipapo_avx2.c
index b3f105520a85..21f5be68c703 100644
--- a/net/netfilter/nft_set_pipapo_avx2.c
+++ b/net/netfilter/nft_set_pipapo_avx2.c
@@ -1134,6 +1134,7 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m,
 	struct nft_pipapo_scratch *scratch;
 	const struct nft_pipapo_field *f;
 	unsigned long *res, *fill, *map;
+	struct nft_pipapo_elem *e;
 	bool map_index;
 	int ret = 0;
 	int i;
@@ -1207,14 +1208,11 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m,
 next_match:
 		if (ret < 0) {
 			scratch->map_index = map_index;
-			kernel_fpu_end();
-			__local_unlock_nested_bh(&scratch->bh_lock);
-			return NULL;
+			e = NULL;
+			goto out;
 		}
 
 		if (last) {
-			struct nft_pipapo_elem *e;
-
 			e = f->mt[ret].e;
 			if (unlikely(__nft_set_elem_expired(&e->ext, tstamp) ||
 				     !nft_set_elem_active(&e->ext, genmask))) {
@@ -1224,9 +1222,7 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m,
 			}
 
 			scratch->map_index = map_index;
-			kernel_fpu_end();
-			__local_unlock_nested_bh(&scratch->bh_lock);
-			return e;
+			goto out;
 		}
 
 		map_index = !map_index;
@@ -1234,9 +1230,12 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m,
 		data += NFT_PIPAPO_GROUPS_PADDED_SIZE(f);
 	}
 
+	e = NULL;
+out:
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 	__local_unlock_nested_bh(&scratch->bh_lock);
-	return NULL;
+	return e;
 }
 
 /**
-- 
2.55.0

Re: [PATCH 6/6] netfilter: nft_set_pipapo_avx2: add missing vzeroupper

[Stefano Brivio]

Eric, thanks for taking care of this.

The patch looks good to me, I just have two questions:

On Sat, 15 Aug 2026 13:57:50 -0700
Eric Biggers <ebiggers@kernel.org> wrote:

> Since pipapo_get_avx2() uses YMM registers, execute vzeroupper before
> returning from it.  This is needed to avoid degrading the performance of
> any later SSE code that may happen to be executed.

Out of curiosity: was this prompted by some observed latency spike in
execution of SSE code, or it's just meant to satisfy the recommendation
from AMD and Intel to use it while transitioning from AVX to SSE modes?

> Fixes: 7400b063969b ("nft_set_pipapo: Introduce AVX2-based lookup implementation")
> Cc: stable@vger.kernel.org

Is this really stable material? Skipping vzeroupper might have a
performance impact, but it's not an issue for correctness.

The main reason why I'm asking is that, while vzeroupper might look
harmless and obviously safe, it actually caused CVE-2023-20593
("ZenBleed") on AMD Zen 2.

I expect systems receiving stable kernel updates to also run the
patched microcode by now, so I'm not overly concerned in any case.

-- 
Stefano

Re: [PATCH 6/6] netfilter: nft_set_pipapo_avx2: add missing vzeroupper

[Eric Biggers]

On Sun, Aug 16, 2026 at 12:38:41PM +0200, Stefano Brivio wrote:
> Eric, thanks for taking care of this.
> 
> The patch looks good to me, I just have two questions:
> 
> On Sat, 15 Aug 2026 13:57:50 -0700
> Eric Biggers <ebiggers@kernel.org> wrote:
> 
> > Since pipapo_get_avx2() uses YMM registers, execute vzeroupper before
> > returning from it.  This is needed to avoid degrading the performance of
> > any later SSE code that may happen to be executed.
> 
> Out of curiosity: was this prompted by some observed latency spike in
> execution of SSE code, or it's just meant to satisfy the recommendation
> from AMD and Intel to use it while transitioning from AVX to SSE modes?

This one was found by code review.  But the latency spike has been
observed in other cases with missing vzeroupper, so it's definitely a
real effect at least on some CPUs.

> > Fixes: 7400b063969b ("nft_set_pipapo: Introduce AVX2-based lookup implementation")
> > Cc: stable@vger.kernel.org
> 
> Is this really stable material? Skipping vzeroupper might have a
> performance impact, but it's not an issue for correctness.
> 
> The main reason why I'm asking is that, while vzeroupper might look
> harmless and obviously safe, it actually caused CVE-2023-20593
> ("ZenBleed") on AMD Zen 2.
> 
> I expect systems receiving stable kernel updates to also run the
> patched microcode by now, so I'm not overly concerned in any case.

It's awkward to mark something as a fix, then not actually want it to be
fixed everywhere.  The stable maintainers know this, and they often
apply fixes anyway regardless of 'Cc stable'.

And if vzeroupper is really not safe due to ZenBleed, then why is it
used everywhere else in the kernel?  This is just one of the few
exceptions that doesn't have it.  If it's not safe then they should all
be alternatives that patch them out to no-ops on affected CPUs.

But since that was never implemented, and userspace almost always uses
vzeroupper too, it seems the only real solution is the microcode.  I
don't think it makes sense to have a middle ground where almost all of
userspace uses vzeroupper, ~97% of the kernel uses vzeroupper, and a
random 3% of the kernel doesn't use it.

- Eric

Re: [PATCH 6/6] netfilter: nft_set_pipapo_avx2: add missing vzeroupper

[Stefano Brivio]

On Sun, 16 Aug 2026 10:15:18 -0700
Eric Biggers <ebiggers@kernel.org> wrote:

> On Sun, Aug 16, 2026 at 12:38:41PM +0200, Stefano Brivio wrote:
> > Eric, thanks for taking care of this.
> > 
> > The patch looks good to me, I just have two questions:
> > 
> > On Sat, 15 Aug 2026 13:57:50 -0700
> > Eric Biggers <ebiggers@kernel.org> wrote:
> >   
> > > Since pipapo_get_avx2() uses YMM registers, execute vzeroupper before
> > > returning from it.  This is needed to avoid degrading the performance of
> > > any later SSE code that may happen to be executed.  
> > 
> > Out of curiosity: was this prompted by some observed latency spike in
> > execution of SSE code, or it's just meant to satisfy the recommendation
> > from AMD and Intel to use it while transitioning from AVX to SSE modes?  
> 
> This one was found by code review.  But the latency spike has been
> observed in other cases with missing vzeroupper, so it's definitely a
> real effect at least on some CPUs.

Ah, interesting, I've been wondering about that.

> > > Fixes: 7400b063969b ("nft_set_pipapo: Introduce AVX2-based lookup implementation")
> > > Cc: stable@vger.kernel.org  
> > 
> > Is this really stable material? Skipping vzeroupper might have a
> > performance impact, but it's not an issue for correctness.
> > 
> > The main reason why I'm asking is that, while vzeroupper might look
> > harmless and obviously safe, it actually caused CVE-2023-20593
> > ("ZenBleed") on AMD Zen 2.
> > 
> > I expect systems receiving stable kernel updates to also run the
> > patched microcode by now, so I'm not overly concerned in any case.  
> 
> It's awkward to mark something as a fix, then not actually want it to be
> fixed everywhere.  The stable maintainers know this, and they often
> apply fixes anyway regardless of 'Cc stable'.

Right, I guess you have point there.

> And if vzeroupper is really not safe due to ZenBleed, then why is it
> used everywhere else in the kernel?  This is just one of the few
> exceptions that doesn't have it.  If it's not safe then they should all
> be alternatives that patch them out to no-ops on affected CPUs.
> 
> But since that was never implemented, and userspace almost always uses
> vzeroupper too, it seems the only real solution is the microcode.  I
> don't think it makes sense to have a middle ground where almost all of
> userspace uses vzeroupper, ~97% of the kernel uses vzeroupper, and a
> random 3% of the kernel doesn't use it.

Don't get me wrong, I wasn't advocating against "fixing" this for
specific paths, rather just pointing out that something seemingly
harmless such as vzeroupper can have nasty side effects.

I used ZenBleed as a mere example, not as a current reason to do or not
do anything specific.

-- 
Stefano

Re: [PATCH 6/6] netfilter: nft_set_pipapo_avx2: add missing vzeroupper

[Stefano Brivio]

On Sat, 15 Aug 2026 13:57:50 -0700
Eric Biggers <ebiggers@kernel.org> wrote:

> Since pipapo_get_avx2() uses YMM registers, execute vzeroupper before
> returning from it.  This is needed to avoid degrading the performance of
> any later SSE code that may happen to be executed.
> 
> Fixes: 7400b063969b ("nft_set_pipapo: Introduce AVX2-based lookup implementation")
> Cc: stable@vger.kernel.org
> Cc: Florian Westphal <fw@strlen.de>
> Cc: Stefano Brivio <sbrivio@redhat.com>
> Signed-off-by: Eric Biggers <ebiggers@kernel.org>

Reviewed-by: Stefano Brivio <sbrivio@redhat.com>

-- 
Stefano