The Linux Kernel Mailing List
 help / color / mirror / Atom feed
* [PATCH] crypto: powerpc/p10-aes-gcm - Fix IPsec stall on bond failover
@ 2026-11-20 15:08 Danny Tsen
  0 siblings, 0 replies; only message in thread
From: Danny Tsen @ 2026-11-20 15:08 UTC (permalink / raw)
  To: linux-crypto
  Cc: stable, herbert, leitao, nayna, appro, linux-kernel, linuxppc-dev,
	mpe, ltcgcw, dtsen

The Power10 AES-GCM driver registers its gcm(aes) and
rfc4106(gcm(aes)) algorithms through the crypto SIMD helper. When
crypto_simd_usable() is false, for example in softirq context, the
helper defers requests to cryptd and completes them asynchronously
with -EINPROGRESS.

This breaks IPsec traffic during a bond failover. With the crypto
offload device active, traffic is switched to a passive device that
has only software encryption, and the ESP requests then go through
aes_gcm_p10_crypto. Under heavy load, some or all of the TCP streams
drop to zero throughput and do not recover. The stack is seen to drop
the ACKs as invalid, which points to the cryptd queueing delaying or
reordering the packets.

Fix this by registering the algorithms directly as aead_alg with
CRYPTO_ALG_NEED_FALLBACK, and allocating a "gcm(aes)" or
"rfc4106(gcm(aes))" fallback tfm in ->init(). When VSX cannot be used
(!crypto_simd_usable()), the key was not set up for SIMD, or
cryptlen < 64, the request is handed to the fallback and completed
synchronously in the caller's context. The driver no longer defers
work to cryptd.

Fixes: c954b252dee9 ("crypto: powerpc/p10-aes-gcm - Register modules as SIMD")

Signed-off-by: Danny Tsen <dtsen@linux.ibm.com>
---
Tested with tcrypt.ko (including the self-tests) and with IPsec traffic
across a bond failover. TCP throughput no longer drops to zero.

 arch/powerpc/crypto/aes-gcm-p10-glue.c | 143 ++++++++++++++++++-------
 1 file changed, 107 insertions(+), 36 deletions(-)

diff --git a/arch/powerpc/crypto/aes-gcm-p10-glue.c b/arch/powerpc/crypto/aes-gcm-p10-glue.c
index f3417436d3f7..13352b838432 100644
--- a/arch/powerpc/crypto/aes-gcm-p10-glue.c
+++ b/arch/powerpc/crypto/aes-gcm-p10-glue.c
@@ -2,7 +2,7 @@
 /*
  * Glue code for accelerated AES-GCM stitched implementation for ppc64le.
  *
- * Copyright 2022- IBM Inc. All rights reserved
+ * Copyright IBM Corp. 2022, 2023, 2024, 2025, 2026
  */
 
 #include <linux/unaligned.h>
@@ -63,8 +63,12 @@ struct Hash_ctx {
 };
 
 struct p10_aes_gcm_ctx {
+	struct crypto_aead *fallback_tfm;
 	struct p10_aes_key enc_key;
-	u8 nonce[RFC4106_NONCE_SIZE];
+	u8 nonce[RFC4106_NONCE_SIZE] __aligned(16);
+	unsigned int keylen;
+	u8 key[AES_MAX_KEYLENGTH] __aligned(16);
+	unsigned int simd_set_key;
 };
 
 static void vsx_begin(void)
@@ -173,6 +177,40 @@ static void finish_tag(struct gcm_ctx *gctx, struct Hash_ctx *hash, int len)
 		hash->Htable[i] ^= gctx->ivtag[i];
 }
 
+static int p10_aead_init(struct crypto_aead *tfm)
+{
+	struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
+
+	ctx->fallback_tfm = crypto_alloc_aead("gcm(aes)", 0, CRYPTO_ALG_NEED_FALLBACK);
+	if (IS_ERR(ctx->fallback_tfm))
+		return PTR_ERR(ctx->fallback_tfm);
+
+	crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) +
+				crypto_aead_reqsize(ctx->fallback_tfm));
+	return 0;
+}
+
+static int rfc4106_init(struct crypto_aead *tfm)
+{
+	struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
+
+	ctx->fallback_tfm = crypto_alloc_aead("rfc4106(gcm(aes))", 0, CRYPTO_ALG_NEED_FALLBACK);
+	if (IS_ERR(ctx->fallback_tfm))
+		return PTR_ERR(ctx->fallback_tfm);
+
+	crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) +
+				crypto_aead_reqsize(ctx->fallback_tfm));
+	return 0;
+}
+
+static void p10_aead_exit(struct crypto_aead *tfm)
+{
+	struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
+
+	if (ctx->fallback_tfm)
+		crypto_free_aead(ctx->fallback_tfm);
+}
+
 static int set_authsize(struct crypto_aead *tfm, unsigned int authsize)
 {
 	switch (authsize) {
@@ -194,13 +232,34 @@ static int set_authsize(struct crypto_aead *tfm, unsigned int authsize)
 static int p10_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
 			      unsigned int keylen)
 {
-	struct crypto_tfm *tfm = crypto_aead_tfm(aead);
-	struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(aead);
+	int ret = 0;
+
+	ctx->simd_set_key = 0;
+	if (crypto_simd_usable()) {
+		vsx_begin();
+		ret = aes_p10_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
+		vsx_end();
+		ctx->simd_set_key = 1;
+		if (ret)
+			ctx->simd_set_key = 0;
+	}
+
+	ctx->keylen = keylen;
+	memcpy(ctx->key, key, keylen);
+
+	return ret ? -EINVAL : 0;
+}
+
+static int gcm_setkey(struct crypto_aead *aead, const u8 *key,
+		      unsigned int keylen)
+{
+	struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(aead);
 	int ret;
 
-	vsx_begin();
-	ret = aes_p10_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
-	vsx_end();
+	ret = p10_aes_gcm_setkey(aead, key, keylen);
+
+	ret |= crypto_aead_setkey(ctx->fallback_tfm, key, keylen);
 
 	return ret ? -EINVAL : 0;
 }
@@ -208,8 +267,8 @@ static int p10_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
 static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv,
 			     int assoclen, int enc)
 {
-	struct crypto_tfm *tfm = req->base.tfm;
-	struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
 	u8 databuf[sizeof(struct gcm_ctx) + PPC_ALIGN];
 	struct gcm_ctx *gctx = PTR_ALIGN((void *)databuf, PPC_ALIGN);
 	u8 hashbuf[sizeof(struct Hash_ctx) + PPC_ALIGN];
@@ -220,12 +279,24 @@ static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv,
 	unsigned int cryptlen = req->cryptlen;
 	unsigned char ivbuf[AES_BLOCK_SIZE+PPC_ALIGN];
 	unsigned char *iv = PTR_ALIGN((void *)ivbuf, PPC_ALIGN);
-	int ret;
-	unsigned long auth_tag_len = crypto_aead_authsize(__crypto_aead_cast(tfm));
+	int ret = 0;
+	unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 	u8 otag[16];
 	int total_processed = 0;
 	int nbytes;
 
+	if (!crypto_simd_usable() || !ctx->simd_set_key || req->cryptlen < 64) {
+		struct aead_request *subreq = aead_request_ctx(req);
+
+		aead_request_set_tfm(subreq, ctx->fallback_tfm);
+		aead_request_set_crypt(subreq, req->src, req->dst,
+					req->cryptlen, req->iv);
+		aead_request_set_ad(subreq, req->assoclen);
+
+		return enc ? crypto_aead_encrypt(subreq) :
+			     crypto_aead_decrypt(subreq);
+	}
+
 	memset(databuf, 0, sizeof(databuf));
 	memset(hashbuf, 0, sizeof(hashbuf));
 	memset(ivbuf, 0, sizeof(ivbuf));
@@ -260,6 +331,7 @@ static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv,
 	if (ret)
 		return ret;
 
+	vsx_begin();
 	while ((nbytes = walk.nbytes) > 0 && ret == 0) {
 		const u8 *src = walk.src.virt.addr;
 		u8 *dst = walk.dst.virt.addr;
@@ -268,7 +340,6 @@ static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv,
 		if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
 			src = dst = memcpy(buf, src, nbytes);
 
-		vsx_begin();
 		if (enc)
 			aes_p10_gcm_encrypt(src, dst, nbytes,
 					    &ctx->enc_key, gctx->iv, hash->Htable);
@@ -283,13 +354,13 @@ static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv,
 
 		total_processed += walk.nbytes;
 		ret = skcipher_walk_done(&walk, 0);
-	}
+		if (ret)
+			return ret;
 
-	if (ret)
-		return ret;
+		vsx_begin();
+	}
 
 	/* Finalize hash */
-	vsx_begin();
 	gcm_update(gctx->iv, hash->Htable);
 	finish_tag(gctx, hash, total_processed);
 	vsx_end();
@@ -305,11 +376,11 @@ static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv,
 
 		if (crypto_memneq(otag, hash->Htable, auth_tag_len)) {
 			memzero_explicit(hash->Htable, 16);
-			return -EBADMSG;
+			ret = -EBADMSG;
 		}
 	}
 
-	return 0;
+	return ret;
 }
 
 static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey,
@@ -318,6 +389,10 @@ static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey,
 	struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
 	int err;
 
+	err = crypto_aead_setkey(ctx->fallback_tfm, inkey, keylen);
+	if (err)
+		return err;
+
 	keylen -= RFC4106_NONCE_SIZE;
 	err = p10_aes_gcm_setkey(tfm, inkey, keylen);
 	if (err)
@@ -373,18 +448,20 @@ static struct aead_alg gcm_aes_algs[] = {{
 	.maxauthsize		= 16,
 
 	.setauthsize		= set_authsize,
-	.setkey			= p10_aes_gcm_setkey,
+	.setkey			= gcm_setkey,
 	.encrypt		= p10_aes_gcm_encrypt,
 	.decrypt		= p10_aes_gcm_decrypt,
+	.init			= p10_aead_init,
+	.exit			= p10_aead_exit,
 
-	.base.cra_name		= "__gcm(aes)",
-	.base.cra_driver_name	= "__aes_gcm_p10",
+	.base.cra_name		= "gcm(aes)",
+	.base.cra_driver_name	= "aes_gcm_p10",
 	.base.cra_priority	= 2100,
 	.base.cra_blocksize	= 1,
 	.base.cra_ctxsize	= sizeof(struct p10_aes_gcm_ctx)+
 				  4 * sizeof(u64[2]),
 	.base.cra_module	= THIS_MODULE,
-	.base.cra_flags		= CRYPTO_ALG_INTERNAL,
+	.base.cra_flags		= CRYPTO_ALG_NEED_FALLBACK,
 }, {
 	.ivsize			= GCM_RFC4106_IV_SIZE,
 	.maxauthsize		= 16,
@@ -392,19 +469,19 @@ static struct aead_alg gcm_aes_algs[] = {{
 	.setauthsize		= rfc4106_setauthsize,
 	.encrypt		= rfc4106_encrypt,
 	.decrypt		= rfc4106_decrypt,
+	.init			= rfc4106_init,
+	.exit			= p10_aead_exit,
 
-	.base.cra_name		= "__rfc4106(gcm(aes))",
-	.base.cra_driver_name	= "__rfc4106_aes_gcm_p10",
+	.base.cra_name		= "rfc4106(gcm(aes))",
+	.base.cra_driver_name	= "rfc4106_aes_gcm_p10",
 	.base.cra_priority	= 2100,
 	.base.cra_blocksize	= 1,
 	.base.cra_ctxsize	= sizeof(struct p10_aes_gcm_ctx) +
 				  4 * sizeof(u64[2]),
 	.base.cra_module	= THIS_MODULE,
-	.base.cra_flags		= CRYPTO_ALG_INTERNAL,
+	.base.cra_flags		= CRYPTO_ALG_NEED_FALLBACK,
 }};
 
-static struct simd_aead_alg *p10_simd_aeads[ARRAY_SIZE(gcm_aes_algs)];
-
 static int __init p10_init(void)
 {
 	int ret;
@@ -412,21 +489,15 @@ static int __init p10_init(void)
 	if (!cpu_has_feature(CPU_FTR_ARCH_31))
 		return 0;
 
-	ret = simd_register_aeads_compat(gcm_aes_algs,
-					 ARRAY_SIZE(gcm_aes_algs),
-					 p10_simd_aeads);
-	if (ret) {
-		simd_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs),
-				      p10_simd_aeads);
+	ret = crypto_register_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs));
+	if (ret)
 		return ret;
-	}
 	return 0;
 }
 
 static void __exit p10_exit(void)
 {
-	simd_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs),
-			      p10_simd_aeads);
+	crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs));
 }
 
 module_init(p10_init);
-- 
2.47.1


^ permalink raw reply related	[flat|nested] only message in thread

only message in thread, other threads:[~2026-11-20 15:39 UTC | newest]

Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-11-20 15:08 [PATCH] crypto: powerpc/p10-aes-gcm - Fix IPsec stall on bond failover Danny Tsen

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox