* [PATCH 05/29] crypto: talitos - Prepare crypto implementation file splitting
From: Paul Louvel @ 2026-05-28 9:08 UTC (permalink / raw)
To: Herbert Xu, David S. Miller
Cc: Thomas Petazzoni, Herve Codina, Christophe Leroy, linux-crypto,
linux-kernel, Paul Louvel
In-Reply-To: <20260528-7-1-rc1_talitos_cleanup-v1-0-cb1ad6cdea49@bootlin.com>
Remove the static qualifier on multiple function that will be called
inside each crypto implementation file.
Add them to the main driver header file.
Add the common structures too.
Signed-off-by: Paul Louvel <paul.louvel@bootlin.com>
---
drivers/crypto/talitos/talitos.c | 123 ++++++++++++++-------------------------
drivers/crypto/talitos/talitos.h | 91 +++++++++++++++++++++++++++++
2 files changed, 135 insertions(+), 79 deletions(-)
diff --git a/drivers/crypto/talitos/talitos.c b/drivers/crypto/talitos/talitos.c
index f5feff8f7d3d..3fc1069062da 100644
--- a/drivers/crypto/talitos/talitos.c
+++ b/drivers/crypto/talitos/talitos.c
@@ -40,8 +40,8 @@
#include "talitos.h"
-static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
- unsigned int len, bool is_sec1)
+void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
+ unsigned int len, bool is_sec1)
{
ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
if (is_sec1) {
@@ -52,8 +52,8 @@ static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
}
}
-static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
- struct talitos_ptr *src_ptr, bool is_sec1)
+void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
+ struct talitos_ptr *src_ptr, bool is_sec1)
{
dst_ptr->ptr = src_ptr->ptr;
if (is_sec1) {
@@ -64,8 +64,8 @@ static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
}
}
-static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
- bool is_sec1)
+unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
+ bool is_sec1)
{
if (is_sec1)
return be16_to_cpu(ptr->len1);
@@ -73,14 +73,14 @@ static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
return be16_to_cpu(ptr->len);
}
-static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
- bool is_sec1)
+void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
+ bool is_sec1)
{
if (!is_sec1)
ptr->j_extent = val;
}
-static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
+void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
{
if (!is_sec1)
ptr->j_extent |= val;
@@ -102,15 +102,15 @@ static void __map_single_talitos_ptr(struct device *dev,
to_talitos_ptr(ptr, dma_addr, len, is_sec1);
}
-static void map_single_talitos_ptr(struct device *dev,
- struct talitos_ptr *ptr,
- unsigned int len, void *data,
- enum dma_data_direction dir)
+void map_single_talitos_ptr(struct device *dev,
+ struct talitos_ptr *ptr,
+ unsigned int len, void *data,
+ enum dma_data_direction dir)
{
__map_single_talitos_ptr(dev, ptr, len, data, dir, 0);
}
-static void map_single_talitos_ptr_nosync(struct device *dev,
+void map_single_talitos_ptr_nosync(struct device *dev,
struct talitos_ptr *ptr,
unsigned int len, void *data,
enum dma_data_direction dir)
@@ -122,9 +122,9 @@ static void map_single_talitos_ptr_nosync(struct device *dev,
/*
* unmap bus single (contiguous) h/w descriptor pointer
*/
-static void unmap_single_talitos_ptr(struct device *dev,
- struct talitos_ptr *ptr,
- enum dma_data_direction dir)
+void unmap_single_talitos_ptr(struct device *dev,
+ struct talitos_ptr *ptr,
+ enum dma_data_direction dir)
{
struct talitos_private *priv = dev_get_drvdata(dev);
bool is_sec1 = has_ftr_sec1(priv);
@@ -303,11 +303,11 @@ static void dma_map_request(struct device *dev, struct talitos_request *request,
* callback must check err and feedback in descriptor header
* for device processing status.
*/
-static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
- void (*callback)(struct device *dev,
- struct talitos_desc *desc,
- void *context, int error),
- void *context)
+int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
+ void (*callback)(struct device *dev,
+ struct talitos_desc *desc,
+ void *context, int error),
+ void *context)
{
struct talitos_private *priv = dev_get_drvdata(dev);
struct talitos_request *request;
@@ -830,24 +830,6 @@ DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
* HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
*/
#define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1)
-#ifdef CONFIG_CRYPTO_DEV_TALITOS2
-#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
-#else
-#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE)
-#endif
-#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
-
-struct talitos_ctx {
- struct device *dev;
- int ch;
- __be32 desc_hdr_template;
- u8 key[TALITOS_MAX_KEY_SIZE];
- u8 iv[TALITOS_MAX_IV_LENGTH];
- dma_addr_t dma_key;
- unsigned int keylen;
- unsigned int enckeylen;
- unsigned int authkeylen;
-};
#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
@@ -939,7 +921,7 @@ static int aead_des3_setkey(struct crypto_aead *authenc,
return err;
}
-static void talitos_sg_unmap(struct device *dev,
+void talitos_sg_unmap(struct device *dev,
struct talitos_edesc *edesc,
struct scatterlist *src,
struct scatterlist *dst,
@@ -1124,7 +1106,7 @@ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
return count;
}
-static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
+int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
unsigned int len, struct talitos_edesc *edesc,
struct talitos_ptr *ptr, int sg_count,
unsigned int offset, int tbl_off, int elen,
@@ -1161,7 +1143,7 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
return sg_count;
}
-static int talitos_sg_map(struct device *dev, struct scatterlist *src,
+int talitos_sg_map(struct device *dev, struct scatterlist *src,
unsigned int len, struct talitos_edesc *edesc,
struct talitos_ptr *ptr, int sg_count,
unsigned int offset, int tbl_off)
@@ -1299,17 +1281,17 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
/*
* allocate and map the extended descriptor
*/
-static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
- struct scatterlist *src,
- struct scatterlist *dst,
- u8 *iv,
- unsigned int assoclen,
- unsigned int cryptlen,
- unsigned int authsize,
- unsigned int ivsize,
- int icv_stashing,
- u32 cryptoflags,
- bool encrypt)
+struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
+ struct scatterlist *src,
+ struct scatterlist *dst,
+ u8 *iv,
+ unsigned int assoclen,
+ unsigned int cryptlen,
+ unsigned int authsize,
+ unsigned int ivsize,
+ int icv_stashing,
+ u32 cryptoflags,
+ bool encrypt)
{
struct talitos_edesc *edesc;
int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
@@ -2172,18 +2154,6 @@ static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
return 0;
}
-
-struct talitos_alg_template {
- u32 type;
- u32 priority;
- union {
- struct skcipher_alg skcipher;
- struct ahash_alg hash;
- struct aead_alg aead;
- } alg;
- __be32 desc_hdr_template;
-};
-
static struct talitos_alg_template driver_algs[] = {
/* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
{ .type = CRYPTO_ALG_TYPE_AEAD,
@@ -2998,14 +2968,8 @@ static struct talitos_alg_template driver_algs[] = {
}
};
-struct talitos_crypto_alg {
- struct list_head entry;
- struct device *dev;
- struct talitos_alg_template algt;
-};
-
-static int talitos_init_common(struct talitos_ctx *ctx,
- struct talitos_crypto_alg *talitos_alg)
+int talitos_init_common(struct talitos_ctx *ctx,
+ struct talitos_crypto_alg *talitos_alg)
{
struct talitos_private *priv;
@@ -3066,7 +3030,7 @@ static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
return talitos_init_common(ctx, talitos_alg);
}
-static void talitos_cra_exit(struct crypto_tfm *tfm)
+void talitos_cra_exit(struct crypto_tfm *tfm)
{
struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
struct device *dev = ctx->dev;
@@ -3080,7 +3044,7 @@ static void talitos_cra_exit(struct crypto_tfm *tfm)
* type and primary/secondary execution units required match the hw
* capabilities description provided in the device tree node.
*/
-static int hw_supports(struct device *dev, __be32 desc_hdr_template)
+int talitos_hw_supports(struct device *dev, __be32 desc_hdr_template)
{
struct talitos_private *priv = dev_get_drvdata(dev);
int ret;
@@ -3117,7 +3081,7 @@ static void talitos_remove(struct platform_device *ofdev)
list_del(&t_alg->entry);
}
- if (hw_supports(dev, DESC_HDR_SEL0_RNG))
+ if (talitos_hw_supports(dev, DESC_HDR_SEL0_RNG))
talitos_unregister_rng(dev);
for (i = 0; i < 2; i++)
@@ -3426,7 +3390,7 @@ static int talitos_probe(struct platform_device *ofdev)
}
/* register the RNG, if available */
- if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
+ if (talitos_hw_supports(dev, DESC_HDR_SEL0_RNG)) {
err = talitos_register_rng(dev);
if (err) {
dev_err(dev, "failed to register hwrng: %d\n", err);
@@ -3437,7 +3401,8 @@ static int talitos_probe(struct platform_device *ofdev)
/* register crypto algorithms the device supports */
for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
- if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
+ if (talitos_hw_supports(dev,
+ driver_algs[i].desc_hdr_template)) {
struct talitos_crypto_alg *t_alg;
struct crypto_alg *alg = NULL;
diff --git a/drivers/crypto/talitos/talitos.h b/drivers/crypto/talitos/talitos.h
index fa8c71b1f90f..1f81d336dae8 100644
--- a/drivers/crypto/talitos/talitos.h
+++ b/drivers/crypto/talitos/talitos.h
@@ -5,7 +5,13 @@
* Copyright (c) 2006-2011 Freescale Semiconductor, Inc.
*/
+#include <crypto/aes.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/hash.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/sha2.h>
#include <linux/device.h>
+#include <linux/dma-mapping.h>
#include <linux/hw_random.h>
#include <linux/interrupt.h>
#include <linux/scatterlist.h>
@@ -19,6 +25,13 @@
#define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf)
#define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf)
+#ifdef CONFIG_CRYPTO_DEV_TALITOS2
+#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
+#else
+#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE)
+#endif
+#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
+
/* descriptor pointer entry */
struct talitos_ptr {
union {
@@ -174,6 +187,35 @@ struct talitos_private {
};
+struct talitos_ctx {
+ struct device *dev;
+ int ch;
+ __be32 desc_hdr_template;
+ u8 key[TALITOS_MAX_KEY_SIZE];
+ u8 iv[TALITOS_MAX_IV_LENGTH];
+ dma_addr_t dma_key;
+ unsigned int keylen;
+ unsigned int enckeylen;
+ unsigned int authkeylen;
+};
+
+struct talitos_alg_template {
+ u32 type;
+ u32 priority;
+ union {
+ struct skcipher_alg skcipher;
+ struct ahash_alg hash;
+ struct aead_alg aead;
+ } alg;
+ __be32 desc_hdr_template;
+};
+
+struct talitos_crypto_alg {
+ struct list_head entry;
+ struct device *dev;
+ struct talitos_alg_template algt;
+};
+
/* .features flag */
#define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
#define TALITOS_FTR_HW_AUTH_CHECK 0x00000002
@@ -432,6 +474,55 @@ static inline bool has_ftr_sec1(struct talitos_private *priv)
#define DESC_PTR_LNKTBL_RET 0x02
#define DESC_PTR_LNKTBL_NEXT 0x01
+void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
+ unsigned int len, bool is_sec1);
+void copy_talitos_ptr(struct talitos_ptr *dst_ptr, struct talitos_ptr *src_ptr,
+ bool is_sec1);
+unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr, bool is_sec1);
+void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val, bool is_sec1);
+void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1);
+
+void map_single_talitos_ptr(struct device *dev, struct talitos_ptr *ptr,
+ unsigned int len, void *data,
+ enum dma_data_direction dir);
+void map_single_talitos_ptr_nosync(struct device *dev, struct talitos_ptr *ptr,
+ unsigned int len, void *data,
+ enum dma_data_direction dir);
+void unmap_single_talitos_ptr(struct device *dev, struct talitos_ptr *ptr,
+ enum dma_data_direction dir);
+
+int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
+ void (*callback)(struct device *dev,
+ struct talitos_desc *desc, void *context,
+ int error),
+ void *context);
+
+void talitos_sg_unmap(struct device *dev, struct talitos_edesc *edesc,
+ struct scatterlist *src, struct scatterlist *dst,
+ unsigned int len, unsigned int offset);
+int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
+ unsigned int len, struct talitos_edesc *edesc,
+ struct talitos_ptr *ptr, int sg_count,
+ unsigned int offset, int tbl_off, int elen, bool force,
+ int align);
+int talitos_sg_map(struct device *dev, struct scatterlist *src,
+ unsigned int len, struct talitos_edesc *edesc,
+ struct talitos_ptr *ptr, int sg_count, unsigned int offset,
+ int tbl_off);
+
+struct talitos_edesc *
+talitos_edesc_alloc(struct device *dev, struct scatterlist *src,
+ struct scatterlist *dst, u8 *iv, unsigned int assoclen,
+ unsigned int cryptlen, unsigned int authsize,
+ unsigned int ivsize, int icv_stashing, u32 cryptoflags,
+ bool encrypt);
+
+int talitos_hw_supports(struct device *dev, __be32 desc_hdr_template);
+
+int talitos_init_common(struct talitos_ctx *ctx,
+ struct talitos_crypto_alg *talitos_alg);
+void talitos_cra_exit(struct crypto_tfm *tfm);
+
/* Hardware RNG */
int talitos_register_rng(struct device *dev);
--
2.54.0
^ permalink raw reply related
* [PATCH 03/29] crypto: talitos - Add missing includes to driver header file
From: Paul Louvel @ 2026-05-28 9:08 UTC (permalink / raw)
To: Herbert Xu, David S. Miller
Cc: Thomas Petazzoni, Herve Codina, Christophe Leroy, linux-crypto,
linux-kernel, Paul Louvel
In-Reply-To: <20260528-7-1-rc1_talitos_cleanup-v1-0-cb1ad6cdea49@bootlin.com>
Add explicit includes for types used by the header file to make
it self-contained and fix implicit include dependencies.
Signed-off-by: Paul Louvel <paul.louvel@bootlin.com>
---
drivers/crypto/talitos/talitos.c | 3 ---
drivers/crypto/talitos/talitos.h | 6 ++++++
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/drivers/crypto/talitos/talitos.c b/drivers/crypto/talitos/talitos.c
index 3610d9f6d5ea..8ca587b98d92 100644
--- a/drivers/crypto/talitos/talitos.c
+++ b/drivers/crypto/talitos/talitos.c
@@ -15,10 +15,7 @@
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mod_devicetable.h>
-#include <linux/device.h>
-#include <linux/interrupt.h>
#include <linux/crypto.h>
-#include <linux/hw_random.h>
#include <linux/of.h>
#include <linux/of_irq.h>
#include <linux/platform_device.h>
diff --git a/drivers/crypto/talitos/talitos.h b/drivers/crypto/talitos/talitos.h
index d4ff8d589f46..56e36a65ddcc 100644
--- a/drivers/crypto/talitos/talitos.h
+++ b/drivers/crypto/talitos/talitos.h
@@ -5,6 +5,12 @@
* Copyright (c) 2006-2011 Freescale Semiconductor, Inc.
*/
+#include <linux/device.h>
+#include <linux/hw_random.h>
+#include <linux/interrupt.h>
+#include <linux/scatterlist.h>
+#include <linux/types.h>
+
#define TALITOS_TIMEOUT 100000
#define TALITOS1_MAX_DATA_LEN 32768
#define TALITOS2_MAX_DATA_LEN 65535
--
2.54.0
^ permalink raw reply related
* [PATCH 02/29] crypto: talitos - Move driver into dedicated directory
From: Paul Louvel @ 2026-05-28 9:08 UTC (permalink / raw)
To: Herbert Xu, David S. Miller
Cc: Thomas Petazzoni, Herve Codina, Christophe Leroy, linux-crypto,
linux-kernel, Paul Louvel
In-Reply-To: <20260528-7-1-rc1_talitos_cleanup-v1-0-cb1ad6cdea49@bootlin.com>
Move the talitos driver files from drivers/crypto/ into
drivers/crypto/talitos/ to accommodate upcoming code
reorganization.
Signed-off-by: Paul Louvel <paul.louvel@bootlin.com>
---
drivers/crypto/Kconfig | 38 +---------------------------------
drivers/crypto/Makefile | 2 +-
drivers/crypto/talitos/Kconfig | 36 ++++++++++++++++++++++++++++++++
drivers/crypto/talitos/Makefile | 1 +
drivers/crypto/{ => talitos}/talitos.c | 0
drivers/crypto/{ => talitos}/talitos.h | 0
6 files changed, 39 insertions(+), 38 deletions(-)
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index d23b58b81ca3..783b5dc42a42 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -253,43 +253,7 @@ config CRYPTO_DEV_HIFN_795X_RNG
on the HIFN 795x crypto adapters.
source "drivers/crypto/caam/Kconfig"
-
-config CRYPTO_DEV_TALITOS
- tristate "Talitos Freescale Security Engine (SEC)"
- select CRYPTO_AEAD
- select CRYPTO_AUTHENC
- select CRYPTO_SKCIPHER
- select CRYPTO_HASH
- select CRYPTO_LIB_DES
- select HW_RANDOM
- depends on FSL_SOC
- help
- Say 'Y' here to use the Freescale Security Engine (SEC)
- to offload cryptographic algorithm computation.
-
- The Freescale SEC is present on PowerQUICC 'E' processors, such
- as the MPC8349E and MPC8548E.
-
- To compile this driver as a module, choose M here: the module
- will be called talitos.
-
-config CRYPTO_DEV_TALITOS1
- bool "SEC1 (SEC 1.0 and SEC Lite 1.2)"
- depends on CRYPTO_DEV_TALITOS
- depends on PPC_8xx || PPC_82xx
- default y
- help
- Say 'Y' here to use the Freescale Security Engine (SEC) version 1.0
- found on MPC82xx or the Freescale Security Engine (SEC Lite)
- version 1.2 found on MPC8xx
-
-config CRYPTO_DEV_TALITOS2
- bool "SEC2+ (SEC version 2.0 or upper)"
- depends on CRYPTO_DEV_TALITOS
- default y if !PPC_8xx
- help
- Say 'Y' here to use the Freescale Security Engine (SEC)
- version 2 and following as found on MPC83xx, MPC85xx, etc ...
+source "drivers/crypto/talitos/Kconfig"
config CRYPTO_DEV_PPC4XX
tristate "Driver AMCC PPC4xx crypto accelerator"
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 283bbc650b5b..a059139d4a75 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -35,7 +35,7 @@ obj-$(CONFIG_CRYPTO_DEV_SA2UL) += sa2ul.o
obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o
obj-$(CONFIG_CRYPTO_DEV_SL3516) += gemini/
obj-y += stm32/
-obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
+obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos/
obj-$(CONFIG_CRYPTO_DEV_TEGRA) += tegra/
obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/
obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/
diff --git a/drivers/crypto/talitos/Kconfig b/drivers/crypto/talitos/Kconfig
new file mode 100644
index 000000000000..c3470553a966
--- /dev/null
+++ b/drivers/crypto/talitos/Kconfig
@@ -0,0 +1,36 @@
+config CRYPTO_DEV_TALITOS
+ tristate "Talitos Freescale Security Engine (SEC)"
+ select CRYPTO_AEAD
+ select CRYPTO_AUTHENC
+ select CRYPTO_SKCIPHER
+ select CRYPTO_HASH
+ select CRYPTO_LIB_DES
+ select HW_RANDOM
+ depends on FSL_SOC
+ help
+ Say 'Y' here to use the Freescale Security Engine (SEC)
+ to offload cryptographic algorithm computation.
+
+ The Freescale SEC is present on PowerQUICC 'E' processors, such
+ as the MPC8349E and MPC8548E.
+
+ To compile this driver as a module, choose M here: the module
+ will be called talitos.
+
+config CRYPTO_DEV_TALITOS1
+ bool "SEC1 (SEC 1.0 and SEC Lite 1.2)"
+ depends on CRYPTO_DEV_TALITOS
+ depends on PPC_8xx || PPC_82xx
+ default y
+ help
+ Say 'Y' here to use the Freescale Security Engine (SEC) version 1.0
+ found on MPC82xx or the Freescale Security Engine (SEC Lite)
+ version 1.2 found on MPC8xx
+
+config CRYPTO_DEV_TALITOS2
+ bool "SEC2+ (SEC version 2.0 or upper)"
+ depends on CRYPTO_DEV_TALITOS
+ default y if !PPC_8xx
+ help
+ Say 'Y' here to use the Freescale Security Engine (SEC)
+ version 2 and following as found on MPC83xx, MPC85xx, etc ...
diff --git a/drivers/crypto/talitos/Makefile b/drivers/crypto/talitos/Makefile
new file mode 100644
index 000000000000..fcc5db5e63c2
--- /dev/null
+++ b/drivers/crypto/talitos/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos/talitos.c
similarity index 100%
rename from drivers/crypto/talitos.c
rename to drivers/crypto/talitos/talitos.c
diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos/talitos.h
similarity index 100%
rename from drivers/crypto/talitos.h
rename to drivers/crypto/talitos/talitos.h
--
2.54.0
^ permalink raw reply related
* [PATCH 01/29] crypto: talitos/hash - Use CRYPTO_AHASH_BLOCK_ONLY API
From: Paul Louvel @ 2026-05-28 9:08 UTC (permalink / raw)
To: Herbert Xu, David S. Miller
Cc: Thomas Petazzoni, Herve Codina, Christophe Leroy, linux-crypto,
linux-kernel, Paul Louvel
In-Reply-To: <20260528-7-1-rc1_talitos_cleanup-v1-0-cb1ad6cdea49@bootlin.com>
The hash implementation maintained a software buffer to accumulate
partial blocks across update() calls, copying data to/from scatterlists
with sg_copy_to_buffer()/sg_pcopy_to_buffer() and chaining in a virtual
scatterlist entry. This is unnecessary now with
CRYPTO_AHASH_ALG_BLOCK_ONLY flag.
Remove unnecessary fields in the request and export structure. On
completion, pass any remaining tail bytes back via
ahash_request_complete() so that the core re-submits them with the next
request.
Signed-off-by: Paul Louvel <paul.louvel@bootlin.com>
---
drivers/crypto/talitos.c | 149 ++++++++++++++++++-----------------------------
1 file changed, 57 insertions(+), 92 deletions(-)
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index 584508963241..3610d9f6d5ea 100644
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -941,25 +941,18 @@ struct talitos_ctx {
struct talitos_ahash_req_ctx {
u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
unsigned int hw_context_size;
- u8 buf[2][HASH_MAX_BLOCK_SIZE];
- int buf_idx;
unsigned int swinit;
unsigned int first_request;
unsigned int last_request;
unsigned int to_hash_later;
- unsigned int nbuf;
- struct scatterlist bufsl[2];
- struct scatterlist *psrc;
};
struct talitos_export_state {
u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
- u8 buf[HASH_MAX_BLOCK_SIZE];
unsigned int swinit;
unsigned int first_request;
unsigned int last_request;
unsigned int to_hash_later;
- unsigned int nbuf;
};
static int aead_setkey(struct crypto_aead *authenc,
@@ -1826,14 +1819,8 @@ static void ahash_done(struct device *dev,
struct talitos_edesc *next;
if (is_sec1) {
- if (!req_ctx->last_request && req_ctx->to_hash_later) {
- /* Position any partial block for next update/final/finup */
- req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1;
- req_ctx->nbuf = req_ctx->to_hash_later;
- }
-
free_edesc_list_from(areq, edesc);
- ahash_request_complete(areq, err);
+ ahash_request_complete(areq, err ?: req_ctx->to_hash_later);
} else {
next = edesc->next_desc;
@@ -1851,14 +1838,9 @@ static void ahash_done(struct device *dev,
return;
}
out:
- if (!req_ctx->last_request && req_ctx->to_hash_later) {
- /* Position any partial block for next update/final/finup */
- req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1;
- req_ctx->nbuf = req_ctx->to_hash_later;
- }
if (err && next)
free_edesc_list_from(areq, next);
- ahash_request_complete(areq, err);
+ ahash_request_complete(areq, err ?: req_ctx->to_hash_later);
}
}
@@ -1978,7 +1960,7 @@ ahash_process_req_prepare(struct ahash_request *areq, unsigned int nbytes,
size_t offset = 0;
do {
- src = scatterwalk_ffwd(tmp, req_ctx->psrc, offset);
+ src = scatterwalk_ffwd(tmp, areq->src, offset);
to_hash_this_desc =
min(nbytes, ALIGN_DOWN(desc_max, blocksize));
@@ -1991,8 +1973,7 @@ ahash_process_req_prepare(struct ahash_request *areq, unsigned int nbytes,
return edesc;
}
- edesc->src =
- scatterwalk_ffwd(edesc->bufsl, req_ctx->psrc, offset);
+ edesc->src = scatterwalk_ffwd(edesc->bufsl, areq->src, offset);
edesc->desc.hdr = ctx->desc_hdr_template;
edesc->first = offset == 0;
edesc->last = nbytes - to_hash_this_desc == 0;
@@ -2045,62 +2026,17 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
bool is_sec1 = has_ftr_sec1(dev_get_drvdata(ctx->dev));
unsigned int nbytes_to_hash;
unsigned int to_hash_later;
- unsigned int nsg;
- int nents;
struct device *dev = ctx->dev;
- u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx];
int ret;
- if (!req_ctx->last_request && (nbytes + req_ctx->nbuf <= blocksize)) {
- /* Buffer up to one whole block */
- nents = sg_nents_for_len(areq->src, nbytes);
- if (nents < 0) {
- dev_err(dev, "Invalid number of src SG.\n");
- return nents;
- }
- sg_copy_to_buffer(areq->src, nents,
- ctx_buf + req_ctx->nbuf, nbytes);
- req_ctx->nbuf += nbytes;
- return 0;
- }
-
- /* At least (blocksize + 1) bytes are available to hash */
- nbytes_to_hash = nbytes + req_ctx->nbuf;
- to_hash_later = nbytes_to_hash & (blocksize - 1);
+ nbytes_to_hash = ALIGN_DOWN(nbytes, blocksize);
+ to_hash_later = nbytes - nbytes_to_hash;
- if (req_ctx->last_request)
+ if (req_ctx->last_request) {
+ nbytes_to_hash = nbytes;
to_hash_later = 0;
- else if (to_hash_later)
- /* There is a partial block. Hash the full block(s) now */
- nbytes_to_hash -= to_hash_later;
- else {
- /* Keep one block buffered */
- nbytes_to_hash -= blocksize;
- to_hash_later = blocksize;
- }
-
- /* Chain in any previously buffered data */
- if (req_ctx->nbuf) {
- nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
- sg_init_table(req_ctx->bufsl, nsg);
- sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf);
- if (nsg > 1)
- sg_chain(req_ctx->bufsl, 2, areq->src);
- req_ctx->psrc = req_ctx->bufsl;
- } else
- req_ctx->psrc = areq->src;
-
- if (to_hash_later) {
- nents = sg_nents_for_len(areq->src, nbytes);
- if (nents < 0) {
- dev_err(dev, "Invalid number of src SG.\n");
- return nents;
- }
- sg_pcopy_to_buffer(areq->src, nents,
- req_ctx->buf[(req_ctx->buf_idx + 1) & 1],
- to_hash_later,
- nbytes - to_hash_later);
}
+
req_ctx->to_hash_later = to_hash_later;
edesc = ahash_process_req_prepare(areq, nbytes_to_hash, blocksize,
@@ -2125,8 +2061,6 @@ static int ahash_init(struct ahash_request *areq)
dma_addr_t dma;
/* Initialize the context */
- req_ctx->buf_idx = 0;
- req_ctx->nbuf = 0;
req_ctx->first_request = 1;
req_ctx->swinit = 0; /* assume h/w init of context */
size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
@@ -2223,12 +2157,10 @@ static int ahash_export(struct ahash_request *areq, void *out)
memcpy(export->hw_context, req_ctx->hw_context,
req_ctx->hw_context_size);
- memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf);
export->swinit = req_ctx->swinit;
export->first_request = req_ctx->first_request;
export->last_request = req_ctx->last_request;
export->to_hash_later = req_ctx->to_hash_later;
- export->nbuf = req_ctx->nbuf;
return 0;
}
@@ -2249,12 +2181,10 @@ static int ahash_import(struct ahash_request *areq, const void *in)
: TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
req_ctx->hw_context_size = size;
memcpy(req_ctx->hw_context, export->hw_context, size);
- memcpy(req_ctx->buf[0], export->buf, export->nbuf);
req_ctx->swinit = export->swinit;
req_ctx->first_request = export->first_request;
req_ctx->last_request = export->last_request;
req_ctx->to_hash_later = export->to_hash_later;
- req_ctx->nbuf = export->nbuf;
dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
DMA_TO_DEVICE);
@@ -2932,8 +2862,11 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "md5",
.cra_driver_name = "md5-talitos",
.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
+ .cra_reqsize = sizeof(struct talitos_ahash_req_ctx),
.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_ALLOCATES_MEMORY,
+ CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_AHASH_ALG_BLOCK_ONLY |
+ CRYPTO_AHASH_ALG_FINAL_NONZERO,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2948,8 +2881,11 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "sha1",
.cra_driver_name = "sha1-talitos",
.cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_reqsize = sizeof(struct talitos_ahash_req_ctx),
.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_ALLOCATES_MEMORY,
+ CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_AHASH_ALG_BLOCK_ONLY |
+ CRYPTO_AHASH_ALG_FINAL_NONZERO,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2964,8 +2900,11 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "sha224",
.cra_driver_name = "sha224-talitos",
.cra_blocksize = SHA224_BLOCK_SIZE,
+ .cra_reqsize = sizeof(struct talitos_ahash_req_ctx),
.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_ALLOCATES_MEMORY,
+ CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_AHASH_ALG_BLOCK_ONLY |
+ CRYPTO_AHASH_ALG_FINAL_NONZERO,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2980,8 +2919,11 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "sha256",
.cra_driver_name = "sha256-talitos",
.cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_reqsize = sizeof(struct talitos_ahash_req_ctx),
.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_ALLOCATES_MEMORY,
+ CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_AHASH_ALG_BLOCK_ONLY |
+ CRYPTO_AHASH_ALG_FINAL_NONZERO,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2996,8 +2938,11 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "sha384",
.cra_driver_name = "sha384-talitos",
.cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_reqsize = sizeof(struct talitos_ahash_req_ctx),
.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_ALLOCATES_MEMORY,
+ CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_AHASH_ALG_BLOCK_ONLY |
+ CRYPTO_AHASH_ALG_FINAL_NONZERO,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -3012,8 +2957,11 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "sha512",
.cra_driver_name = "sha512-talitos",
.cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_reqsize = sizeof(struct talitos_ahash_req_ctx),
.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_ALLOCATES_MEMORY,
+ CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_AHASH_ALG_BLOCK_ONLY |
+ CRYPTO_AHASH_ALG_FINAL_NONZERO,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -3028,8 +2976,11 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "hmac(md5)",
.cra_driver_name = "hmac-md5-talitos",
.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
+ .cra_reqsize = sizeof(struct talitos_ahash_req_ctx),
.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_ALLOCATES_MEMORY,
+ CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_AHASH_ALG_BLOCK_ONLY |
+ CRYPTO_AHASH_ALG_FINAL_NONZERO,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -3044,8 +2995,11 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "hmac(sha1)",
.cra_driver_name = "hmac-sha1-talitos",
.cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_reqsize = sizeof(struct talitos_ahash_req_ctx),
.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_ALLOCATES_MEMORY,
+ CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_AHASH_ALG_BLOCK_ONLY |
+ CRYPTO_AHASH_ALG_FINAL_NONZERO,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -3060,8 +3014,11 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "hmac(sha224)",
.cra_driver_name = "hmac-sha224-talitos",
.cra_blocksize = SHA224_BLOCK_SIZE,
+ .cra_reqsize = sizeof(struct talitos_ahash_req_ctx),
.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_ALLOCATES_MEMORY,
+ CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_AHASH_ALG_BLOCK_ONLY |
+ CRYPTO_AHASH_ALG_FINAL_NONZERO,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -3076,8 +3033,11 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "hmac(sha256)",
.cra_driver_name = "hmac-sha256-talitos",
.cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_reqsize = sizeof(struct talitos_ahash_req_ctx),
.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_ALLOCATES_MEMORY,
+ CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_AHASH_ALG_BLOCK_ONLY |
+ CRYPTO_AHASH_ALG_FINAL_NONZERO,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -3092,8 +3052,11 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "hmac(sha384)",
.cra_driver_name = "hmac-sha384-talitos",
.cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_reqsize = sizeof(struct talitos_ahash_req_ctx),
.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_ALLOCATES_MEMORY,
+ CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_AHASH_ALG_BLOCK_ONLY |
+ CRYPTO_AHASH_ALG_FINAL_NONZERO,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -3108,8 +3071,11 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "hmac(sha512)",
.cra_driver_name = "hmac-sha512-talitos",
.cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_reqsize = sizeof(struct talitos_ahash_req_ctx),
.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_ALLOCATES_MEMORY,
+ CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_AHASH_ALG_BLOCK_ONLY |
+ CRYPTO_AHASH_ALG_FINAL_NONZERO,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -3181,7 +3147,6 @@ static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
algt.alg.hash);
ctx->keylen = 0;
- crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
sizeof(struct talitos_ahash_req_ctx));
return talitos_init_common(ctx, talitos_alg);
--
2.54.0
^ permalink raw reply related
* [PATCH 00/29] crypto: talitos - Driver cleanup
From: Paul Louvel @ 2026-05-28 9:08 UTC (permalink / raw)
To: Herbert Xu, David S. Miller
Cc: Thomas Petazzoni, Herve Codina, Christophe Leroy, linux-crypto,
linux-kernel, Paul Louvel
The Freescale Integrated Security Engine (SEC) aka "Talitos" driver
implementation is a monolithic ~3800-line file that mixes SEC1 and SEC2
hardware variants with hash, skcipher, aead and hwrng algorithm.
This series reorganises the driver to improve readability and
maintainability.
One of the main motivation for this series is to eleminate all the
conditionals around the has_ftr_sec1(). Some checks still remains in
crypto algorithm implementation at the end of the series.
Patch 1 adds the CRYPTO_AHASH_ALG_BLOCK_ONLY flag for the ahash
implementation to eliminate manual partial-block buffering.
Driver reorganisation (patches 2-9):
Move the driver into a dedicated directory, split the different crypto
implementation into dedicated translation units.
Algorithm definition cleanup (patches 10-17):
Remove algorithm property mutations from the registration loop, delete
the now-unused priority field in struct talitos_alg_template, and
convert hash, skcipher and aead to the type-specific init_tfm/exit_tfm
API, replacing the deprecated cra_init/cra_exit fields.
Use preprocessor macros to deduplicate the hash, skcipher and aead
algorithm definitions.
SEC1/SEC2 ops abstraction (patches 18-27):
Introduce struct talitos_ops, split SEC1/SEC2-specific
code into separate function variants, and replace runtime is_sec1
conditionals with indirect calls through the ops table.
Export common channel and error handling routines, and move SEC1 and
SEC2 ops into dedicated translation units.
Introduce struct talitos_ptr_ops to abstract SEC1/SEC2 pointer
helpers behind per-SEC-version ops, then remove the now-unused
global pointer helper functions.
Introduce per-SEC-version descriptor structures and ops.
Patch 28 cleans up the includes in the core driver file now that all
crypto implementation code has been moved out.
Patch 29 removes a now-useless macro.
No functional changes are intended for patches 2-29.
This series depends on the "crypto: talitos - bug fixes" series :
https://patch.msgid.link/20260507-bootlin_test-7-1-rc1_sec_bugfix-v3-0-c98d7589b942@bootlin.com
Signed-off-by: Paul Louvel <paul.louvel@bootlin.com>
---
Paul Louvel (29):
crypto: talitos/hash - Use CRYPTO_AHASH_BLOCK_ONLY API
crypto: talitos - Move driver into dedicated directory
crypto: talitos - Add missing includes to driver header file
crypto: talitos/hwrng - Move into separate file
crypto: talitos - Prepare crypto implementation file splitting
crypto: talitos - Introduce registration helper
crypto: talitos/hash - Move into separate file
crypto: talitos/skcipher - Move into separate file
crypto: talitos/aead - Move into separate file
crypto: talitos - Remove alg settings in talitos_register_common()
crypto: talitos - Remove unused priority field in struct talitos_alg_template
crypto: talitos/hash - Convert to init_tfm/exit_tfm type-specific API
crypto: talitos/skcipher - Convert to init/exit type-specific API
crypto: talitos/aead - Convert to init/exit type-specific API
crypto: talitos/hash - Use macro for algorithm definitions
crypto: talitos/skcipher - Use macro for algorithm definitions
crypto: talitos/aead - Use macro for algorithm definitions
crypto: talitos - Split SEC1/SEC2 code into separate function variants
crypto: talitos - Introduce struct talitos_ops
crypto: talitos - Replace SEC1/SEC2 conditionals with ops dispatch
crypto: talitos - Export common channel and error handling routines
crypto: talitos - Move SEC1 ops into talitos-sec1.c
crypto: talitos - Move SEC2 ops into talitos-sec2.c
crypto: talitos - Introduce per-SEC-version pointer helper ops
crypto: talitos - Dispatch pointer helpers through ptr_ops
crypto: talitos - Remove now-unused global pointer helpers
crypto: talitos - Introduce per-SEC-version descriptor structures and ops
crypto: talitos - Clean up includes in core driver file
crypto: talitos - Remove TALITOS_DESC_SIZE macro
drivers/crypto/Kconfig | 38 +-
drivers/crypto/Makefile | 2 +-
drivers/crypto/talitos.c | 3640 -----------------------------
drivers/crypto/talitos/Kconfig | 36 +
drivers/crypto/talitos/Makefile | 6 +
drivers/crypto/talitos/talitos-aead.c | 677 ++++++
drivers/crypto/talitos/talitos-hash.c | 711 ++++++
drivers/crypto/talitos/talitos-rng.c | 93 +
drivers/crypto/talitos/talitos-sec1.c | 374 +++
drivers/crypto/talitos/talitos-sec2.c | 404 ++++
drivers/crypto/talitos/talitos-skcipher.c | 364 +++
drivers/crypto/talitos/talitos.c | 917 ++++++++
drivers/crypto/{ => talitos}/talitos.h | 255 +-
13 files changed, 3810 insertions(+), 3707 deletions(-)
---
base-commit: db8b9f227833e729faf44a512aa1e88a625b5ad8
change-id: 20260518-7-1-rc1_talitos_cleanup-9231a64e29fa
prerequisite-change-id: 20260504-bootlin_test-7-1-rc1_sec_bugfix-13169ed07ddc:v3
prerequisite-patch-id: 7b364911e4b8d1c1033eb14e67ed24dac6a4bc13
prerequisite-patch-id: 2c1cd7fdd003d9a116a697efa25d1716d548389f
prerequisite-patch-id: b12bdbf565747609e0cfe0609a42cf69b5d816a1
prerequisite-patch-id: 72cb2bc0fc2a48a5a029b049c199f4c86085cf04
prerequisite-patch-id: 5f1f5ad6add760161bd48875df48c0893aa12613
prerequisite-patch-id: 934931086968229434d15a2f2358aeb7e6975a1d
prerequisite-patch-id: 8a0b4828fc0690e0c841bc9adcc6568bb522e0e8
prerequisite-patch-id: 1d870f32e7dbf9a8bd3b8979558544107693e0f4
prerequisite-patch-id: 758c18d7c9fabb14bd90df62e5e8a62a6f880db4
prerequisite-patch-id: ce6e9e585f8edc1861ae6bb8fbdd836c20cbd290
prerequisite-patch-id: 9446dc03e442ea81c5f5b39e802e01b37da29971
Best regards,
--
Paul Louvel, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
^ permalink raw reply
* Re: [PATCH v5 2/7] x86/msr: add wrmsrq_on_cpus helper
From: Borislav Petkov @ 2026-05-28 0:43 UTC (permalink / raw)
To: Dave Hansen
Cc: Ashish Kalra, tglx, mingo, dave.hansen, x86, hpa, seanjc, peterz,
thomas.lendacky, herbert, davem, ardb, pbonzini, aik,
Michael.Roth, KPrateek.Nayak, Tycho.Andersen, Nathan.Fontenot,
ackerleytng, jackyli, pgonda, rientjes, jacobhxu, xin,
pawan.kumar.gupta, babu.moger, dyoung, nikunj, john.allen, darwi,
linux-kernel, linux-crypto, kvm, linux-coco
In-Reply-To: <eea0497f-6930-43e3-947d-dae139e657ad@intel.com>
On Wed, May 27, 2026 at 02:38:05PM -0700, Dave Hansen wrote:
> This one is my doing.
I know.
But hey, maybe we should not disagree on the public ML because the submitter
might disappear like the last one. :-P
> wrmsr_on_cpus() is kinda a mess. I think it only has a single user. It's
> also not very flexible because it needs a 'struct msr __percpu *msrs'
> argument where each MSR has a value in memory.
Right, we did that a looong time ago.
The only reason I'd have for per-CPU MSR structs is reading different MSR
values on different cores, modifying only the bits you need and then *keeping*
the remaining values as they were. And that interface allows you to do that
while this new thing won't.
And I'm going to venture a guess here that adding a simpler interface which
simply forces a new value ontop of a whole MSR could cause a lot of subtle
bugs when people don't pay attention to keep the old values.
> The use case for RMPOPT is that all CPUs get the same value. It'd be a
> little awkward to go create a percpu data structure to duplcate the same
> value to call wrmsr_on_cpus(). The RMPOPT case is also arguably
> performance sensitive since it's done during boot. It should do the IPIs
> in parallel.
Oh sure, my meaning was to create something that serves both purposes.
> toggle_ecc_err_reporting(), on the other hand, is done at module init
> time. It's not really performance sensitive. It's probably pretty easy
> to zap wrmsr_on_cpus() and just have toggle_ecc_err_reporting() do
> something slightly less efficient.
Sure. That's fine.
> Yeah, the
>
> wrmsr_on_cpus()
> wrmsrq_on_cpus()
>
> naming pain is real. There's little chance of bugs coming from it
> because the function signatures are *SO* different. But, it certainly
> could confuse humans for a minute.
Yap.
> But the real solution to this is axing wrmsr_on_cpus().
Yap, for example. Basically reingeneering the whole
write-MSRs-on-multiple-CPUs functionality is what I meant.
> Which I think we could do after killing its one user which the attached
> (completely untested) patch does. The only downside of the patch is that it
> does RDMSR via IPIs one CPU at a time. But, looking at the code, I'm not
> sure anyone would care. If anyone did, I _think_ all those MSRs have the
> same value and the code could be simplified further. But that would take
> more than 3 minutes.
>
> It's also possible that my grepping was bad or I'm completely
> misunderstanding amd64_edac.c. Cluebat welcome if I'm being dense.
Looks ok to me, we can surely do that. I even hw to test it. I think...
> BTW, I also don't feel the need to make Ashish go do any of this edac
> cleanup. I think it can just be done in parallel. But I wouldn't stop
> him if he volunteered.
Why not?
It has always been the case: cleanups and bug fixes first, new features ontop.
So yeah, modulo figuring out how to redefine the *msr_on_cpus() interface,
I think this all makes sense.
Thx.
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette
^ permalink raw reply
* Re: [PATCH v5 2/7] x86/msr: add wrmsrq_on_cpus helper
From: Dave Hansen @ 2026-05-27 21:38 UTC (permalink / raw)
To: Borislav Petkov, Ashish Kalra
Cc: tglx, mingo, dave.hansen, x86, hpa, seanjc, peterz,
thomas.lendacky, herbert, davem, ardb, pbonzini, aik,
Michael.Roth, KPrateek.Nayak, Tycho.Andersen, Nathan.Fontenot,
ackerleytng, jackyli, pgonda, rientjes, jacobhxu, xin,
pawan.kumar.gupta, babu.moger, dyoung, nikunj, john.allen, darwi,
linux-kernel, linux-crypto, kvm, linux-coco
In-Reply-To: <20260527210603.GCahdcu8zvVjfKfGEL@fat_crate.local>
[-- Attachment #1: Type: text/plain, Size: 2757 bytes --]
On 5/27/26 14:06, Borislav Petkov wrote:
> On Mon, May 18, 2026 at 09:42:15PM +0000, Ashish Kalra wrote:
>> From: Ashish Kalra <ashish.kalra@amd.com>
>>
>> The existing wrmsr_on_cpus() takes a per-cpu struct msr array, requiring
>> callers to allocate and populate per-cpu storage even when every CPU
>> receives the same value. This is unnecessary overhead for the common
>> case of writing a single uniform u64 to a per-CPU MSR across multiple
>> CPUs.
>>
>> Add wrmsrq_on_cpus() which writes the same u64 value to the specified
>> MSR on all CPUs in the given cpumask.
>
> So let's add yet another function which name differs from the other one by
> a single letter and have people go look at the implementation to know which is
> which...?
>
> Instead of unifying what's there and extending this one to do what you want it
> to do?
>
> And now you have a wrmsrQ_on_cpus() but no rdmsrQ_on_cpus()?
>
> Because if you look at the code, you'll see how those are used: first you
> rdmsr on CPUs, modify each value and then wrmsr on same CPUs.
This one is my doing.
wrmsr_on_cpus() is kinda a mess. I think it only has a single user. It's
also not very flexible because it needs a 'struct msr __percpu *msrs'
argument where each MSR has a value in memory.
The use case for RMPOPT is that all CPUs get the same value. It'd be a
little awkward to go create a percpu data structure to duplcate the same
value to call wrmsr_on_cpus(). The RMPOPT case is also arguably
performance sensitive since it's done during boot. It should do the IPIs
in parallel.
toggle_ecc_err_reporting(), on the other hand, is done at module init
time. It's not really performance sensitive. It's probably pretty easy
to zap wrmsr_on_cpus() and just have toggle_ecc_err_reporting() do
something slightly less efficient.
Yeah, the
wrmsr_on_cpus()
wrmsrq_on_cpus()
naming pain is real. There's little chance of bugs coming from it
because the function signatures are *SO* different. But, it certainly
could confuse humans for a minute.
But the real solution to this is axing wrmsr_on_cpus(). Which I think we
could do after killing its one user which the attached (completely
untested) patch does. The only downside of the patch is that it does
RDMSR via IPIs one CPU at a time. But, looking at the code, I'm not sure
anyone would care. If anyone did, I _think_ all those MSRs have the same
value and the code could be simplified further. But that would take more
than 3 minutes.
It's also possible that my grepping was bad or I'm completely
misunderstanding amd64_edac.c. Cluebat welcome if I'm being dense.
BTW, I also don't feel the need to make Ashish go do any of this edac
cleanup. I think it can just be done in parallel. But I wouldn't stop
him if he volunteered.
[-- Attachment #2: axe-wrmsr_on_cpus.patch --]
[-- Type: text/x-patch, Size: 2609 bytes --]
---
b/drivers/edac/amd64_edac.c | 35 +++++++++++------------------------
1 file changed, 11 insertions(+), 24 deletions(-)
diff -puN drivers/edac/amd64_edac.c~axe-wrmsr_on_cpus drivers/edac/amd64_edac.c
--- a/drivers/edac/amd64_edac.c~axe-wrmsr_on_cpus 2026-05-27 14:25:07.881694152 -0700
+++ b/drivers/edac/amd64_edac.c 2026-05-27 14:33:03.278139821 -0700
@@ -14,8 +14,6 @@ static struct edac_pci_ctl_info *pci_ctl
static int ecc_enable_override;
module_param(ecc_enable_override, int, 0644);
-static struct msr __percpu *msrs;
-
static inline u32 get_umc_reg(struct amd64_pvt *pvt, u32 reg)
{
if (!pvt->flags.zn_regs_v2)
@@ -3215,14 +3213,14 @@ static bool nb_mce_bank_enabled_on_node(
get_cpus_on_this_dct_cpumask(mask, nid);
- rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
-
for_each_cpu(cpu, mask) {
- struct msr *reg = per_cpu_ptr(msrs, cpu);
- nbe = reg->l & MSR_MCGCTL_NBE;
+ u64 mcg_ctl;
+
+ rdmsrq_on_cpu(cpu, MSR_IA32_MCG_CTL, &mcg_ctl);
+ nbe = mcg_ctl & MSR_MCGCTL_NBE;
edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
- cpu, reg->q, str_enabled_disabled(nbe));
+ cpu, mcg_ctl, str_enabled_disabled(nbe));
if (!nbe)
goto out;
@@ -3246,26 +3244,25 @@ static int toggle_ecc_err_reporting(stru
get_cpus_on_this_dct_cpumask(cmask, nid);
- rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
-
for_each_cpu(cpu, cmask) {
+ u64 mcg_ctl;
- struct msr *reg = per_cpu_ptr(msrs, cpu);
+ rdmsrq_on_cpu(cpu, MSR_IA32_MCG_CTL, &mcg_ctl);
if (on) {
- if (reg->l & MSR_MCGCTL_NBE)
+ if (mcg_ctl & MSR_MCGCTL_NBE)
s->flags.nb_mce_enable = 1;
- reg->l |= MSR_MCGCTL_NBE;
+ mcg_ctl |= MSR_MCGCTL_NBE;
} else {
/*
* Turn off NB MCE reporting only when it was off before
*/
if (!s->flags.nb_mce_enable)
- reg->l &= ~MSR_MCGCTL_NBE;
+ mcg_ctl &= ~MSR_MCGCTL_NBE;
}
+ wrmsrq_on_cpu(cpu, MSR_IA32_MCG_CTL, mcg_ctl);
}
- wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
free_cpumask_var(cmask);
@@ -4153,10 +4150,6 @@ static int __init amd64_edac_init(void)
if (!ecc_stngs)
goto err_free;
- msrs = msrs_alloc();
- if (!msrs)
- goto err_free;
-
for (i = 0; i < amd_nb_num(); i++) {
err = probe_one_instance(i);
if (err) {
@@ -4190,9 +4183,6 @@ static int __init amd64_edac_init(void)
err_pci:
pci_ctl_dev = NULL;
- msrs_free(msrs);
- msrs = NULL;
-
err_free:
kfree(ecc_stngs);
ecc_stngs = NULL;
@@ -4220,9 +4210,6 @@ static void __exit amd64_edac_exit(void)
ecc_stngs = NULL;
pci_ctl_dev = NULL;
-
- msrs_free(msrs);
- msrs = NULL;
}
module_init(amd64_edac_init);
_
^ permalink raw reply
* Re: [PATCH v5 2/7] x86/msr: add wrmsrq_on_cpus helper
From: Borislav Petkov @ 2026-05-27 21:06 UTC (permalink / raw)
To: Ashish Kalra
Cc: tglx, mingo, dave.hansen, x86, hpa, seanjc, peterz,
thomas.lendacky, herbert, davem, ardb, pbonzini, aik,
Michael.Roth, KPrateek.Nayak, Tycho.Andersen, Nathan.Fontenot,
ackerleytng, jackyli, pgonda, rientjes, jacobhxu, xin,
pawan.kumar.gupta, babu.moger, dyoung, nikunj, john.allen, darwi,
linux-kernel, linux-crypto, kvm, linux-coco
In-Reply-To: <c9fe5c2fef063f5006cc9bfa03eec824ac015db7.1779133590.git.ashish.kalra@amd.com>
On Mon, May 18, 2026 at 09:42:15PM +0000, Ashish Kalra wrote:
> From: Ashish Kalra <ashish.kalra@amd.com>
>
> The existing wrmsr_on_cpus() takes a per-cpu struct msr array, requiring
> callers to allocate and populate per-cpu storage even when every CPU
> receives the same value. This is unnecessary overhead for the common
> case of writing a single uniform u64 to a per-CPU MSR across multiple
> CPUs.
>
> Add wrmsrq_on_cpus() which writes the same u64 value to the specified
> MSR on all CPUs in the given cpumask.
So let's add yet another function which name differs from the other one by
a single letter and have people go look at the implementation to know which is
which...?
Instead of unifying what's there and extending this one to do what you want it
to do?
And now you have a wrmsrQ_on_cpus() but no rdmsrQ_on_cpus()?
Because if you look at the code, you'll see how those are used: first you
rdmsr on CPUs, modify each value and then wrmsr on same CPUs.
So no, try again pls.
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette
^ permalink raw reply
* Re: [PATCH v5 1/7] x86/cpufeatures: Add X86_FEATURE_AMD_RMPOPT feature flag
From: Borislav Petkov @ 2026-05-27 20:17 UTC (permalink / raw)
To: Ashish Kalra
Cc: tglx, mingo, dave.hansen, x86, hpa, seanjc, peterz,
thomas.lendacky, herbert, davem, ardb, pbonzini, aik,
Michael.Roth, KPrateek.Nayak, Tycho.Andersen, Nathan.Fontenot,
ackerleytng, jackyli, pgonda, rientjes, jacobhxu, xin,
pawan.kumar.gupta, babu.moger, dyoung, nikunj, john.allen, darwi,
linux-kernel, linux-crypto, kvm, linux-coco
In-Reply-To: <305b625c0528f16a95542001c66e643fbd3a2622.1779133590.git.ashish.kalra@amd.com>
On Mon, May 18, 2026 at 09:41:53PM +0000, Ashish Kalra wrote:
> From: Ashish Kalra <ashish.kalra@amd.com>
>
> Add a flag indicating whether RMPOPT instruction is supported.
>
> RMPOPT is a new instruction designed to minimize the performance
> overhead of RMP checks on the hypervisor and on non-SNP guests by
> allowing RMP checks to be skipped when 1G regions of memory are known
> not to contain any SEV-SNP guest memory.
Streamline:
"RMPOPT is a new instruction that reduces the performance overhead of RMP
checks for the hypervisor and non‑SNP guests by allowing those checks to be
skipped when 1‑GB memory regions are known to contain no SEV‑SNP guest
memory."
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette
^ permalink raw reply
* [PATCH] crypto: qat - simplify adf_service_mask_to_string helper
From: Thorsten Blum @ 2026-05-27 17:46 UTC (permalink / raw)
To: Giovanni Cabiddu, Herbert Xu, David S. Miller,
Suman Kumar Chakraborty, Karthikeyan Gopal
Cc: Thorsten Blum, qat-linux, linux-crypto, linux-kernel
Use a single scnprintf() for each set bit and drop the offset in the
else branch to simplify adf_service_mask_to_string().
Signed-off-by: Thorsten Blum <thorsten.blum@linux.dev>
---
drivers/crypto/intel/qat/qat_common/adf_cfg_services.c | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c
index 21b21ac78e53..baf563c6f9b7 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c
@@ -93,10 +93,9 @@ static int adf_service_mask_to_string(unsigned long mask, char *buf, size_t len)
for_each_set_bit(bit, &mask, SVC_COUNT) {
if (offset)
offset += scnprintf(buf + offset, len - offset,
- ADF_SERVICES_DELIMITER);
-
- offset += scnprintf(buf + offset, len - offset, "%s",
- adf_cfg_services[bit]);
+ ADF_SERVICES_DELIMITER "%s", adf_cfg_services[bit]);
+ else
+ offset += scnprintf(buf, len, "%s", adf_cfg_services[bit]);
}
return 0;
^ permalink raw reply related
* Re: [PATCH v2 4/4] dm crypt: batch all sectors of a bio per crypto request
From: Mikulas Patocka @ 2026-05-27 17:32 UTC (permalink / raw)
To: Leonid Ravich
Cc: Herbert Xu, David S . Miller, Mike Snitzer, Alasdair Kergon,
Ard Biesheuvel, Eric Biggers, Jens Axboe, Horia Geanta,
Gilad Ben-Yossef, linux-crypto, dm-devel, linux-block
In-Reply-To: <20260527065021.19525-5-lravich@amazon.com>
Hi
On Wed, 27 May 2026, Leonid Ravich wrote:
> +/*
> + * Multi-data-unit variant of crypt_convert_block_skcipher. Submits all
> + * remaining sectors of the current bio in one skcipher request whose
> + * data_unit_size is cc->sector_size. The cipher walks the IV between
> + * data units (see crypto_skcipher_set_data_unit_size()).
> + *
> + * Returns the same set of values as crypt_convert_block_skcipher:
> + * 0 on synchronous success (full chunk processed),
> + * -EINPROGRESS / -EBUSY on asynchronous dispatch,
> + * -EAGAIN if the per-bio scatterlist allocation cannot be made. The
> + * caller MUST disable multi-data-unit batching for the rest
> + * of this bio and re-enter the per-sector path, which uses
> + * only mempool reserves and is therefore safe even on the
> + * swap-out-to-dm-crypt path under total memory exhaustion.
> + * negative errno otherwise.
> + *
> + * On success the bio iterators have been advanced by the chunk size.
> + *
> + * Walks the bio with __bio_for_each_bvec so that multi-page folios
> + * produce one scatterlist entry rather than N (one per PAGE_SIZE).
> + */
> +static int crypt_convert_block_skcipher_multi(struct crypt_config *cc,
> + struct convert_context *ctx,
> + struct skcipher_request *req,
> + unsigned int *out_processed)
> +{
> + const unsigned int sector_size = cc->sector_size;
> + const gfp_t gfp = GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN;
> + unsigned int total_in = ctx->iter_in.bi_size;
> + unsigned int total_out = ctx->iter_out.bi_size;
> + unsigned int total = min(total_in, total_out);
> + unsigned int n_sectors;
> + unsigned int n_sg_in = 0, n_sg_out = 0;
> + struct dm_crypt_request *dmreq = dmreq_of_req(cc, req);
> + struct scatterlist *sg_in = NULL, *sg_out = NULL;
> + struct bvec_iter iter_in, iter_out;
> + struct bio_vec bv;
> + u8 *iv, *org_iv;
> + int r;
> +
> + if (unlikely(total < sector_size))
> + return -EIO;
> + n_sectors = total / sector_size;
> + total = n_sectors * sector_size;
Division is slow. There should be this:
n_sectors = total >> cc->sector_shift;
> + if (unlikely(total < sector_size))
> + return -EIO;
The condition total < sector_size is true if total is small but it goes
through if total is bigger but unaligned. I think that it should be:
if (unlikely(total & (sector_size - 1)))
return -EIO;
(then, we can drop the line "total = n_sectors * sector_size")
ctx->iter_in.bi_size is supposed to be the same as ctx->iter_out.bi_size,
so do we really need total = min(total_in, total_out)? Should it instead
warn if they differ? (where the warning would indicate a bug in the code)
Mikulas
^ permalink raw reply
* Re: [PATCH 7/8] lib/raid6: Include asm/neon-intrinsics.h rather than arm_neon.h
From: Ard Biesheuvel @ 2026-05-27 15:42 UTC (permalink / raw)
To: Eric Biggers, Christoph Hellwig
Cc: Ard Biesheuvel, linux-arm-kernel, linux-crypto, linux-raid,
Russell King, Arnd Bergmann
In-Reply-To: <20260527015754.GA13078@sol>
On Wed, 27 May 2026, at 03:57, Eric Biggers wrote:
> On Sat, May 09, 2026 at 01:23:54PM -0700, Eric Biggers wrote:
>> On Thu, Apr 23, 2026 at 09:47:12AM +0200, Christoph Hellwig wrote:
>> > On Wed, Apr 22, 2026 at 07:17:03PM +0200, Ard Biesheuvel wrote:
>> > > From: Ard Biesheuvel <ardb@kernel.org>
>> > >
>> > > arm_neon.h is a compiler header which needs some scaffolding to work
>> > > correctly in the linux context, and so it is better not to include it
>> > > directly. Both ARM and arm64 now provide asm/neon-intrinsics.h which
>> > > takes care of this.
>> >
>> >
>> > This could potentially clash with the raid6 library rework I'm doing
>> > for 7.2. Although git has become pretty good about renamed files, so
>> > maybe it won't be so bad.
>> >
>>
>> I think this patch also breaks the userspace build of lib/raid6/. Which
>> is going away in Christoph's series anyway, but maybe it would make
>> sense to drop this patch (and patch 8 which depends on this, I think)
>> from this series for now? That would make it a bit easier to take the
>> rest through crc-next.
>
> Ard, are you okay with me applying just patches 1-6 to crc-next?
Yes that's fine - thanks.
^ permalink raw reply
* [PATCH] crypto: powerpc/aes - use min in ppc_{ecb,cbc,ctr,xts}_crypt
From: Thorsten Blum @ 2026-05-27 14:11 UTC (permalink / raw)
To: Herbert Xu, David S. Miller, Madhavan Srinivasan,
Michael Ellerman, Nicholas Piggin, Christophe Leroy (CS GROUP)
Cc: Thorsten Blum, linux-crypto, linuxppc-dev, linux-kernel
Replace min_t() with the simpler min() macro since the values are
unsigned and compatible.
Signed-off-by: Thorsten Blum <thorsten.blum@linux.dev>
---
arch/powerpc/crypto/aes-spe-glue.c | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/arch/powerpc/crypto/aes-spe-glue.c b/arch/powerpc/crypto/aes-spe-glue.c
index 7d2827e65240..e038488087e6 100644
--- a/arch/powerpc/crypto/aes-spe-glue.c
+++ b/arch/powerpc/crypto/aes-spe-glue.c
@@ -9,6 +9,7 @@
*/
#include <crypto/aes.h>
+#include <linux/minmax.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/types.h>
@@ -140,7 +141,7 @@ static int ppc_ecb_crypt(struct skcipher_request *req, bool enc)
err = skcipher_walk_virt(&walk, req, false);
while ((nbytes = walk.nbytes) != 0) {
- nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
+ nbytes = min(nbytes, MAX_BYTES);
nbytes = round_down(nbytes, AES_BLOCK_SIZE);
spe_begin();
@@ -179,7 +180,7 @@ static int ppc_cbc_crypt(struct skcipher_request *req, bool enc)
err = skcipher_walk_virt(&walk, req, false);
while ((nbytes = walk.nbytes) != 0) {
- nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
+ nbytes = min(nbytes, MAX_BYTES);
nbytes = round_down(nbytes, AES_BLOCK_SIZE);
spe_begin();
@@ -220,7 +221,7 @@ static int ppc_ctr_crypt(struct skcipher_request *req)
err = skcipher_walk_virt(&walk, req, false);
while ((nbytes = walk.nbytes) != 0) {
- nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
+ nbytes = min(nbytes, MAX_BYTES);
if (nbytes < walk.total)
nbytes = round_down(nbytes, AES_BLOCK_SIZE);
@@ -248,7 +249,7 @@ static int ppc_xts_crypt(struct skcipher_request *req, bool enc)
twk = ctx->key_twk;
while ((nbytes = walk.nbytes) != 0) {
- nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
+ nbytes = min(nbytes, MAX_BYTES);
nbytes = round_down(nbytes, AES_BLOCK_SIZE);
spe_begin();
^ permalink raw reply related
* Re: [PATCH 2/3] crypto: inside-secure: add EIP93 ESP packet backend
From: Simon Horman @ 2026-05-27 10:08 UTC (permalink / raw)
To: Jihong Min
Cc: Christian Marangi, Antoine Tenart, Herbert Xu, David S . Miller,
Lorenzo Bianconi, Andrew Lunn, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Steffen Klassert, linux-kernel, linux-crypto,
linux-arm-kernel, linux-mediatek, netdev
In-Reply-To: <20260523121522.3023992-3-hurryman2212@gmail.com>
On Sat, May 23, 2026 at 09:15:21PM +0900, Jihong Min wrote:
> Expose an EIP93 packet-mode IPsec backend for netdev drivers that need
> ESP encapsulation and decapsulation offload without advertising EIP93
> itself as a netdev.
>
> Add provider selection, capability reporting, SA lifecycle management,
> IPsec request completion, and provider fault notification around the
> existing EIP93 descriptor path.
>
> Assisted-by: Codex:gpt-5.5
> Signed-off-by: Jihong Min <hurryman2212@gmail.com>
...
> diff --git a/drivers/crypto/inside-secure/eip93/eip93-ipsec.c b/drivers/crypto/inside-secure/eip93/eip93-ipsec.c
...
> +static void eip93_ipsec_abort_requests(struct eip93_ipsec *ipsec, int err)
> +{
> + struct eip93_ipsec_sa *sa;
> +
> + while (true) {
> + bool found = false;
> +
> + spin_lock_bh(&ipsec->lock);
> + list_for_each_entry(sa, &ipsec->sa_list, node) {
> + spin_lock(&sa->lock);
> + if (sa->aborting) {
> + spin_unlock(&sa->lock);
> + continue;
> + }
> +
> + sa->aborting = true;
> + found = refcount_inc_not_zero(&sa->refcnt);
> + spin_unlock(&sa->lock);
> + if (found)
> + break;
> + }
> + spin_unlock_bh(&ipsec->lock);
> + if (!found)
> + return;
> +
> + eip93_ipsec_abort_sa(sa, err);
> + eip93_ipsec_sa_put(sa);
sa is the iterator for the list_for_each_entry loop.
However, here it is used outside of that context.
"If list_for_each_entry, etc complete a traversal of the list, the
iterator variable ends up pointing to an address at an offset from
the list head, and not a meaningful structure. Thus this value
should not be used after the end of the iterator.
https://www.spinics.net/lists/linux-kernel-janitors/msg11994.html
Flagged by Coccinelle.
> + }
> +}
...
^ permalink raw reply
* [PATCH RESEND 1/6] sock: add sock_kzalloc helper
From: Thorsten Blum @ 2026-05-27 8:25 UTC (permalink / raw)
To: Herbert Xu, David S. Miller, Eric Dumazet, Kuniyuki Iwashima,
Paolo Abeni, Willem de Bruijn, Jakub Kicinski, Simon Horman
Cc: linux-crypto, linux-kernel, netdev, Thorsten Blum
Add sock_kzalloc() helper - the sock equivalent to kzalloc().
Reviewed-by: Kuniyuki Iwashima <kuniyu@google.com>
Signed-off-by: Thorsten Blum <thorsten.blum@linux.dev>
---
Patch 1/6 needs an Acked-by: from netdev maintainers for the series to
go through Herbert's crypto tree:
https://lore.kernel.org/lkml/ahVkZOxZtFes6Huf@gondor.apana.org.au/
---
include/net/sock.h | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/include/net/sock.h b/include/net/sock.h
index 76bfd3e56d63..b521bd34ac9f 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -1913,6 +1913,11 @@ void sock_kfree_s(struct sock *sk, void *mem, int size);
void sock_kzfree_s(struct sock *sk, void *mem, int size);
void sk_send_sigurg(struct sock *sk);
+static inline void *sock_kzalloc(struct sock *sk, int size, gfp_t priority)
+{
+ return sock_kmalloc(sk, size, priority | __GFP_ZERO);
+}
+
static inline void sock_replace_proto(struct sock *sk, struct proto *proto)
{
if (sk->sk_socket)
^ permalink raw reply related
* [PATCH RESEND 6/6] crypto: algif_skcipher - use sock_kzalloc in accept_parent_nokey
From: Thorsten Blum @ 2026-05-27 8:25 UTC (permalink / raw)
To: Herbert Xu, David S. Miller, Eric Dumazet, Kuniyuki Iwashima,
Paolo Abeni, Willem de Bruijn, Jakub Kicinski, Simon Horman
Cc: linux-crypto, linux-kernel, netdev, Thorsten Blum
In-Reply-To: <20260527082509.1133816-8-thorsten.blum@linux.dev>
Replace sock_kmalloc() followed by memset(0) with sock_kzalloc() to
simplify skcipher_accept_parent_nokey().
Reviewed-by: Kuniyuki Iwashima <kuniyu@google.com>
Signed-off-by: Thorsten Blum <thorsten.blum@linux.dev>
---
crypto/algif_skcipher.c | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c
index ba0a17fd95ac..f29a304e1268 100644
--- a/crypto/algif_skcipher.c
+++ b/crypto/algif_skcipher.c
@@ -383,18 +383,15 @@ static int skcipher_accept_parent_nokey(void *private, struct sock *sk)
struct crypto_skcipher *tfm = private;
unsigned int len = sizeof(*ctx);
- ctx = sock_kmalloc(sk, len, GFP_KERNEL);
+ ctx = sock_kzalloc(sk, len, GFP_KERNEL);
if (!ctx)
return -ENOMEM;
- memset(ctx, 0, len);
- ctx->iv = sock_kmalloc(sk, crypto_skcipher_ivsize(tfm),
- GFP_KERNEL);
+ ctx->iv = sock_kzalloc(sk, crypto_skcipher_ivsize(tfm), GFP_KERNEL);
if (!ctx->iv) {
sock_kfree_s(sk, ctx, len);
return -ENOMEM;
}
- memset(ctx->iv, 0, crypto_skcipher_ivsize(tfm));
INIT_LIST_HEAD(&ctx->tsgl_list);
ctx->len = len;
^ permalink raw reply related
* [PATCH RESEND 5/6] crypto: algif_rng - use sock_kzalloc in rng_accept_parent
From: Thorsten Blum @ 2026-05-27 8:25 UTC (permalink / raw)
To: Herbert Xu, David S. Miller, Eric Dumazet, Kuniyuki Iwashima,
Paolo Abeni, Willem de Bruijn, Jakub Kicinski, Simon Horman
Cc: linux-crypto, linux-kernel, netdev, Thorsten Blum
In-Reply-To: <20260527082509.1133816-8-thorsten.blum@linux.dev>
Replace sock_kmalloc() followed by memset(0) with sock_kzalloc() to
simplify rng_accept_parent().
Reviewed-by: Kuniyuki Iwashima <kuniyu@google.com>
Signed-off-by: Thorsten Blum <thorsten.blum@linux.dev>
---
crypto/algif_rng.c | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/crypto/algif_rng.c b/crypto/algif_rng.c
index a9fb492e929a..f609463f9e14 100644
--- a/crypto/algif_rng.c
+++ b/crypto/algif_rng.c
@@ -244,11 +244,10 @@ static int rng_accept_parent(void *private, struct sock *sk)
struct alg_sock *ask = alg_sk(sk);
unsigned int len = sizeof(*ctx);
- ctx = sock_kmalloc(sk, len, GFP_KERNEL);
+ ctx = sock_kzalloc(sk, len, GFP_KERNEL);
if (!ctx)
return -ENOMEM;
- memset(ctx, 0, len);
ctx->len = len;
/*
^ permalink raw reply related
* [PATCH RESEND 4/6] crypto: af_alg - use sock_kzalloc in alloc_result + accept_parent_nokey
From: Thorsten Blum @ 2026-05-27 8:25 UTC (permalink / raw)
To: Herbert Xu, David S. Miller, Eric Dumazet, Kuniyuki Iwashima,
Paolo Abeni, Willem de Bruijn, Jakub Kicinski, Simon Horman
Cc: linux-crypto, linux-kernel, netdev, Thorsten Blum
In-Reply-To: <20260527082509.1133816-8-thorsten.blum@linux.dev>
Replace sock_kmalloc() followed by memset(0) with sock_kzalloc() to
simplify hash_alloc_result() and hash_accept_parent_nokey().
Reviewed-by: Kuniyuki Iwashima <kuniyu@google.com>
Signed-off-by: Thorsten Blum <thorsten.blum@linux.dev>
---
crypto/algif_hash.c | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c
index 4d3dfc60a16a..02c0e448390d 100644
--- a/crypto/algif_hash.c
+++ b/crypto/algif_hash.c
@@ -38,12 +38,10 @@ static int hash_alloc_result(struct sock *sk, struct hash_ctx *ctx)
ds = crypto_ahash_digestsize(crypto_ahash_reqtfm(&ctx->req));
- ctx->result = sock_kmalloc(sk, ds, GFP_KERNEL);
+ ctx->result = sock_kzalloc(sk, ds, GFP_KERNEL);
if (!ctx->result)
return -ENOMEM;
- memset(ctx->result, 0, ds);
-
return 0;
}
@@ -412,11 +410,10 @@ static int hash_accept_parent_nokey(void *private, struct sock *sk)
struct hash_ctx *ctx;
unsigned int len = sizeof(*ctx) + crypto_ahash_reqsize(tfm);
- ctx = sock_kmalloc(sk, len, GFP_KERNEL);
+ ctx = sock_kzalloc(sk, len, GFP_KERNEL);
if (!ctx)
return -ENOMEM;
- memset(ctx, 0, len);
ctx->len = len;
crypto_init_wait(&ctx->wait);
^ permalink raw reply related
* [PATCH RESEND 3/6] crypto: algif_aead - use sock_kzalloc in aead_accept_parent_nokey
From: Thorsten Blum @ 2026-05-27 8:25 UTC (permalink / raw)
To: Herbert Xu, David S. Miller, Eric Dumazet, Kuniyuki Iwashima,
Paolo Abeni, Willem de Bruijn, Jakub Kicinski, Simon Horman
Cc: linux-crypto, linux-kernel, netdev, Thorsten Blum
In-Reply-To: <20260527082509.1133816-8-thorsten.blum@linux.dev>
Replace sock_kmalloc() followed by memset(0) with sock_kzalloc() to
simplify aead_accept_parent_nokey().
Reviewed-by: Kuniyuki Iwashima <kuniyu@google.com>
Signed-off-by: Thorsten Blum <thorsten.blum@linux.dev>
---
crypto/algif_aead.c | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c
index c6c2ce21895d..6ebc2f9f741d 100644
--- a/crypto/algif_aead.c
+++ b/crypto/algif_aead.c
@@ -408,17 +408,15 @@ static int aead_accept_parent_nokey(void *private, struct sock *sk)
unsigned int len = sizeof(*ctx);
unsigned int ivlen = crypto_aead_ivsize(tfm);
- ctx = sock_kmalloc(sk, len, GFP_KERNEL);
+ ctx = sock_kzalloc(sk, len, GFP_KERNEL);
if (!ctx)
return -ENOMEM;
- memset(ctx, 0, len);
- ctx->iv = sock_kmalloc(sk, ivlen, GFP_KERNEL);
+ ctx->iv = sock_kzalloc(sk, ivlen, GFP_KERNEL);
if (!ctx->iv) {
sock_kfree_s(sk, ctx, len);
return -ENOMEM;
}
- memset(ctx->iv, 0, ivlen);
INIT_LIST_HEAD(&ctx->tsgl_list);
ctx->len = len;
^ permalink raw reply related
* [PATCH RESEND 2/6] crypto: af_alg - use sock_kzalloc in af_alg_alloc_areq
From: Thorsten Blum @ 2026-05-27 8:25 UTC (permalink / raw)
To: Herbert Xu, David S. Miller, Eric Dumazet, Kuniyuki Iwashima,
Paolo Abeni, Willem de Bruijn, Jakub Kicinski, Simon Horman
Cc: linux-crypto, linux-kernel, netdev, Thorsten Blum
In-Reply-To: <20260527082509.1133816-8-thorsten.blum@linux.dev>
Replace sock_kmalloc() followed by memset(0) with sock_kzalloc() to
simplify af_alg_alloc_areq().
Reviewed-by: Kuniyuki Iwashima <kuniyu@google.com>
Signed-off-by: Thorsten Blum <thorsten.blum@linux.dev>
---
crypto/af_alg.c | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/crypto/af_alg.c b/crypto/af_alg.c
index 48c53f488e0f..9438a874c1f1 100644
--- a/crypto/af_alg.c
+++ b/crypto/af_alg.c
@@ -1158,12 +1158,10 @@ struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk,
if (ctx->inflight)
return ERR_PTR(-EBUSY);
- areq = sock_kmalloc(sk, areqlen, GFP_KERNEL);
+ areq = sock_kzalloc(sk, areqlen, GFP_KERNEL);
if (unlikely(!areq))
return ERR_PTR(-ENOMEM);
- memset(areq, 0, areqlen);
-
ctx->inflight = true;
areq->areqlen = areqlen;
^ permalink raw reply related
* Re: [PATCH 1/3] net: Remove support for AIO on sockets
From: Christoph Hellwig @ 2026-05-27 8:13 UTC (permalink / raw)
To: Jens Axboe
Cc: Christoph Hellwig, demiobenour, Herbert Xu, David S. Miller,
Eric Dumazet, Kuniyuki Iwashima, Paolo Abeni, Willem de Bruijn,
Jakub Kicinski, Simon Horman, Peter Zijlstra, Ingo Molnar,
Arnaldo Carvalho de Melo, Namhyung Kim, Mark Rutland,
Alexander Shishkin, Jiri Olsa, Ian Rogers, Adrian Hunter,
James Clark, Jonathan Corbet, Shuah Khan, Eric Biggers,
Ard Biesheuvel, linux-crypto, linux-kernel, io-uring, netdev,
linux-perf-users, linux-doc, Toke Høiland-Jørgensen,
linux-api, David Howells
In-Reply-To: <92db3ff0-8f0b-4b61-a167-5004ffcf9025@kernel.dk>
On Tue, May 26, 2026 at 09:58:27AM -0600, Jens Axboe wrote:
> > The current TCP zerocopy implementation provides completion notification
> > through the socket error code, which is freaking weird and doesn't
> > integrate well with either io_uring or in-kernel callers.
>
> We already have that via io_uring
Where? And how do make that available to in-kernel users like
storage protocols and network file system, which really suffer from
the current MSG_SPLICE_PAGES semantics.
> , and without needing msg_kiocb or the
What do you think is the downside of using a kiocb here like for
everything else with async notifications?
^ permalink raw reply
* [PATCH v2 4/4] dm crypt: batch all sectors of a bio per crypto request
From: Leonid Ravich @ 2026-05-27 6:50 UTC (permalink / raw)
To: Herbert Xu
Cc: David S . Miller, Mike Snitzer, Mikulas Patocka, Alasdair Kergon,
Ard Biesheuvel, Eric Biggers, Jens Axboe, Horia Geanta,
Gilad Ben-Yossef, linux-crypto, dm-devel, linux-block
In-Reply-To: <20260527065021.19525-1-lravich@amazon.com>
When the underlying skcipher driver advertises support for multiple
data units in a single request (CRYPTO_ALG_SKCIPHER_MULTI_DATA_UNIT),
configure the cipher with cc->sector_size as data_unit_size and
submit one request per bio instead of one request per sector. This
removes per-sector overhead in the crypto API hot path: request
allocation, callback dispatch, completion handling, and SG setup.
The optimisation is enabled automatically at table load when all
of the following hold:
- the cipher is non-aead (i.e. skcipher);
- tfms_count is 1 (interleaved per-sector keys would break batching);
- the IV mode is plain or plain64 (the only modes whose generator
produces a sequential 64-bit little-endian counter that the cipher
can extend by adding the data-unit index, matching the convention
documented in crypto_skcipher_set_data_unit_size());
- the iv_gen_ops->post() hook is unset (lmk and tcw use it; both are
already excluded by the IV-mode test, but the explicit check makes
the assumption durable against future IV modes);
- dm-integrity is not stacked (no integrity tag or integrity IV);
- the cipher driver advertises multi-data-unit support.
A new CRYPT_MULTI_DATA_UNIT cipher_flag, set once at construction
time, gates the multi-data-unit path. The existing per-sector path
in crypt_convert_block_skcipher() is unchanged; the new
crypt_convert_block_skcipher_multi() is reached from a small dispatch
in crypt_convert() and shares the same backlog/-EBUSY/-EINPROGRESS
flow control with the per-sector path.
Heap-allocated scatterlists are stashed in dm_crypt_request and freed
in crypt_free_req_skcipher() to avoid races between the synchronous-
success free path and async-completion reuse from the request pool.
On -ENOMEM during scatterlist allocation, the bio is requeued via
BLK_STS_DEV_RESOURCE rather than failed, matching the behaviour of
the existing -ENOMEM path for crypto request allocation.
Verified end-to-end with a byte-equivalence test: encrypted output of
plain64 dm-crypt with the multi-data-unit path matches output of the
single-data-unit path bit-for-bit over a 256 MB device.
Signed-off-by: Leonid Ravich <lravich@amazon.com>
---
drivers/md/dm-crypt.c | 272 ++++++++++++++++++++++++++++++++++++++++--
1 file changed, 265 insertions(+), 7 deletions(-)
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 608b617fb817..e3cc88cf0095 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -101,6 +101,14 @@ struct dm_crypt_request {
struct scatterlist sg_in[4];
struct scatterlist sg_out[4];
u64 iv_sector;
+ /*
+ * Heap-allocated scatterlists used by the multi-data-unit path
+ * when one bio is processed in a single skcipher request. NULL
+ * when the inline sg_in[]/sg_out[] arrays above are sufficient
+ * (single-data-unit path). Freed in crypt_free_req_skcipher().
+ */
+ struct scatterlist *sg_in_ext;
+ struct scatterlist *sg_out_ext;
};
struct crypt_config;
@@ -151,6 +159,7 @@ enum cipher_flags {
CRYPT_IV_LARGE_SECTORS, /* Calculate IV from sector_size, not 512B sectors */
CRYPT_ENCRYPT_PREPROCESS, /* Must preprocess data for encryption (elephant) */
CRYPT_KEY_MAC_SIZE_SET, /* The integrity_key_size option was used */
+ CRYPT_MULTI_DATA_UNIT, /* Batch all sectors of a bio per crypto request */
};
/*
@@ -1426,12 +1435,153 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
return r;
}
+/*
+ * Multi-data-unit variant of crypt_convert_block_skcipher. Submits all
+ * remaining sectors of the current bio in one skcipher request whose
+ * data_unit_size is cc->sector_size. The cipher walks the IV between
+ * data units (see crypto_skcipher_set_data_unit_size()).
+ *
+ * Returns the same set of values as crypt_convert_block_skcipher:
+ * 0 on synchronous success (full chunk processed),
+ * -EINPROGRESS / -EBUSY on asynchronous dispatch,
+ * -EAGAIN if the per-bio scatterlist allocation cannot be made. The
+ * caller MUST disable multi-data-unit batching for the rest
+ * of this bio and re-enter the per-sector path, which uses
+ * only mempool reserves and is therefore safe even on the
+ * swap-out-to-dm-crypt path under total memory exhaustion.
+ * negative errno otherwise.
+ *
+ * On success the bio iterators have been advanced by the chunk size.
+ *
+ * Walks the bio with __bio_for_each_bvec so that multi-page folios
+ * produce one scatterlist entry rather than N (one per PAGE_SIZE).
+ */
+static int crypt_convert_block_skcipher_multi(struct crypt_config *cc,
+ struct convert_context *ctx,
+ struct skcipher_request *req,
+ unsigned int *out_processed)
+{
+ const unsigned int sector_size = cc->sector_size;
+ const gfp_t gfp = GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN;
+ unsigned int total_in = ctx->iter_in.bi_size;
+ unsigned int total_out = ctx->iter_out.bi_size;
+ unsigned int total = min(total_in, total_out);
+ unsigned int n_sectors;
+ unsigned int n_sg_in = 0, n_sg_out = 0;
+ struct dm_crypt_request *dmreq = dmreq_of_req(cc, req);
+ struct scatterlist *sg_in = NULL, *sg_out = NULL;
+ struct bvec_iter iter_in, iter_out;
+ struct bio_vec bv;
+ u8 *iv, *org_iv;
+ int r;
+
+ if (unlikely(total < sector_size))
+ return -EIO;
+ n_sectors = total / sector_size;
+ total = n_sectors * sector_size;
+
+ /*
+ * Walk the bio_vec iterators to count how many SG entries we need
+ * for exactly @total bytes. bi_size of the iterators is at least
+ * @total by construction above.
+ */
+ iter_in = ctx->iter_in;
+ iter_in.bi_size = total;
+ __bio_for_each_bvec(bv, ctx->bio_in, iter_in, iter_in)
+ n_sg_in++;
+
+ iter_out = ctx->iter_out;
+ iter_out.bi_size = total;
+ __bio_for_each_bvec(bv, ctx->bio_out, iter_out, iter_out)
+ n_sg_out++;
+
+ sg_in = kmalloc_array(n_sg_in, sizeof(*sg_in), gfp);
+ sg_out = (ctx->bio_in == ctx->bio_out) ? sg_in :
+ kmalloc_array(n_sg_out, sizeof(*sg_out), gfp);
+ if (!sg_in || !sg_out) {
+ /*
+ * Allocation may legitimately fail under memory pressure on
+ * the swap-out-to-dm-crypt path. Return -EAGAIN so the
+ * caller falls back to the per-sector path for this bio
+ * rather than looping forever in the allocator or requeueing
+ * the bio just to fail again.
+ */
+ kfree(sg_in);
+ if (sg_out != sg_in)
+ kfree(sg_out);
+ return -EAGAIN;
+ }
+
+ sg_init_table(sg_in, n_sg_in);
+ {
+ unsigned int i = 0;
+
+ iter_in = ctx->iter_in;
+ iter_in.bi_size = total;
+ __bio_for_each_bvec(bv, ctx->bio_in, iter_in, iter_in)
+ sg_set_page(&sg_in[i++], bv.bv_page, bv.bv_len,
+ bv.bv_offset);
+ }
+
+ if (sg_out != sg_in) {
+ unsigned int i = 0;
+
+ sg_init_table(sg_out, n_sg_out);
+ iter_out = ctx->iter_out;
+ iter_out.bi_size = total;
+ __bio_for_each_bvec(bv, ctx->bio_out, iter_out, iter_out)
+ sg_set_page(&sg_out[i++], bv.bv_page, bv.bv_len,
+ bv.bv_offset);
+ }
+
+ /*
+ * Compute the IV for the first data unit. The cipher will derive
+ * IVs for subsequent data units by treating this one as a 128-bit
+ * little-endian counter and adding the data-unit index, which
+ * matches the layout produced by plain and plain64.
+ */
+ dmreq->iv_sector = ctx->cc_sector;
+ if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
+ dmreq->iv_sector >>= cc->sector_shift;
+ dmreq->ctx = ctx;
+
+ iv = iv_of_dmreq(cc, dmreq);
+ org_iv = org_iv_of_dmreq(cc, dmreq);
+ r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
+ if (r < 0)
+ goto out_free_sg;
+ memcpy(iv, org_iv, cc->iv_size);
+
+ /* Stash the SG arrays for cleanup on completion / free. */
+ dmreq->sg_in_ext = sg_in;
+ dmreq->sg_out_ext = (sg_out == sg_in) ? NULL : sg_out;
+
+ skcipher_request_set_crypt(req, sg_in, sg_out, total, iv);
+
+ if (bio_data_dir(ctx->bio_in) == WRITE)
+ r = crypto_skcipher_encrypt(req);
+ else
+ r = crypto_skcipher_decrypt(req);
+
+ *out_processed = total;
+ return r;
+
+out_free_sg:
+ kfree(sg_in);
+ if (sg_out != sg_in)
+ kfree(sg_out);
+ dmreq->sg_in_ext = NULL;
+ dmreq->sg_out_ext = NULL;
+ return r;
+}
+
static void kcryptd_async_done(void *async_req, int error);
static int crypt_alloc_req_skcipher(struct crypt_config *cc,
struct convert_context *ctx)
{
unsigned int key_index = ctx->cc_sector & (cc->tfms_count - 1);
+ struct dm_crypt_request *dmreq;
if (!ctx->r.req) {
ctx->r.req = mempool_alloc(&cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO);
@@ -1441,6 +1591,18 @@ static int crypt_alloc_req_skcipher(struct crypt_config *cc,
skcipher_request_set_tfm(ctx->r.req, cc->cipher_tfm.tfms[key_index]);
+ /*
+ * Initialise the heap-allocated scatterlist pointers so that
+ * crypt_free_req_skcipher() does not read uninitialised memory
+ * for paths that don't take the multi-data-unit branch. The
+ * dmreq trailer lives in the per-bio data area which is not
+ * zeroed by the dm core, and the request is reused from the
+ * mempool across many bios.
+ */
+ dmreq = dmreq_of_req(cc, ctx->r.req);
+ dmreq->sg_in_ext = NULL;
+ dmreq->sg_out_ext = NULL;
+
/*
* Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
* requests if driver request queue is full.
@@ -1487,6 +1649,12 @@ static void crypt_free_req_skcipher(struct crypt_config *cc,
struct skcipher_request *req, struct bio *base_bio)
{
struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
+ struct dm_crypt_request *dmreq = dmreq_of_req(cc, req);
+
+ kfree(dmreq->sg_in_ext);
+ dmreq->sg_in_ext = NULL;
+ kfree(dmreq->sg_out_ext);
+ dmreq->sg_out_ext = NULL;
if ((struct skcipher_request *)(io + 1) != req)
mempool_free(req, &cc->req_pool);
@@ -1515,7 +1683,9 @@ static void crypt_free_req(struct crypt_config *cc, void *req, struct bio *base_
static blk_status_t crypt_convert(struct crypt_config *cc,
struct convert_context *ctx, bool atomic, bool reset_pending)
{
- unsigned int sector_step = cc->sector_size >> SECTOR_SHIFT;
+ const unsigned int sector_step = cc->sector_size >> SECTOR_SHIFT;
+ bool multi_du = test_bit(CRYPT_MULTI_DATA_UNIT, &cc->cipher_flags);
+ unsigned int processed;
int r;
/*
@@ -1536,8 +1706,13 @@ static blk_status_t crypt_convert(struct crypt_config *cc,
atomic_inc(&ctx->cc_pending);
+ processed = cc->sector_size;
if (crypt_integrity_aead(cc))
r = crypt_convert_block_aead(cc, ctx, ctx->r.req_aead, ctx->tag_offset);
+ else if (multi_du)
+ r = crypt_convert_block_skcipher_multi(cc, ctx,
+ ctx->r.req,
+ &processed);
else
r = crypt_convert_block_skcipher(cc, ctx, ctx->r.req, ctx->tag_offset);
@@ -1559,8 +1734,19 @@ static blk_status_t crypt_convert(struct crypt_config *cc,
* exit and continue processing in a workqueue
*/
ctx->r.req = NULL;
- ctx->tag_offset++;
- ctx->cc_sector += sector_step;
+ if (!multi_du) {
+ ctx->tag_offset++;
+ ctx->cc_sector += sector_step;
+ } else {
+ bio_advance_iter(ctx->bio_in,
+ &ctx->iter_in,
+ processed);
+ bio_advance_iter(ctx->bio_out,
+ &ctx->iter_out,
+ processed);
+ ctx->cc_sector +=
+ processed >> SECTOR_SHIFT;
+ }
return BLK_STS_DEV_RESOURCE;
}
} else {
@@ -1574,19 +1760,52 @@ static blk_status_t crypt_convert(struct crypt_config *cc,
*/
case -EINPROGRESS:
ctx->r.req = NULL;
- ctx->tag_offset++;
- ctx->cc_sector += sector_step;
+ if (!multi_du) {
+ ctx->tag_offset++;
+ ctx->cc_sector += sector_step;
+ } else {
+ bio_advance_iter(ctx->bio_in, &ctx->iter_in,
+ processed);
+ bio_advance_iter(ctx->bio_out, &ctx->iter_out,
+ processed);
+ ctx->cc_sector += processed >> SECTOR_SHIFT;
+ }
continue;
/*
* The request was already processed (synchronously).
*/
case 0:
atomic_dec(&ctx->cc_pending);
- ctx->cc_sector += sector_step;
- ctx->tag_offset++;
+ if (!multi_du) {
+ ctx->cc_sector += sector_step;
+ ctx->tag_offset++;
+ } else {
+ bio_advance_iter(ctx->bio_in, &ctx->iter_in,
+ processed);
+ bio_advance_iter(ctx->bio_out, &ctx->iter_out,
+ processed);
+ ctx->cc_sector += processed >> SECTOR_SHIFT;
+ }
if (!atomic)
cond_resched();
continue;
+ /*
+ * Multi-data-unit scatterlist allocation failed. This can
+ * happen on the swap-out-to-dm-crypt path under memory
+ * pressure, where retrying with the same allocation policy
+ * could loop forever. Disable multi-data-unit batching for
+ * the rest of this crypt_convert() invocation and re-enter
+ * the per-sector path, which uses only mempool reserves and
+ * is guaranteed to make forward progress even under total
+ * memory exhaustion. The per-tfm data_unit_size is left
+ * unchanged, so subsequent bios (which start a fresh
+ * crypt_convert() and re-read cipher_flags) will retry the
+ * multi-data-unit path once memory pressure eases.
+ */
+ case -EAGAIN:
+ atomic_dec(&ctx->cc_pending);
+ multi_du = false;
+ continue;
/*
* There was a data integrity error.
*/
@@ -3063,6 +3282,45 @@ static int crypt_ctr_cipher(struct dm_target *ti, char *cipher_in, char *key)
}
}
+ /*
+ * Enable multi-data-unit batching when the cipher supports it and
+ * the IV layout is one we can derive per-DU from a single starting
+ * IV: plain or plain64 produce a sequential 64-bit little-endian
+ * counter, which matches the convention of
+ * crypto_skcipher_set_data_unit_size(). Restrict to the simple
+ * case (single tfm, no integrity, no per-sector post() callback)
+ * to keep the consumer path small; modes like essiv, lmk, tcw,
+ * eboiv, plain64be, random, null, benbi, and elephant are
+ * deliberately excluded because their generators or post-IV hooks
+ * cannot be re-derived by the cipher between data units.
+ */
+ if (!crypt_integrity_aead(cc) && cc->tfms_count == 1 &&
+ cc->iv_gen_ops &&
+ (cc->iv_gen_ops == &crypt_iv_plain_ops ||
+ cc->iv_gen_ops == &crypt_iv_plain64_ops) &&
+ !cc->iv_gen_ops->post &&
+ !cc->integrity_tag_size && !cc->integrity_iv_size &&
+ crypto_skcipher_supports_multi_data_unit(cc->cipher_tfm.tfms[0])) {
+ ret = crypto_skcipher_set_data_unit_size(cc->cipher_tfm.tfms[0],
+ cc->sector_size);
+ if (!ret) {
+ set_bit(CRYPT_MULTI_DATA_UNIT, &cc->cipher_flags);
+ DMINFO("Using multi-data-unit crypto offload (du=%u)",
+ cc->sector_size);
+ } else {
+ /*
+ * The driver advertised the capability via cra_flags
+ * but rejected the requested data unit size. This is
+ * a driver bug worth seeing in dmesg; fall back to
+ * the per-sector path so the device still activates.
+ */
+ DMWARN_LIMIT("multi-DU offload disabled: %s rejected du=%u (%d)",
+ crypto_skcipher_driver_name(cc->cipher_tfm.tfms[0]),
+ cc->sector_size, ret);
+ ret = 0;
+ }
+ }
+
/* wipe the kernel key payload copy */
if (cc->key_string)
memset(cc->key, 0, cc->key_size * sizeof(u8));
--
2.47.3
^ permalink raw reply related
* [PATCH v2 3/4] crypto: testmgr - exercise multi-data-unit path for skcipher
From: Leonid Ravich @ 2026-05-27 6:50 UTC (permalink / raw)
To: Herbert Xu
Cc: David S . Miller, Mike Snitzer, Mikulas Patocka, Alasdair Kergon,
Ard Biesheuvel, Eric Biggers, Jens Axboe, Horia Geanta,
Gilad Ben-Yossef, linux-crypto, dm-devel, linux-block
In-Reply-To: <20260527065021.19525-1-lravich@amazon.com>
Add a self-comparison test that runs whenever an skcipher algorithm
advertises CRYPTO_ALG_SKCIPHER_MULTI_DATA_UNIT in cra_flags. The test
encrypts the same random plaintext two ways:
1. as one batched request with data_unit_size set, and
2. as N back-to-back single-data-unit requests with IVs derived from
the original IV by adding the data-unit index (treated as a
128-bit little-endian counter, matching the convention documented
in crypto_skcipher_set_data_unit_size()).
Both encrypts must produce byte-identical ciphertext, otherwise the
algorithm's multi-DU implementation is inconsistent with its single-DU
behaviour. Iterates over a fixed set of typical data unit sizes
(512, 1024, 2048, 4096) which cover the dm-crypt sector-size range.
The test is gated on ivsize == 16 (XTS, the only multi-DU consumer in
the kernel today) and on the algorithm advertising the capability,
so it costs nothing for the existing fleet of skcipher drivers.
Signed-off-by: Leonid Ravich <lravich@amazon.com>
---
crypto/testmgr.c | 129 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 129 insertions(+)
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 4d86efae65b2..8ca92ee6b37c 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -3211,6 +3211,123 @@ static int test_skcipher(int enc, const struct cipher_test_suite *suite,
return 0;
}
+/*
+ * For algorithms that advertise CRYPTO_ALG_SKCIPHER_MULTI_DATA_UNIT,
+ * verify that one request batching N data units produces the same
+ * ciphertext as N back-to-back single-data-unit requests with IVs
+ * derived from the original IV by adding the data-unit index (treated
+ * as a 128-bit little-endian counter).
+ *
+ * This is a self-comparison: it does not depend on test-vector
+ * authoritativeness, only on the algorithm being internally consistent
+ * between its single-DU and multi-DU paths.
+ */
+#define TEST_MDU_NR_UNITS 4
+static int test_skcipher_multi_du(struct crypto_skcipher *tfm,
+ unsigned int du_size)
+{
+ const char *driver = crypto_skcipher_driver_name(tfm);
+ const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
+ const unsigned int total = du_size * TEST_MDU_NR_UNITS;
+ struct skcipher_request *req = NULL;
+ struct scatterlist sg_in, sg_out;
+ DECLARE_CRYPTO_WAIT(wait);
+ u8 iv_orig[16] = {0};
+ u8 iv_work[16];
+ u8 *plain = NULL, *batched = NULL, *unit = NULL;
+ unsigned int i;
+ int err;
+
+ if (ivsize != 16)
+ return 0;
+
+ plain = kmalloc(total, GFP_KERNEL);
+ batched = kmalloc(total, GFP_KERNEL);
+ unit = kmalloc(total, GFP_KERNEL);
+ req = skcipher_request_alloc(tfm, GFP_KERNEL);
+ if (!plain || !batched || !unit || !req) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ get_random_bytes(plain, total);
+ get_random_bytes(iv_orig, ivsize);
+
+ /* Pass 1: one batched encrypt with data_unit_size set. */
+ err = crypto_skcipher_set_data_unit_size(tfm, du_size);
+ if (err) {
+ pr_err("alg: skcipher: %s set_data_unit_size(%u) failed: %d\n",
+ driver, du_size, err);
+ goto out;
+ }
+ memcpy(batched, plain, total);
+ memcpy(iv_work, iv_orig, ivsize);
+ sg_init_one(&sg_in, batched, total);
+ sg_out = sg_in;
+ skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG |
+ CRYPTO_TFM_REQ_MAY_SLEEP,
+ crypto_req_done, &wait);
+ skcipher_request_set_crypt(req, &sg_in, &sg_out, total, iv_work);
+ err = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
+ if (err) {
+ pr_err("alg: skcipher: %s multi-DU batched encrypt failed: %d\n",
+ driver, err);
+ goto out_clear_du;
+ }
+
+ /* Pass 2: TEST_MDU_NR_UNITS single-DU encrypts with derived IVs. */
+ err = crypto_skcipher_set_data_unit_size(tfm, 0);
+ if (err)
+ goto out;
+ memcpy(unit, plain, total);
+ memcpy(iv_work, iv_orig, ivsize);
+ for (i = 0; i < TEST_MDU_NR_UNITS; i++) {
+ sg_init_one(&sg_in, unit + i * du_size, du_size);
+ sg_out = sg_in;
+ skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG |
+ CRYPTO_TFM_REQ_MAY_SLEEP,
+ crypto_req_done, &wait);
+ skcipher_request_set_crypt(req, &sg_in, &sg_out, du_size,
+ iv_work);
+ err = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
+ if (err) {
+ pr_err("alg: skcipher: %s single-DU[%u] encrypt failed: %d\n",
+ driver, i, err);
+ goto out;
+ }
+ /* Increment iv_work as a 128-bit little-endian counter. */
+ {
+ __le64 lo_le, hi_le;
+ u64 lo;
+
+ memcpy(&lo_le, iv_work, 8);
+ memcpy(&hi_le, iv_work + 8, 8);
+ lo = le64_to_cpu(lo_le) + 1;
+ lo_le = cpu_to_le64(lo);
+ memcpy(iv_work, &lo_le, 8);
+ if (lo == 0) {
+ hi_le = cpu_to_le64(le64_to_cpu(hi_le) + 1);
+ memcpy(iv_work + 8, &hi_le, 8);
+ }
+ }
+ }
+
+ if (memcmp(batched, unit, total) != 0) {
+ pr_err("alg: skcipher: %s multi-DU mismatch (du=%u, n=%u)\n",
+ driver, du_size, TEST_MDU_NR_UNITS);
+ err = -EINVAL;
+ }
+
+out_clear_du:
+ (void)crypto_skcipher_set_data_unit_size(tfm, 0);
+out:
+ skcipher_request_free(req);
+ kfree(unit);
+ kfree(batched);
+ kfree(plain);
+ return err;
+}
+
static int alg_test_skcipher(const struct alg_test_desc *desc,
const char *driver, u32 type, u32 mask)
{
@@ -3259,6 +3376,18 @@ static int alg_test_skcipher(const struct alg_test_desc *desc,
if (err)
goto out;
+ if (crypto_skcipher_supports_multi_data_unit(tfm)) {
+ static const unsigned int du_sizes[] = { 512, 1024, 2048, 4096 };
+ unsigned int j;
+
+ for (j = 0; j < ARRAY_SIZE(du_sizes); j++) {
+ err = test_skcipher_multi_du(tfm, du_sizes[j]);
+ if (err)
+ goto out;
+ cond_resched();
+ }
+ }
+
err = test_skcipher_vs_generic_impl(desc->generic_driver, req, tsgls);
out:
free_cipher_test_sglists(tsgls);
--
2.47.3
^ permalink raw reply related
* [PATCH v2 2/4] crypto: xts - support multiple data units per request in template
From: Leonid Ravich @ 2026-05-27 6:50 UTC (permalink / raw)
To: Herbert Xu
Cc: David S . Miller, Mike Snitzer, Mikulas Patocka, Alasdair Kergon,
Ard Biesheuvel, Eric Biggers, Jens Axboe, Horia Geanta,
Gilad Ben-Yossef, linux-crypto, dm-devel, linux-block
In-Reply-To: <20260527065021.19525-1-lravich@amazon.com>
Teach the generic xts() template to consume cryptlen larger than one
data unit when the caller has configured a non-zero data_unit_size on
the tfm. Each data unit is processed with its own IV, derived from
the caller-supplied IV by treating it as a 128-bit little-endian
counter and adding the data-unit index. This matches the
sector-indexed XTS used by dm-crypt's plain64 IV mode and by typical
inline-encryption hardware.
The single-data-unit body is unchanged and is now reached via a thin
xts_crypt_multi() dispatcher that skips straight to the body when
data_unit_size is zero (the legacy default), so existing users see
no extra cost.
Advertise CRYPTO_ALG_SKCIPHER_MULTI_DATA_UNIT in cra_flags only when
the inner cipher is synchronous. An async inner cipher would require
a per-DU completion chain which is out of scope for the slow software
template; consumers that need multi-DU on async hardware will use one
of the arch-specific drivers added later in this series.
Signed-off-by: Leonid Ravich <lravich@amazon.com>
---
crypto/xts.c | 25 +++++++++++++++++++++++--
1 file changed, 23 insertions(+), 2 deletions(-)
diff --git a/crypto/xts.c b/crypto/xts.c
index ad97c8091582..f0585ea9d6d5 100644
--- a/crypto/xts.c
+++ b/crypto/xts.c
@@ -258,7 +258,7 @@ static int xts_init_crypt(struct skcipher_request *req,
return 0;
}
-static int xts_encrypt(struct skcipher_request *req)
+static int xts_encrypt_one(struct skcipher_request *req)
{
struct xts_request_ctx *rctx = skcipher_request_ctx(req);
struct skcipher_request *subreq = &rctx->subreq;
@@ -275,7 +275,7 @@ static int xts_encrypt(struct skcipher_request *req)
return xts_cts_final(req, crypto_skcipher_encrypt);
}
-static int xts_decrypt(struct skcipher_request *req)
+static int xts_decrypt_one(struct skcipher_request *req)
{
struct xts_request_ctx *rctx = skcipher_request_ctx(req);
struct skcipher_request *subreq = &rctx->subreq;
@@ -292,6 +292,16 @@ static int xts_decrypt(struct skcipher_request *req)
return xts_cts_final(req, crypto_skcipher_decrypt);
}
+static int xts_encrypt(struct skcipher_request *req)
+{
+ return skcipher_walk_data_units(req, xts_encrypt_one);
+}
+
+static int xts_decrypt(struct skcipher_request *req)
+{
+ return skcipher_walk_data_units(req, xts_decrypt_one);
+}
+
static int xts_init_tfm(struct crypto_skcipher *tfm)
{
struct skcipher_instance *inst = skcipher_alg_instance(tfm);
@@ -427,6 +437,17 @@ static int xts_create(struct crypto_template *tmpl, struct rtattr **tb)
inst->alg.base.cra_alignmask = alg->base.cra_alignmask |
(__alignof__(u64) - 1);
+ /*
+ * Advertise multi-data-unit support only when the inner cipher is
+ * synchronous. The dispatcher in skcipher_walk_data_units() calls
+ * the single-DU body in a loop and assumes synchronous completion;
+ * supporting async would require a per-DU callback chain, which
+ * the slow software template does not need.
+ */
+ if (!(alg->base.cra_flags & CRYPTO_ALG_ASYNC))
+ inst->alg.base.cra_flags |=
+ CRYPTO_ALG_SKCIPHER_MULTI_DATA_UNIT;
+
inst->alg.ivsize = XTS_BLOCK_SIZE;
inst->alg.min_keysize = alg->min_keysize * 2;
inst->alg.max_keysize = alg->max_keysize * 2;
--
2.47.3
^ permalink raw reply related
* [PATCH v2 1/4] crypto: skcipher - add per-tfm data_unit_size for batched requests
From: Leonid Ravich @ 2026-05-27 6:50 UTC (permalink / raw)
To: Herbert Xu
Cc: David S . Miller, Mike Snitzer, Mikulas Patocka, Alasdair Kergon,
Ard Biesheuvel, Eric Biggers, Jens Axboe, Horia Geanta,
Gilad Ben-Yossef, linux-crypto, dm-devel, linux-block
In-Reply-To: <20260527065021.19525-1-lravich@amazon.com>
Add a per-tfm data_unit_size and an algorithm capability flag that
together allow a caller to submit several data units in a single
skcipher request. The IV passed in the request applies to the first
data unit; the algorithm advances the tweak between data units
according to the mode specification (e.g., LE128 multiply for XTS per
IEEE 1619).
This mirrors the data_unit_size concept already exposed by
struct blk_crypto_config for inline encryption hardware, but at the
software skcipher layer. The first user is dm-crypt, which today
issues one request per sector and so pays a per-sector cost in
request allocation, IV generation, callback dispatch, and completion
handling. Allowing the cipher to consume a whole bio per request
removes that overhead for drivers that can chain across data units
internally.
The data_unit_size lives on struct crypto_skcipher rather than on
struct skcipher_request because it does not change between requests
for any plausible consumer: dm-crypt picks one sector size per
mapped target at table load time; fscrypt would pick one per master
key. Anchoring it to the tfm also lets the driver validate it once
at setkey() time and avoids per-request initialisation hazards on
mempool-recycled requests.
Capability is advertised with CRYPTO_ALG_SKCIPHER_MULTI_DATA_UNIT
in cra_flags (type-specific high-byte range, mirroring the
CRYPTO_AHASH_ALG_* convention). This makes the capability visible
in /proc/crypto and lets templates OR it into their derived
algorithms.
crypto_skcipher_set_data_unit_size() returns -EOPNOTSUPP if the
algorithm does not advertise the flag, and accepts 0 (the default)
unconditionally so callers can re-disable batching cheaply.
crypto_skcipher_encrypt()/decrypt() reject requests whose cryptlen
is not a multiple of the configured data_unit_size with -EINVAL.
The check is gated on data_unit_size != 0 so it costs nothing for
the common single-data-unit case.
No in-tree algorithm advertises the flag yet; subsequent patches
add the generic xts() template, arm64, and x86 producers as well
as the dm-crypt consumer.
Signed-off-by: Leonid Ravich <lravich@amazon.com>
---
crypto/skcipher.c | 120 +++++++++++++++++++++++++++++
include/crypto/internal/skcipher.h | 34 ++++++++
include/crypto/skcipher.h | 85 ++++++++++++++++++++
3 files changed, 239 insertions(+)
diff --git a/crypto/skcipher.c b/crypto/skcipher.c
index 2b31d1d5d268..bc37bd554aec 100644
--- a/crypto/skcipher.c
+++ b/crypto/skcipher.c
@@ -432,13 +432,119 @@ int crypto_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
}
EXPORT_SYMBOL_GPL(crypto_skcipher_setkey);
+int crypto_skcipher_set_data_unit_size(struct crypto_skcipher *tfm,
+ unsigned int data_unit_size)
+{
+ unsigned int blocksize;
+
+ if (!data_unit_size) {
+ tfm->data_unit_size = 0;
+ return 0;
+ }
+
+ if (!crypto_skcipher_supports_multi_data_unit(tfm))
+ return -EOPNOTSUPP;
+
+ blocksize = crypto_skcipher_blocksize(tfm);
+ if (data_unit_size < blocksize || data_unit_size % blocksize)
+ return -EINVAL;
+
+ tfm->data_unit_size = data_unit_size;
+ return 0;
+}
+EXPORT_SYMBOL_GPL(crypto_skcipher_set_data_unit_size);
+
+static int crypto_skcipher_check_data_unit_size(struct crypto_skcipher *tfm,
+ struct skcipher_request *req)
+{
+ unsigned int du = tfm->data_unit_size;
+
+ if (likely(!du))
+ return 0;
+ if (req->cryptlen % du)
+ return -EINVAL;
+ return 0;
+}
+
+/*
+ * Increment a 16-byte little-endian counter held in @iv. See
+ * crypto_skcipher_set_data_unit_size() for the convention.
+ */
+static inline void skcipher_iv_inc_le128(u8 *iv)
+{
+ __le64 lo_le, hi_le;
+ u64 lo;
+
+ memcpy(&lo_le, iv, 8);
+ memcpy(&hi_le, iv + 8, 8);
+ lo = le64_to_cpu(lo_le) + 1;
+ lo_le = cpu_to_le64(lo);
+ memcpy(iv, &lo_le, 8);
+ if (unlikely(lo == 0)) {
+ hi_le = cpu_to_le64(le64_to_cpu(hi_le) + 1);
+ memcpy(iv + 8, &hi_le, 8);
+ }
+}
+
+int skcipher_walk_data_units(struct skcipher_request *req,
+ int (*body)(struct skcipher_request *))
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ const unsigned int du = tfm->data_unit_size;
+ const unsigned int total = req->cryptlen;
+ struct scatterlist *orig_src = req->src;
+ struct scatterlist *orig_dst = req->dst;
+ struct scatterlist src_sg[2], dst_sg[2];
+ u8 iv_save[16];
+ unsigned int off;
+ int err = 0;
+
+ if (likely(!du))
+ return body(req);
+
+ /*
+ * Registration of an algorithm advertising
+ * CRYPTO_ALG_SKCIPHER_MULTI_DATA_UNIT enforces ivsize == 16
+ * (see skcipher_prepare_alg_common()), so this is purely
+ * defensive against algorithm-registration bugs.
+ */
+ if (WARN_ON_ONCE(crypto_skcipher_ivsize(tfm) != 16))
+ return -EINVAL;
+
+ memcpy(iv_save, req->iv, 16);
+
+ for (off = 0; off < total; off += du) {
+ req->cryptlen = du;
+ req->src = scatterwalk_ffwd(src_sg, orig_src, off);
+ req->dst = (orig_src == orig_dst) ? req->src :
+ scatterwalk_ffwd(dst_sg, orig_dst, off);
+
+ err = body(req);
+ if (err)
+ break;
+
+ skcipher_iv_inc_le128(iv_save);
+ memcpy(req->iv, iv_save, 16);
+ }
+
+ req->src = orig_src;
+ req->dst = orig_dst;
+ req->cryptlen = total;
+ return err;
+}
+EXPORT_SYMBOL_GPL(skcipher_walk_data_units);
+
int crypto_skcipher_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
+ int err;
if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
return -ENOKEY;
+ err = crypto_skcipher_check_data_unit_size(tfm, req);
+ if (err)
+ return err;
if (alg->co.base.cra_type != &crypto_skcipher_type)
return crypto_lskcipher_encrypt_sg(req);
return alg->encrypt(req);
@@ -449,9 +555,13 @@ int crypto_skcipher_decrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
+ int err;
if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
return -ENOKEY;
+ err = crypto_skcipher_check_data_unit_size(tfm, req);
+ if (err)
+ return err;
if (alg->co.base.cra_type != &crypto_skcipher_type)
return crypto_lskcipher_decrypt_sg(req);
return alg->decrypt(req);
@@ -680,6 +790,16 @@ int skcipher_prepare_alg_common(struct skcipher_alg_common *alg)
(alg->ivsize + alg->statesize) > PAGE_SIZE / 2)
return -EINVAL;
+ /*
+ * Algorithms advertising multi-data-unit support must use the
+ * 16-byte little-endian counter convention documented in
+ * crypto_skcipher_set_data_unit_size(); see also
+ * skcipher_walk_data_units().
+ */
+ if ((base->cra_flags & CRYPTO_ALG_SKCIPHER_MULTI_DATA_UNIT) &&
+ alg->ivsize != 16)
+ return -EINVAL;
+
if (!alg->chunksize)
alg->chunksize = base->cra_blocksize;
diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h
index a965b6aabf61..bed1b1f1bbdc 100644
--- a/include/crypto/internal/skcipher.h
+++ b/include/crypto/internal/skcipher.h
@@ -21,6 +21,40 @@
*/
#define CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE CRYPTO_ALG_OPTIONAL_KEY
+/**
+ * skcipher_walk_data_units - dispatch a request as one body call per data unit
+ * @req: the caller's skcipher request
+ * @body: the algorithm's single-data-unit encrypt or decrypt function
+ *
+ * When tfm->data_unit_size is zero this is a tail call into @body with
+ * @req unchanged. Otherwise the request is split into
+ * cryptlen / data_unit_size sub-ranges and @body is called once per
+ * sub-range with req->cryptlen, req->src, req->dst, and req->iv adjusted
+ * for that sub-range. The IV passed to data unit n is the caller-
+ * supplied IV plus n, where + is a 128-bit little-endian add — this
+ * matches the convention documented in
+ * crypto_skcipher_set_data_unit_size().
+ *
+ * Many single-data-unit XTS bodies modify the IV buffer in place during
+ * processing (the tweak is walked block by block). This helper saves
+ * the caller's IV before each call and rewrites the next data unit's
+ * IV from the saved value, so the body always sees a fresh per-DU IV
+ * regardless of any in-place mutation it performs.
+ *
+ * The body MUST run to completion synchronously. Drivers that use this
+ * helper therefore advertise CRYPTO_ALG_SKCIPHER_MULTI_DATA_UNIT only
+ * for synchronous configurations.
+ *
+ * After the call returns, the contents of req->iv are unspecified per
+ * the documented contract. src/dst/cryptlen are restored to the
+ * caller's values to keep skcipher request post-conditions intact.
+ *
+ * Return: 0 on success, or the body's negative errno on the first
+ * data unit that returned non-zero.
+ */
+int skcipher_walk_data_units(struct skcipher_request *req,
+ int (*body)(struct skcipher_request *));
+
struct aead_request;
struct rtattr;
diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h
index 4efe2ca8c4d1..5941b6b24b98 100644
--- a/include/crypto/skcipher.h
+++ b/include/crypto/skcipher.h
@@ -26,6 +26,15 @@
/* Set this bit if the skcipher operation is not final. */
#define CRYPTO_SKCIPHER_REQ_NOTFINAL 0x00000002
+/*
+ * Set in cra_flags by an skcipher algorithm that supports processing
+ * multiple data units in a single request. See
+ * crypto_skcipher_set_data_unit_size().
+ *
+ * Type-specific flag in the 0xff000000 reserved range.
+ */
+#define CRYPTO_ALG_SKCIPHER_MULTI_DATA_UNIT 0x01000000
+
struct scatterlist;
/**
@@ -53,6 +62,22 @@ struct skcipher_request {
struct crypto_skcipher {
unsigned int reqsize;
+ /*
+ * Number of bytes in one data unit when batching multiple data units
+ * per request. 0 means "single data unit per request" (legacy
+ * behaviour). Set via crypto_skcipher_set_data_unit_size().
+ *
+ * When non-zero, cryptlen must be a multiple of data_unit_size. The
+ * IV passed in skcipher_request::iv applies to the first data unit;
+ * the algorithm advances the tweak between data units according to
+ * the mode specification (e.g., LE128 multiply for XTS per
+ * IEEE 1619).
+ *
+ * Only algorithms that advertise CRYPTO_ALG_SKCIPHER_MULTI_DATA_UNIT
+ * in cra_flags accept a non-zero value.
+ */
+ unsigned int data_unit_size;
+
struct crypto_tfm base;
};
@@ -492,6 +517,66 @@ static inline unsigned int crypto_lskcipher_chunksize(
return crypto_lskcipher_alg(tfm)->co.chunksize;
}
+/**
+ * crypto_skcipher_supports_multi_data_unit() - test multi-data-unit support
+ * @tfm: cipher handle
+ *
+ * Return: true if the algorithm advertises that it can process multiple
+ * data units in a single skcipher_request.
+ */
+static inline bool
+crypto_skcipher_supports_multi_data_unit(struct crypto_skcipher *tfm)
+{
+ return crypto_skcipher_alg_common(tfm)->base.cra_flags &
+ CRYPTO_ALG_SKCIPHER_MULTI_DATA_UNIT;
+}
+
+/**
+ * crypto_skcipher_set_data_unit_size() - set data unit size for the tfm
+ * @tfm: cipher handle
+ * @data_unit_size: data unit size in bytes; 0 disables multi-data-unit mode
+ *
+ * Configure the tfm to process multiple data units per request. When set
+ * to a non-zero value, every subsequent encrypt/decrypt request must have
+ * cryptlen that is a multiple of @data_unit_size. Each data unit is
+ * processed as if it were a separate request whose IV is derived from the
+ * preceding data unit's IV by the algorithm-specific tweak update rule:
+ * the implementation treats the caller-supplied IV as a 128-bit
+ * little-endian counter and adds the data-unit index for each subsequent
+ * data unit.
+ *
+ * The contents of req->iv after a multi-data-unit request returns are
+ * unspecified — callers MUST NOT rely on it being either the original
+ * value or the final-data-unit value. Set a fresh IV before every
+ * request.
+ *
+ * The algorithm must advertise CRYPTO_ALG_SKCIPHER_MULTI_DATA_UNIT in its
+ * cra_flags. @data_unit_size must be a positive multiple of the
+ * algorithm's cra_blocksize, otherwise -EINVAL is returned.
+ *
+ * Setting @data_unit_size to 0 reverts the tfm to single-data-unit
+ * behaviour and is always permitted.
+ *
+ * Return: 0 on success; -EOPNOTSUPP if the algorithm does not advertise
+ * multi-data-unit support; -EINVAL if @data_unit_size is not a
+ * positive multiple of the cipher block size.
+ */
+int crypto_skcipher_set_data_unit_size(struct crypto_skcipher *tfm,
+ unsigned int data_unit_size);
+
+/**
+ * crypto_skcipher_data_unit_size() - obtain data unit size
+ * @tfm: cipher handle
+ *
+ * Return: configured data unit size in bytes; 0 if multi-data-unit mode
+ * is disabled.
+ */
+static inline unsigned int
+crypto_skcipher_data_unit_size(struct crypto_skcipher *tfm)
+{
+ return tfm->data_unit_size;
+}
+
/**
* crypto_skcipher_statesize() - obtain state size
* @tfm: cipher handle
--
2.47.3
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox