* [PATCH v2 08/19] crypto: cmh - add AES skcipher/aead/cmac
From: Saravanakrishnan Krishnamoorthy @ 2026-07-09 20:30 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260709203037.1884436-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register AES algorithms using the CMH AES core (core ID 0x03):
- skcipher: AES-ECB, AES-CBC, AES-CTR, AES-XTS, AES-CFB
- aead: AES-GCM, AES-CCM
- ahash: AES-CMAC
Supports 128, 192, and 256-bit keys. AEAD algorithms handle
associated data, payload, and authentication tag with correct
encrypt/decrypt separation.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 5 +-
drivers/crypto/cmh/cmh_aes.c | 736 ++++++++++++++++++++
drivers/crypto/cmh/cmh_aes_aead.c | 987 +++++++++++++++++++++++++++
drivers/crypto/cmh/cmh_aes_cmac.c | 537 +++++++++++++++
drivers/crypto/cmh/cmh_main.c | 25 +
drivers/crypto/cmh/include/cmh_aes.h | 24 +
6 files changed, 2313 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_aes.c
create mode 100644 drivers/crypto/cmh/cmh_aes_aead.c
create mode 100644 drivers/crypto/cmh/cmh_aes_cmac.c
create mode 100644 drivers/crypto/cmh/include/cmh_aes.h
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index b3018fbcf211..ced8d1748e6c 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -19,7 +19,10 @@ cmh-y := \
cmh_hmac.o \
cmh_cshake.o \
cmh_kmac.o \
- cmh_sm3.o
+ cmh_sm3.o \
+ cmh_aes.o \
+ cmh_aes_aead.o \
+ cmh_aes_cmac.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_aes.c b/drivers/crypto/cmh/cmh_aes.c
new file mode 100644
index 000000000000..b36295763e33
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_aes.c
@@ -0,0 +1,736 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API AES (skcipher) Driver
+ *
+ * Registers skcipher algorithms with the Linux crypto subsystem:
+ * ecb(aes), cbc(aes), ctr(aes), cfb(aes), xts(aes)
+ *
+ * Uses the CMH AES Core via VCQ commands:
+ * [SYS_CMD_WRITE] + AES_CMD_INIT + [AES_CMD_UPDATE] + AES_CMD_FINAL
+ * + VCQ_CMD_FLUSH
+ *
+ * The AES core requires bidirectional DMA -- both input and output
+ * buffers are mapped and passed in a single AES_CMD_FINAL command.
+ *
+ * Raw-key atomicity: SYS_CMD_WRITE to SYS_REF_TEMP is packed into
+ * the same VCQ as AES commands (see cmh_key.h for details).
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <crypto/xts.h>
+#include <crypto/scatterwalk.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/unaligned.h>
+
+#include "cmh_aes.h"
+#include "cmh_vcq.h"
+#include "cmh_aes_abi.h"
+#include "cmh_sys_abi.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+/* Algorithm Table */
+
+struct cmh_aes_alg_info {
+ u32 aes_mode; /* AES_MODE_* */
+ u32 ivsize; /* bytes (0 for ECB) */
+ u32 min_keysize; /* minimum key bytes */
+ u32 max_keysize; /* maximum key bytes */
+ const char *alg_name; /* Linux crypto name: "ecb(aes)" */
+ const char *drv_name; /* driver name: "cri-cmh-ecb-aes" */
+};
+
+static const struct cmh_aes_alg_info aes_algs[] = {
+ { AES_MODE_ECB, 0, AES_KEYSIZE_128, AES_KEYSIZE_256,
+ "ecb(aes)", "cri-cmh-ecb-aes" },
+ { AES_MODE_CBC, CMH_AES_IV_SIZE, AES_KEYSIZE_128, AES_KEYSIZE_256,
+ "cbc(aes)", "cri-cmh-cbc-aes" },
+ { AES_MODE_CTR, CMH_AES_IV_SIZE, AES_KEYSIZE_128, AES_KEYSIZE_256,
+ "ctr(aes)", "cri-cmh-ctr-aes" },
+ { AES_MODE_CFB, CMH_AES_IV_SIZE, AES_KEYSIZE_128, AES_KEYSIZE_256,
+ "cfb(aes)", "cri-cmh-cfb-aes" },
+ { AES_MODE_XTS, CMH_AES_IV_SIZE, 2 * AES_KEYSIZE_128, 2 * AES_KEYSIZE_256,
+ "xts(aes)", "cri-cmh-xts-aes" },
+};
+
+/* Per-transform context (allocated by crypto framework) */
+
+struct cmh_aes_tfm_ctx {
+ struct cmh_key_ctx key;
+};
+
+/* Per-request context (lives in skcipher_request::__ctx) */
+
+/*
+ * Maximum payload commands:
+ * [SYS_CMD_WRITE] + AES_CMD_INIT + [AES_CMD_UPDATE] + AES_CMD_FINAL
+ * + VCQ_CMD_FLUSH = 5
+ * UPDATE is used for XTS data > 2 blocks (see cmh_aes_crypt).
+ */
+#define CMH_AES_MAX_PAYLOAD 5
+#define CMH_AES_MAX_PACKED (CMH_AES_MAX_PAYLOAD * 2)
+
+struct cmh_aes_reqctx {
+ dma_addr_t in_dma;
+ dma_addr_t out_dma;
+ dma_addr_t iv_dma;
+ dma_addr_t iv2_dma;
+ dma_addr_t key_dma;
+ u8 *in_buf;
+ u8 *out_buf;
+ u8 *iv_buf;
+ u8 *iv2_buf;
+ u32 cryptlen;
+ u32 ivsize;
+ u32 keylen;
+ u32 aes_mode;
+ u32 aes_op;
+ /* CTR counter-wrap split state */
+ u32 ctr_chunk1_len;
+ u32 core_id;
+ s32 target_mbx;
+ u64 key_ref;
+ struct vcq_cmd packed[CMH_AES_MAX_PACKED];
+};
+
+/* VCQ Builders -- AES-specific */
+
+static void vcq_add_aes_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, u64 iv_dma,
+ u32 keylen, u32 ivlen, u32 mode, u32 op,
+ u32 iolen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_INIT);
+ slot->hwc.aes.cmd_init.key = key_ref;
+ slot->hwc.aes.cmd_init.iv = iv_dma;
+ slot->hwc.aes.cmd_init.keylen = keylen;
+ slot->hwc.aes.cmd_init.ivlen = ivlen;
+ slot->hwc.aes.cmd_init.mode = mode;
+ slot->hwc.aes.cmd_init.op = op;
+ slot->hwc.aes.cmd_init.aadlen = 0;
+ slot->hwc.aes.cmd_init.iolen = iolen;
+ slot->hwc.aes.cmd_init.taglen = 0;
+}
+
+static void vcq_add_aes_update(struct vcq_cmd *slot, u32 core_id, u64 input_dma,
+ u64 output_dma, u32 iolen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_UPDATE);
+ slot->hwc.aes.cmd_update.input = input_dma;
+ slot->hwc.aes.cmd_update.output = output_dma;
+ slot->hwc.aes.cmd_update.iolen = iolen;
+}
+
+static void vcq_add_aes_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma,
+ u64 output_dma, u32 iolen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_FINAL);
+ slot->hwc.aes.cmd_final.input = input_dma;
+ slot->hwc.aes.cmd_final.output = output_dma;
+ slot->hwc.aes.cmd_final.iolen = iolen;
+ slot->hwc.aes.cmd_final.tag = 0;
+ slot->hwc.aes.cmd_final.taglen = 0;
+}
+
+/*
+ * We wrap each skcipher_alg with its info pointer in a compound struct,
+ * then use container_of() in cmh_aes_get_info() to recover it.
+ * This is the same pattern used by hash, hmac, cshake, kmac.
+ */
+struct cmh_aes_alg_drv {
+ struct skcipher_alg alg;
+ const struct cmh_aes_alg_info *info;
+};
+
+static bool aes_is_stream_mode(u32 mode)
+{
+ return mode == AES_MODE_CTR || mode == AES_MODE_CFB;
+}
+
+/*
+ * Update req->iv after a successful encrypt/decrypt.
+ *
+ * The Linux skcipher API contract requires that req->iv is updated to
+ * reflect the state needed to continue processing in a chained call:
+ * CBC encrypt: IV <- last ciphertext block
+ * CBC decrypt: IV <- last ciphertext block of the *input*
+ * CTR: IV <- counter incremented by ceil(cryptlen / blocksize)
+ * CFB: IV <- last ciphertext block
+ */
+static void cmh_aes_update_iv(struct skcipher_request *req, u32 mode,
+ u32 op, const u8 *in_buf, const u8 *out_buf)
+{
+ u32 bs = CMH_AES_BLOCK_SIZE;
+ u32 nblocks;
+
+ switch (mode) {
+ case AES_MODE_CBC:
+ if (op == AES_OP_ENCRYPT)
+ memcpy(req->iv, out_buf + req->cryptlen - bs, bs);
+ else
+ memcpy(req->iv, in_buf + req->cryptlen - bs, bs);
+ break;
+ case AES_MODE_CTR:
+ /*
+ * Arithmetic big-endian 128-bit counter increment.
+ * Process from the least-significant byte (index 15)
+ * upward, carrying as needed.
+ */
+ nblocks = DIV_ROUND_UP(req->cryptlen, bs);
+ {
+ u8 *iv = req->iv;
+ int i;
+
+ for (i = bs - 1; i >= 0 && nblocks; i--) {
+ u32 sum = (u32)iv[i] + (nblocks & 0xff);
+
+ iv[i] = (u8)sum;
+ nblocks = (nblocks >> 8) + (sum >> 8);
+ }
+ }
+ break;
+ case AES_MODE_CFB:
+ /*
+ * CFB-128 chains on the last ciphertext block. On encrypt,
+ * that is out_buf; on decrypt, it is in_buf.
+ *
+ * For sub-block requests (cryptlen < 16), there is no
+ * complete ciphertext block to chain, so the IV is left
+ * unchanged -- CFB-128 has no defined chaining semantic
+ * for partial blocks (shift-register CFB-n is a different
+ * mode). Without this guard the pointer arithmetic
+ * underflows and reads before the buffer.
+ */
+ if (req->cryptlen >= bs) {
+ if (op == AES_OP_ENCRYPT)
+ memcpy(req->iv, out_buf + req->cryptlen - bs,
+ bs);
+ else
+ memcpy(req->iv, in_buf + req->cryptlen - bs,
+ bs);
+ }
+ break;
+ default:
+ break;
+ }
+}
+
+/* skcipher Operations */
+
+static const struct cmh_aes_alg_info *
+cmh_aes_get_info(struct crypto_skcipher *tfm)
+{
+ struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
+
+ return container_of(alg, struct cmh_aes_alg_drv, alg)->info;
+}
+
+static int cmh_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+ const struct cmh_aes_alg_info *info = cmh_aes_get_info(tfm);
+
+ if (info->aes_mode == AES_MODE_XTS) {
+ int err;
+
+ /* XTS: double key (32, 48, or 64 bytes) */
+ if (keylen != 2 * AES_KEYSIZE_128 &&
+ keylen != 2 * AES_KEYSIZE_192 &&
+ keylen != 2 * AES_KEYSIZE_256)
+ return -EINVAL;
+ err = xts_verify_key(tfm, key, keylen);
+ if (err)
+ return err;
+ } else {
+ /* Standard: 16, 24, or 32 bytes */
+ if (keylen != AES_KEYSIZE_128 &&
+ keylen != AES_KEYSIZE_192 &&
+ keylen != AES_KEYSIZE_256)
+ return -EINVAL;
+ }
+
+ return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_AES);
+}
+
+static int cmh_aes_init_tfm(struct crypto_skcipher *tfm)
+{
+ struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+
+ memset(tctx, 0, sizeof(*tctx));
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct cmh_aes_reqctx));
+ return 0;
+}
+
+static void cmh_aes_exit_tfm(struct crypto_skcipher *tfm)
+{
+ struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+
+ cmh_key_destroy(&tctx->key);
+}
+
+#define CMH_AES_MAX_CRYPTLEN SZ_32M
+
+/* DMA unmap helper */
+static void cmh_aes_unmap_dma(struct cmh_aes_reqctx *rctx)
+{
+ if (rctx->iv2_buf)
+ cmh_dma_unmap_single(rctx->iv2_dma, rctx->ivsize,
+ DMA_TO_DEVICE);
+ if (rctx->ivsize > 0)
+ cmh_dma_unmap_single(rctx->iv_dma, rctx->ivsize,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen, DMA_FROM_DEVICE);
+ cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen, DMA_TO_DEVICE);
+}
+
+static void cmh_aes_free_bufs(struct cmh_aes_reqctx *rctx)
+{
+ kfree(rctx->iv2_buf);
+ rctx->iv2_buf = NULL;
+ kfree(rctx->iv_buf);
+ rctx->iv_buf = NULL;
+ kfree_sensitive(rctx->out_buf);
+ rctx->out_buf = NULL;
+ kfree_sensitive(rctx->in_buf);
+ rctx->in_buf = NULL;
+}
+
+/*
+ * Submit the second CTR chunk after the first completes.
+ * Called from cmh_aes_complete when ctr_chunk1_len > 0.
+ */
+static void cmh_aes_complete(void *data, int error);
+
+static int cmh_aes_ctr_submit_chunk2(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+ struct cmh_aes_reqctx *rctx = skcipher_request_ctx(req);
+ struct vcq_cmd cmds[CMH_AES_MAX_PAYLOAD];
+ u32 chunk1 = rctx->ctr_chunk1_len;
+ u32 chunk2 = rctx->cryptlen - chunk1;
+ u64 key_ref;
+ u32 keylen;
+ u32 idx = 0;
+
+ /* Clear split flag so next completion is final */
+ rctx->ctr_chunk1_len = 0;
+
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP,
+ (u64)rctx->key_dma, SYS_REF_NONE,
+ tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ keylen = tctx->key.raw.len;
+
+ vcq_add_aes_init(&cmds[idx++], rctx->core_id, key_ref,
+ (u64)rctx->iv2_dma, keylen, rctx->ivsize,
+ rctx->aes_mode, rctx->aes_op, 0);
+ vcq_add_aes_final(&cmds[idx++], rctx->core_id,
+ (u64)(rctx->in_dma + chunk1),
+ (u64)(rctx->out_dma + chunk1), chunk2);
+ vcq_add_flush(&cmds[idx++], rctx->core_id);
+
+ return cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_AES_MAX_PACKED,
+ rctx->target_mbx,
+ cmh_aes_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+}
+
+/*
+ * Async completion callback -- fires from RH threaded IRQ context.
+ *
+ * Unmaps DMA buffers, copies output to req->dst scatterlist,
+ * updates the IV state, frees temporaries, and completes the request.
+ *
+ * For CTR counter-wrap splits, the first chunk completion chains
+ * into a second VCQ submission rather than finalizing immediately.
+ */
+static void cmh_aes_complete(void *data, int error)
+{
+ struct skcipher_request *req = data;
+ struct cmh_aes_reqctx *rctx = skcipher_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ /*
+ * CTR counter-wrap: first chunk completed, submit second.
+ * DMA mappings remain valid (they cover the full buffer).
+ *
+ * Recursion depth bounded: chunk2 clears ctr_chunk1_len before
+ * submission, so the second cmh_aes_complete invocation sees 0
+ * and finalizes (max depth = 2).
+ */
+ if (rctx->ctr_chunk1_len && !error) {
+ int ret;
+
+ ret = cmh_aes_ctr_submit_chunk2(req);
+
+ if (!ret || ret == -EBUSY)
+ return;
+ /* Submission failed; clean up below */
+ error = ret;
+ }
+
+ cmh_aes_unmap_dma(rctx);
+
+ if (!error) {
+ scatterwalk_map_and_copy(rctx->out_buf, req->dst,
+ 0, rctx->cryptlen, 1);
+ cmh_aes_update_iv(req, rctx->aes_mode, rctx->aes_op,
+ rctx->in_buf, rctx->out_buf);
+ }
+
+ cmh_aes_free_bufs(rctx);
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * Core encrypt/decrypt -- builds a VCQ transaction and submits async.
+ *
+ * Returns -EINPROGRESS on successful submission (completion callback
+ * will fire later). Returns 0 for trivial cases (zero-length).
+ * Returns negative errno on pre-submission errors.
+ */
+static int cmh_aes_crypt(struct skcipher_request *req, u32 aes_op)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+ const struct cmh_aes_alg_info *info = cmh_aes_get_info(tfm);
+ struct cmh_aes_reqctx *rctx = skcipher_request_ctx(req);
+ struct vcq_cmd cmds[CMH_AES_MAX_PAYLOAD];
+ u64 key_ref;
+ u32 keylen;
+ struct core_dispatch d;
+ s32 target_mbx;
+ u32 core_id;
+ u32 idx;
+ int ret;
+ gfp_t gfp;
+
+ if (tctx->key.mode == CMH_KEY_NONE)
+ return -ENOKEY;
+
+ if (!req->cryptlen)
+ return 0;
+
+ if (req->cryptlen > CMH_AES_MAX_CRYPTLEN)
+ return -EINVAL;
+
+ switch (info->aes_mode) {
+ case AES_MODE_CTR:
+ case AES_MODE_CFB:
+ break;
+ case AES_MODE_XTS:
+ if (req->cryptlen < CMH_AES_BLOCK_SIZE)
+ return -EINVAL;
+ break;
+ default:
+ if (req->cryptlen & (CMH_AES_BLOCK_SIZE - 1))
+ return -EINVAL;
+ break;
+ }
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ /* Initialise reqctx */
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->cryptlen = req->cryptlen;
+ rctx->ivsize = info->ivsize;
+ rctx->aes_mode = info->aes_mode;
+ rctx->aes_op = aes_op;
+ rctx->iv2_buf = NULL;
+
+ /* Linearise input from scatterlist */
+ rctx->in_buf = kmalloc(req->cryptlen, gfp);
+ if (!rctx->in_buf)
+ return -ENOMEM;
+
+ scatterwalk_map_and_copy(rctx->in_buf, req->src, 0, req->cryptlen, 0);
+
+ rctx->in_dma = cmh_dma_map_single(rctx->in_buf, req->cryptlen,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->in_dma)) {
+ ret = -ENOMEM;
+ goto out_free_in;
+ }
+
+ /* Allocate and map output buffer */
+ rctx->out_buf = kmalloc(req->cryptlen, gfp);
+ if (!rctx->out_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_in;
+ }
+
+ rctx->out_dma = cmh_dma_map_single(rctx->out_buf, req->cryptlen,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->out_dma)) {
+ ret = -ENOMEM;
+ goto out_free_out;
+ }
+
+ /* Map IV if required */
+ if (info->ivsize > 0) {
+ rctx->iv_buf = kmemdup(req->iv, info->ivsize, gfp);
+ if (!rctx->iv_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_out;
+ }
+ rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, info->ivsize,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->iv_dma)) {
+ ret = -ENOMEM;
+ goto out_free_iv;
+ }
+ }
+
+ /* Resolve key reference */
+ idx = 0;
+
+ rctx->key_dma = tctx->key.raw.dma;
+ rctx->keylen = tctx->key.raw.len;
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP,
+ (u64)rctx->key_dma, SYS_REF_NONE,
+ tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ keylen = tctx->key.raw.len;
+ d = cmh_core_select_instance(CMH_CORE_AES);
+ target_mbx = d.mbx_idx;
+ core_id = d.core_id;
+
+ /*
+ * iolen in INIT: XTS needs total length upfront for tweak
+ * computation; all other modes use 0 (streaming).
+ */
+ vcq_add_aes_init(&cmds[idx++], core_id, key_ref, (u64)rctx->iv_dma,
+ keylen, info->ivsize, info->aes_mode, aes_op,
+ info->aes_mode == AES_MODE_XTS ?
+ req->cryptlen : 0);
+
+ if (info->aes_mode == AES_MODE_XTS &&
+ req->cryptlen > 2 * CMH_AES_BLOCK_SIZE) {
+ u32 final_len, update_len;
+
+ if (req->cryptlen & (CMH_AES_BLOCK_SIZE - 1))
+ final_len = CMH_AES_BLOCK_SIZE +
+ (req->cryptlen & (CMH_AES_BLOCK_SIZE - 1));
+ else
+ final_len = 2 * CMH_AES_BLOCK_SIZE;
+
+ update_len = req->cryptlen - final_len;
+
+ vcq_add_aes_update(&cmds[idx++], core_id,
+ (u64)rctx->in_dma,
+ (u64)rctx->out_dma, update_len);
+ vcq_add_aes_final(&cmds[idx++], core_id,
+ (u64)(rctx->in_dma + update_len),
+ (u64)(rctx->out_dma + update_len),
+ final_len);
+ } else if (info->aes_mode == AES_MODE_CTR) {
+ /*
+ * CTR counter-wrap workaround:
+ * The AES-SCA hardware uses a 64-bit block counter.
+ * If the lower 64 bits of the IV would wrap during
+ * this operation, split into two separate VCQ
+ * transactions -- the completion callback for the
+ * first chunk submits the second.
+ */
+ u64 lower64 = get_unaligned_be64(rctx->iv_buf + 8);
+ u32 nblocks = DIV_ROUND_UP(req->cryptlen,
+ CMH_AES_BLOCK_SIZE);
+ u64 bwrap = lower64 ? (~lower64 + 1ULL) : U64_MAX;
+
+ if (nblocks > bwrap) {
+ u32 chunk1 = (u32)bwrap * CMH_AES_BLOCK_SIZE;
+ u64 upper64;
+
+ /* Prepare second IV for chained submission */
+ rctx->iv2_buf = kmalloc(info->ivsize, gfp);
+ if (!rctx->iv2_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_iv;
+ }
+ upper64 = get_unaligned_be64(rctx->iv_buf);
+ put_unaligned_be64(upper64 + 1, rctx->iv2_buf);
+ put_unaligned_be64(0, rctx->iv2_buf + 8);
+
+ rctx->iv2_dma =
+ cmh_dma_map_single(rctx->iv2_buf,
+ info->ivsize,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->iv2_dma)) {
+ ret = -ENOMEM;
+ goto out_free_iv2;
+ }
+
+ /* Store state for the chained second submission */
+ rctx->ctr_chunk1_len = chunk1;
+ rctx->core_id = core_id;
+ rctx->target_mbx = target_mbx;
+ rctx->key_ref = key_ref;
+
+ /* First transaction: only chunk1 */
+ vcq_add_aes_final(&cmds[idx++], core_id,
+ (u64)rctx->in_dma,
+ (u64)rctx->out_dma, chunk1);
+ } else {
+ /* No wrap: single FINAL with all data */
+ vcq_add_aes_final(&cmds[idx++], core_id,
+ (u64)rctx->in_dma,
+ (u64)rctx->out_dma,
+ req->cryptlen);
+ }
+ } else {
+ vcq_add_aes_final(&cmds[idx++], core_id,
+ (u64)rctx->in_dma,
+ (u64)rctx->out_dma, req->cryptlen);
+ }
+
+ vcq_add_flush(&cmds[idx++], core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_AES_MAX_PACKED, target_mbx,
+ cmh_aes_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_cleanup_all;
+
+ return -EINPROGRESS;
+
+out_cleanup_all:
+ if (rctx->iv2_buf) {
+ cmh_dma_unmap_single(rctx->iv2_dma, info->ivsize,
+ DMA_TO_DEVICE);
+ }
+out_free_iv2:
+ kfree(rctx->iv2_buf);
+out_unmap_iv:
+ if (info->ivsize > 0)
+ cmh_dma_unmap_single(rctx->iv_dma, info->ivsize,
+ DMA_TO_DEVICE);
+out_free_iv:
+ kfree(rctx->iv_buf);
+out_unmap_out:
+ cmh_dma_unmap_single(rctx->out_dma, req->cryptlen, DMA_FROM_DEVICE);
+out_free_out:
+ kfree_sensitive(rctx->out_buf);
+out_unmap_in:
+ cmh_dma_unmap_single(rctx->in_dma, req->cryptlen, DMA_TO_DEVICE);
+out_free_in:
+ kfree_sensitive(rctx->in_buf);
+ return ret;
+}
+
+static int cmh_aes_encrypt(struct skcipher_request *req)
+{
+ return cmh_aes_crypt(req, AES_OP_ENCRYPT);
+}
+
+static int cmh_aes_decrypt(struct skcipher_request *req)
+{
+ return cmh_aes_crypt(req, AES_OP_DECRYPT);
+}
+
+/* Registration */
+
+static struct cmh_aes_alg_drv aes_drv_algs[ARRAY_SIZE(aes_algs)];
+
+/**
+ * cmh_aes_register() - Register AES-CBC/CTR/ECB/XTS skcipher algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_aes_register(void)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
+ const struct cmh_aes_alg_info *info = &aes_algs[i];
+ struct cmh_aes_alg_drv *drv = &aes_drv_algs[i];
+ struct skcipher_alg *alg = &drv->alg;
+
+ drv->info = info;
+
+ memset(alg, 0, sizeof(*alg));
+
+ alg->setkey = cmh_aes_setkey;
+ alg->encrypt = cmh_aes_encrypt;
+ alg->decrypt = cmh_aes_decrypt;
+ alg->init = cmh_aes_init_tfm;
+ alg->exit = cmh_aes_exit_tfm;
+ alg->min_keysize = info->min_keysize;
+ alg->max_keysize = info->max_keysize;
+ alg->ivsize = info->ivsize;
+
+ strscpy(alg->base.cra_name, info->alg_name,
+ CRYPTO_MAX_ALG_NAME);
+ strscpy(alg->base.cra_driver_name, info->drv_name,
+ CRYPTO_MAX_ALG_NAME);
+ alg->base.cra_priority = 300;
+ alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC;
+ alg->base.cra_blocksize = aes_is_stream_mode(info->aes_mode)
+ ? 1 : CMH_AES_BLOCK_SIZE;
+ alg->base.cra_ctxsize = sizeof(struct cmh_aes_tfm_ctx);
+ alg->base.cra_module = THIS_MODULE;
+
+ ret = crypto_register_skcipher(alg);
+ if (ret) {
+ dev_err(cmh_dev(), "cmh_aes: failed to register %s (rc=%d)\n",
+ info->alg_name, ret);
+ goto err_unregister;
+ }
+
+ dev_dbg(cmh_dev(), "cmh_aes: registered %s\n", info->alg_name);
+ }
+
+ return 0;
+
+err_unregister:
+ while (i--)
+ crypto_unregister_skcipher(&aes_drv_algs[i].alg);
+ return ret;
+}
+
+/**
+ * cmh_aes_unregister() - Unregister AES skcipher algorithms from the crypto framework
+ */
+void cmh_aes_unregister(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
+ crypto_unregister_skcipher(&aes_drv_algs[i].alg);
+ dev_dbg(cmh_dev(), "cmh_aes: unregistered %s\n", aes_algs[i].alg_name);
+ }
+}
diff --git a/drivers/crypto/cmh/cmh_aes_aead.c b/drivers/crypto/cmh/cmh_aes_aead.c
new file mode 100644
index 000000000000..0b59c5f7d474
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_aes_aead.c
@@ -0,0 +1,987 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API AES AEAD Driver (GCM/CCM)
+ *
+ * Registers AEAD algorithms with the Linux crypto subsystem:
+ * gcm(aes), ccm(aes)
+ *
+ * GCM: AES_CMD_INIT(mode=GCM) + [AAD_FINAL] + AES_CMD_FINAL + FLUSH
+ * - Standard 12-byte IV (nonce), 16-byte tag
+ * - AES_CMD_INIT carries aadlen/iolen/taglen
+ * - AES_CMD_FINAL carries tag DMA for encrypt (produce) / decrypt (verify)
+ *
+ * CCM: AES_CMD_CCM_INIT + [AAD_FINAL] + AES_CMD_FINAL + FLUSH
+ * - Variable nonce (7--13 bytes), variable tag (4--16 bytes)
+ * - Uses AES_CMD_CCM_INIT (0x0A) with aes_cmd_init struct
+ * - Nonce passed via IV field, taglen in init
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/cipher.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/utils.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "cmh_aes.h"
+#include "cmh_vcq.h"
+#include "cmh_aes_abi.h"
+#include "cmh_sys_abi.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+/*
+ * GCM IV contract:
+ *
+ * The AES core requires exactly 16 bytes loaded into its IV register.
+ * For standard 96-bit nonce GCM, the driver passes:
+ *
+ * IV[0..11] = user-supplied 12-byte nonce
+ * IV[12..15] = 0x00000000
+ *
+ * The hardware internally sets the last 32 bits to the big-endian
+ * counter value 1 (forming J0 = nonce || 0x00000001) before
+ * processing AAD. The driver must NOT pre-set the counter.
+ *
+ * If the IV format is incorrect, GCM authentication will fail
+ * (encrypt produces wrong ciphertext/tag, decrypt rejects).
+ */
+#define AES_GCM_IV_SIZE 12U /* GCM nonce size (standard) */
+#define AES_GCM_HW_IV_SIZE 16U /* HW requires 16-byte IV buffer */
+#define AES_GCM_TAG_SIZE 16U
+
+/* CCM: callers pass a 16-byte IV in RFC 3610 format:
+ * iv[0] = L-1, iv[1..14-iv[0]] = nonce, rest = counter (zeroed).
+ * Nonce length = 14 - iv[0], range 7..13.
+ */
+#define AES_CCM_IV_SIZE 16U
+
+enum cmh_aes_aead_type {
+ CMH_AES_AEAD_GCM,
+ CMH_AES_AEAD_CCM,
+};
+
+struct cmh_aes_aead_info {
+ enum cmh_aes_aead_type type;
+ u32 aes_mode; /* AES_MODE_GCM or AES_MODE_CCM */
+ u32 ivsize;
+ u32 maxauthsize;
+ const char *alg_name;
+ const char *drv_name;
+};
+
+static const struct cmh_aes_aead_info aes_aead_algs[] = {
+ { CMH_AES_AEAD_GCM, AES_MODE_GCM, AES_GCM_IV_SIZE,
+ AES_GCM_TAG_SIZE, "gcm(aes)", "cri-cmh-gcm-aes" },
+ { CMH_AES_AEAD_CCM, AES_MODE_CCM, AES_CCM_IV_SIZE,
+ AES_GCM_TAG_SIZE, "ccm(aes)", "cri-cmh-ccm-aes" },
+};
+
+struct cmh_aes_aead_tfm_ctx {
+ struct cmh_key_ctx key;
+ u32 authsize; /* tag length set by setauthsize */
+ struct crypto_cipher *sw_cipher; /* CCM empty-input fallback */
+ struct crypto_aead *fallback; /* CCM authsize=10 fallback */
+};
+
+/* Per-request context (lives in aead_request::__ctx) */
+
+/*
+ * Maximum payload commands:
+ * [SYS_CMD_WRITE] + AES_CMD_INIT + AAD_FINAL + AES_CMD_FINAL + FLUSH = 5
+ */
+#define CMH_AES_AEAD_MAX_PAYLOAD 5
+#define CMH_AES_AEAD_MAX_PACKED (CMH_AES_AEAD_MAX_PAYLOAD * 2)
+
+struct cmh_aes_aead_reqctx {
+ dma_addr_t in_dma;
+ dma_addr_t out_dma;
+ dma_addr_t iv_dma;
+ dma_addr_t key_dma;
+ dma_addr_t aad_dma;
+ dma_addr_t tag_dma;
+ u8 *in_buf;
+ u8 *out_buf;
+ u8 *iv_buf;
+ u8 *aad_buf;
+ u8 *tag_buf;
+ u32 cryptlen;
+ u32 assoclen;
+ u32 authsize;
+ u32 iv_map_len;
+ u32 keylen;
+ bool encrypting;
+ bool empty_gcm_fallback;
+ struct vcq_cmd packed[CMH_AES_AEAD_MAX_PACKED];
+};
+
+struct cmh_aes_aead_drv {
+ struct aead_alg alg;
+ const struct cmh_aes_aead_info *info;
+};
+
+static const struct cmh_aes_aead_info *
+cmh_aes_aead_get_info(struct crypto_aead *tfm)
+{
+ struct aead_alg *alg = crypto_aead_alg(tfm);
+
+ return container_of(alg, struct cmh_aes_aead_drv, alg)->info;
+}
+
+/* VCQ Builders -- AEAD-specific */
+
+static void vcq_add_aes_aead_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref,
+ u64 iv_dma, u32 keylen, u32 ivlen,
+ u32 mode, u32 op, u32 aadlen, u32 iolen,
+ u32 taglen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_INIT);
+ slot->hwc.aes.cmd_init.key = key_ref;
+ slot->hwc.aes.cmd_init.iv = iv_dma;
+ slot->hwc.aes.cmd_init.keylen = keylen;
+ slot->hwc.aes.cmd_init.ivlen = ivlen;
+ slot->hwc.aes.cmd_init.mode = mode;
+ slot->hwc.aes.cmd_init.op = op;
+ slot->hwc.aes.cmd_init.aadlen = aadlen;
+ slot->hwc.aes.cmd_init.iolen = iolen;
+ slot->hwc.aes.cmd_init.taglen = taglen;
+}
+
+static void vcq_add_aes_ccm_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref,
+ u64 nonce_dma, u32 keylen, u32 noncelen,
+ u32 op, u32 aadlen, u32 iolen, u32 taglen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_CCM_INIT);
+ slot->hwc.aes.cmd_init.key = key_ref;
+ slot->hwc.aes.cmd_init.iv = nonce_dma;
+ slot->hwc.aes.cmd_init.keylen = keylen;
+ slot->hwc.aes.cmd_init.ivlen = noncelen;
+ slot->hwc.aes.cmd_init.mode = AES_MODE_CCM;
+ slot->hwc.aes.cmd_init.op = op;
+ slot->hwc.aes.cmd_init.aadlen = aadlen;
+ slot->hwc.aes.cmd_init.iolen = iolen;
+ slot->hwc.aes.cmd_init.taglen = taglen;
+}
+
+static void vcq_add_aes_aad_final(struct vcq_cmd *slot, u32 core_id, u64 aad_dma,
+ u32 aadlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_AAD_FINAL);
+ slot->hwc.aes.cmd_aad_final.data = aad_dma;
+ slot->hwc.aes.cmd_aad_final.datalen = aadlen;
+}
+
+static void vcq_add_aes_aead_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma,
+ u64 output_dma, u64 tag_dma,
+ u32 iolen, u32 taglen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_FINAL);
+ slot->hwc.aes.cmd_final.input = input_dma;
+ slot->hwc.aes.cmd_final.output = output_dma;
+ slot->hwc.aes.cmd_final.tag = tag_dma;
+ slot->hwc.aes.cmd_final.iolen = iolen;
+ slot->hwc.aes.cmd_final.taglen = taglen;
+}
+
+/* setkey */
+static int cmh_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ int ret;
+
+ if (keylen != 16 && keylen != 24 && keylen != 32)
+ return -EINVAL;
+
+ /* Keep SW fallback ciphers in sync for CCM edge cases */
+ if (tctx->sw_cipher) {
+ ret = crypto_cipher_setkey(tctx->sw_cipher, key, keylen);
+ if (ret)
+ return ret;
+ }
+ if (tctx->fallback) {
+ ret = crypto_aead_setkey(tctx->fallback, key, keylen);
+ if (ret)
+ return ret;
+ }
+
+ ret = cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_AES);
+
+ return ret;
+}
+
+static int cmh_aes_aead_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ const struct cmh_aes_aead_info *info = cmh_aes_aead_get_info(tfm);
+ int ret;
+
+ if (info->type == CMH_AES_AEAD_GCM) {
+ /* GCM: accept 4, 8, 12, 13, 14, 15, 16 per NIST SP 800-38D */
+ if (authsize < 4 || authsize > 16 ||
+ (authsize > 4 && authsize < 8) ||
+ (authsize > 8 && authsize < 12))
+ return -EINVAL;
+ } else {
+ /* CCM: accept all RFC 3610 values {4,6,8,10,12,14,16} */
+ if (authsize < 4 || authsize > 16 || (authsize & 1))
+ return -EINVAL;
+ /* Forward to SW fallback for authsize=10 (HW unsupported) */
+ if (tctx->fallback) {
+ ret = crypto_aead_setauthsize(tctx->fallback,
+ authsize);
+ if (ret)
+ return ret;
+ }
+ }
+
+ tctx->authsize = authsize;
+ return 0;
+}
+
+static int cmh_aes_aead_init_tfm(struct crypto_aead *tfm)
+{
+ struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ const struct cmh_aes_aead_info *info = cmh_aes_aead_get_info(tfm);
+
+ memset(tctx, 0, sizeof(*tctx));
+ tctx->authsize = info->maxauthsize;
+
+ if (info->type == CMH_AES_AEAD_CCM) {
+ struct crypto_aead *fb;
+ struct crypto_cipher *ci;
+
+ ci = crypto_alloc_cipher("aes", 0, 0);
+ if (IS_ERR(ci))
+ return PTR_ERR(ci);
+ tctx->sw_cipher = ci;
+
+ fb = crypto_alloc_aead("ccm(aes)", 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(fb)) {
+ crypto_free_cipher(ci);
+ tctx->sw_cipher = NULL;
+ return PTR_ERR(fb);
+ }
+ tctx->fallback = fb;
+
+ /*
+ * Subreq lives at (rctx + 1). Alignment is guaranteed
+ * by the crypto framework's __ctx ALIGN mechanism.
+ */
+ crypto_aead_set_reqsize(tfm,
+ sizeof(struct cmh_aes_aead_reqctx) +
+ sizeof(struct aead_request) +
+ crypto_aead_reqsize(fb));
+ } else {
+ crypto_aead_set_reqsize(tfm,
+ sizeof(struct cmh_aes_aead_reqctx));
+ }
+
+ return 0;
+}
+
+static void cmh_aes_aead_exit_tfm(struct crypto_aead *tfm)
+{
+ struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+
+ if (tctx->fallback)
+ crypto_free_aead(tctx->fallback);
+ if (tctx->sw_cipher)
+ crypto_free_cipher(tctx->sw_cipher);
+ cmh_key_destroy(&tctx->key);
+}
+
+/* DMA unmap helper */
+static void cmh_aes_aead_unmap_dma(struct cmh_aes_aead_reqctx *rctx)
+{
+ u32 tag_map_len;
+
+ cmh_dma_unmap_single(rctx->iv_dma, rctx->iv_map_len, DMA_TO_DEVICE);
+ /*
+ * The empty-GCM fallback maps a full AES block (16 bytes) for the
+ * ECB output regardless of authsize, so unmap with the mapped size.
+ */
+ tag_map_len = rctx->empty_gcm_fallback ?
+ AES_GCM_HW_IV_SIZE : rctx->authsize;
+ cmh_dma_unmap_single(rctx->tag_dma, tag_map_len,
+ (rctx->encrypting || rctx->empty_gcm_fallback) ?
+ DMA_FROM_DEVICE : DMA_TO_DEVICE);
+ if (rctx->cryptlen > 0) {
+ cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen,
+ DMA_FROM_DEVICE);
+ cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen,
+ DMA_TO_DEVICE);
+ }
+ if (rctx->assoclen > 0)
+ cmh_dma_unmap_single(rctx->aad_dma, rctx->assoclen,
+ DMA_TO_DEVICE);
+}
+
+static void cmh_aes_aead_free_bufs(struct cmh_aes_aead_reqctx *rctx)
+{
+ kfree(rctx->iv_buf);
+ rctx->iv_buf = NULL;
+ kfree(rctx->tag_buf);
+ rctx->tag_buf = NULL;
+ kfree_sensitive(rctx->out_buf);
+ rctx->out_buf = NULL;
+ kfree_sensitive(rctx->in_buf);
+ rctx->in_buf = NULL;
+ kfree(rctx->aad_buf);
+ rctx->aad_buf = NULL;
+}
+
+static void cmh_aes_aead_complete(void *data, int error)
+{
+ struct aead_request *req = data;
+ struct cmh_aes_aead_reqctx *rctx = aead_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ cmh_aes_aead_unmap_dma(rctx);
+
+ /*
+ * Map HW error on decrypt to -EBADMSG. The eSW AES core uses a
+ * single error code (-EIO) for both authentication failures and
+ * other core errors (e.g. DMA timeout), so we cannot distinguish
+ * them from the MBX_STATUS alone. In practice the only error
+ * during a well-formed AEAD decrypt is auth-tag mismatch; a DMA
+ * timeout would indicate a fatal HW problem where -EBADMSG vs
+ * -EIO is moot. The kernel crypto API requires -EBADMSG for
+ * AEAD authentication failures.
+ */
+ if (error == -EIO && !rctx->encrypting)
+ error = -EBADMSG;
+
+ if (!error) {
+ /* GCM empty-input decrypt: compare computed tag with expected */
+ if (rctx->empty_gcm_fallback && !rctx->encrypting) {
+ if (crypto_memneq(rctx->tag_buf, rctx->in_buf,
+ rctx->authsize))
+ error = -EBADMSG;
+ }
+ if (!error && rctx->cryptlen > 0)
+ scatterwalk_map_and_copy(rctx->out_buf, req->dst,
+ req->assoclen,
+ rctx->cryptlen, 1);
+ if (!error && rctx->encrypting)
+ scatterwalk_map_and_copy(rctx->tag_buf, req->dst,
+ req->assoclen +
+ rctx->cryptlen,
+ rctx->authsize, 1);
+ }
+
+ cmh_aes_aead_free_bufs(rctx);
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * GCM empty-input fallback.
+ *
+ * When both AAD and plaintext are empty, GCM reduces to:
+ * tag = E(K, J0) where J0 = nonce || 0x00000001
+ *
+ * The eSW GCM engine rejects this degenerate case, so we compute it
+ * via a single ECB block encryption of J0.
+ *
+ * VCQ: [SYS_CMD_WRITE] + AES_CMD_INIT(ECB) + AES_CMD_FINAL + FLUSH
+ */
+static int cmh_aes_gcm_empty(struct aead_request *req, u32 aes_op)
+{
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ struct cmh_aes_aead_reqctx *rctx = aead_request_ctx(req);
+ struct vcq_cmd cmds[CMH_AES_AEAD_MAX_PAYLOAD];
+ u64 key_ref;
+ u32 keylen, authsize;
+ struct core_dispatch d;
+ s32 target_mbx;
+ u32 core_id;
+ u32 idx;
+ int ret;
+ gfp_t gfp;
+
+ authsize = tctx->authsize;
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->cryptlen = 0;
+ rctx->assoclen = 0;
+ rctx->authsize = authsize;
+ rctx->encrypting = (aes_op == AES_OP_ENCRYPT);
+ rctx->empty_gcm_fallback = true;
+
+ /* Build J0 = nonce || 0x00000001 in iv_buf */
+ rctx->iv_buf = kzalloc(AES_GCM_HW_IV_SIZE, gfp);
+ if (!rctx->iv_buf)
+ return -ENOMEM;
+ memcpy(rctx->iv_buf, req->iv, AES_GCM_IV_SIZE);
+ rctx->iv_buf[15] = 0x01; /* big-endian counter = 1 */
+ rctx->iv_map_len = AES_GCM_HW_IV_SIZE;
+
+ rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, AES_GCM_HW_IV_SIZE,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->iv_dma)) {
+ ret = -ENOMEM;
+ goto out_free_iv;
+ }
+
+ /* Tag buffer -- receives E(K, J0) output */
+ rctx->tag_buf = kzalloc(AES_GCM_HW_IV_SIZE, gfp);
+ if (!rctx->tag_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_iv;
+ }
+ rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, AES_GCM_HW_IV_SIZE,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->tag_dma)) {
+ ret = -ENOMEM;
+ goto out_free_tag;
+ }
+
+ /* For decrypt: read expected tag from request for later comparison */
+ if (!rctx->encrypting) {
+ rctx->in_buf = kmalloc(authsize, gfp);
+ if (!rctx->in_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_tag;
+ }
+ scatterwalk_map_and_copy(rctx->in_buf, req->src, 0,
+ authsize, 0);
+ }
+
+ /* Resolve key */
+ idx = 0;
+ rctx->key_dma = tctx->key.raw.dma;
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP,
+ (u64)rctx->key_dma, SYS_REF_NONE,
+ tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ keylen = tctx->key.raw.len;
+ d = cmh_core_select_instance(CMH_CORE_AES);
+ target_mbx = d.mbx_idx;
+ core_id = d.core_id;
+
+ /* ECB INIT: single block encryption of J0 */
+ vcq_add_aes_aead_init(&cmds[idx++], core_id, key_ref,
+ 0, keylen, 0, AES_MODE_ECB, AES_OP_ENCRYPT,
+ 0, AES_GCM_HW_IV_SIZE, 0);
+
+ /* FINAL: J0 in, E(K,J0) out */
+ vcq_add_aes_aead_final(&cmds[idx++], core_id,
+ (u64)rctx->iv_dma, (u64)rctx->tag_dma,
+ 0, AES_GCM_HW_IV_SIZE, 0);
+
+ vcq_add_flush(&cmds[idx++], core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_AES_AEAD_MAX_PACKED,
+ target_mbx,
+ cmh_aes_aead_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_free_in;
+
+ return -EINPROGRESS;
+
+out_free_in:
+ kfree_sensitive(rctx->in_buf);
+out_unmap_tag:
+ cmh_dma_unmap_single(rctx->tag_dma, AES_GCM_HW_IV_SIZE,
+ DMA_FROM_DEVICE);
+out_free_tag:
+ kfree(rctx->tag_buf);
+out_unmap_iv:
+ cmh_dma_unmap_single(rctx->iv_dma, AES_GCM_HW_IV_SIZE, DMA_TO_DEVICE);
+out_free_iv:
+ kfree(rctx->iv_buf);
+ return ret;
+}
+
+/*
+ * CCM empty-input fallback.
+ *
+ * When both AAD and plaintext are empty, CCM reduces to:
+ * T = E(K, B0) -- CBC-MAC of the single formatting block
+ * S0 = E(K, A0) -- CTR block zero
+ * tag = (T XOR S0)[0..authsize-1]
+ *
+ * The eSW rejects this degenerate case, so the driver computes it
+ * synchronously via two crypto_cipher single-block encryptions.
+ */
+static int cmh_aes_ccm_empty(struct aead_request *req, u32 aes_op)
+{
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ u32 authsize = tctx->authsize;
+ u8 b0[CMH_AES_BLOCK_SIZE], a0[CMH_AES_BLOCK_SIZE];
+ u8 t[CMH_AES_BLOCK_SIZE], s0[CMH_AES_BLOCK_SIZE];
+ u8 tag[CMH_AES_BLOCK_SIZE];
+ u8 L;
+ u32 i;
+
+ /* Defense-in-depth: iv[0] = L-1, valid L is 2..8 per RFC 3610 S2.1 */
+ if (WARN_ON_ONCE(req->iv[0] < 1 || req->iv[0] > 7))
+ return -EINVAL;
+
+ L = req->iv[0] + 1;
+
+ if (tctx->key.mode != CMH_KEY_RAW)
+ return -EOPNOTSUPP;
+
+ /* B0: flags || nonce || Q(=0). Adata=0, t=authsize, q=L. */
+ memset(b0, 0, CMH_AES_BLOCK_SIZE);
+ b0[0] = (u8)(8 * ((authsize - 2) / 2) + (L - 1));
+ memcpy(&b0[1], &req->iv[1], 15 - L);
+
+ /* A0: (L-1) || nonce || counter(=0) */
+ memset(a0, 0, CMH_AES_BLOCK_SIZE);
+ a0[0] = (u8)(L - 1);
+ memcpy(&a0[1], &req->iv[1], 15 - L);
+
+ crypto_cipher_encrypt_one(tctx->sw_cipher, t, b0);
+ crypto_cipher_encrypt_one(tctx->sw_cipher, s0, a0);
+
+ for (i = 0; i < authsize; i++)
+ tag[i] = t[i] ^ s0[i];
+
+ if (aes_op == AES_OP_ENCRYPT) {
+ scatterwalk_map_and_copy(tag, req->dst,
+ req->assoclen, authsize, 1);
+ } else {
+ u8 expected[CMH_AES_BLOCK_SIZE];
+
+ scatterwalk_map_and_copy(expected, req->src,
+ req->assoclen, authsize, 0);
+ if (crypto_memneq(tag, expected, authsize))
+ return -EBADMSG;
+ }
+
+ return 0;
+}
+
+/*
+ * CCM authsize=10 fallback.
+ *
+ * The eSW AES CCM core does not support authsize=10 (valid per RFC 3610).
+ * Forward the entire request to the generic CCM implementation.
+ */
+static void cmh_aes_ccm_fb_done(void *data, int err)
+{
+ struct aead_request *req = data;
+
+ cmh_complete(&req->base, err);
+}
+
+static int cmh_aes_ccm_fallback(struct aead_request *req, u32 aes_op)
+{
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ struct cmh_aes_aead_reqctx *rctx = aead_request_ctx(req);
+ struct aead_request *subreq = (void *)(rctx + 1);
+
+ aead_request_set_tfm(subreq, tctx->fallback);
+ aead_request_set_callback(subreq, req->base.flags,
+ cmh_aes_ccm_fb_done, req);
+ aead_request_set_crypt(subreq, req->src, req->dst,
+ req->cryptlen, req->iv);
+ aead_request_set_ad(subreq, req->assoclen);
+
+ return (aes_op == AES_OP_ENCRYPT) ?
+ crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq);
+}
+
+/*
+ * Core AEAD encrypt/decrypt -- async path.
+ *
+ * Encrypt: plaintext -> ciphertext + tag appended
+ * Decrypt: ciphertext + tag -> plaintext (tag verified by eSW)
+ *
+ * VCQ: [SYS_CMD_WRITE] + INIT/CCM_INIT + [AAD_FINAL] + FINAL + FLUSH
+ */
+static int cmh_aes_aead_crypt(struct aead_request *req, u32 aes_op)
+{
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ const struct cmh_aes_aead_info *info = cmh_aes_aead_get_info(tfm);
+ struct cmh_aes_aead_reqctx *rctx = aead_request_ctx(req);
+ struct vcq_cmd cmds[CMH_AES_AEAD_MAX_PAYLOAD];
+ u64 key_ref;
+ u32 keylen, authsize, cryptlen;
+ struct core_dispatch d;
+ s32 target_mbx;
+ u32 core_id;
+ u32 idx;
+ int ret;
+ gfp_t gfp;
+
+ if (tctx->key.mode == CMH_KEY_NONE)
+ return -ENOKEY;
+
+ authsize = tctx->authsize;
+
+ if (aes_op == AES_OP_ENCRYPT) {
+ cryptlen = req->cryptlen;
+ } else {
+ if (req->cryptlen < authsize)
+ return -EINVAL;
+ cryptlen = req->cryptlen - authsize;
+ }
+
+ /*
+ * Validate CCM IV format early -- the empty-input fallback and
+ * nonce extraction both depend on iv[0] being in range [1,7].
+ */
+ if (info->type == CMH_AES_AEAD_CCM) {
+ if (req->iv[0] < 1 || req->iv[0] > 7)
+ return -EINVAL;
+ }
+
+ /*
+ * The CMH eSW rejects GCM/CCM when both aadlen and iolen are zero.
+ * For GCM, the tag is simply E(K, J0) -- handle with ECB fallback.
+ * For CCM, compute tag = E(K,B0) XOR E(K,A0) in software.
+ */
+ if (cryptlen == 0 && req->assoclen == 0) {
+ if (info->type == CMH_AES_AEAD_GCM)
+ return cmh_aes_gcm_empty(req, aes_op);
+ return cmh_aes_ccm_empty(req, aes_op);
+ }
+
+ /*
+ * HW does not support authsize=10 for CCM. Forward the entire
+ * request to the generic CCM implementation.
+ */
+ if (info->type == CMH_AES_AEAD_CCM && authsize == 10)
+ return cmh_aes_ccm_fallback(req, aes_op);
+
+ /*
+ * HW uses a proprietary LLI scatter-gather format that is
+ * incompatible with struct scatterlist, so the payload is
+ * linearised into contiguous buffers for DMA. Cap total
+ * size to prevent excessive memory consumption.
+ */
+ if ((u64)cryptlen + req->assoclen > SZ_1M)
+ return -EINVAL;
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->cryptlen = cryptlen;
+ rctx->assoclen = req->assoclen;
+ rctx->authsize = authsize;
+ rctx->encrypting = (aes_op == AES_OP_ENCRYPT);
+
+ /* Linearise AAD */
+ if (req->assoclen > 0) {
+ rctx->aad_buf = kmalloc(req->assoclen, gfp);
+ if (!rctx->aad_buf)
+ return -ENOMEM;
+ scatterwalk_map_and_copy(rctx->aad_buf, req->src,
+ 0, req->assoclen, 0);
+ rctx->aad_dma = cmh_dma_map_single(rctx->aad_buf,
+ req->assoclen,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->aad_dma)) {
+ ret = -ENOMEM;
+ goto out_free_aad;
+ }
+ }
+
+ /* Linearise input */
+ if (cryptlen > 0) {
+ rctx->in_buf = kmalloc(cryptlen, gfp);
+ if (!rctx->in_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_aad;
+ }
+ scatterwalk_map_and_copy(rctx->in_buf, req->src,
+ req->assoclen, cryptlen, 0);
+ rctx->in_dma = cmh_dma_map_single(rctx->in_buf, cryptlen,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->in_dma)) {
+ ret = -ENOMEM;
+ goto out_free_in;
+ }
+ }
+
+ /* Allocate output buffer */
+ if (cryptlen > 0) {
+ rctx->out_buf = kmalloc(cryptlen, gfp);
+ if (!rctx->out_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_in;
+ }
+ rctx->out_dma = cmh_dma_map_single(rctx->out_buf, cryptlen,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->out_dma)) {
+ ret = -ENOMEM;
+ goto out_free_out;
+ }
+ }
+
+ /* Tag buffer */
+ rctx->tag_buf = kmalloc(authsize, gfp);
+ if (!rctx->tag_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_out;
+ }
+
+ if (!rctx->encrypting) {
+ scatterwalk_map_and_copy(rctx->tag_buf, req->src,
+ req->assoclen + cryptlen,
+ authsize, 0);
+ } else {
+ memset(rctx->tag_buf, 0, authsize);
+ }
+
+ rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, authsize,
+ rctx->encrypting ?
+ DMA_FROM_DEVICE : DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->tag_dma)) {
+ ret = -ENOMEM;
+ goto out_free_tag;
+ }
+
+ /* Map IV/nonce */
+ if (info->type == CMH_AES_AEAD_GCM) {
+ rctx->iv_buf = kzalloc(AES_GCM_HW_IV_SIZE, gfp);
+ if (!rctx->iv_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_tag;
+ }
+ memcpy(rctx->iv_buf, req->iv, AES_GCM_IV_SIZE);
+ rctx->iv_map_len = AES_GCM_HW_IV_SIZE;
+ rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf,
+ rctx->iv_map_len,
+ DMA_TO_DEVICE);
+ } else {
+ u32 noncelen;
+
+ if (req->iv[0] < 1 || req->iv[0] > 7) {
+ ret = -EINVAL;
+ goto out_unmap_tag;
+ }
+ noncelen = 14 - req->iv[0];
+
+ rctx->iv_buf = kmemdup(req->iv + 1, noncelen, gfp);
+ if (!rctx->iv_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_tag;
+ }
+ rctx->iv_map_len = noncelen;
+ rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf,
+ rctx->iv_map_len,
+ DMA_TO_DEVICE);
+ }
+ if (cmh_dma_map_error(rctx->iv_dma)) {
+ ret = -ENOMEM;
+ goto out_free_iv;
+ }
+
+ /* Resolve key reference */
+ idx = 0;
+
+ rctx->key_dma = tctx->key.raw.dma;
+ rctx->keylen = tctx->key.raw.len;
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP,
+ (u64)rctx->key_dma, SYS_REF_NONE,
+ tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ keylen = tctx->key.raw.len;
+ d = cmh_core_select_instance(CMH_CORE_AES);
+ target_mbx = d.mbx_idx;
+ core_id = d.core_id;
+
+ /* Build INIT command */
+ if (info->type == CMH_AES_AEAD_CCM) {
+ vcq_add_aes_ccm_init(&cmds[idx++], core_id, key_ref,
+ (u64)rctx->iv_dma, keylen,
+ rctx->iv_map_len, aes_op,
+ req->assoclen, cryptlen, authsize);
+ } else {
+ vcq_add_aes_aead_init(&cmds[idx++], core_id, key_ref,
+ (u64)rctx->iv_dma, keylen,
+ AES_GCM_HW_IV_SIZE, info->aes_mode,
+ aes_op, req->assoclen, cryptlen,
+ authsize);
+ }
+
+ if (req->assoclen > 0)
+ vcq_add_aes_aad_final(&cmds[idx++], core_id,
+ (u64)rctx->aad_dma, req->assoclen);
+
+ vcq_add_aes_aead_final(&cmds[idx++], core_id,
+ cryptlen > 0 ? (u64)rctx->in_dma : 0,
+ cryptlen > 0 ? (u64)rctx->out_dma : 0,
+ (u64)rctx->tag_dma, cryptlen, authsize);
+
+ vcq_add_flush(&cmds[idx++], core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_AES_AEAD_MAX_PACKED,
+ target_mbx,
+ cmh_aes_aead_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_cleanup_all;
+
+ return -EINPROGRESS;
+
+out_cleanup_all:
+ cmh_dma_unmap_single(rctx->iv_dma, rctx->iv_map_len, DMA_TO_DEVICE);
+out_free_iv:
+ kfree(rctx->iv_buf);
+out_unmap_tag:
+ cmh_dma_unmap_single(rctx->tag_dma, authsize,
+ rctx->encrypting ? DMA_FROM_DEVICE :
+ DMA_TO_DEVICE);
+out_free_tag:
+ kfree(rctx->tag_buf);
+out_unmap_out:
+ if (cryptlen > 0)
+ cmh_dma_unmap_single(rctx->out_dma, cryptlen, DMA_FROM_DEVICE);
+out_free_out:
+ kfree_sensitive(rctx->out_buf);
+out_unmap_in:
+ if (cryptlen > 0)
+ cmh_dma_unmap_single(rctx->in_dma, cryptlen, DMA_TO_DEVICE);
+out_free_in:
+ kfree_sensitive(rctx->in_buf);
+out_unmap_aad:
+ if (req->assoclen > 0)
+ cmh_dma_unmap_single(rctx->aad_dma, req->assoclen,
+ DMA_TO_DEVICE);
+out_free_aad:
+ kfree(rctx->aad_buf);
+ return ret;
+}
+
+static int cmh_aes_aead_encrypt(struct aead_request *req)
+{
+ return cmh_aes_aead_crypt(req, AES_OP_ENCRYPT);
+}
+
+static int cmh_aes_aead_decrypt(struct aead_request *req)
+{
+ return cmh_aes_aead_crypt(req, AES_OP_DECRYPT);
+}
+
+/* Registration */
+
+static struct cmh_aes_aead_drv aes_aead_drv_algs[ARRAY_SIZE(aes_aead_algs)];
+
+/**
+ * cmh_aes_aead_register() - Register AES-GCM/CCM AEAD algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_aes_aead_register(void)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < ARRAY_SIZE(aes_aead_algs); i++) {
+ const struct cmh_aes_aead_info *info = &aes_aead_algs[i];
+ struct cmh_aes_aead_drv *drv = &aes_aead_drv_algs[i];
+ struct aead_alg *alg = &drv->alg;
+
+ drv->info = info;
+
+ memset(alg, 0, sizeof(*alg));
+
+ alg->setkey = cmh_aes_aead_setkey;
+ alg->setauthsize = cmh_aes_aead_setauthsize;
+ alg->encrypt = cmh_aes_aead_encrypt;
+ alg->decrypt = cmh_aes_aead_decrypt;
+ alg->init = cmh_aes_aead_init_tfm;
+ alg->exit = cmh_aes_aead_exit_tfm;
+ alg->ivsize = info->ivsize;
+ alg->maxauthsize = info->maxauthsize;
+
+ strscpy(alg->base.cra_name, info->alg_name,
+ CRYPTO_MAX_ALG_NAME);
+ strscpy(alg->base.cra_driver_name, info->drv_name,
+ CRYPTO_MAX_ALG_NAME);
+ alg->base.cra_priority = 300;
+ alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC;
+ if (info->type == CMH_AES_AEAD_CCM) {
+ alg->base.cra_flags |= CRYPTO_ALG_NEED_FALLBACK;
+ /*
+ * Bump priority above 300 so we beat the generic
+ * ccm_base template instance. That template inherits
+ * priority (ctr + cbcmac) / 2 = 300 when both
+ * constituents are at 300, and list ordering would
+ * otherwise let it shadow our driver.
+ */
+ alg->base.cra_priority = 301;
+ }
+ alg->base.cra_blocksize = 1;
+ alg->base.cra_ctxsize = sizeof(struct cmh_aes_aead_tfm_ctx);
+ alg->base.cra_module = THIS_MODULE;
+
+ ret = crypto_register_aead(alg);
+ if (ret) {
+ dev_err(cmh_dev(), "cmh_aes_aead: failed to register %s (rc=%d)\n",
+ info->alg_name, ret);
+ goto err_unregister;
+ }
+
+ dev_dbg(cmh_dev(), "cmh_aes_aead: registered %s\n", info->alg_name);
+ }
+
+ return 0;
+
+err_unregister:
+ while (i--)
+ crypto_unregister_aead(&aes_aead_drv_algs[i].alg);
+ return ret;
+}
+
+/**
+ * cmh_aes_aead_unregister() - Unregister AES AEAD algorithms from the crypto framework
+ */
+void cmh_aes_aead_unregister(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(aes_aead_algs); i++) {
+ crypto_unregister_aead(&aes_aead_drv_algs[i].alg);
+ dev_dbg(cmh_dev(), "cmh_aes_aead: unregistered %s\n",
+ aes_aead_algs[i].alg_name);
+ }
+}
diff --git a/drivers/crypto/cmh/cmh_aes_cmac.c b/drivers/crypto/cmh/cmh_aes_cmac.c
new file mode 100644
index 000000000000..a711c575398d
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_aes_cmac.c
@@ -0,0 +1,537 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API AES-CMAC (ahash) Driver
+ *
+ * Registers cmac(aes) as an ahash algorithm.
+ *
+ * CMAC produces a 16-byte tag (MAC) from a key and message.
+ * VCQ sequence: [SYS_CMD_WRITE] + AES_CMD_INIT(CMAC) +
+ * AES_CMD_AAD_FINAL_AUTH + FLUSH
+ *
+ * The ahash interface accumulates data in a kernel buffer via .update(),
+ * then .final() builds and submits the VCQ asynchronously.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/hash.h>
+#include <crypto/scatterwalk.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "cmh_aes.h"
+#include "cmh_vcq.h"
+#include "cmh_aes_abi.h"
+#include "cmh_sys_abi.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+#define AES_CMAC_DIGEST_SIZE 16U
+#define AES_CMAC_BLOCK_SIZE 16U
+
+/*
+ * Maximum accumulated data for CMAC -- driver-imposed, not HW.
+ *
+ * The AES core does not expose external save/restore VCQ commands,
+ * so the driver must accumulate all data in kernel memory via
+ * .update() and submit it atomically in .final(). This cap limits
+ * the per-request kernel allocation.
+ */
+#define AES_CMAC_MAX_DATA (64 * 1024)
+
+/* Per-transform context */
+struct cmh_aes_cmac_tfm_ctx {
+ struct cmh_key_ctx key;
+ spinlock_t chunk_lock; /* protects all_chunks */
+ struct list_head all_chunks; /* orphan-safe chunk tracking */
+};
+
+/* One chunk per .update() call -- data is embedded via flexible array */
+struct cmh_aes_cmac_chunk {
+ struct list_head list;
+ struct list_head tfm_node; /* per-tfm orphan tracking */
+ u32 len;
+ u8 data[];
+};
+
+/* Per-request context (lives in ahash_request::__ctx) */
+
+/*
+ * Maximum payload commands:
+ * [SYS_CMD_WRITE] + AES_CMD_INIT + AES_CMD_AAD_FINAL_AUTH + FLUSH = 4
+ */
+#define CMH_AES_CMAC_MAX_PAYLOAD 4
+#define CMH_AES_CMAC_MAX_PACKED (CMH_AES_CMAC_MAX_PAYLOAD * 2)
+
+struct cmh_aes_cmac_reqctx {
+ struct list_head chunks;
+ u32 total_len;
+ u8 *buf; /* linearised in final() for DMA */
+ /* DMA state for async final */
+ dma_addr_t key_dma;
+ dma_addr_t in_dma;
+ dma_addr_t tag_dma;
+ u8 *tag_buf;
+ u32 keylen;
+ struct vcq_cmd packed[CMH_AES_CMAC_MAX_PACKED];
+};
+
+/* Flat state for export/import -- holds accumulated input data only */
+struct cmh_aes_cmac_export_state {
+ u32 total_len;
+ u8 data[];
+};
+
+/*
+ * Flat state buffer for export/import. The CMH AES core does not
+ * support save/restore of intermediate CMAC state, so this driver
+ * accumulates input in SW and serialises the buffer on export.
+ *
+ * PAGE_SIZE (4096) caps the exportable accumulated-data window.
+ * Full-range export is not feasible because the crypto subsystem
+ * pre-allocates statesize bytes per request. Export returns -EINVAL
+ * if the caller has accumulated more than CMH_AES_CMAC_EXPORT_MAX.
+ */
+#define CMH_AES_CMAC_STATE_SIZE 4096
+#define CMH_AES_CMAC_EXPORT_MAX \
+ (CMH_AES_CMAC_STATE_SIZE - sizeof(struct cmh_aes_cmac_export_state))
+
+/*
+ * Export/import: not supported.
+ *
+ * The AES core lacks external save/restore VCQ commands, so there is
+ * no way to checkpoint intermediate CMAC state to host memory.
+ * Pending eSW ABI extension to add save/restore for the AES core.
+ */
+
+static int cmh_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+
+ if (keylen != 16 && keylen != 24 && keylen != 32)
+ return -EINVAL;
+
+ return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_AES);
+}
+
+static void cmh_aes_cmac_free_chunks(struct cmh_aes_cmac_reqctx *rctx,
+ struct cmh_aes_cmac_tfm_ctx *tctx)
+{
+ struct cmh_aes_cmac_chunk *c, *tmp;
+
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(c, tmp, &rctx->chunks, list) {
+ list_del(&c->list);
+ list_del(&c->tfm_node);
+ kfree_sensitive(c);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->total_len = 0;
+}
+
+static int cmh_aes_cmac_init(struct ahash_request *req)
+{
+ struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req);
+
+ memset(rctx, 0, sizeof(*rctx));
+ INIT_LIST_HEAD(&rctx->chunks);
+ return 0;
+}
+
+static int cmh_aes_cmac_update(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_aes_cmac_chunk *chunk;
+ gfp_t gfp;
+ int ret;
+
+ if (!req->nbytes)
+ return 0;
+
+ if (req->nbytes > AES_CMAC_MAX_DATA - rctx->total_len) {
+ ret = -EINVAL;
+ goto err_free_chunks;
+ }
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ chunk = kmalloc(sizeof(*chunk) + req->nbytes, gfp);
+ if (!chunk) {
+ ret = -ENOMEM;
+ goto err_free_chunks;
+ }
+
+ chunk->len = req->nbytes;
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(chunk->data, req->svirt, req->nbytes);
+ else
+ scatterwalk_map_and_copy(chunk->data, req->src,
+ 0, req->nbytes, 0);
+
+ list_add_tail(&chunk->list, &rctx->chunks);
+ spin_lock_bh(&tctx->chunk_lock);
+ list_add_tail(&chunk->tfm_node, &tctx->all_chunks);
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->total_len += req->nbytes;
+ return 0;
+
+err_free_chunks:
+ /*
+ * Terminal error -- free all previously accumulated chunks.
+ * callers may not call .final() on error, so they would leak.
+ */
+ cmh_aes_cmac_free_chunks(rctx, tctx);
+ return ret;
+}
+
+static void cmh_aes_cmac_complete(void *data, int error)
+{
+ struct ahash_request *req = data;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ /* Unmap DMA */
+ if (rctx->total_len > 0)
+ cmh_dma_unmap_single(rctx->in_dma, rctx->total_len,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->tag_dma, AES_CMAC_DIGEST_SIZE,
+ DMA_FROM_DEVICE);
+
+ if (!error)
+ memcpy(req->result, rctx->tag_buf, AES_CMAC_DIGEST_SIZE);
+
+ kfree(rctx->tag_buf);
+ rctx->tag_buf = NULL;
+ kfree_sensitive(rctx->buf);
+ rctx->buf = NULL;
+ cmh_aes_cmac_free_chunks(rctx, tctx);
+ cmh_complete(&req->base, error);
+}
+
+static int cmh_aes_cmac_final(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req);
+ struct vcq_cmd cmds[CMH_AES_CMAC_MAX_PAYLOAD];
+ u64 key_ref;
+ u32 keylen;
+ struct core_dispatch d;
+ s32 target_mbx;
+ u32 core_id;
+ u32 idx;
+ int ret;
+ gfp_t gfp;
+
+ if (tctx->key.mode == CMH_KEY_NONE) {
+ ret = -ENOKEY;
+ goto out_free_buf;
+ }
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ /* Linearise accumulated chunks into a contiguous buffer for DMA */
+ if (rctx->total_len > 0) {
+ struct cmh_aes_cmac_chunk *c;
+ u32 off = 0;
+
+ rctx->buf = kmalloc(rctx->total_len, gfp);
+ if (!rctx->buf) {
+ ret = -ENOMEM;
+ goto out_free_chunks;
+ }
+ list_for_each_entry(c, &rctx->chunks, list) {
+ memcpy(rctx->buf + off, c->data, c->len);
+ off += c->len;
+ }
+ }
+
+ /* Tag output buffer */
+ rctx->tag_buf = kzalloc(AES_CMAC_DIGEST_SIZE, gfp);
+ if (!rctx->tag_buf) {
+ ret = -ENOMEM;
+ goto out_free_buf;
+ }
+
+ rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf,
+ AES_CMAC_DIGEST_SIZE,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->tag_dma)) {
+ ret = -ENOMEM;
+ goto out_free_tag;
+ }
+
+ /* Map input data (may be zero-length for empty CMAC) */
+ if (rctx->total_len > 0) {
+ rctx->in_dma = cmh_dma_map_single(rctx->buf, rctx->total_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->in_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap_tag;
+ }
+ }
+
+ /* Resolve key */
+ idx = 0;
+
+ rctx->key_dma = tctx->key.raw.dma;
+ rctx->keylen = tctx->key.raw.len;
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP,
+ (u64)rctx->key_dma, SYS_REF_NONE,
+ tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ keylen = tctx->key.raw.len;
+ d = cmh_core_select_instance(CMH_CORE_AES);
+ target_mbx = d.mbx_idx;
+ core_id = d.core_id;
+
+ /*
+ * INIT: mode=CMAC, op=ENCRYPT (CMAC always "encrypts")
+ * CMAC data goes through the AAD path:
+ * aadlen = total data length, iolen = 0
+ */
+ {
+ struct vcq_cmd *slot = &cmds[idx++];
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_INIT);
+ slot->hwc.aes.cmd_init.key = key_ref;
+ slot->hwc.aes.cmd_init.iv = 0;
+ slot->hwc.aes.cmd_init.keylen = keylen;
+ slot->hwc.aes.cmd_init.ivlen = 0;
+ slot->hwc.aes.cmd_init.mode = AES_MODE_CMAC;
+ slot->hwc.aes.cmd_init.op = AES_OP_ENCRYPT;
+ slot->hwc.aes.cmd_init.aadlen = rctx->total_len;
+ slot->hwc.aes.cmd_init.iolen = 0;
+ slot->hwc.aes.cmd_init.taglen = AES_CMAC_DIGEST_SIZE;
+ }
+
+ /* AAD_FINAL_AUTH: final AAD + tag extraction in one atomic step */
+ {
+ struct vcq_cmd *slot = &cmds[idx++];
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_AAD_FINAL_AUTH);
+ slot->hwc.aes.cmd_aad_final_auth.data =
+ rctx->total_len > 0 ? (u64)rctx->in_dma : 0;
+ slot->hwc.aes.cmd_aad_final_auth.datalen = rctx->total_len;
+ slot->hwc.aes.cmd_aad_final_auth.tag = (u64)rctx->tag_dma;
+ slot->hwc.aes.cmd_aad_final_auth.taglen = AES_CMAC_DIGEST_SIZE;
+ }
+
+ vcq_add_flush(&cmds[idx++], core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_AES_CMAC_MAX_PACKED,
+ target_mbx,
+ cmh_aes_cmac_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ /* -EBUSY = backlogged; ownership transferred to callback. */
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_cleanup_all;
+
+ return -EINPROGRESS;
+
+out_cleanup_all:
+ if (rctx->total_len > 0 && !cmh_dma_map_error(rctx->in_dma))
+ cmh_dma_unmap_single(rctx->in_dma, rctx->total_len,
+ DMA_TO_DEVICE);
+out_unmap_tag:
+ cmh_dma_unmap_single(rctx->tag_dma, AES_CMAC_DIGEST_SIZE,
+ DMA_FROM_DEVICE);
+out_free_tag:
+ kfree(rctx->tag_buf);
+out_free_buf:
+out_free_chunks:
+ cmh_aes_cmac_free_chunks(rctx, tctx);
+ kfree_sensitive(rctx->buf);
+ rctx->buf = NULL;
+ rctx->total_len = 0;
+ return ret;
+}
+
+/*
+ * ahash .export()/.import(): serialize/deserialize the software
+ * accumulation buffer. No HW state is involved -- the AES core
+ * does not support save/restore, but we only export the input queue.
+ */
+
+static int cmh_aes_cmac_export(struct ahash_request *req, void *out)
+{
+ struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_aes_cmac_export_state *state = out;
+ struct cmh_aes_cmac_chunk *chunk;
+ u32 offset = 0;
+
+ if (rctx->total_len > CMH_AES_CMAC_EXPORT_MAX)
+ return -ENOSPC;
+
+ state->total_len = rctx->total_len;
+ list_for_each_entry(chunk, &rctx->chunks, list) {
+ memcpy(state->data + offset, chunk->data, chunk->len);
+ offset += chunk->len;
+ }
+ return 0;
+}
+
+static int cmh_aes_cmac_import(struct ahash_request *req, const void *in)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_aes_cmac_export_state *state = in;
+ struct cmh_aes_cmac_chunk *chunk;
+
+ /*
+ * Do NOT call free_chunks() here: the crypto API does not
+ * guarantee the request context is in a valid state before
+ * import(), so the list pointers may be stale or invalid.
+ * Re-initialize from scratch instead. Any pre-existing chunks
+ * are tracked on tctx->all_chunks and freed in exit_tfm.
+ */
+ memset(rctx, 0, sizeof(*rctx));
+ INIT_LIST_HEAD(&rctx->chunks);
+
+ if (state->total_len > CMH_AES_CMAC_EXPORT_MAX)
+ return -EINVAL;
+
+ if (state->total_len) {
+ chunk = kmalloc(sizeof(*chunk) + state->total_len, GFP_KERNEL);
+ if (!chunk)
+ return -ENOMEM;
+ chunk->len = state->total_len;
+ memcpy(chunk->data, state->data, state->total_len);
+ list_add_tail(&chunk->list, &rctx->chunks);
+ spin_lock_bh(&tctx->chunk_lock);
+ list_add_tail(&chunk->tfm_node, &tctx->all_chunks);
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->total_len = state->total_len;
+ }
+ return 0;
+}
+
+static int cmh_aes_cmac_finup(struct ahash_request *req)
+{
+ int err;
+
+ err = cmh_aes_cmac_update(req);
+ if (err)
+ return err;
+ return cmh_aes_cmac_final(req);
+}
+
+static int cmh_aes_cmac_digest(struct ahash_request *req)
+{
+ int err;
+
+ err = cmh_aes_cmac_init(req);
+ if (err)
+ return err;
+ return cmh_aes_cmac_finup(req);
+}
+
+static int cmh_aes_cmac_init_tfm(struct crypto_ahash *tfm)
+{
+ struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+
+ memset(tctx, 0, sizeof(*tctx));
+ spin_lock_init(&tctx->chunk_lock);
+ INIT_LIST_HEAD(&tctx->all_chunks);
+ crypto_ahash_set_reqsize(tfm, sizeof(struct cmh_aes_cmac_reqctx));
+ return 0;
+}
+
+static void cmh_aes_cmac_exit_tfm(struct crypto_ahash *tfm)
+{
+ struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_aes_cmac_chunk *c, *tmp;
+
+ /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(c, tmp, &tctx->all_chunks, tfm_node) {
+ list_del(&c->tfm_node);
+ kfree_sensitive(c);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+
+ cmh_key_destroy(&tctx->key);
+}
+
+static struct ahash_alg cmh_aes_cmac_alg = {
+ .init = cmh_aes_cmac_init,
+ .update = cmh_aes_cmac_update,
+ .final = cmh_aes_cmac_final,
+ .finup = cmh_aes_cmac_finup,
+ .digest = cmh_aes_cmac_digest,
+ .export = cmh_aes_cmac_export,
+ .import = cmh_aes_cmac_import,
+ .setkey = cmh_aes_cmac_setkey,
+ .init_tfm = cmh_aes_cmac_init_tfm,
+ .exit_tfm = cmh_aes_cmac_exit_tfm,
+ .halg = {
+ .digestsize = AES_CMAC_DIGEST_SIZE,
+ .statesize = CMH_AES_CMAC_STATE_SIZE,
+ .base = {
+ .cra_name = "cmac(aes)",
+ .cra_driver_name = "cri-cmh-cmac-aes",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NO_FALLBACK |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_REQ_VIRT,
+ .cra_blocksize = AES_CMAC_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct cmh_aes_cmac_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
+
+/**
+ * cmh_aes_cmac_register() - Register AES-CMAC hash algorithm with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_aes_cmac_register(void)
+{
+ int ret;
+
+ ret = crypto_register_ahash(&cmh_aes_cmac_alg);
+ if (ret)
+ dev_err(cmh_dev(), "cmh_aes_cmac: failed to register cmac(aes) (rc=%d)\n",
+ ret);
+ else
+ dev_dbg(cmh_dev(), "cmh_aes_cmac: registered cmac(aes)\n");
+
+ return ret;
+}
+
+/**
+ * cmh_aes_cmac_unregister() - Unregister AES-CMAC hash algorithm from the crypto framework
+ */
+void cmh_aes_cmac_unregister(void)
+{
+ crypto_unregister_ahash(&cmh_aes_cmac_alg);
+ dev_dbg(cmh_dev(), "cmh_aes_cmac: unregistered cmac(aes)\n");
+}
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index bc326a9d7626..535a6ff551a2 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -34,6 +34,7 @@
#include "cmh_cshake.h"
#include "cmh_kmac.h"
#include "cmh_sm3.h"
+#include "cmh_aes.h"
#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
@@ -227,6 +228,21 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_sm3_register;
+ /* Register AES skcipher algorithms */
+ ret = cmh_aes_register();
+ if (ret)
+ goto err_aes_register;
+
+ /* Register AES AEAD algorithms (GCM, CCM) */
+ ret = cmh_aes_aead_register();
+ if (ret)
+ goto err_aes_aead_register;
+
+ /* Register AES CMAC algorithm */
+ ret = cmh_aes_cmac_register();
+ if (ret)
+ goto err_aes_cmac_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -239,6 +255,12 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_aes_cmac_unregister();
+err_aes_cmac_register:
+ cmh_aes_aead_unregister();
+err_aes_aead_register:
+ cmh_aes_unregister();
+err_aes_register:
cmh_sm3_unregister();
err_sm3_register:
cmh_kmac_unregister();
@@ -275,6 +297,9 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_aes_cmac_unregister();
+ cmh_aes_aead_unregister();
+ cmh_aes_unregister();
cmh_sm3_unregister();
cmh_kmac_unregister();
cmh_cshake_unregister();
diff --git a/drivers/crypto/cmh/include/cmh_aes.h b/drivers/crypto/cmh/include/cmh_aes.h
new file mode 100644
index 000000000000..591afaa36f85
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_aes.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- AES Crypto API Drivers
+ *
+ * Registers AES algorithms with the Linux crypto subsystem:
+ * skcipher: ecb/cbc/ctr/cfb/xts(aes)
+ * aead: gcm/ccm(aes)
+ * shash: cmac(aes)
+ */
+
+#ifndef CMH_AES_H
+#define CMH_AES_H
+
+int cmh_aes_register(void);
+void cmh_aes_unregister(void);
+
+int cmh_aes_aead_register(void);
+void cmh_aes_aead_unregister(void);
+
+int cmh_aes_cmac_register(void);
+void cmh_aes_cmac_unregister(void);
+
+#endif /* CMH_AES_H */
--
2.43.7
^ permalink raw reply related
* [PATCH v2 12/19] crypto: cmh - add RSA akcipher
From: Saravanakrishnan Krishnamoorthy @ 2026-07-09 20:30 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260709203037.1884436-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register the RSA akcipher algorithm using the CMH PKE core (core ID
0x0a). Supports encrypt, decrypt, sign, and verify operations with
2048, 3072, and 4096-bit keys. 512- and 1024-bit keys are also
accepted for legacy/test interoperability. Includes common PKE
helpers shared by subsequent ECDSA and ECDH patches.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 4 +-
drivers/crypto/cmh/cmh_main.c | 9 +
drivers/crypto/cmh/cmh_pke_common.c | 578 +++++++++++++++++++++++++
drivers/crypto/cmh/cmh_pke_rsa.c | 642 ++++++++++++++++++++++++++++
4 files changed, 1232 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_pke_common.c
create mode 100644 drivers/crypto/cmh/cmh_pke_rsa.c
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index 1c4cb817424c..7afd9852c337 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -29,7 +29,9 @@ cmh-y := \
cmh_ccp.o \
cmh_ccp_aead.o \
cmh_ccp_poly.o \
- cmh_rng.o
+ cmh_rng.o \
+ cmh_pke_common.o \
+ cmh_pke_rsa.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index 40440e2b73b4..07f26b0dd2ef 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -38,6 +38,7 @@
#include "cmh_aes.h"
#include "cmh_sm4.h"
#include "cmh_ccp.h"
+#include "cmh_pke.h"
#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
@@ -281,6 +282,11 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_ccp_poly_register;
+ /* Register PKE RSA akcipher */
+ ret = cmh_pke_rsa_register();
+ if (ret)
+ goto err_pke_rsa_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -293,6 +299,8 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_pke_rsa_unregister();
+err_pke_rsa_register:
cmh_ccp_poly_unregister();
err_ccp_poly_register:
cmh_ccp_aead_unregister();
@@ -349,6 +357,7 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_pke_rsa_unregister();
cmh_ccp_poly_unregister();
cmh_ccp_aead_unregister();
cmh_ccp_unregister();
diff --git a/drivers/crypto/cmh/cmh_pke_common.c b/drivers/crypto/cmh/cmh_pke_common.c
new file mode 100644
index 000000000000..ab3e2eb7d3f8
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pke_common.c
@@ -0,0 +1,578 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- PKE Common VCQ Builders
+ *
+ * VCQ builder functions for all PKE core commands. Each builder
+ * populates a single vcq_cmd slot with the appropriate magic,
+ * command ID, byte-swap flags, and command-specific payload.
+ *
+ * RSA commands always use PKE_SWAP_FLAGS (VCQ_FLAG_SWAP_BYTES |
+ * VCQ_FLAG_SWAP_WORDS). EC Weierstrass curves (NIST P-*, Brainpool,
+ * secp256k1, SM2) use PKE_SWAP_FLAGS; Edwards curves (25519, 448)
+ * use no swap flags. SM2 commands use per-command flags documented
+ * in the eSW ABI.
+ *
+ * Callers combine these with vcq_set_header() + vcq_add_flush()
+ * and submit via cmh_tm_submit_sync().
+ */
+
+#include <linux/string.h>
+
+#include "cmh_pke.h"
+
+/**
+ * vcq_add_pke_flush() - Add a PKE flush command to a VCQ slot
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ *
+ * Populates @slot with a flush command for the specified PKE core.
+ */
+void vcq_add_pke_flush(struct vcq_cmd *slot, u32 core_id)
+{
+ vcq_add_flush(slot, core_id);
+}
+
+/* RSA */
+
+/**
+ * vcq_add_pke_rsa_enc() - Build a VCQ command for RSA public-key encryption
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @bits: RSA key size in bits
+ * @e_len: Length of the public exponent in bytes
+ * @e_dma: DMA address of public exponent buffer
+ * @n_dma: DMA address of modulus buffer
+ * @m_dma: DMA address of plaintext message buffer
+ * @c_dma: DMA address of ciphertext output buffer
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_rsa_enc(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len,
+ u64 e_dma, u64 n_dma, u64 m_dma, u64 c_dma,
+ u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_RSA_ENC);
+ slot->hwc.pke.cmd_rsa_enc.bits = bits;
+ slot->hwc.pke.cmd_rsa_enc.e_len = e_len;
+ slot->hwc.pke.cmd_rsa_enc.e = e_dma;
+ slot->hwc.pke.cmd_rsa_enc.n = n_dma;
+ slot->hwc.pke.cmd_rsa_enc.m = m_dma;
+ slot->hwc.pke.cmd_rsa_enc.c = c_dma;
+}
+
+/**
+ * vcq_add_pke_rsa_dec() - Build a VCQ command for RSA private-key decryption
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @bits: RSA key size in bits
+ * @e_len: Length of the public exponent in bytes
+ * @e_dma: DMA address of public exponent buffer
+ * @n_dma: DMA address of modulus buffer
+ * @c_dma: DMA address of ciphertext input buffer
+ * @m_dma: DMA address of plaintext output buffer
+ * @d_ref: Datastore reference for the private exponent
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_rsa_dec(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len,
+ u64 e_dma, u64 n_dma, u64 c_dma, u64 m_dma,
+ u64 d_ref, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_RSA_DEC);
+ slot->hwc.pke.cmd_rsa_dec.bits = bits;
+ slot->hwc.pke.cmd_rsa_dec.e_len = e_len;
+ slot->hwc.pke.cmd_rsa_dec.e = e_dma;
+ slot->hwc.pke.cmd_rsa_dec.n = n_dma;
+ slot->hwc.pke.cmd_rsa_dec.c = c_dma;
+ slot->hwc.pke.cmd_rsa_dec.m = m_dma;
+ slot->hwc.pke.cmd_rsa_dec.d = d_ref;
+}
+
+/**
+ * vcq_add_pke_rsa_crt_dec() - Build a VCQ command for RSA-CRT decryption
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @bits: RSA key size in bits
+ * @e_len: Length of the public exponent in bytes
+ * @e_dma: DMA address of public exponent buffer
+ * @n_dma: DMA address of modulus buffer
+ * @c_dma: DMA address of ciphertext input buffer
+ * @m_dma: DMA address of plaintext output buffer
+ * @crt_ref: Datastore reference for CRT private key components
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_rsa_crt_dec(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len,
+ u64 e_dma, u64 n_dma, u64 c_dma, u64 m_dma,
+ u64 crt_ref, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_RSA_CRT_DEC);
+ slot->hwc.pke.cmd_rsa_crt_dec.bits = bits;
+ slot->hwc.pke.cmd_rsa_crt_dec.e_len = e_len;
+ slot->hwc.pke.cmd_rsa_crt_dec.e = e_dma;
+ slot->hwc.pke.cmd_rsa_crt_dec.n = n_dma;
+ slot->hwc.pke.cmd_rsa_crt_dec.c = c_dma;
+ slot->hwc.pke.cmd_rsa_crt_dec.m = m_dma;
+ slot->hwc.pke.cmd_rsa_crt_dec.crt = crt_ref;
+}
+
+/* ECDSA */
+
+/**
+ * vcq_add_pke_ecdsa_verify() - Build a VCQ command for ECDSA signature verification
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521)
+ * @dlen: Digest length in bytes
+ * @pk_dma: DMA address of public key buffer
+ * @dig_dma: DMA address of digest buffer
+ * @sig_dma: DMA address of signature buffer
+ * @rp_dma: DMA address of r-prime verification output buffer
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_ecdsa_verify(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 dlen,
+ u64 pk_dma, u64 dig_dma, u64 sig_dma,
+ u64 rp_dma, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDSA_VERIFY);
+ slot->hwc.pke.cmd_ecdsa_verify.curve = curve;
+ slot->hwc.pke.cmd_ecdsa_verify.digest_len = dlen;
+ slot->hwc.pke.cmd_ecdsa_verify.public_key = pk_dma;
+ slot->hwc.pke.cmd_ecdsa_verify.digest = dig_dma;
+ slot->hwc.pke.cmd_ecdsa_verify.signature = sig_dma;
+ slot->hwc.pke.cmd_ecdsa_verify.rprime = rp_dma;
+}
+
+/**
+ * vcq_add_pke_ecdsa_sign() - Build a VCQ command for ECDSA signing
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521)
+ * @sklen: Secret key length in bytes
+ * @dig_dma: DMA address of digest buffer
+ * @sig_dma: DMA address of signature output buffer
+ * @sk_ref: Datastore reference for the secret key
+ * @dlen: Digest length in bytes
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_ecdsa_sign(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 dig_dma, u64 sig_dma, u64 sk_ref,
+ u32 dlen, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDSA_SIGN);
+ slot->hwc.pke.cmd_ecdsa_sign.curve = curve;
+ slot->hwc.pke.cmd_ecdsa_sign.secret_key_len = sklen;
+ slot->hwc.pke.cmd_ecdsa_sign.digest = dig_dma;
+ slot->hwc.pke.cmd_ecdsa_sign.signature = sig_dma;
+ slot->hwc.pke.cmd_ecdsa_sign.secret_key = sk_ref;
+ slot->hwc.pke.cmd_ecdsa_sign.digest_len = dlen;
+}
+
+/**
+ * vcq_add_pke_ecdsa_pubgen() - Build a VCQ command for ECDSA public key generation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521)
+ * @sklen: Secret key length in bytes
+ * @pk_dma: DMA address of public key output buffer
+ * @sk_ref: Datastore reference for the secret key
+ * @flags: VCQ command flags
+ *
+ * Generates the public key from an existing private key stored in the
+ * datastore.
+ */
+void vcq_add_pke_ecdsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 pk_dma, u64 sk_ref, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDSA_PUBGEN);
+ slot->hwc.pke.cmd_ecdsa_pubgen.curve = curve;
+ slot->hwc.pke.cmd_ecdsa_pubgen.secret_key_len = sklen;
+ slot->hwc.pke.cmd_ecdsa_pubgen.public_key = pk_dma;
+ slot->hwc.pke.cmd_ecdsa_pubgen.secret_key = sk_ref;
+}
+
+/**
+ * vcq_add_pke_ecdsa_keygen() - Build a VCQ command for ECDSA key pair generation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521)
+ * @sklen: Secret key length in bytes
+ * @sk_ref: Datastore reference for the generated secret key
+ * @sk_type: Datastore type for the secret key object
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_ecdsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 sk_ref, u32 sk_type, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDSA_KEYGEN);
+ slot->hwc.pke.cmd_ecdsa_keygen.curve = curve;
+ slot->hwc.pke.cmd_ecdsa_keygen.secret_key_len = sklen;
+ slot->hwc.pke.cmd_ecdsa_keygen.secret_key = sk_ref;
+ slot->hwc.pke.cmd_ecdsa_keygen.secret_key_type = sk_type;
+}
+
+/* ECDH */
+
+/**
+ * vcq_add_pke_ecdh_keygen() - Build a VCQ command for ECDH key pair generation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521, X25519, X448)
+ * @sklen: Secret key length in bytes
+ * @pkx_dma: DMA address of public key X-coordinate output buffer
+ * @sk_ref: Datastore reference for the generated secret key
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_ecdh_keygen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 pkx_dma, u64 sk_ref, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDH_KEYGEN);
+ slot->hwc.pke.cmd_ecdh_keygen.curve = curve;
+ slot->hwc.pke.cmd_ecdh_keygen.secret_key_len = sklen;
+ slot->hwc.pke.cmd_ecdh_keygen.public_key_x = pkx_dma;
+ slot->hwc.pke.cmd_ecdh_keygen.secret_key = sk_ref;
+}
+
+/**
+ * vcq_add_pke_ecdh() - Build a VCQ command for ECDH shared secret computation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521, X25519, X448)
+ * @sklen: Secret key length in bytes
+ * @sslen: Shared secret length in bytes
+ * @ss_type: Datastore type for the shared secret object
+ * @peer_dma: DMA address of peer public key buffer
+ * @sk_ref: Datastore reference for the local secret key
+ * @ss_ref: Datastore reference for the computed shared secret
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_ecdh(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u32 sslen, u32 ss_type, u64 peer_dma, u64 sk_ref,
+ u64 ss_ref, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDH);
+ slot->hwc.pke.cmd_ecdh.curve = curve;
+ slot->hwc.pke.cmd_ecdh.secret_key_len = sklen;
+ slot->hwc.pke.cmd_ecdh.shared_secret_len = sslen;
+ slot->hwc.pke.cmd_ecdh.shared_secret_type = ss_type;
+ slot->hwc.pke.cmd_ecdh.peer_key_x = peer_dma;
+ slot->hwc.pke.cmd_ecdh.secret_key = sk_ref;
+ slot->hwc.pke.cmd_ecdh.shared_secret = ss_ref;
+}
+
+/* EdDSA */
+
+/**
+ * vcq_add_pke_eddsa_verify() - Build a VCQ command for EdDSA signature verification
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (Ed25519 or Ed448)
+ * @dlen: Digest (message) length in bytes
+ * @pky_dma: DMA address of public key Y-coordinate buffer
+ * @dig_dma: DMA address of digest buffer
+ * @sig_dma: DMA address of signature buffer
+ * @rp_dma: DMA address of r-prime verification output buffer
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_eddsa_verify(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 dlen,
+ u64 pky_dma, u64 dig_dma, u64 sig_dma,
+ u64 rp_dma, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_EDDSA_VERIFY);
+ slot->hwc.pke.cmd_eddsa_verify.curve = curve;
+ slot->hwc.pke.cmd_eddsa_verify.digest_len = dlen;
+ slot->hwc.pke.cmd_eddsa_verify.public_key_y = pky_dma;
+ slot->hwc.pke.cmd_eddsa_verify.digest = dig_dma;
+ slot->hwc.pke.cmd_eddsa_verify.signature = sig_dma;
+ slot->hwc.pke.cmd_eddsa_verify.rprime = rp_dma;
+}
+
+/**
+ * vcq_add_pke_eddsa_sign() - Build a VCQ command for EdDSA signing
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (Ed25519 or Ed448)
+ * @sklen: Secret key length in bytes
+ * @dig_dma: DMA address of digest (message) buffer
+ * @sig_dma: DMA address of signature output buffer
+ * @sk_ref: Datastore reference for the secret key
+ * @dlen: Digest (message) length in bytes
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_eddsa_sign(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 dig_dma, u64 sig_dma, u64 sk_ref,
+ u32 dlen, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_EDDSA_SIGN);
+ slot->hwc.pke.cmd_eddsa_sign.curve = curve;
+ slot->hwc.pke.cmd_eddsa_sign.secret_key_len = sklen;
+ slot->hwc.pke.cmd_eddsa_sign.digest = dig_dma;
+ slot->hwc.pke.cmd_eddsa_sign.signature = sig_dma;
+ slot->hwc.pke.cmd_eddsa_sign.secret_key = sk_ref;
+ slot->hwc.pke.cmd_eddsa_sign.digest_len = dlen;
+}
+
+/**
+ * vcq_add_pke_eddsa_pubgen() - Build a VCQ command for EdDSA public key generation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (Ed25519 or Ed448)
+ * @sklen: Secret key length in bytes
+ * @pky_dma: DMA address of public key Y-coordinate output buffer
+ * @sk_ref: Datastore reference for the secret key
+ * @flags: VCQ command flags
+ *
+ * Generates the public key from an existing private key stored in the
+ * datastore.
+ */
+void vcq_add_pke_eddsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 pky_dma, u64 sk_ref, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_EDDSA_PUBGEN);
+ slot->hwc.pke.cmd_eddsa_pubgen.curve = curve;
+ slot->hwc.pke.cmd_eddsa_pubgen.secret_key_len = sklen;
+ slot->hwc.pke.cmd_eddsa_pubgen.public_key_y = pky_dma;
+ slot->hwc.pke.cmd_eddsa_pubgen.secret_key = sk_ref;
+}
+
+/**
+ * vcq_add_pke_eddsa_keygen_sca() - Build a VCQ command for EdDSA SCA key generation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (Ed448)
+ * @sk_ref: Datastore reference for the input secret key
+ * @sca_sk_ref: Datastore reference for the SCA-masked output key
+ *
+ * Blinds an Ed448 private key into a side-channel-protected masked
+ * form. No byte-swap flags are used (CRI reference uses flags=0).
+ */
+void vcq_add_pke_eddsa_keygen_sca(struct vcq_cmd *slot, u32 core_id, u32 curve,
+ u64 sk_ref, u64 sca_sk_ref)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1,
+ PKE_CMD_EDDSA_PRIV_KEYGEN_SCA);
+ slot->hwc.pke.cmd_eddsa_keygen_sca.curve = curve;
+ slot->hwc.pke.cmd_eddsa_keygen_sca.secret_key = sk_ref;
+ slot->hwc.pke.cmd_eddsa_keygen_sca.sca_secret_key = sca_sk_ref;
+}
+
+/* SM2 */
+
+/**
+ * vcq_add_pke_sm2_ecdh_keygen() - Build a VCQ command for SM2 ECDH ephemeral key generation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @nonce_dma: DMA address of nonce input buffer
+ * @session_key_dma: DMA address of session key output buffer
+ * @nonce_len: Nonce length in bytes
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_ecdh_keygen(struct vcq_cmd *slot, u32 core_id, u64 nonce_dma,
+ u64 session_key_dma, u32 nonce_len, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1,
+ PKE_CMD_SM2_ECDH_KEYGEN);
+ slot->hwc.pke.cmd_sm2_ecdh_keygen.nonce = nonce_dma;
+ slot->hwc.pke.cmd_sm2_ecdh_keygen.session_key = session_key_dma;
+ slot->hwc.pke.cmd_sm2_ecdh_keygen.nonce_len = nonce_len;
+}
+
+/**
+ * vcq_add_pke_sm2_ecdh() - Build a VCQ command for SM2 ECDH shared secret computation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @nonce_len: Nonce length in bytes
+ * @private_key_len: Private key length in bytes
+ * @nonce_dma: DMA address of nonce buffer
+ * @peer_pk_dma: DMA address of peer public key buffer
+ * @peer_sk_dma: DMA address of peer session key buffer
+ * @priv_ref: Datastore reference for the local private key
+ * @sp_ref: Datastore reference for the shared point output
+ * @sp_type: Datastore type for the shared point object
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_ecdh(struct vcq_cmd *slot, u32 core_id, u32 nonce_len,
+ u32 private_key_len, u64 nonce_dma,
+ u64 peer_pk_dma, u64 peer_sk_dma,
+ u64 priv_ref, u64 sp_ref, u32 sp_type, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ECDH);
+ slot->hwc.pke.cmd_sm2_ecdh.nonce_len = nonce_len;
+ slot->hwc.pke.cmd_sm2_ecdh.private_key_len = private_key_len;
+ slot->hwc.pke.cmd_sm2_ecdh.nonce = nonce_dma;
+ slot->hwc.pke.cmd_sm2_ecdh.peer_public_key = peer_pk_dma;
+ slot->hwc.pke.cmd_sm2_ecdh.peer_session_key = peer_sk_dma;
+ slot->hwc.pke.cmd_sm2_ecdh.private_key = priv_ref;
+ slot->hwc.pke.cmd_sm2_ecdh.shared_point = sp_ref;
+ slot->hwc.pke.cmd_sm2_ecdh.shared_point_type = sp_type;
+}
+
+/**
+ * vcq_add_pke_sm2_dec_point() - Build a VCQ command for SM2 decryption point multiplication
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @ct_len: Ciphertext length in bytes
+ * @pk_len: Private key length in bytes
+ * @ct_dma: DMA address of ciphertext input buffer
+ * @dp_dma: DMA address of decryption point output buffer
+ * @priv_ref: Datastore reference for the private key
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_dec_point(struct vcq_cmd *slot, u32 core_id, u32 ct_len,
+ u32 pk_len, u64 ct_dma, u64 dp_dma,
+ u64 priv_ref, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_DEC_POINT);
+ slot->hwc.pke.cmd_sm2_dec_point.ciphertext_len = ct_len;
+ slot->hwc.pke.cmd_sm2_dec_point.private_key_len = pk_len;
+ slot->hwc.pke.cmd_sm2_dec_point.ciphertext = ct_dma;
+ slot->hwc.pke.cmd_sm2_dec_point.dec_point = dp_dma;
+ slot->hwc.pke.cmd_sm2_dec_point.private_key = priv_ref;
+}
+
+/**
+ * vcq_add_pke_sm2_enc_point() - Build a VCQ command for SM2 encryption point multiplication
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @nonce_dma: DMA address of nonce buffer
+ * @pk_dma: DMA address of public key buffer
+ * @ct_dma: DMA address of ciphertext header output buffer
+ * @ep_dma: DMA address of encryption point output buffer
+ * @nonce_len: Nonce length in bytes
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_enc_point(struct vcq_cmd *slot, u32 core_id, u64 nonce_dma,
+ u64 pk_dma, u64 ct_dma, u64 ep_dma,
+ u32 nonce_len, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ENC_POINT);
+ slot->hwc.pke.cmd_sm2_enc_point.nonce = nonce_dma;
+ slot->hwc.pke.cmd_sm2_enc_point.public_key = pk_dma;
+ slot->hwc.pke.cmd_sm2_enc_point.ciphertext = ct_dma;
+ slot->hwc.pke.cmd_sm2_enc_point.enc_point = ep_dma;
+ slot->hwc.pke.cmd_sm2_enc_point.nonce_len = nonce_len;
+}
+
+/**
+ * vcq_add_pke_sm2_id_digest() - Build a VCQ command for SM2 identity digest computation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @id_dma: DMA address of identity string buffer
+ * @pk_dma: DMA address of public key buffer
+ * @dig_dma: DMA address of digest output buffer
+ * @id_len: Identity string length in bytes
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_id_digest(struct vcq_cmd *slot, u32 core_id, u64 id_dma,
+ u64 pk_dma, u64 dig_dma, u32 id_len,
+ u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ID_DIGEST);
+ slot->hwc.pke.cmd_sm2_id_digest.id = id_dma;
+ slot->hwc.pke.cmd_sm2_id_digest.public_key = pk_dma;
+ slot->hwc.pke.cmd_sm2_id_digest.digest = dig_dma;
+ slot->hwc.pke.cmd_sm2_id_digest.id_len = id_len;
+}
+
+/**
+ * vcq_add_pke_sm2_ecdh_hash() - Build a VCQ command for SM2 ECDH key derivation hash
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @peer_dig_dma: DMA address of peer identity digest buffer
+ * @dig_dma: DMA address of local identity digest buffer
+ * @sp_ref: Datastore reference for the shared point
+ * @sk_ref: Datastore reference for the derived shared key output
+ * @sk_type: Datastore type for the shared key object
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_ecdh_hash(struct vcq_cmd *slot, u32 core_id, u64 peer_dig_dma,
+ u64 dig_dma, u64 sp_ref, u64 sk_ref,
+ u32 sk_type, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ECDH_HASH);
+ slot->hwc.pke.cmd_sm2_ecdh_hash.peer_id_digest = peer_dig_dma;
+ slot->hwc.pke.cmd_sm2_ecdh_hash.id_digest = dig_dma;
+ slot->hwc.pke.cmd_sm2_ecdh_hash.shared_point = sp_ref;
+ slot->hwc.pke.cmd_sm2_ecdh_hash.shared_key = sk_ref;
+ slot->hwc.pke.cmd_sm2_ecdh_hash.shared_key_type = sk_type;
+}
+
+/**
+ * vcq_add_pke_sm2_dec_hash() - Build a VCQ command for SM2 decryption hash verification
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @ct_dma: DMA address of ciphertext input buffer
+ * @dp_dma: DMA address of decryption point buffer
+ * @pt_dma: DMA address of plaintext output buffer
+ * @ct_len: Ciphertext length in bytes
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_dec_hash(struct vcq_cmd *slot, u32 core_id, u64 ct_dma,
+ u64 dp_dma, u64 pt_dma, u32 ct_len, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_DEC_HASH);
+ slot->hwc.pke.cmd_sm2_dec_hash.ciphertext = ct_dma;
+ slot->hwc.pke.cmd_sm2_dec_hash.dec_point = dp_dma;
+ slot->hwc.pke.cmd_sm2_dec_hash.plaintext = pt_dma;
+ slot->hwc.pke.cmd_sm2_dec_hash.ciphertext_len = ct_len;
+}
+
+/**
+ * vcq_add_pke_sm2_enc_hash() - Build a VCQ command for SM2 encryption hash computation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @msg_dma: DMA address of plaintext message buffer
+ * @ep_dma: DMA address of encryption point buffer
+ * @ct_dma: DMA address of ciphertext output buffer
+ * @msg_len: Message length in bytes
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_enc_hash(struct vcq_cmd *slot, u32 core_id, u64 msg_dma,
+ u64 ep_dma, u64 ct_dma, u32 msg_len, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ENC_HASH);
+ slot->hwc.pke.cmd_sm2_enc_hash.message = msg_dma;
+ slot->hwc.pke.cmd_sm2_enc_hash.enc_point = ep_dma;
+ slot->hwc.pke.cmd_sm2_enc_hash.ciphertext = ct_dma;
+ slot->hwc.pke.cmd_sm2_enc_hash.message_len = msg_len;
+}
diff --git a/drivers/crypto/cmh/cmh_pke_rsa.c b/drivers/crypto/cmh/cmh_pke_rsa.c
new file mode 100644
index 000000000000..010f8bd98f0d
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pke_rsa.c
@@ -0,0 +1,642 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- RSA akcipher Driver
+ *
+ * Registers "rsa" akcipher algorithm with the Linux crypto subsystem
+ * (priority 300, overrides software rsa-generic at 100).
+ *
+ * Raw RSA operations only (m^e mod n / c^d mod n). The kernel's
+ * pkcs1pad() template wraps this for PKCS#1 v1.5 / PSS / OAEP.
+ *
+ * Key format: DER-encoded ASN.1, parsed by kernel rsa_parse_pub_key()
+ * / rsa_parse_priv_key() helpers.
+ *
+ * Private key via cmh_key_ctx: raw keys written via SYS_REF_TEMP.
+ * Datastore-referenced keys are only reachable through the ioctl
+ * path (cmh_mgmt.c).
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/scatterlist.h>
+#include <crypto/akcipher.h>
+#include <crypto/internal/akcipher.h>
+#include <crypto/internal/rsa.h>
+
+#include "cmh_pke.h"
+#include "cmh_sys.h"
+#include "cmh_sys_abi.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+struct cmh_rsa_tfm_ctx {
+ struct cmh_key_ctx key; /* private key (raw d only) */
+ u8 *n; /* modulus (big-endian) */
+ u8 *e; /* public exponent (big-endian) */
+ size_t n_sz;
+ size_t e_sz;
+ u32 bits; /* key size in bits */
+};
+
+static inline struct cmh_rsa_tfm_ctx *cmh_rsa_ctx(struct crypto_akcipher *tfm)
+{
+ return akcipher_tfm_ctx(tfm);
+}
+
+struct cmh_rsa_reqctx {
+ u8 *e_buf;
+ u8 *n_buf;
+ u8 *m_buf;
+ u8 *c_buf;
+ u8 *d_buf; /* dec only: private key copy */
+ dma_addr_t e_dma;
+ dma_addr_t n_dma;
+ dma_addr_t m_dma;
+ dma_addr_t c_dma;
+ dma_addr_t d_dma;
+ u32 key_bytes;
+ u32 e_padded;
+ u32 n_sz;
+ u32 d_len; /* dec only */
+};
+
+static u32 cmh_rsa_key_bits(size_t n_sz)
+{
+ /*
+ * Only accept exact modulus sizes supported by the hardware.
+ * The programmed RSA width must match the actual modulus buffer
+ * length; rounding a shorter modulus up to the next size would
+ * let the device read past the end of the DMA buffer.
+ */
+ switch (n_sz) {
+ case 64:
+ return 512;
+ case 128:
+ return 1024;
+ case 256:
+ return 2048;
+ case 384:
+ return 3072;
+ case 512:
+ return 4096;
+ default:
+ return 0;
+ }
+}
+
+static void cmh_rsa_enc_complete(void *data, int error)
+{
+ struct akcipher_request *req = data;
+ struct cmh_rsa_reqctx *rctx = akcipher_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ if (!cmh_dma_map_error(rctx->c_dma))
+ cmh_dma_unmap_single(rctx->c_dma, rctx->key_bytes,
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(rctx->m_dma))
+ cmh_dma_unmap_single(rctx->m_dma, rctx->key_bytes,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->n_dma))
+ cmh_dma_unmap_single(rctx->n_dma, rctx->n_sz,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->e_dma))
+ cmh_dma_unmap_single(rctx->e_dma, rctx->e_padded,
+ DMA_TO_DEVICE);
+
+ if (!error) {
+ int nents;
+
+ nents = sg_nents_for_len(req->dst, rctx->key_bytes);
+ if (nents < 0 ||
+ sg_copy_from_buffer(req->dst, nents,
+ rctx->c_buf,
+ rctx->key_bytes) != rctx->key_bytes)
+ error = -EINVAL;
+ else
+ req->dst_len = rctx->key_bytes;
+ }
+
+ kfree(rctx->c_buf);
+ rctx->c_buf = NULL;
+ kfree_sensitive(rctx->m_buf);
+ rctx->m_buf = NULL;
+ kfree(rctx->n_buf);
+ rctx->n_buf = NULL;
+ kfree(rctx->e_buf);
+ rctx->e_buf = NULL;
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * RSA encrypt: c = m^e mod n (public key operation)
+ * Also used for signature verification (verify = encrypt for raw RSA).
+ */
+static int cmh_rsa_enc(struct akcipher_request *req)
+{
+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+ struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm);
+ struct cmh_rsa_reqctx *rctx = akcipher_request_ctx(req);
+ u32 key_bytes = ctx->bits / 8;
+ u32 e_padded = ALIGN(ctx->e_sz, 4);
+ struct core_dispatch d = cmh_core_select_instance(CMH_CORE_PKE);
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ int ret, nents;
+ gfp_t gfp;
+
+ if (!ctx->n || !ctx->e)
+ return -EINVAL;
+ if (req->src_len > key_bytes || req->dst_len < key_bytes)
+ return -EINVAL;
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->key_bytes = key_bytes;
+ rctx->e_padded = e_padded;
+ rctx->n_sz = ctx->n_sz;
+ rctx->e_dma = DMA_MAPPING_ERROR;
+ rctx->n_dma = DMA_MAPPING_ERROR;
+ rctx->m_dma = DMA_MAPPING_ERROR;
+ rctx->c_dma = DMA_MAPPING_ERROR;
+
+ rctx->e_buf = kzalloc(e_padded, gfp);
+ rctx->n_buf = kmemdup(ctx->n, ctx->n_sz, gfp);
+ rctx->m_buf = kzalloc(key_bytes, gfp);
+ rctx->c_buf = kzalloc(key_bytes, gfp);
+ if (!rctx->e_buf || !rctx->n_buf || !rctx->m_buf || !rctx->c_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ memcpy(rctx->e_buf + e_padded - ctx->e_sz, ctx->e, ctx->e_sz);
+
+ nents = sg_nents_for_len(req->src, req->src_len);
+ if (nents < 0 ||
+ sg_pcopy_to_buffer(req->src, nents,
+ rctx->m_buf + key_bytes - req->src_len,
+ req->src_len, 0) != req->src_len) {
+ ret = -EINVAL;
+ goto out_free;
+ }
+
+ rctx->e_dma = cmh_dma_map_single(rctx->e_buf, e_padded,
+ DMA_TO_DEVICE);
+ rctx->n_dma = cmh_dma_map_single(rctx->n_buf, ctx->n_sz,
+ DMA_TO_DEVICE);
+ rctx->m_dma = cmh_dma_map_single(rctx->m_buf, key_bytes,
+ DMA_TO_DEVICE);
+ rctx->c_dma = cmh_dma_map_single(rctx->c_buf, key_bytes,
+ DMA_FROM_DEVICE);
+
+ if (cmh_dma_map_error(rctx->e_dma) ||
+ cmh_dma_map_error(rctx->n_dma) ||
+ cmh_dma_map_error(rctx->m_dma) ||
+ cmh_dma_map_error(rctx->c_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_rsa_enc(&vcq[1], d.core_id, ctx->bits, e_padded,
+ rctx->e_dma, rctx->n_dma, rctx->m_dma,
+ rctx->c_dma, PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[2], d.core_id);
+
+ ret = cmh_tm_submit_async(vcq, PKE_VCQ_CMDS_MIN, 1, d.mbx_idx,
+ cmh_rsa_enc_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG), 0);
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (!ret)
+ return -EINPROGRESS;
+
+out_unmap:
+ if (!cmh_dma_map_error(rctx->c_dma))
+ cmh_dma_unmap_single(rctx->c_dma, key_bytes,
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(rctx->m_dma))
+ cmh_dma_unmap_single(rctx->m_dma, key_bytes,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->n_dma))
+ cmh_dma_unmap_single(rctx->n_dma, ctx->n_sz,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->e_dma))
+ cmh_dma_unmap_single(rctx->e_dma, e_padded,
+ DMA_TO_DEVICE);
+
+out_free:
+ kfree(rctx->c_buf);
+ kfree_sensitive(rctx->m_buf);
+ kfree(rctx->n_buf);
+ kfree(rctx->e_buf);
+ return ret;
+}
+
+static void cmh_rsa_dec_complete(void *data, int error)
+{
+ struct akcipher_request *req = data;
+ struct cmh_rsa_reqctx *rctx = akcipher_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ if (!cmh_dma_map_error(rctx->d_dma))
+ cmh_dma_unmap_single(rctx->d_dma, rctx->d_len,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->m_dma))
+ cmh_dma_unmap_single(rctx->m_dma, rctx->key_bytes,
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(rctx->c_dma))
+ cmh_dma_unmap_single(rctx->c_dma, rctx->key_bytes,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->n_dma))
+ cmh_dma_unmap_single(rctx->n_dma, rctx->n_sz,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->e_dma))
+ cmh_dma_unmap_single(rctx->e_dma, rctx->e_padded,
+ DMA_TO_DEVICE);
+
+ if (!error) {
+ int nents;
+
+ nents = sg_nents_for_len(req->dst, rctx->key_bytes);
+ if (nents < 0 ||
+ sg_copy_from_buffer(req->dst, nents,
+ rctx->m_buf,
+ rctx->key_bytes) != rctx->key_bytes)
+ error = -EINVAL;
+ else
+ req->dst_len = rctx->key_bytes;
+ }
+
+ kfree_sensitive(rctx->d_buf);
+ rctx->d_buf = NULL;
+ kfree_sensitive(rctx->m_buf);
+ rctx->m_buf = NULL;
+ kfree(rctx->c_buf);
+ rctx->c_buf = NULL;
+ kfree(rctx->n_buf);
+ rctx->n_buf = NULL;
+ kfree(rctx->e_buf);
+ rctx->e_buf = NULL;
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * RSA decrypt: m = c^d mod n (private key operation)
+ * Also used for signing (sign = decrypt for raw RSA).
+ *
+ * Private key 'd' is written via SYS_REF_TEMP inline.
+ */
+static int cmh_rsa_dec(struct akcipher_request *req)
+{
+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+ struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm);
+ struct cmh_rsa_reqctx *rctx = akcipher_request_ctx(req);
+ u32 key_bytes = ctx->bits / 8;
+ u32 e_padded = ALIGN(ctx->e_sz, 4);
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MAX];
+ struct core_dispatch dd;
+ int ret, idx, nents;
+ gfp_t gfp;
+
+ if (ctx->key.mode != CMH_KEY_RAW)
+ return -EINVAL;
+ if (!ctx->n || !ctx->e)
+ return -EINVAL;
+ if (req->src_len > key_bytes || req->dst_len < key_bytes)
+ return -EINVAL;
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->key_bytes = key_bytes;
+ rctx->e_padded = e_padded;
+ rctx->n_sz = ctx->n_sz;
+ rctx->e_dma = DMA_MAPPING_ERROR;
+ rctx->n_dma = DMA_MAPPING_ERROR;
+ rctx->m_dma = DMA_MAPPING_ERROR;
+ rctx->c_dma = DMA_MAPPING_ERROR;
+ rctx->d_dma = DMA_MAPPING_ERROR;
+
+ rctx->e_buf = kzalloc(e_padded, gfp);
+ rctx->n_buf = kmemdup(ctx->n, ctx->n_sz, gfp);
+ rctx->c_buf = kzalloc(key_bytes, gfp);
+ rctx->m_buf = kzalloc(key_bytes, gfp);
+ if (!rctx->e_buf || !rctx->n_buf || !rctx->c_buf || !rctx->m_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ memcpy(rctx->e_buf + e_padded - ctx->e_sz, ctx->e, ctx->e_sz);
+
+ nents = sg_nents_for_len(req->src, req->src_len);
+ if (nents < 0 ||
+ sg_pcopy_to_buffer(req->src, nents,
+ rctx->c_buf + key_bytes - req->src_len,
+ req->src_len, 0) != req->src_len) {
+ ret = -EINVAL;
+ goto out_free;
+ }
+
+ rctx->e_dma = cmh_dma_map_single(rctx->e_buf, e_padded,
+ DMA_TO_DEVICE);
+ rctx->n_dma = cmh_dma_map_single(rctx->n_buf, ctx->n_sz,
+ DMA_TO_DEVICE);
+ rctx->c_dma = cmh_dma_map_single(rctx->c_buf, key_bytes,
+ DMA_TO_DEVICE);
+ rctx->m_dma = cmh_dma_map_single(rctx->m_buf, key_bytes,
+ DMA_FROM_DEVICE);
+
+ if (cmh_dma_map_error(rctx->e_dma) ||
+ cmh_dma_map_error(rctx->n_dma) ||
+ cmh_dma_map_error(rctx->c_dma) ||
+ cmh_dma_map_error(rctx->m_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ dd = cmh_core_select_instance(CMH_CORE_PKE);
+
+ rctx->d_buf = kmemdup(ctx->key.raw.data, ctx->key.raw.len, gfp);
+ if (!rctx->d_buf) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ rctx->d_len = ctx->key.raw.len;
+
+ rctx->d_dma = cmh_dma_map_single(rctx->d_buf, ctx->key.raw.len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->d_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ idx = 1;
+ vcq_add_sys_write(&vcq[idx], SYS_REF_TEMP, rctx->d_dma,
+ SYS_REF_NONE, ctx->key.raw.len,
+ ctx->key.raw.sys_type);
+ vcq[idx].id |= PKE_SWAP_FLAGS;
+ idx++;
+ vcq_add_pke_rsa_dec(&vcq[idx++], dd.core_id, ctx->bits, e_padded,
+ rctx->e_dma, rctx->n_dma, rctx->c_dma,
+ rctx->m_dma, SYS_REF_TEMP, PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[idx++], dd.core_id);
+ vcq_set_header(&vcq[0], idx);
+
+ ret = cmh_tm_submit_async(vcq, idx, 1, dd.mbx_idx,
+ cmh_rsa_dec_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG), 0);
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (!ret)
+ return -EINPROGRESS;
+
+out_unmap:
+ if (!cmh_dma_map_error(rctx->d_dma))
+ cmh_dma_unmap_single(rctx->d_dma, rctx->d_len,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->m_dma))
+ cmh_dma_unmap_single(rctx->m_dma, key_bytes,
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(rctx->c_dma))
+ cmh_dma_unmap_single(rctx->c_dma, key_bytes,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->n_dma))
+ cmh_dma_unmap_single(rctx->n_dma, ctx->n_sz,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->e_dma))
+ cmh_dma_unmap_single(rctx->e_dma, e_padded,
+ DMA_TO_DEVICE);
+
+out_free:
+ kfree_sensitive(rctx->d_buf);
+ kfree_sensitive(rctx->m_buf);
+ kfree(rctx->c_buf);
+ kfree(rctx->n_buf);
+ kfree(rctx->e_buf);
+ return ret;
+}
+
+static int cmh_rsa_set_pub_key(struct crypto_akcipher *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm);
+ struct rsa_key rsa = {};
+ int ret;
+
+ ret = rsa_parse_pub_key(&rsa, key, keylen);
+ if (ret)
+ return ret;
+
+ /* Strip ASN.1 leading zero padding from modulus */
+ while (rsa.n_sz > 0 && rsa.n[0] == 0) {
+ rsa.n++;
+ rsa.n_sz--;
+ }
+
+ ctx->bits = cmh_rsa_key_bits(rsa.n_sz);
+ if (!ctx->bits)
+ return -EINVAL;
+
+ kfree(ctx->n);
+ kfree(ctx->e);
+ ctx->n = NULL;
+ ctx->e = NULL;
+ ctx->n_sz = 0;
+ ctx->e_sz = 0;
+
+ ctx->n = kmemdup(rsa.n, rsa.n_sz, GFP_KERNEL);
+ ctx->e = kmemdup(rsa.e, rsa.e_sz, GFP_KERNEL);
+ if (!ctx->n || !ctx->e) {
+ kfree(ctx->n);
+ kfree(ctx->e);
+ ctx->n = NULL;
+ ctx->e = NULL;
+ return -ENOMEM;
+ }
+
+ ctx->n_sz = rsa.n_sz;
+ ctx->e_sz = rsa.e_sz;
+
+ return 0;
+}
+
+static int cmh_rsa_set_priv_key(struct crypto_akcipher *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm);
+ struct rsa_key rsa = {};
+ u32 key_bytes;
+ u8 *d_padded;
+ int ret;
+
+ ret = rsa_parse_priv_key(&rsa, key, keylen);
+ if (ret)
+ return ret;
+
+ /* Strip ASN.1 leading zero padding from modulus */
+ while (rsa.n_sz > 0 && rsa.n[0] == 0) {
+ rsa.n++;
+ rsa.n_sz--;
+ }
+
+ ctx->bits = cmh_rsa_key_bits(rsa.n_sz);
+ if (!ctx->bits || !rsa.d_sz)
+ return -EINVAL;
+
+ key_bytes = ctx->bits / 8;
+
+ /* Strip ASN.1 leading zero padding from private exponent */
+ while (rsa.d_sz > 0 && rsa.d[0] == 0) {
+ rsa.d++;
+ rsa.d_sz--;
+ }
+
+ if (!rsa.d_sz || rsa.d_sz > key_bytes)
+ return -EINVAL;
+
+ kfree(ctx->n);
+ kfree(ctx->e);
+ ctx->n = NULL;
+ ctx->e = NULL;
+ ctx->n_sz = 0;
+ ctx->e_sz = 0;
+
+ ctx->n = kmemdup(rsa.n, rsa.n_sz, GFP_KERNEL);
+ ctx->e = kmemdup(rsa.e, rsa.e_sz, GFP_KERNEL);
+ if (!ctx->n || !ctx->e) {
+ ret = -ENOMEM;
+ goto err;
+ }
+
+ ctx->n_sz = rsa.n_sz;
+ ctx->e_sz = rsa.e_sz;
+
+ /*
+ * Left-pad d to key_bytes (big-endian alignment).
+ * The CMH eSW resolves SYS_REF_TEMP by checking
+ * hdr->len >= key_bytes, so the written buffer must
+ * be at least key_bytes wide.
+ */
+ d_padded = kzalloc(key_bytes, GFP_KERNEL);
+ if (!d_padded) {
+ ret = -ENOMEM;
+ goto err;
+ }
+ memcpy(d_padded + key_bytes - rsa.d_sz, rsa.d, rsa.d_sz);
+
+ ret = cmh_key_setkey_raw(&ctx->key, d_padded, key_bytes,
+ CORE_ID_PKE);
+ kfree_sensitive(d_padded);
+ if (ret)
+ goto err;
+
+ return 0;
+err:
+ kfree(ctx->n);
+ kfree(ctx->e);
+ ctx->n = NULL;
+ ctx->e = NULL;
+ ctx->n_sz = 0;
+ ctx->e_sz = 0;
+ ctx->bits = 0;
+ return ret;
+}
+
+static unsigned int cmh_rsa_max_size(struct crypto_akcipher *tfm)
+{
+ struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm);
+
+ return ctx->n_sz;
+}
+
+static int cmh_rsa_init_tfm(struct crypto_akcipher *tfm)
+{
+ struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ tfm->reqsize = sizeof(struct cmh_rsa_reqctx);
+ return 0;
+}
+
+static void cmh_rsa_exit_tfm(struct crypto_akcipher *tfm)
+{
+ struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm);
+
+ cmh_key_destroy(&ctx->key);
+ kfree(ctx->n);
+ kfree(ctx->e);
+ ctx->n = NULL;
+ ctx->e = NULL;
+}
+
+/*
+ * Raw RSA stays as akcipher (encrypt/decrypt only). The kernel's
+ * rsassa-pkcs1 sig template wraps our akcipher for sign/verify,
+ * matching the upstream split (rsa.c = akcipher,
+ * rsassa-pkcs1.c = sig template).
+ */
+static struct akcipher_alg cmh_rsa_alg = {
+ .encrypt = cmh_rsa_enc,
+ .decrypt = cmh_rsa_dec,
+ .set_pub_key = cmh_rsa_set_pub_key,
+ .set_priv_key = cmh_rsa_set_priv_key,
+ .max_size = cmh_rsa_max_size,
+ .init = cmh_rsa_init_tfm,
+ .exit = cmh_rsa_exit_tfm,
+ .base = {
+ .cra_name = "rsa",
+ .cra_driver_name = "cri-cmh-rsa",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_rsa_tfm_ctx),
+ },
+};
+
+static bool cmh_rsa_registered;
+
+/**
+ * cmh_pke_rsa_register() - Register RSA akcipher algorithm with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_pke_rsa_register(void)
+{
+ int ret;
+
+ ret = crypto_register_akcipher(&cmh_rsa_alg);
+ if (ret) {
+ dev_err(cmh_dev(),
+ "cmh: failed to register rsa akcipher (%d)\n",
+ ret);
+ return ret;
+ }
+
+ cmh_rsa_registered = true;
+ return 0;
+}
+
+/**
+ * cmh_pke_rsa_unregister() - Unregister RSA akcipher algorithm from the crypto framework
+ */
+void cmh_pke_rsa_unregister(void)
+{
+ if (cmh_rsa_registered)
+ crypto_unregister_akcipher(&cmh_rsa_alg);
+ cmh_rsa_registered = false;
+}
--
2.43.7
^ permalink raw reply related
* [PATCH v2 06/19] crypto: cmh - add CSHAKE/KMAC ahash
From: Saravanakrishnan Krishnamoorthy @ 2026-07-09 20:30 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260709203037.1884436-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register ahash algorithms for cSHAKE128, cSHAKE256, KMAC128, and
KMAC256 using the CMH hash core. cSHAKE supports incremental
update and export/import. KMAC has a 64KB data cap imposed by the
hardware.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 4 +-
drivers/crypto/cmh/cmh_cshake.c | 808 ++++++++++++++++++++++++
drivers/crypto/cmh/cmh_kmac.c | 630 ++++++++++++++++++
drivers/crypto/cmh/cmh_main.c | 18 +
drivers/crypto/cmh/include/cmh_cshake.h | 16 +
drivers/crypto/cmh/include/cmh_kmac.h | 16 +
6 files changed, 1491 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_cshake.c
create mode 100644 drivers/crypto/cmh/cmh_kmac.c
create mode 100644 drivers/crypto/cmh/include/cmh_cshake.h
create mode 100644 drivers/crypto/cmh/include/cmh_kmac.h
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index 1f760c0214ef..2bb240b97f31 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -16,7 +16,9 @@ cmh-y := \
cmh_key.o \
cmh_sys.o \
cmh_hash.o \
- cmh_hmac.o
+ cmh_hmac.o \
+ cmh_cshake.o \
+ cmh_kmac.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_cshake.c b/drivers/crypto/cmh/cmh_cshake.c
new file mode 100644
index 000000000000..02f9b853dd33
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_cshake.c
@@ -0,0 +1,808 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API CSHAKE Driver
+ *
+ * Registers cSHAKE-128 and cSHAKE-256 as ahash algorithms using the
+ * CMH Hash Core (HC) via HC_CMD_CSHAKE.
+ *
+ * CSHAKE (NIST SP 800-185) extends SHAKE with two domain separation
+ * parameters: function name N and customization string S. When both
+ * are empty, cSHAKE reduces to plain SHAKE -- the driver falls back to
+ * HC_CMD_INIT in that case (per SP 800-185 S6.2).
+ *
+ * N and S are set via .setkey() using a self-describing binary header
+ * (matching the upstream authenc precedent):
+ *
+ * struct cshake_cfg { __be32 n_len; __be32 s_len; };
+ * setkey blob: cshake_cfg || N[n_len] || S[s_len]
+ *
+ * If .setkey() is never called, the driver defaults to plain SHAKE
+ * (N="" S=""). .setkey() is per-tfm, not per-request.
+ *
+ * N is embedded inline in the HC_CMD_CSHAKE struct (max 36 bytes).
+ * S is passed as VCQ inline data following the command slot (multi-span).
+ *
+ * Uses the same self-contained transaction model as cmh_hash.c:
+ * .init() -> software-only
+ * .update() -> software-only (accumulate chunks)
+ * .final() -> CSHAKE [+ inline S] [+ RESTORE] [+ GATHER] + FINAL + FLUSH
+ * .export() -> CSHAKE [+ inline S] [+ RESTORE] [+ GATHER] + SAVE + FLUSH
+ * .import() -> restore HC context checkpoint (software-only)
+ *
+ * The HC core supports HC_CMD_SAVE / HC_CMD_RESTORE for cSHAKE mode.
+ * The cSHAKE domain-separation prefix (function name N, customization
+ * string S) is absorbed into the Keccak sponge state by HC_CMD_CSHAKE
+ * on the first submission, and preserved through save/restore.
+ * Export/import enables crypto API transform cloning.
+ *
+ * .setkey() here configures public domain-separation parameters (N, S),
+ * not a secret key.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/hash.h>
+#include <linux/scatterlist.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <asm/byteorder.h>
+
+#include "cmh_cshake.h"
+#include "cmh_vcq.h"
+#include "cmh_hc_abi.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+
+/* Algorithm Table */
+
+struct cmh_cshake_alg_info {
+ u32 hc_algo;
+ u32 digest_size;
+ const char *alg_name;
+ const char *drv_name;
+};
+
+static const struct cmh_cshake_alg_info cmh_cshake_algs_info[] = {
+ {
+ .hc_algo = HC_ALGO_SHAKE128,
+ .digest_size = CMH_SHAKE128_DIGEST_SIZE,
+ .alg_name = "cshake128",
+ .drv_name = "cri-cmh-cshake128",
+ },
+ {
+ .hc_algo = HC_ALGO_SHAKE256,
+ .digest_size = CMH_SHAKE256_DIGEST_SIZE,
+ .alg_name = "cshake256",
+ .drv_name = "cri-cmh-cshake256",
+ },
+};
+
+#define CMH_CSHAKE_ALG_COUNT ARRAY_SIZE(cmh_cshake_algs_info)
+
+/* Per-Request State */
+
+struct cmh_cshake_chunk {
+ struct list_head list;
+ struct list_head tfm_node; /* per-tfm orphan tracking */
+ u32 len;
+ u8 data[];
+};
+
+/*
+ * Max payload slots for CSHAKE:
+ * CSHAKE (1) + inline S (ceil(S_len/64)) + GATHER (1) + FINAL (1) + FLUSH (1)
+ * S can be up to SHAKE-128 block (168 bytes) = 3 inline slots.
+ * Conservative: 1 + 3 + 1 + 1 + 1 = 7, plus headers.
+ *
+ * Or INIT + GATHER + FINAL + FLUSH = 4 (plain SHAKE fallback).
+ */
+#define CMH_CSHAKE_MAX_PAYLOAD 8
+#define CMH_CSHAKE_MAX_PACKED (CMH_CSHAKE_MAX_PAYLOAD * 2)
+
+/*
+ * Checkpoint embedded inline: the kernel ahash API has no per-request
+ * destructor, so a heap-allocated checkpoint leaks if a request is
+ * abandoned without .final().
+ */
+struct cmh_cshake_reqctx {
+ const struct cmh_cshake_alg_info *info;
+ int error;
+ struct list_head chunks;
+ u32 num_chunks;
+ u32 total_len;
+ u32 has_checkpoint;
+ u8 checkpoint[HC_CONTEXT_SIZE];
+ /* DMA state for async final */
+ dma_addr_t digest_dma;
+ dma_addr_t ckpt_dma;
+ u8 *digest_buf;
+ struct cmh_sg_map *sgm;
+ struct vcq_cmd packed[CMH_CSHAKE_MAX_PACKED];
+};
+
+/* Per-Transform State (carries N and S across requests) */
+
+struct cmh_cshake_tfm_ctx {
+ u8 *func_name; /* N (function name), NULL if empty */
+ u32 func_name_len;
+ u8 *custom; /* S (customization string), NULL if empty */
+ u32 custom_len;
+ spinlock_t chunk_lock; /* protects all_chunks */
+ struct list_head all_chunks; /* orphan-safe chunk tracking */
+};
+
+/* VCQ Builders */
+
+/* VCQ Builders (cSHAKE-specific; shared builders in cmh_hc_abi.h / cmh_vcq.h) */
+
+static void vcq_add_hc_save(struct vcq_cmd *slot, u32 core_id,
+ u64 output_phys, u32 outlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_SAVE);
+ slot->hwc.hc.cmd_save.output = output_phys;
+ slot->hwc.hc.cmd_save.outlen = outlen;
+}
+
+static void vcq_add_hc_restore(struct vcq_cmd *slot, u32 core_id,
+ u64 input_phys, u32 inlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_RESTORE);
+ slot->hwc.hc.cmd_restore.input = input_phys;
+ slot->hwc.hc.cmd_restore.inlen = inlen;
+}
+
+static void vcq_add_hc_cshake(struct vcq_cmd *slot, u32 core_id, u32 algo,
+ const u8 *name, u32 namelen,
+ u32 customlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_CSHAKE);
+ slot->hwc.hc.cmd_cshake.custom = 0; /* inline -- CMH eSW reads from next slot(s) */
+ slot->hwc.hc.cmd_cshake.customlen = customlen;
+ slot->hwc.hc.cmd_cshake.algo = algo;
+ slot->hwc.hc.cmd_cshake.namelen = namelen;
+ if (namelen > 0 && name)
+ memcpy(slot->hwc.hc.cmd_cshake.name, name,
+ min_t(u32, namelen, HC_CSHAKE_MAX_NAMELEN));
+}
+
+/* Request Context Cleanup */
+
+static void cmh_cshake_free_chunks(struct cmh_cshake_reqctx *rctx,
+ struct cmh_cshake_tfm_ctx *tctx)
+{
+ struct cmh_cshake_chunk *chunk, *tmp;
+
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(chunk, tmp, &rctx->chunks, list) {
+ list_del(&chunk->list);
+ list_del(&chunk->tfm_node);
+ kfree(chunk);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->num_chunks = 0;
+ rctx->total_len = 0;
+}
+
+static void cmh_cshake_free_reqctx(struct cmh_cshake_reqctx *rctx,
+ struct cmh_cshake_tfm_ctx *tctx)
+{
+ cmh_cshake_free_chunks(rctx, tctx);
+ rctx->has_checkpoint = 0;
+}
+
+static struct cmh_sg_map *
+cmh_cshake_build_sg(struct cmh_cshake_reqctx *rctx, gfp_t gfp)
+{
+ struct cmh_dma_buf *bufs;
+ struct cmh_cshake_chunk *chunk;
+ struct cmh_sg_map *sgm;
+ u32 i;
+
+ bufs = kcalloc(rctx->num_chunks, sizeof(*bufs), gfp);
+ if (!bufs)
+ return NULL;
+
+ i = 0;
+ list_for_each_entry(chunk, &rctx->chunks, list) {
+ bufs[i].data = chunk->data;
+ bufs[i].len = chunk->len;
+ i++;
+ }
+
+ sgm = cmh_dma_build_sg(bufs, rctx->num_chunks, gfp);
+ kfree(bufs);
+ return sgm;
+}
+
+/* VCQ Packing + Submit */
+
+/* ahash Operations */
+
+struct cmh_cshake_alg_drv {
+ struct ahash_alg alg;
+ const struct cmh_cshake_alg_info *info;
+};
+
+static const struct cmh_cshake_alg_info *
+cmh_cshake_get_info(struct crypto_ahash *tfm)
+{
+ struct ahash_alg *alg = crypto_ahash_alg(tfm);
+
+ return container_of(alg, struct cmh_cshake_alg_drv, alg)->info;
+}
+
+/*
+ * .setkey() -- parse N and S from the self-describing cshake_cfg header.
+ *
+ * Blob format: cshake_cfg { __be32 n_len; __be32 s_len; } || N || S
+ * If never called, the driver defaults to plain SHAKE (N="" S="").
+ */
+struct cshake_cfg {
+ __be32 n_len;
+ __be32 s_len;
+};
+
+static int cmh_cshake_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cshake_cfg cfg;
+ u32 n_len, s_len;
+ const u8 *ptr;
+
+ if (keylen < sizeof(cfg))
+ return -EINVAL;
+
+ memcpy(&cfg, key, sizeof(cfg));
+ n_len = be32_to_cpu(cfg.n_len);
+ s_len = be32_to_cpu(cfg.s_len);
+
+ if (keylen != sizeof(cfg) + n_len + s_len)
+ return -EINVAL;
+
+ if (n_len > HC_CSHAKE_MAX_NAMELEN)
+ return -EINVAL;
+
+ if (s_len > HC_CSHAKE_MAX_CUSTOMLEN)
+ return -EINVAL;
+
+ /* Free previous N and S */
+ kfree(tctx->func_name);
+ kfree(tctx->custom);
+ tctx->func_name = NULL;
+ tctx->func_name_len = 0;
+ tctx->custom = NULL;
+ tctx->custom_len = 0;
+
+ ptr = key + sizeof(cfg);
+
+ if (n_len > 0) {
+ tctx->func_name = kmemdup(ptr, n_len, GFP_KERNEL);
+ if (!tctx->func_name)
+ return -ENOMEM;
+ tctx->func_name_len = n_len;
+ ptr += n_len;
+ }
+
+ if (s_len > 0) {
+ tctx->custom = kmemdup(ptr, s_len, GFP_KERNEL);
+ if (!tctx->custom) {
+ kfree(tctx->func_name);
+ tctx->func_name = NULL;
+ tctx->func_name_len = 0;
+ return -ENOMEM;
+ }
+ tctx->custom_len = s_len;
+ }
+
+ return 0;
+}
+
+static int cmh_cshake_init(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req);
+
+ rctx->info = cmh_cshake_get_info(tfm);
+ rctx->error = 0;
+ INIT_LIST_HEAD(&rctx->chunks);
+ rctx->num_chunks = 0;
+ rctx->total_len = 0;
+ rctx->has_checkpoint = 0;
+
+ return 0;
+}
+
+static int cmh_cshake_update(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_cshake_chunk *chunk;
+ int nents;
+
+ if (rctx->error)
+ return rctx->error;
+
+ if (!req->nbytes)
+ return 0;
+
+ chunk = kmalloc(sizeof(*chunk) + req->nbytes,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC);
+ if (!chunk) {
+ rctx->error = -ENOMEM;
+ goto err_free_chunks;
+ }
+
+ chunk->len = req->nbytes;
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) {
+ memcpy(chunk->data, req->svirt, req->nbytes);
+ } else {
+ nents = sg_nents_for_len(req->src, req->nbytes);
+ if (nents < 0 ||
+ sg_copy_to_buffer(req->src, nents,
+ chunk->data, req->nbytes) != req->nbytes) {
+ kfree(chunk);
+ rctx->error = -EINVAL;
+ goto err_free_chunks;
+ }
+ }
+
+ list_add_tail(&chunk->list, &rctx->chunks);
+ spin_lock_bh(&tctx->chunk_lock);
+ list_add_tail(&chunk->tfm_node, &tctx->all_chunks);
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->num_chunks++;
+ rctx->total_len += req->nbytes;
+
+ return 0;
+
+err_free_chunks:
+ /*
+ * Terminal error -- free all previously accumulated chunks.
+ * The crypto API hash path does not call .final() on error,
+ * so chunks would be orphaned otherwise.
+ */
+ cmh_cshake_free_chunks(rctx, tctx);
+ return rctx->error;
+}
+
+static void cmh_cshake_complete(void *data, int error)
+{
+ struct ahash_request *req = data;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->digest_dma, rctx->info->digest_size,
+ DMA_FROM_DEVICE);
+
+ if (!error)
+ memcpy(req->result, rctx->digest_buf,
+ rctx->info->digest_size);
+
+ kfree(rctx->digest_buf);
+ rctx->digest_buf = NULL;
+ cmh_dma_free_sg(rctx->sgm);
+ rctx->sgm = NULL;
+ cmh_cshake_free_reqctx(rctx, tctx);
+ cmh_complete(&req->base, error);
+}
+
+static int cmh_cshake_final(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_cshake_alg_info *info = rctx->info;
+ struct core_dispatch d;
+ struct vcq_cmd cmds[CMH_CSHAKE_MAX_PAYLOAD];
+ struct cmh_sg_map *sgm = NULL;
+ dma_addr_t digest_dma = DMA_MAPPING_ERROR;
+ dma_addr_t ckpt_dma = DMA_MAPPING_ERROR;
+ u8 *digest_buf;
+ u32 idx;
+ int ret;
+ gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ if (rctx->error) {
+ ret = rctx->error;
+ goto out_free;
+ }
+
+ if (rctx->num_chunks > 0) {
+ sgm = cmh_cshake_build_sg(rctx, gfp);
+ if (!sgm) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ }
+
+ digest_buf = kzalloc(info->digest_size, gfp);
+ if (!digest_buf) {
+ ret = -ENOMEM;
+ goto out_free_sg;
+ }
+ digest_dma = cmh_dma_map_single(digest_buf, info->digest_size,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(digest_dma)) {
+ ret = -ENOMEM;
+ goto out_free_digest;
+ }
+
+ /* Map checkpoint buffer if present (CMH eSW reads it) */
+ if (rctx->has_checkpoint) {
+ ckpt_dma = cmh_dma_map_single(rctx->checkpoint,
+ HC_CONTEXT_SIZE, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(ckpt_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap_digest;
+ }
+ }
+
+ d = cmh_core_select_instance(CMH_CORE_HC);
+ idx = 0;
+
+ if (rctx->has_checkpoint) {
+ /*
+ * Resuming from a saved checkpoint (after export/import):
+ * INIT + RESTORE [+ GATHER] + FINAL + FLUSH
+ * The cSHAKE prefix (N,S) is already absorbed in the
+ * saved Keccak state -- no need to replay HC_CMD_CSHAKE.
+ */
+ vcq_add_hc_init(&cmds[idx++], d.core_id, info->hc_algo);
+ vcq_add_hc_restore(&cmds[idx++], d.core_id, (u64)ckpt_dma,
+ HC_CONTEXT_SIZE);
+ } else {
+ bool use_cshake = (tctx->func_name_len > 0 ||
+ tctx->custom_len > 0);
+
+ if (use_cshake) {
+ u32 span;
+
+ vcq_add_hc_cshake(&cmds[idx], d.core_id,
+ info->hc_algo,
+ tctx->func_name,
+ tctx->func_name_len,
+ tctx->custom_len);
+ span = vcq_add_inline_data(&cmds[idx],
+ tctx->custom,
+ tctx->custom_len);
+ idx += span;
+ } else {
+ vcq_add_hc_init(&cmds[idx++], d.core_id,
+ info->hc_algo);
+ }
+ }
+
+ if (sgm)
+ vcq_add_hc_gather(&cmds[idx++], d.core_id, (u64)sgm->items_dma,
+ HC_CMD_UPDATE);
+
+ vcq_add_hc_final(&cmds[idx++], d.core_id, (u64)digest_dma, info->digest_size);
+ vcq_add_flush(&cmds[idx++], d.core_id);
+
+ rctx->digest_buf = digest_buf;
+ rctx->digest_dma = digest_dma;
+ rctx->ckpt_dma = ckpt_dma;
+ rctx->sgm = sgm;
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_CSHAKE_MAX_PACKED,
+ d.mbx_idx,
+ cmh_cshake_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_cleanup_all;
+
+ return -EINPROGRESS;
+
+out_cleanup_all:
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(ckpt_dma, HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+out_unmap_digest:
+ cmh_dma_unmap_single(digest_dma, info->digest_size,
+ DMA_FROM_DEVICE);
+out_free_digest:
+ kfree(digest_buf);
+
+out_free_sg:
+ cmh_dma_free_sg(sgm);
+
+out_free:
+ cmh_cshake_free_reqctx(rctx, tctx);
+ return ret;
+}
+
+static int cmh_cshake_finup(struct ahash_request *req)
+{
+ int ret;
+
+ ret = cmh_cshake_update(req);
+ if (ret)
+ return ret;
+
+ return cmh_cshake_final(req);
+}
+
+static int cmh_cshake_digest(struct ahash_request *req)
+{
+ int ret;
+
+ ret = cmh_cshake_init(req);
+ if (ret)
+ return ret;
+
+ return cmh_cshake_finup(req);
+}
+
+static int cmh_cshake_export(struct ahash_request *req, void *out)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_cshake_alg_info *info = rctx->info;
+ struct core_dispatch d;
+ struct vcq_cmd cmds[CMH_CSHAKE_MAX_PAYLOAD];
+ struct cmh_sg_map *sgm = NULL;
+ dma_addr_t save_dma = DMA_MAPPING_ERROR;
+ dma_addr_t ckpt_dma = DMA_MAPPING_ERROR;
+ u8 *save_buf;
+ u32 idx;
+ int ret;
+
+ if (rctx->num_chunks > 0) {
+ sgm = cmh_cshake_build_sg(rctx, GFP_KERNEL);
+ if (!sgm)
+ return -ENOMEM;
+ }
+
+ save_buf = kzalloc(HC_CONTEXT_SIZE, GFP_KERNEL);
+ if (!save_buf) {
+ cmh_dma_free_sg(sgm);
+ return -ENOMEM;
+ }
+ save_dma = cmh_dma_map_single(save_buf, HC_CONTEXT_SIZE,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(save_dma)) {
+ kfree(save_buf);
+ cmh_dma_free_sg(sgm);
+ return -ENOMEM;
+ }
+
+ /* Map checkpoint buffer if present (CMH eSW reads it) */
+ if (rctx->has_checkpoint) {
+ ckpt_dma = cmh_dma_map_single(rctx->checkpoint,
+ HC_CONTEXT_SIZE, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(ckpt_dma)) {
+ cmh_dma_unmap_single(save_dma, HC_CONTEXT_SIZE,
+ DMA_FROM_DEVICE);
+ kfree(save_buf);
+ cmh_dma_free_sg(sgm);
+ return -ENOMEM;
+ }
+ }
+
+ d = cmh_core_select_instance(CMH_CORE_HC);
+ idx = 0;
+
+ if (rctx->has_checkpoint) {
+ /*
+ * Resuming from a saved checkpoint:
+ * INIT + RESTORE [+ GATHER] + SAVE + FLUSH
+ */
+ vcq_add_hc_init(&cmds[idx++], d.core_id, info->hc_algo);
+ vcq_add_hc_restore(&cmds[idx++], d.core_id, (u64)ckpt_dma,
+ HC_CONTEXT_SIZE);
+ } else {
+ bool use_cshake = (tctx->func_name_len > 0 ||
+ tctx->custom_len > 0);
+
+ if (use_cshake) {
+ u32 span;
+
+ vcq_add_hc_cshake(&cmds[idx], d.core_id,
+ info->hc_algo,
+ tctx->func_name,
+ tctx->func_name_len,
+ tctx->custom_len);
+ span = vcq_add_inline_data(&cmds[idx],
+ tctx->custom,
+ tctx->custom_len);
+ idx += span;
+ } else {
+ vcq_add_hc_init(&cmds[idx++], d.core_id,
+ info->hc_algo);
+ }
+ }
+
+ if (sgm)
+ vcq_add_hc_gather(&cmds[idx++], d.core_id, (u64)sgm->items_dma,
+ HC_CMD_UPDATE);
+
+ vcq_add_hc_save(&cmds[idx++], d.core_id, (u64)save_dma,
+ HC_CONTEXT_SIZE);
+ vcq_add_flush(&cmds[idx++], d.core_id);
+
+ ret = cmh_vcq_pack_and_submit(cmds, idx, rctx->packed, CMH_CSHAKE_MAX_PACKED,
+ d.mbx_idx);
+
+ /* Unmap before CPU read */
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(ckpt_dma, HC_CONTEXT_SIZE, DMA_TO_DEVICE);
+ cmh_dma_unmap_single(save_dma, HC_CONTEXT_SIZE, DMA_FROM_DEVICE);
+
+ if (!ret) {
+ memcpy(out, save_buf, HC_CONTEXT_SIZE);
+ /* Checkpoint now represents all accumulated state */
+ memcpy(rctx->checkpoint, save_buf, HC_CONTEXT_SIZE);
+ rctx->has_checkpoint = 1;
+ /* Accumulated chunks are now captured in checkpoint */
+ cmh_cshake_free_chunks(rctx, tctx);
+ }
+
+ kfree(save_buf);
+ cmh_dma_free_sg(sgm);
+ return ret;
+}
+
+static int cmh_cshake_import(struct ahash_request *req, const void *in)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req);
+
+ rctx->info = cmh_cshake_get_info(tfm);
+ rctx->error = 0;
+ INIT_LIST_HEAD(&rctx->chunks);
+ rctx->num_chunks = 0;
+ rctx->total_len = 0;
+
+ memcpy(rctx->checkpoint, in, HC_CONTEXT_SIZE);
+ rctx->has_checkpoint = 1;
+
+ return 0;
+}
+
+/* Transform init/exit */
+
+static int cmh_cshake_cra_init(struct crypto_tfm *tfm)
+{
+ struct cmh_cshake_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
+
+ tctx->func_name = NULL;
+ tctx->func_name_len = 0;
+ tctx->custom = NULL;
+ tctx->custom_len = 0;
+ spin_lock_init(&tctx->chunk_lock);
+ INIT_LIST_HEAD(&tctx->all_chunks);
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct cmh_cshake_reqctx));
+ return 0;
+}
+
+static void cmh_cshake_cra_exit(struct crypto_tfm *tfm)
+{
+ struct cmh_cshake_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
+ struct cmh_cshake_chunk *chunk, *tmp;
+
+ /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(chunk, tmp, &tctx->all_chunks, tfm_node) {
+ list_del(&chunk->tfm_node);
+ kfree(chunk);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+
+ kfree(tctx->func_name);
+ kfree(tctx->custom);
+ tctx->func_name = NULL;
+ tctx->custom = NULL;
+}
+
+/* Registration */
+
+static struct cmh_cshake_alg_drv cmh_cshake_drvs[CMH_CSHAKE_ALG_COUNT];
+
+/**
+ * cmh_cshake_register() - Register cSHAKE-128/256 hash algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_cshake_register(void)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < CMH_CSHAKE_ALG_COUNT; i++) {
+ const struct cmh_cshake_alg_info *info =
+ &cmh_cshake_algs_info[i];
+ struct cmh_cshake_alg_drv *drv = &cmh_cshake_drvs[i];
+ struct ahash_alg *alg = &drv->alg;
+
+ drv->info = info;
+
+ alg->init = cmh_cshake_init;
+ alg->update = cmh_cshake_update;
+ alg->final = cmh_cshake_final;
+ alg->finup = cmh_cshake_finup;
+ alg->digest = cmh_cshake_digest;
+ alg->export = cmh_cshake_export;
+ alg->import = cmh_cshake_import;
+ alg->setkey = cmh_cshake_setkey;
+
+ alg->halg.digestsize = info->digest_size;
+ alg->halg.statesize = HC_CONTEXT_SIZE;
+
+ strscpy(alg->halg.base.cra_name, info->alg_name,
+ CRYPTO_MAX_ALG_NAME);
+ strscpy(alg->halg.base.cra_driver_name, info->drv_name,
+ CRYPTO_MAX_ALG_NAME);
+ alg->halg.base.cra_priority = 300;
+ alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NO_FALLBACK |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_OPTIONAL_KEY |
+ CRYPTO_ALG_REQ_VIRT;
+ alg->halg.base.cra_blocksize = 1; /* XOF */
+ alg->halg.base.cra_ctxsize = sizeof(struct cmh_cshake_tfm_ctx);
+ alg->halg.base.cra_init = cmh_cshake_cra_init;
+ alg->halg.base.cra_exit = cmh_cshake_cra_exit;
+ alg->halg.base.cra_module = THIS_MODULE;
+
+ ret = crypto_register_ahash(alg);
+ if (ret) {
+ dev_err(cmh_dev(), "cshake: failed to register %s (rc=%d)\n",
+ info->drv_name, ret);
+ while (i--)
+ crypto_unregister_ahash(&cmh_cshake_drvs[i].alg);
+ return ret;
+ }
+
+ dev_dbg(cmh_dev(), "cshake: registered %s (priority 300)\n",
+ info->drv_name);
+ }
+
+ dev_info(cmh_dev(), "cshake: %zu algorithm(s) registered\n",
+ CMH_CSHAKE_ALG_COUNT);
+ return 0;
+}
+
+/**
+ * cmh_cshake_unregister() - Unregister cSHAKE hash algorithms from the crypto framework
+ */
+void cmh_cshake_unregister(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < CMH_CSHAKE_ALG_COUNT; i++) {
+ crypto_unregister_ahash(&cmh_cshake_drvs[i].alg);
+ dev_dbg(cmh_dev(), "cshake: unregistered %s\n",
+ cmh_cshake_algs_info[i].drv_name);
+ }
+
+ dev_info(cmh_dev(), "cshake: cleaned up\n");
+}
diff --git a/drivers/crypto/cmh/cmh_kmac.c b/drivers/crypto/cmh/cmh_kmac.c
new file mode 100644
index 000000000000..7177a2558e97
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_kmac.c
@@ -0,0 +1,630 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API KMAC Driver
+ *
+ * Registers KMAC-128 and KMAC-256 as keyed ahash algorithms using the
+ * CMH Hash Core (HC) via HC_CMD_KMAC.
+ *
+ * KMAC (NIST SP 800-185) is a keyed variant of cSHAKE. The function
+ * name N is always "KMAC" (hardcoded by the CMH eSW). The user sets:
+ * - A key via .setkey() (raw bytes + optional S)
+ * - An optional customization string S via the setkey blob
+ *
+ * setkey blob format:
+ * struct kmac_key_param { __be32 keylen; __be32 s_len; };
+ * blob: kmac_key_param || key[keylen] || S[s_len]
+ *
+ * Uses the same self-contained transaction model as cmh_hmac.c:
+ * .setkey() -> store raw key (+ S)
+ * .init() -> software-only
+ * .update() -> software-only (accumulate chunks)
+ * .final() -> [SYS_CMD_WRITE] + HC_CMD_KMAC [+ inline S] +
+ * [GATHER] + FINAL + FLUSH
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/hash.h>
+#include <linux/scatterlist.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <asm/byteorder.h>
+
+#include "cmh_kmac.h"
+#include "cmh_vcq.h"
+#include "cmh_hc_abi.h"
+#include "cmh_sys_abi.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+/*
+ * Maximum data that can be accumulated across .update() calls.
+ * The CMH eSW rejects HC_CMD_SAVE when ctx->outlen != 0, which is
+ * always the case for KMAC (eip59_hc_kmac() sets ctx->outlen for
+ * right_encode(outlen) at finalization). All data must be buffered
+ * in kernel memory and submitted atomically in .final().
+ *
+ * The CMH eSW does not serialize outlen into the external save
+ * context, so HC_CMD_SAVE fails for KMAC mode.
+ */
+#define KMAC_MAX_DATA (64 * 1024)
+
+/* Algorithm Table */
+
+struct cmh_kmac_alg_info {
+ u32 hc_algo;
+ u32 digest_size;
+ const char *alg_name;
+ const char *drv_name;
+};
+
+static const struct cmh_kmac_alg_info cmh_kmac_algs_info[] = {
+ {
+ .hc_algo = HC_ALGO_SHAKE128,
+ .digest_size = CMH_SHAKE128_DIGEST_SIZE,
+ .alg_name = "kmac128",
+ .drv_name = "cri-cmh-kmac128",
+ },
+ {
+ .hc_algo = HC_ALGO_SHAKE256,
+ .digest_size = CMH_SHAKE256_DIGEST_SIZE,
+ .alg_name = "kmac256",
+ .drv_name = "cri-cmh-kmac256",
+ },
+};
+
+#define CMH_KMAC_ALG_COUNT ARRAY_SIZE(cmh_kmac_algs_info)
+
+/* Per-Request State */
+
+struct cmh_kmac_chunk {
+ struct list_head list;
+ struct list_head tfm_node; /* per-tfm orphan tracking */
+ u32 len;
+ u8 data[];
+};
+
+/*
+ * Max payload slots for KMAC:
+ * SYS_CMD_WRITE (1) + KMAC (1) + inline S (3 max) + GATHER (1) +
+ * FINAL (1) + FLUSH (1) = 8
+ */
+#define CMH_KMAC_MAX_PAYLOAD 9
+#define CMH_KMAC_MAX_PACKED (CMH_KMAC_MAX_PAYLOAD * 2)
+
+struct cmh_kmac_reqctx {
+ const struct cmh_kmac_alg_info *info;
+ int error;
+ struct list_head chunks;
+ u32 num_chunks;
+ u32 total_len;
+ /* DMA state for async final */
+ dma_addr_t digest_dma;
+ dma_addr_t key_dma;
+ u8 *digest_buf;
+ struct cmh_sg_map *sgm;
+ u32 keylen;
+ struct vcq_cmd packed[CMH_KMAC_MAX_PACKED];
+};
+
+/* Per-Transform State (carries key + S across requests) */
+
+struct cmh_kmac_tfm_ctx {
+ struct cmh_key_ctx key;
+ u8 *custom; /* S (customization string), NULL if empty */
+ u32 custom_len;
+ spinlock_t chunk_lock; /* protects all_chunks */
+ struct list_head all_chunks; /* orphan-safe chunk tracking */
+};
+
+/* VCQ Builders (KMAC-specific; shared builders in cmh_hc_abi.h / cmh_vcq.h) */
+
+static void vcq_add_hc_kmac(struct vcq_cmd *slot, u32 core_id, u64 key_ref, u32 keylen,
+ u32 customlen, u32 algo, u32 outlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_KMAC);
+ slot->hwc.hc.cmd_kmac.key = key_ref;
+ slot->hwc.hc.cmd_kmac.custom = 0; /* inline */
+ slot->hwc.hc.cmd_kmac.keylen = keylen;
+ slot->hwc.hc.cmd_kmac.customlen = customlen;
+ slot->hwc.hc.cmd_kmac.algo = algo;
+ slot->hwc.hc.cmd_kmac.outlen = outlen;
+}
+
+/* Request Context Cleanup */
+
+static void cmh_kmac_free_chunks(struct cmh_kmac_reqctx *rctx,
+ struct cmh_kmac_tfm_ctx *tctx)
+{
+ struct cmh_kmac_chunk *chunk, *tmp;
+
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(chunk, tmp, &rctx->chunks, list) {
+ list_del(&chunk->list);
+ list_del(&chunk->tfm_node);
+ kfree(chunk);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->num_chunks = 0;
+ rctx->total_len = 0;
+}
+
+static struct cmh_sg_map *
+cmh_kmac_build_sg(struct cmh_kmac_reqctx *rctx, gfp_t gfp)
+{
+ struct cmh_dma_buf *bufs;
+ struct cmh_kmac_chunk *chunk;
+ struct cmh_sg_map *sgm;
+ u32 i;
+
+ bufs = kcalloc(rctx->num_chunks, sizeof(*bufs), gfp);
+ if (!bufs)
+ return NULL;
+
+ i = 0;
+ list_for_each_entry(chunk, &rctx->chunks, list) {
+ bufs[i].data = chunk->data;
+ bufs[i].len = chunk->len;
+ i++;
+ }
+
+ sgm = cmh_dma_build_sg(bufs, rctx->num_chunks, gfp);
+ kfree(bufs);
+ return sgm;
+}
+
+/* VCQ Packing + Submit */
+
+/* ahash Operations */
+
+struct cmh_kmac_alg_drv {
+ struct ahash_alg alg;
+ const struct cmh_kmac_alg_info *info;
+};
+
+static const struct cmh_kmac_alg_info *
+cmh_kmac_get_info(struct crypto_ahash *tfm)
+{
+ struct ahash_alg *alg = crypto_ahash_alg(tfm);
+
+ return container_of(alg, struct cmh_kmac_alg_drv, alg)->info;
+}
+
+/*
+ * setkey blob for KMAC (raw key path):
+ * struct kmac_key_param { __be32 keylen; __be32 s_len; };
+ * blob: kmac_key_param || key[keylen] || S[s_len]
+ */
+struct kmac_key_param {
+ __be32 keylen;
+ __be32 s_len;
+};
+
+static int cmh_kmac_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ /* raw key bytes with optional S */
+ {
+ struct kmac_key_param hdr;
+ u32 raw_keylen, s_len;
+ const u8 *ptr;
+
+ if (keylen < sizeof(hdr))
+ return -EINVAL;
+
+ memcpy(&hdr, key, sizeof(hdr));
+ raw_keylen = be32_to_cpu(hdr.keylen);
+ s_len = be32_to_cpu(hdr.s_len);
+
+ if (keylen != sizeof(hdr) + raw_keylen + s_len)
+ return -EINVAL;
+
+ if (raw_keylen == 0)
+ return -EINVAL;
+
+ if (s_len > HC_CSHAKE_MAX_CUSTOMLEN)
+ return -EINVAL;
+
+ ptr = key + sizeof(hdr);
+
+ /* Store raw key */
+ {
+ int ret = cmh_key_setkey_raw(&tctx->key, ptr,
+ raw_keylen, CORE_ID_HC);
+ if (ret)
+ return ret;
+ }
+ ptr += raw_keylen;
+
+ /* Store S */
+ kfree(tctx->custom);
+ tctx->custom = NULL;
+ tctx->custom_len = 0;
+
+ if (s_len > 0) {
+ tctx->custom = kmemdup(ptr, s_len, GFP_KERNEL);
+ if (!tctx->custom) {
+ cmh_key_destroy(&tctx->key);
+ return -ENOMEM;
+ }
+ tctx->custom_len = s_len;
+ }
+
+ return 0;
+ }
+}
+
+static int cmh_kmac_init(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req);
+
+ rctx->info = cmh_kmac_get_info(tfm);
+ rctx->error = 0;
+ INIT_LIST_HEAD(&rctx->chunks);
+ rctx->num_chunks = 0;
+ rctx->total_len = 0;
+
+ return 0;
+}
+
+static int cmh_kmac_update(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_kmac_chunk *chunk;
+ int nents;
+
+ if (rctx->error)
+ return rctx->error;
+
+ if (!req->nbytes)
+ return 0;
+
+ if (req->nbytes > KMAC_MAX_DATA - rctx->total_len) {
+ rctx->error = -EINVAL;
+ goto err_free_chunks;
+ }
+
+ chunk = kmalloc(sizeof(*chunk) + req->nbytes,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC);
+ if (!chunk) {
+ rctx->error = -ENOMEM;
+ goto err_free_chunks;
+ }
+
+ chunk->len = req->nbytes;
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) {
+ memcpy(chunk->data, req->svirt, req->nbytes);
+ } else {
+ nents = sg_nents_for_len(req->src, req->nbytes);
+ if (nents < 0 ||
+ sg_copy_to_buffer(req->src, nents,
+ chunk->data, req->nbytes) != req->nbytes) {
+ kfree(chunk);
+ rctx->error = -EINVAL;
+ goto err_free_chunks;
+ }
+ }
+
+ list_add_tail(&chunk->list, &rctx->chunks);
+ spin_lock_bh(&tctx->chunk_lock);
+ list_add_tail(&chunk->tfm_node, &tctx->all_chunks);
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->num_chunks++;
+ rctx->total_len += req->nbytes;
+
+ return 0;
+
+err_free_chunks:
+ /*
+ * Terminal error -- free all previously accumulated chunks.
+ * The crypto API hash path does not call .final() on error,
+ * so chunks would be orphaned otherwise.
+ */
+ cmh_kmac_free_chunks(rctx, tctx);
+ return rctx->error;
+}
+
+static void cmh_kmac_complete(void *data, int error)
+{
+ struct ahash_request *req = data;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ cmh_dma_unmap_single(rctx->digest_dma, rctx->info->digest_size,
+ DMA_FROM_DEVICE);
+
+ if (!error)
+ memcpy(req->result, rctx->digest_buf,
+ rctx->info->digest_size);
+
+ kfree(rctx->digest_buf);
+ rctx->digest_buf = NULL;
+ cmh_dma_free_sg(rctx->sgm);
+ rctx->sgm = NULL;
+ cmh_kmac_free_chunks(rctx, tctx);
+ cmh_complete(&req->base, error);
+}
+
+static int cmh_kmac_final(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_kmac_alg_info *info = rctx->info;
+ struct vcq_cmd cmds[CMH_KMAC_MAX_PAYLOAD];
+ struct cmh_sg_map *sgm = NULL;
+ dma_addr_t digest_dma = DMA_MAPPING_ERROR, key_dma = DMA_MAPPING_ERROR;
+ u8 *digest_buf;
+ u64 key_ref;
+ u32 key_len;
+ struct core_dispatch d;
+ s32 target_mbx;
+ u32 core_id;
+ u32 idx;
+ int ret;
+ gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ if (rctx->error) {
+ ret = rctx->error;
+ goto out_free;
+ }
+
+ if (tctx->key.mode == CMH_KEY_NONE) {
+ ret = -ENOKEY;
+ goto out_free;
+ }
+
+ if (rctx->num_chunks > 0) {
+ sgm = cmh_kmac_build_sg(rctx, gfp);
+ if (!sgm) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ }
+
+ digest_buf = kzalloc(info->digest_size, gfp);
+ if (!digest_buf) {
+ ret = -ENOMEM;
+ goto out_free_sg;
+ }
+ digest_dma = cmh_dma_map_single(digest_buf, info->digest_size,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(digest_dma)) {
+ ret = -ENOMEM;
+ goto out_free_digest;
+ }
+
+ /* Resolve key reference */
+ idx = 0;
+
+ key_dma = tctx->key.raw.dma;
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, (u64)key_dma,
+ SYS_REF_NONE, tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ key_len = tctx->key.raw.len;
+ d = cmh_core_select_instance(CMH_CORE_HC);
+
+ target_mbx = d.mbx_idx;
+
+ core_id = d.core_id;
+
+ {
+ u32 span;
+
+ vcq_add_hc_kmac(&cmds[idx], core_id, key_ref, key_len,
+ tctx->custom_len, info->hc_algo,
+ info->digest_size);
+
+ /* Add inline S data after the KMAC slot */
+ span = vcq_add_inline_data(&cmds[idx], tctx->custom,
+ tctx->custom_len);
+ idx += span;
+ }
+
+ if (sgm)
+ vcq_add_hc_gather(&cmds[idx++], core_id, (u64)sgm->items_dma,
+ HC_CMD_UPDATE);
+
+ vcq_add_hc_final(&cmds[idx++], core_id, (u64)digest_dma, info->digest_size);
+ vcq_add_flush(&cmds[idx++], core_id);
+
+ rctx->digest_buf = digest_buf;
+ rctx->digest_dma = digest_dma;
+ rctx->sgm = sgm;
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_KMAC_MAX_PACKED,
+ target_mbx,
+ cmh_kmac_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_cleanup_all;
+
+ return -EINPROGRESS;
+
+out_cleanup_all:
+ cmh_dma_unmap_single(digest_dma, info->digest_size,
+ DMA_FROM_DEVICE);
+out_free_digest:
+ kfree(digest_buf);
+
+out_free_sg:
+ cmh_dma_free_sg(sgm);
+
+out_free:
+ cmh_kmac_free_chunks(rctx, tctx);
+ return ret;
+}
+
+static int cmh_kmac_finup(struct ahash_request *req)
+{
+ int ret;
+
+ ret = cmh_kmac_update(req);
+ if (ret)
+ return ret;
+
+ return cmh_kmac_final(req);
+}
+
+static int cmh_kmac_digest(struct ahash_request *req)
+{
+ int ret;
+
+ ret = cmh_kmac_init(req);
+ if (ret)
+ return ret;
+
+ return cmh_kmac_finup(req);
+}
+
+static int cmh_kmac_export(struct ahash_request *req, void *out)
+{
+ return -EOPNOTSUPP;
+}
+
+static int cmh_kmac_import(struct ahash_request *req, const void *in)
+{
+ return -EOPNOTSUPP;
+}
+
+/* Transform init/exit */
+
+static int cmh_kmac_cra_init(struct crypto_tfm *tfm)
+{
+ struct cmh_kmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
+
+ tctx->key.mode = CMH_KEY_NONE;
+ tctx->custom = NULL;
+ tctx->custom_len = 0;
+ spin_lock_init(&tctx->chunk_lock);
+ INIT_LIST_HEAD(&tctx->all_chunks);
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct cmh_kmac_reqctx));
+ return 0;
+}
+
+static void cmh_kmac_cra_exit(struct crypto_tfm *tfm)
+{
+ struct cmh_kmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
+ struct cmh_kmac_chunk *chunk, *tmp;
+
+ /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(chunk, tmp, &tctx->all_chunks, tfm_node) {
+ list_del(&chunk->tfm_node);
+ kfree(chunk);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+
+ cmh_key_destroy(&tctx->key);
+ kfree(tctx->custom);
+ tctx->custom = NULL;
+}
+
+/* Registration */
+
+static struct cmh_kmac_alg_drv cmh_kmac_drvs[CMH_KMAC_ALG_COUNT];
+
+/**
+ * cmh_kmac_register() - Register KMAC-128/256 hash algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_kmac_register(void)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < CMH_KMAC_ALG_COUNT; i++) {
+ const struct cmh_kmac_alg_info *info =
+ &cmh_kmac_algs_info[i];
+ struct cmh_kmac_alg_drv *drv = &cmh_kmac_drvs[i];
+ struct ahash_alg *alg = &drv->alg;
+
+ drv->info = info;
+
+ alg->init = cmh_kmac_init;
+ alg->update = cmh_kmac_update;
+ alg->final = cmh_kmac_final;
+ alg->finup = cmh_kmac_finup;
+ alg->digest = cmh_kmac_digest;
+ alg->export = cmh_kmac_export;
+ alg->import = cmh_kmac_import;
+ alg->setkey = cmh_kmac_setkey;
+
+ alg->halg.digestsize = info->digest_size;
+ alg->halg.statesize = sizeof(struct cmh_kmac_reqctx);
+
+ strscpy(alg->halg.base.cra_name, info->alg_name,
+ CRYPTO_MAX_ALG_NAME);
+ strscpy(alg->halg.base.cra_driver_name, info->drv_name,
+ CRYPTO_MAX_ALG_NAME);
+ alg->halg.base.cra_priority = 300;
+ alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NO_FALLBACK |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_REQ_VIRT;
+ alg->halg.base.cra_blocksize = 1; /* XOF/keyed XOF */
+ alg->halg.base.cra_ctxsize = sizeof(struct cmh_kmac_tfm_ctx);
+ alg->halg.base.cra_init = cmh_kmac_cra_init;
+ alg->halg.base.cra_exit = cmh_kmac_cra_exit;
+ alg->halg.base.cra_module = THIS_MODULE;
+
+ ret = crypto_register_ahash(alg);
+ if (ret) {
+ dev_err(cmh_dev(), "kmac: failed to register %s (rc=%d)\n",
+ info->drv_name, ret);
+ while (i--)
+ crypto_unregister_ahash(&cmh_kmac_drvs[i].alg);
+ return ret;
+ }
+
+ dev_dbg(cmh_dev(), "kmac: registered %s (priority 300)\n",
+ info->drv_name);
+ }
+
+ dev_info(cmh_dev(), "kmac: %zu algorithm(s) registered\n",
+ CMH_KMAC_ALG_COUNT);
+ return 0;
+}
+
+/**
+ * cmh_kmac_unregister() - Unregister KMAC hash algorithms from the crypto framework
+ */
+void cmh_kmac_unregister(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < CMH_KMAC_ALG_COUNT; i++) {
+ crypto_unregister_ahash(&cmh_kmac_drvs[i].alg);
+ dev_dbg(cmh_dev(), "kmac: unregistered %s\n",
+ cmh_kmac_algs_info[i].drv_name);
+ }
+
+ dev_info(cmh_dev(), "kmac: cleaned up\n");
+}
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index 618c981e3997..08d14aae912d 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -31,6 +31,8 @@
#include "cmh_rh.h"
#include "cmh_hash.h"
#include "cmh_hmac.h"
+#include "cmh_cshake.h"
+#include "cmh_kmac.h"
#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
@@ -209,6 +211,16 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_hmac_register;
+ /* Register CSHAKE hash algorithms */
+ ret = cmh_cshake_register();
+ if (ret)
+ goto err_cshake_register;
+
+ /* Register KMAC hash algorithms */
+ ret = cmh_kmac_register();
+ if (ret)
+ goto err_kmac_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -221,6 +233,10 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_kmac_unregister();
+err_kmac_register:
+ cmh_cshake_unregister();
+err_cshake_register:
cmh_hmac_unregister();
err_hmac_register:
cmh_hash_unregister();
@@ -251,6 +267,8 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_kmac_unregister();
+ cmh_cshake_unregister();
cmh_hmac_unregister();
cmh_hash_unregister();
cmh_rh_cleanup(cfg);
diff --git a/drivers/crypto/cmh/include/cmh_cshake.h b/drivers/crypto/cmh/include/cmh_cshake.h
new file mode 100644
index 000000000000..9bafe0baf52f
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_cshake.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API CSHAKE Driver
+ *
+ * Registers cSHAKE-128 and cSHAKE-256 ahash algorithms using
+ * HC_CMD_CSHAKE with inline customization string S.
+ */
+
+#ifndef CMH_CSHAKE_H
+#define CMH_CSHAKE_H
+
+int cmh_cshake_register(void);
+void cmh_cshake_unregister(void);
+
+#endif /* CMH_CSHAKE_H */
diff --git a/drivers/crypto/cmh/include/cmh_kmac.h b/drivers/crypto/cmh/include/cmh_kmac.h
new file mode 100644
index 000000000000..b3c92d71a0b6
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_kmac.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API KMAC Driver
+ *
+ * Registers KMAC-128 and KMAC-256 ahash algorithms using
+ * HC_CMD_KMAC with inline customization string S.
+ */
+
+#ifndef CMH_KMAC_H
+#define CMH_KMAC_H
+
+int cmh_kmac_register(void);
+void cmh_kmac_unregister(void);
+
+#endif /* CMH_KMAC_H */
--
2.43.7
^ permalink raw reply related
* [PATCH v2 04/19] crypto: cmh - add SHA-2/SHA-3/SHAKE ahash
From: Saravanakrishnan Krishnamoorthy @ 2026-07-09 20:30 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260709203037.1884436-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register ahash algorithms for SHA-224, SHA-256, SHA-384, SHA-512,
SHA3-224, SHA3-256, SHA3-384, SHA3-512, SHAKE128, and SHAKE256
using the CMH hash core (core ID 0x02).
Supports incremental update/finup/final, init/export/import for
request cloning, and the CRYPTO_AHASH_REQ_VIRT flag for zero-copy
from virtual buffers.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 3 +-
drivers/crypto/cmh/cmh_hash.c | 860 ++++++++++++++++++++++++++
drivers/crypto/cmh/cmh_main.c | 9 +
drivers/crypto/cmh/include/cmh_hash.h | 26 +
4 files changed, 897 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_hash.c
create mode 100644 drivers/crypto/cmh/include/cmh_hash.h
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index 1492e575598c..c0531f416229 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -14,7 +14,8 @@ cmh-y := \
cmh_dma.o \
cmh_sysfs.o \
cmh_key.o \
- cmh_sys.o
+ cmh_sys.o \
+ cmh_hash.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_hash.c b/drivers/crypto/cmh/cmh_hash.c
new file mode 100644
index 000000000000..2256bf4314c3
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_hash.c
@@ -0,0 +1,860 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API Hash Driver
+ *
+ * Registers asynchronous hash (ahash) algorithms with the Linux crypto
+ * subsystem. Implements SHA-2 (224/256/384/512), SHA-3
+ * (224/256/384/512), and SHAKE (128/256) families using the CMH Hash
+ * Core (HC).
+ *
+ * Incremental HW update model -- each .update() with enough data for
+ * at least one full block submits a self-contained VCQ transaction:
+ *
+ * .init() -> software-only: zero per-request context
+ * .update() -> buffer data in holdback; when >= block_size bytes:
+ * INIT [+ RESTORE] + UPDATE(full blocks) + SAVE + FLUSH
+ * -> return -EINPROGRESS (else return 0, data in holdback)
+ * .final() -> INIT [+ RESTORE] [+ UPDATE(residual)] + FINAL + FLUSH
+ * .finup() -> linearise holdback + new data, then final path
+ * .digest() -> INIT + UPDATE + FINAL + FLUSH (single-shot, zero-copy)
+ * .export() -> software-only: copy checkpoint + holdback to out
+ * .import() -> software-only: restore checkpoint + holdback from in
+ *
+ * The FLUSH after each .update() releases the HC core, so no lockout.
+ * Two hash sessions interleave fine on the same MBX -- each saves its
+ * own state via SAVE and restores via RESTORE on the next call.
+ *
+ * Export/import is purely software (no HW interaction), enabling
+ * crypto API transform clone for all plain-hash algorithms.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/hash.h>
+#include <crypto/scatterwalk.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "cmh_hash.h"
+#include "cmh_vcq.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+
+/* Algorithm Table */
+
+struct cmh_hash_alg_info {
+ u32 hc_algo; /* HC_ALGO_* (SHA2, SHA3, SHAKE) */
+ u32 digest_size; /* bytes */
+ u32 block_size; /* cra_blocksize for Linux crypto API */
+ const char *alg_name; /* Linux crypto name: "sha256" */
+ const char *drv_name; /* driver name: "cri-cmh-sha256" */
+};
+
+static const struct cmh_hash_alg_info cmh_hash_algs_info[] = {
+ /* SHA-2 family */
+ {
+ .hc_algo = HC_ALGO_SHA2_224,
+ .digest_size = CMH_SHA224_DIGEST_SIZE,
+ .block_size = 64,
+ .alg_name = "sha224",
+ .drv_name = "cri-cmh-sha224",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA2_256,
+ .digest_size = CMH_SHA256_DIGEST_SIZE,
+ .block_size = 64,
+ .alg_name = "sha256",
+ .drv_name = "cri-cmh-sha256",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA2_384,
+ .digest_size = CMH_SHA384_DIGEST_SIZE,
+ .block_size = 128,
+ .alg_name = "sha384",
+ .drv_name = "cri-cmh-sha384",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA2_512,
+ .digest_size = CMH_SHA512_DIGEST_SIZE,
+ .block_size = 128,
+ .alg_name = "sha512",
+ .drv_name = "cri-cmh-sha512",
+ },
+ /* SHA-3 family */
+ {
+ .hc_algo = HC_ALGO_SHA3_224,
+ .digest_size = CMH_SHA3_224_DIGEST_SIZE,
+ .block_size = 144, /* rate = 1600/8 - 2*224/8 = 144 */
+ .alg_name = "sha3-224",
+ .drv_name = "cri-cmh-sha3-224",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA3_256,
+ .digest_size = CMH_SHA3_256_DIGEST_SIZE,
+ .block_size = 136, /* rate = 1600/8 - 2*256/8 = 136 */
+ .alg_name = "sha3-256",
+ .drv_name = "cri-cmh-sha3-256",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA3_384,
+ .digest_size = CMH_SHA3_384_DIGEST_SIZE,
+ .block_size = 104, /* rate = 1600/8 - 2*384/8 = 104 */
+ .alg_name = "sha3-384",
+ .drv_name = "cri-cmh-sha3-384",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA3_512,
+ .digest_size = CMH_SHA3_512_DIGEST_SIZE,
+ .block_size = 72, /* rate = 1600/8 - 2*512/8 = 72 */
+ .alg_name = "sha3-512",
+ .drv_name = "cri-cmh-sha3-512",
+ },
+ /*
+ * SHAKE (XOF) family -- fixed-output ahash registration.
+ *
+ * cra_blocksize = 1: SHAKE is a sponge/XOF, not Merkle-Damgaard.
+ * The Keccak rate (168 for SHAKE-128, 136 for SHAKE-256) exceeds
+ * MAX_ALGAPI_BLOCKSIZE (160) on Linux <=6.7. Using 1 signals
+ * "byte-oriented" which is correct for XOF consumers. The kernel
+ * raised the limit to 208 in 6.8 (commit 2f3a22704889).
+ */
+ {
+ .hc_algo = HC_ALGO_SHAKE128,
+ .digest_size = CMH_SHAKE128_DIGEST_SIZE,
+ .block_size = 1, /* XOF: no meaningful block for crypto API */
+ .alg_name = "shake128",
+ .drv_name = "cri-cmh-shake128",
+ },
+ {
+ .hc_algo = HC_ALGO_SHAKE256,
+ .digest_size = CMH_SHAKE256_DIGEST_SIZE,
+ .block_size = 1, /* XOF: no meaningful block for crypto API */
+ .alg_name = "shake256",
+ .drv_name = "cri-cmh-shake256",
+ },
+};
+
+#define CMH_HASH_ALG_COUNT ARRAY_SIZE(cmh_hash_algs_info)
+
+/* Per-Request State */
+
+/* Maximum cra_blocksize across all registered algorithms (SHA3-224) */
+#define CMH_HASH_MAX_BLOCK 144
+
+/*
+ * Exported hash state -- serialised by .export(), deserialised by
+ * .import(). This is what statesize advertises to the crypto subsystem.
+ */
+struct cmh_hash_export_state {
+ u8 checkpoint[HC_CONTEXT_SIZE]; /* HC context from last SAVE */
+ u8 buf[CMH_HASH_MAX_BLOCK]; /* holdback buffer */
+ u32 buf_len; /* valid bytes in buf[] */
+ u32 hw_started; /* non-zero if checkpoint valid */
+};
+
+/*
+ * Maximum payload commands any hash transaction can produce:
+ * INIT + RESTORE + UPDATE + SAVE/FINAL + FLUSH = 5
+ * Worst-case packed output (stride=7, 1 payload per VCQ):
+ * 5 VCQs x 2 entries = 10
+ */
+#define CMH_HASH_MAX_PAYLOAD 5
+#define CMH_HASH_MAX_PACKED (CMH_HASH_MAX_PAYLOAD * 2)
+
+/*
+ * Stored in ahash_request_ctx(). Tracks the algorithm, a holdback
+ * buffer for partial blocks, an HC context checkpoint from the last
+ * SAVE, and DMA state for the current in-flight async operation.
+ *
+ * The checkpoint is embedded inline rather than heap-allocated because
+ * the kernel ahash API has no per-request destructor. If a request is
+ * abandoned without .final() (e.g. transform freed early), a heap
+ * checkpoint would leak unconditionally.
+ */
+struct cmh_hash_reqctx {
+ const struct cmh_hash_alg_info *info;
+ int error;
+ u32 hw_started; /* non-zero after first HW submission */
+ u32 buf_len; /* bytes in holdback buf[] */
+ u32 has_checkpoint; /* non-zero if checkpoint[] valid */
+ /* DMA state for current async operation */
+ dma_addr_t ckpt_dma; /* RESTORE input */
+ dma_addr_t save_dma; /* SAVE output (update only) */
+ dma_addr_t data_dma; /* UPDATE input */
+ dma_addr_t digest_dma; /* FINAL output (final/digest only) */
+ u8 *save_buf; /* SAVE output buffer */
+ u8 *data_buf; /* linearised data for DMA */
+ u32 data_len; /* bytes in data_buf */
+ u8 *digest_buf; /* digest output buffer */
+ u8 buf[CMH_HASH_MAX_BLOCK]; /* holdback for partial block */
+ u8 checkpoint[HC_CONTEXT_SIZE]; /* HC context from last SAVE */
+ struct vcq_cmd packed[CMH_HASH_MAX_PACKED];
+};
+
+/* VCQ Builders (HC-specific; shared builders in cmh_hc_abi.h / cmh_vcq.h) */
+
+/* Add an HC_CMD_UPDATE entry */
+static void vcq_add_hc_update(struct vcq_cmd *slot, u32 core_id, u64 input_phys, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_UPDATE);
+ slot->hwc.hc.cmd_update.input = input_phys;
+ slot->hwc.hc.cmd_update.inlen = len;
+}
+
+/* Add an HC_CMD_SAVE entry */
+static void vcq_add_hc_save(struct vcq_cmd *slot, u32 core_id, u64 output_phys, u32 outlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_SAVE);
+ slot->hwc.hc.cmd_save.output = output_phys;
+ slot->hwc.hc.cmd_save.outlen = outlen;
+}
+
+/* Add an HC_CMD_RESTORE entry */
+static void vcq_add_hc_restore(struct vcq_cmd *slot, u32 core_id, u64 input_phys, u32 inlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_RESTORE);
+ slot->hwc.hc.cmd_restore.input = input_phys;
+ slot->hwc.hc.cmd_restore.inlen = inlen;
+}
+
+/* Request Context Cleanup */
+
+static void cmh_hash_free_reqctx(struct cmh_hash_reqctx *rctx)
+{
+ rctx->has_checkpoint = 0;
+}
+
+/* VCQ Packing + Submit */
+
+/* ahash Operations */
+
+/*
+ * Wrapper struct: embeds ahash_alg + a pointer to our alg_info table
+ * entry so we can recover it in the tfm callbacks.
+ */
+struct cmh_hash_alg_drv {
+ struct ahash_alg alg;
+ const struct cmh_hash_alg_info *info;
+};
+
+/*
+ * Find the cmh_hash_alg_info from the crypto_ahash (embedded in our
+ * registered template). We stash the info pointer in the algorithm's
+ * driver-private area at registration time (see cmh_hash_register).
+ */
+static const struct cmh_hash_alg_info *
+cmh_hash_get_info(struct crypto_ahash *tfm)
+{
+ struct ahash_alg *alg = crypto_ahash_alg(tfm);
+
+ return container_of(alg, struct cmh_hash_alg_drv, alg)->info;
+}
+
+static int cmh_hash_init(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->info = cmh_hash_get_info(tfm);
+ return 0;
+}
+
+/*
+ * Update completion -- called from threaded IRQ after SAVE completes.
+ * Takes ownership of save_buf as the new checkpoint.
+ */
+static void cmh_hash_update_complete(void *data, int error)
+{
+ struct ahash_request *req = data;
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ /* Unmap DMA buffers */
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->save_dma, HC_CONTEXT_SIZE,
+ DMA_FROM_DEVICE);
+ cmh_dma_unmap_single(rctx->data_dma, rctx->data_len,
+ DMA_TO_DEVICE);
+
+ if (!error) {
+ memcpy(rctx->checkpoint, rctx->save_buf, HC_CONTEXT_SIZE);
+ rctx->has_checkpoint = 1;
+ kfree(rctx->save_buf);
+ rctx->save_buf = NULL;
+ rctx->hw_started = 1;
+ } else {
+ kfree(rctx->save_buf);
+ rctx->save_buf = NULL;
+ rctx->error = error;
+ }
+
+ kfree(rctx->data_buf);
+ rctx->data_buf = NULL;
+ rctx->data_len = 0;
+
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * .update -- buffer incoming data, submit full blocks to HW.
+ *
+ * Maintains a partial-block holdback buffer in rctx->buf[]. When
+ * enough data is available for at least one full block, the full
+ * blocks are linearised and submitted as:
+ * INIT [+ RESTORE] + UPDATE(full_blocks) + SAVE + FLUSH
+ *
+ * The tail (< block_size) stays in the holdback for the next call.
+ * Returns -EINPROGRESS on HW submission, 0 if only buffering.
+ */
+static int cmh_hash_update(struct ahash_request *req)
+{
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_hash_alg_info *info = rctx->info;
+ struct vcq_cmd cmds[CMH_HASH_MAX_PAYLOAD];
+ struct core_dispatch d;
+ u32 block_size = info->block_size;
+ u32 total_avail, full_len, tail_len, from_src;
+ u32 idx;
+ int ret;
+ gfp_t gfp;
+
+ if (rctx->error)
+ return rctx->error;
+
+ if (!req->nbytes)
+ return 0;
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ total_avail = rctx->buf_len + req->nbytes;
+
+ /* Not enough for a full block -- just buffer */
+ if (total_avail < block_size) {
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(rctx->buf + rctx->buf_len,
+ req->svirt, req->nbytes);
+ else
+ scatterwalk_map_and_copy(rctx->buf + rctx->buf_len,
+ req->src, 0,
+ req->nbytes, 0);
+ rctx->buf_len = total_avail;
+ return 0;
+ }
+
+ /* Have at least one full block -- submit to HW */
+ full_len = total_avail - total_avail % block_size;
+ tail_len = total_avail - full_len;
+ from_src = full_len - rctx->buf_len;
+
+ /* Linearise: holdback prefix + full blocks from scatterlist */
+ rctx->data_buf = kmalloc(full_len, gfp);
+ if (!rctx->data_buf)
+ return -ENOMEM;
+
+ if (rctx->buf_len > 0)
+ memcpy(rctx->data_buf, rctx->buf, rctx->buf_len);
+
+ if (from_src > 0) {
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(rctx->data_buf + rctx->buf_len,
+ req->svirt, from_src);
+ else
+ scatterwalk_map_and_copy(rctx->data_buf + rctx->buf_len,
+ req->src, 0,
+ from_src, 0);
+ }
+
+ /* Move tail to holdback */
+ if (tail_len > 0) {
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(rctx->buf, req->svirt + from_src,
+ tail_len);
+ else
+ scatterwalk_map_and_copy(rctx->buf, req->src,
+ from_src, tail_len,
+ 0);
+ }
+ rctx->buf_len = tail_len;
+ rctx->data_len = full_len;
+
+ /* Allocate SAVE output buffer */
+ rctx->save_buf = kzalloc(HC_CONTEXT_SIZE, gfp);
+ if (!rctx->save_buf) {
+ ret = -ENOMEM;
+ goto err_free;
+ }
+
+ /* DMA map data, save output, and checkpoint */
+ rctx->data_dma = cmh_dma_map_single(rctx->data_buf, full_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->data_dma)) {
+ ret = -ENOMEM;
+ goto err_free;
+ }
+
+ rctx->save_dma = cmh_dma_map_single(rctx->save_buf, HC_CONTEXT_SIZE,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->save_dma)) {
+ ret = -ENOMEM;
+ goto err_unmap_data;
+ }
+
+ rctx->ckpt_dma = DMA_MAPPING_ERROR;
+ if (rctx->has_checkpoint) {
+ rctx->ckpt_dma = cmh_dma_map_single(rctx->checkpoint,
+ HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->ckpt_dma)) {
+ ret = -ENOMEM;
+ goto err_unmap_save;
+ }
+ }
+
+ /* Build VCQ: INIT [+ RESTORE] + UPDATE + SAVE + FLUSH */
+ d = cmh_core_select_instance(CMH_CORE_HC);
+ idx = 0;
+
+ vcq_add_hc_init(&cmds[idx++], d.core_id, info->hc_algo);
+
+ if (rctx->has_checkpoint)
+ vcq_add_hc_restore(&cmds[idx++], d.core_id,
+ (u64)rctx->ckpt_dma, HC_CONTEXT_SIZE);
+
+ vcq_add_hc_update(&cmds[idx++], d.core_id,
+ (u64)rctx->data_dma, full_len);
+
+ vcq_add_hc_save(&cmds[idx++], d.core_id,
+ (u64)rctx->save_dma, HC_CONTEXT_SIZE);
+
+ vcq_add_flush(&cmds[idx++], d.core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_HASH_MAX_PACKED,
+ d.mbx_idx,
+ cmh_hash_update_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto err_unmap_ckpt;
+
+ return -EINPROGRESS;
+
+err_unmap_ckpt:
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+err_unmap_save:
+ cmh_dma_unmap_single(rctx->save_dma, HC_CONTEXT_SIZE,
+ DMA_FROM_DEVICE);
+err_unmap_data:
+ cmh_dma_unmap_single(rctx->data_dma, full_len, DMA_TO_DEVICE);
+err_free:
+ kfree(rctx->save_buf);
+ rctx->save_buf = NULL;
+ kfree(rctx->data_buf);
+ rctx->data_buf = NULL;
+ rctx->data_len = 0;
+ return ret;
+}
+
+/*
+ * Final completion -- unmap all DMA, copy digest, signal done.
+ */
+static void cmh_hash_final_complete(void *data, int error)
+{
+ struct ahash_request *req = data;
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+ if (rctx->data_buf)
+ cmh_dma_unmap_single(rctx->data_dma, rctx->data_len,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->digest_dma, rctx->info->digest_size,
+ DMA_FROM_DEVICE);
+
+ if (!error)
+ memcpy(req->result, rctx->digest_buf,
+ rctx->info->digest_size);
+
+ kfree(rctx->digest_buf);
+ rctx->digest_buf = NULL;
+ kfree(rctx->data_buf);
+ rctx->data_buf = NULL;
+ cmh_hash_free_reqctx(rctx);
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * Submit the final VCQ transaction:
+ * INIT [+ RESTORE] [+ UPDATE(residual)] + FINAL + FLUSH
+ *
+ * @data_buf: linearised residual data, or NULL for empty-hash.
+ * Ownership transferred -- callback frees it.
+ * @data_len: bytes in data_buf.
+ */
+static int cmh_hash_submit_final(struct ahash_request *req,
+ u8 *data_buf, u32 data_len)
+{
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_hash_alg_info *info = rctx->info;
+ struct vcq_cmd cmds[CMH_HASH_MAX_PAYLOAD];
+ struct core_dispatch d;
+ u32 idx;
+ int ret;
+ gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ rctx->data_buf = data_buf;
+ rctx->data_len = data_len;
+
+ /* Allocate digest output buffer */
+ rctx->digest_buf = kzalloc(info->digest_size, gfp);
+ if (!rctx->digest_buf) {
+ ret = -ENOMEM;
+ goto err_free_data;
+ }
+
+ rctx->digest_dma = cmh_dma_map_single(rctx->digest_buf,
+ info->digest_size,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->digest_dma)) {
+ ret = -ENOMEM;
+ goto err_free_digest;
+ }
+
+ /* Map residual data for UPDATE */
+ rctx->data_dma = DMA_MAPPING_ERROR;
+ if (data_buf && data_len > 0) {
+ rctx->data_dma = cmh_dma_map_single(data_buf, data_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->data_dma)) {
+ ret = -ENOMEM;
+ goto err_unmap_digest;
+ }
+ }
+
+ /* Map checkpoint for RESTORE */
+ rctx->ckpt_dma = DMA_MAPPING_ERROR;
+ if (rctx->has_checkpoint) {
+ rctx->ckpt_dma = cmh_dma_map_single(rctx->checkpoint,
+ HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->ckpt_dma)) {
+ ret = -ENOMEM;
+ goto err_unmap_data;
+ }
+ }
+
+ /* Build VCQ: INIT [+ RESTORE] [+ UPDATE] + FINAL + FLUSH */
+ d = cmh_core_select_instance(CMH_CORE_HC);
+ idx = 0;
+
+ vcq_add_hc_init(&cmds[idx++], d.core_id, info->hc_algo);
+
+ if (rctx->has_checkpoint)
+ vcq_add_hc_restore(&cmds[idx++], d.core_id,
+ (u64)rctx->ckpt_dma, HC_CONTEXT_SIZE);
+
+ if (data_buf && data_len > 0)
+ vcq_add_hc_update(&cmds[idx++], d.core_id,
+ (u64)rctx->data_dma, data_len);
+
+ vcq_add_hc_final(&cmds[idx++], d.core_id,
+ (u64)rctx->digest_dma, info->digest_size);
+
+ vcq_add_flush(&cmds[idx++], d.core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_HASH_MAX_PACKED,
+ d.mbx_idx,
+ cmh_hash_final_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto err_unmap_ckpt;
+
+ return -EINPROGRESS;
+
+err_unmap_ckpt:
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+err_unmap_data:
+ if (data_buf && data_len > 0)
+ cmh_dma_unmap_single(rctx->data_dma, data_len,
+ DMA_TO_DEVICE);
+err_unmap_digest:
+ cmh_dma_unmap_single(rctx->digest_dma, info->digest_size,
+ DMA_FROM_DEVICE);
+err_free_digest:
+ kfree(rctx->digest_buf);
+ rctx->digest_buf = NULL;
+err_free_data:
+ kfree(data_buf);
+ rctx->data_buf = NULL;
+ cmh_hash_free_reqctx(rctx);
+ return ret;
+}
+
+static int cmh_hash_final(struct ahash_request *req)
+{
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+ u8 *data_buf = NULL;
+ u32 data_len = 0;
+ gfp_t gfp;
+
+ if (rctx->error)
+ return rctx->error;
+
+ if (rctx->buf_len > 0) {
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+ data_buf = kmalloc(rctx->buf_len, gfp);
+ if (!data_buf)
+ return -ENOMEM;
+ memcpy(data_buf, rctx->buf, rctx->buf_len);
+ data_len = rctx->buf_len;
+ rctx->buf_len = 0;
+ }
+
+ return cmh_hash_submit_final(req, data_buf, data_len);
+}
+
+static int cmh_hash_finup(struct ahash_request *req);
+
+/*
+ * One-shot digest -- delegates to init + finup so that all data is
+ * linearised and mapped through cmh_dma_map_single(), which is the
+ * only DMA mapping path aware of all supported DMA backends.
+ */
+static int cmh_hash_digest(struct ahash_request *req)
+{
+ int ret;
+
+ ret = cmh_hash_init(req);
+ if (ret)
+ return ret;
+ return cmh_hash_finup(req);
+}
+
+/*
+ * .finup -- update + final combined into a single transaction.
+ *
+ * Linearises the holdback buffer + new data and submits everything
+ * through the final path. Avoids the kernel's ahash_def_finup()
+ * which would allocate a subrequest and clone via export/import.
+ */
+static int cmh_hash_finup(struct ahash_request *req)
+{
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+ u32 data_len;
+ u8 *data_buf;
+ gfp_t gfp;
+
+ if (rctx->error)
+ return rctx->error;
+
+ data_len = rctx->buf_len + req->nbytes;
+
+ if (data_len == 0)
+ return cmh_hash_submit_final(req, NULL, 0);
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ data_buf = kmalloc(data_len, gfp);
+ if (!data_buf)
+ return -ENOMEM;
+
+ if (rctx->buf_len > 0)
+ memcpy(data_buf, rctx->buf, rctx->buf_len);
+
+ if (req->nbytes > 0) {
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(data_buf + rctx->buf_len,
+ req->svirt, req->nbytes);
+ else
+ scatterwalk_map_and_copy(data_buf + rctx->buf_len,
+ req->src, 0,
+ req->nbytes, 0);
+ }
+
+ rctx->buf_len = 0;
+ return cmh_hash_submit_final(req, data_buf, data_len);
+}
+
+/*
+ * Export -- purely software.
+ *
+ * Serialise the HC checkpoint (if any) and holdback buffer into the
+ * export state structure. No HW interaction needed because the
+ * incremental model keeps checkpoint up-to-date after each .update().
+ */
+static int cmh_hash_export(struct ahash_request *req, void *out)
+{
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_hash_export_state *state = out;
+
+ if (rctx->hw_started && rctx->has_checkpoint)
+ memcpy(state->checkpoint, rctx->checkpoint, HC_CONTEXT_SIZE);
+ else
+ memset(state->checkpoint, 0, HC_CONTEXT_SIZE);
+
+ if (rctx->buf_len > 0)
+ memcpy(state->buf, rctx->buf, rctx->buf_len);
+
+ state->buf_len = rctx->buf_len;
+ state->hw_started = rctx->hw_started;
+
+ return 0;
+}
+
+/*
+ * Import -- purely software.
+ *
+ * Restore checkpoint and holdback from a previously exported state.
+ * The next .update() or .final() will RESTORE the checkpoint into HW.
+ */
+static int cmh_hash_import(struct ahash_request *req, const void *in)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_hash_export_state *state = in;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->info = cmh_hash_get_info(tfm);
+
+ if (state->buf_len > CMH_HASH_MAX_BLOCK)
+ return -EINVAL;
+
+ rctx->hw_started = state->hw_started;
+ rctx->buf_len = state->buf_len;
+ memcpy(rctx->buf, state->buf, state->buf_len);
+
+ if (state->hw_started) {
+ memcpy(rctx->checkpoint, state->checkpoint, HC_CONTEXT_SIZE);
+ rctx->has_checkpoint = 1;
+ }
+
+ return 0;
+}
+
+/* Transform init (cra_init) -- set per-request context size */
+
+static int cmh_hash_cra_init(struct crypto_tfm *tfm)
+{
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct cmh_hash_reqctx));
+ return 0;
+}
+
+/* Registration */
+
+static struct cmh_hash_alg_drv cmh_hash_drvs[CMH_HASH_ALG_COUNT];
+
+/**
+ * cmh_hash_register() - Register SHA-256/384/512/3-256/3-384/3-512 hash algorithms
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_hash_register(void)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < CMH_HASH_ALG_COUNT; i++) {
+ const struct cmh_hash_alg_info *info = &cmh_hash_algs_info[i];
+ struct cmh_hash_alg_drv *drv = &cmh_hash_drvs[i];
+ struct ahash_alg *alg = &drv->alg;
+
+ drv->info = info;
+
+ alg->init = cmh_hash_init;
+ alg->update = cmh_hash_update;
+ alg->final = cmh_hash_final;
+ alg->finup = cmh_hash_finup;
+ alg->digest = cmh_hash_digest;
+ alg->export = cmh_hash_export;
+ alg->import = cmh_hash_import;
+
+ alg->halg.digestsize = info->digest_size;
+ alg->halg.statesize = sizeof(struct cmh_hash_export_state);
+
+ strscpy(alg->halg.base.cra_name, info->alg_name,
+ CRYPTO_MAX_ALG_NAME);
+ strscpy(alg->halg.base.cra_driver_name, info->drv_name,
+ CRYPTO_MAX_ALG_NAME);
+ alg->halg.base.cra_priority = 300;
+ alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NO_FALLBACK |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_REQ_VIRT;
+ alg->halg.base.cra_blocksize = info->block_size;
+ alg->halg.base.cra_ctxsize = 0;
+ alg->halg.base.cra_init = cmh_hash_cra_init;
+ alg->halg.base.cra_module = THIS_MODULE;
+
+ ret = crypto_register_ahash(alg);
+ if (ret) {
+ dev_err(cmh_dev(), "hash: failed to register %s (rc=%d)\n",
+ info->drv_name, ret);
+ /* Unregister any already-registered algorithms */
+ while (i--)
+ crypto_unregister_ahash(&cmh_hash_drvs[i].alg);
+ return ret;
+ }
+
+ dev_dbg(cmh_dev(), "hash: registered %s (priority 300)\n",
+ info->drv_name);
+ }
+
+ dev_info(cmh_dev(), "hash: %zu algorithm(s) registered\n",
+ CMH_HASH_ALG_COUNT);
+ return 0;
+}
+
+/**
+ * cmh_hash_unregister() - Unregister SHA hash algorithms from the crypto framework
+ */
+void cmh_hash_unregister(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < CMH_HASH_ALG_COUNT; i++) {
+ crypto_unregister_ahash(&cmh_hash_drvs[i].alg);
+ dev_dbg(cmh_dev(), "hash: unregistered %s\n",
+ cmh_hash_algs_info[i].drv_name);
+ }
+
+ dev_info(cmh_dev(), "hash: cleaned up\n");
+}
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index 7673ed3e5861..dea019346567 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -29,6 +29,7 @@
#include "cmh_mqi.h"
#include "cmh_txn.h"
#include "cmh_rh.h"
+#include "cmh_hash.h"
#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
@@ -197,6 +198,11 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_rh_init;
+ /* Register hash algorithms with the kernel crypto API */
+ ret = cmh_hash_register();
+ if (ret)
+ goto err_hash_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -209,6 +215,8 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_hash_unregister();
+err_hash_register:
cmh_rh_cleanup(cfg);
err_rh_init:
cmh_tm_cleanup();
@@ -235,6 +243,7 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_hash_unregister();
cmh_rh_cleanup(cfg);
cmh_tm_cleanup();
cmh_mqi_cleanup(cfg);
diff --git a/drivers/crypto/cmh/include/cmh_hash.h b/drivers/crypto/cmh/include/cmh_hash.h
new file mode 100644
index 000000000000..bf17d3af7787
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_hash.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API Hash Driver
+ *
+ * Registers ahash algorithms (SHA-2, SHA-3, and SHAKE families) with the
+ * Linux crypto subsystem. Uses an incremental HW update model:
+ *
+ * .init() -> software-only: zero per-request context
+ * .update() -> holdback partial blocks; submit full blocks via
+ * INIT [+ RESTORE] + UPDATE + SAVE + FLUSH
+ * .final() -> INIT [+ RESTORE] [+ UPDATE(residual)] + FINAL + FLUSH
+ * .digest() -> INIT + UPDATE + FINAL + FLUSH (single-shot)
+ * .export() -> software-only: copy checkpoint + holdback
+ * .import() -> software-only: restore checkpoint + holdback
+ */
+
+#ifndef CMH_HASH_H
+#define CMH_HASH_H
+
+#include "cmh_config.h"
+
+int cmh_hash_register(void);
+void cmh_hash_unregister(void);
+
+#endif /* CMH_HASH_H */
--
2.43.7
^ permalink raw reply related
* [PATCH v2 15/19] crypto: cmh - add ML-KEM/ML-DSA (QSE)
From: Saravanakrishnan Krishnamoorthy @ 2026-07-09 20:30 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260709203037.1884436-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register ML-KEM (Kyber) and ML-DSA (Dilithium) algorithms using
the CMH QSE core (core ID 0x09). ML-KEM is ioctl-only (keygen,
encaps, decaps). ML-DSA is registered as a sig algorithm with
priority 5001 to override the kernel's verify-only mldsa
implementation at priority 5000. This follows the established
pattern where hardware drivers override software-only fallbacks
(e.g. ccp at 300 over generic AES at 100, qat similarly). The
CMH driver provides full HW-accelerated sign + verify vs the
kernel's verify-only software implementation.
Includes cmh_pqc_sizes.c with compile-time tables of PQC key and
signature sizes for all supported parameter sets.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 5 +-
drivers/crypto/cmh/cmh_main.c | 9 +
drivers/crypto/cmh/cmh_pqc_mldsa.c | 394 +++++++++++++++++++++++++++++
drivers/crypto/cmh/cmh_pqc_sizes.c | 39 +++
drivers/crypto/cmh/cmh_qse.c | 211 +++++++++++++++
5 files changed, 657 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_pqc_mldsa.c
create mode 100644 drivers/crypto/cmh/cmh_pqc_sizes.c
create mode 100644 drivers/crypto/cmh/cmh_qse.c
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index a4cea0a56fc1..3425eb65d653 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -33,7 +33,10 @@ cmh-y := \
cmh_pke_common.o \
cmh_pke_rsa.o \
cmh_pke_ecdsa.o \
- cmh_pke_ecdh.o
+ cmh_pke_ecdh.o \
+ cmh_qse.o \
+ cmh_pqc_mldsa.o \
+ cmh_pqc_sizes.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index dd4e8812c457..bb81e2767974 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -39,6 +39,7 @@
#include "cmh_sm4.h"
#include "cmh_ccp.h"
#include "cmh_pke.h"
+#include "cmh_pqc.h"
#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
@@ -297,6 +298,11 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_pke_ecdh_register;
+ /* Register PQC ML-KEM/ML-DSA */
+ ret = cmh_pqc_mldsa_register();
+ if (ret)
+ goto err_pqc_mldsa_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -309,6 +315,8 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_pqc_mldsa_unregister();
+err_pqc_mldsa_register:
cmh_pke_ecdh_unregister();
err_pke_ecdh_register:
cmh_pke_ecdsa_unregister();
@@ -371,6 +379,7 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_pqc_mldsa_unregister();
cmh_pke_ecdh_unregister();
cmh_pke_ecdsa_unregister();
cmh_pke_rsa_unregister();
diff --git a/drivers/crypto/cmh/cmh_pqc_mldsa.c b/drivers/crypto/cmh/cmh_pqc_mldsa.c
new file mode 100644
index 000000000000..cbe63c34a1c8
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pqc_mldsa.c
@@ -0,0 +1,394 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- ML-DSA Signature Driver (sig_alg, synchronous)
+ *
+ * Registers "mldsa44", "mldsa65", "mldsa87" sig algorithms
+ * with sign, verify, set_pub_key, and set_priv_key callbacks.
+ *
+ * Key format:
+ * Public key = raw pk bytes (1312 / 1952 / 2592 bytes)
+ * Private key = raw sk bytes (2560 / 4032 / 4896 bytes)
+ *
+ * Sign: src = message bytes (up to 10240 bytes), dst = raw signature
+ * Verify: src = raw signature, digest = message bytes
+ *
+ * Non-masked mode only for sig_alg API.
+ * Masked mode available through /dev/cmh_mgmt ioctl.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <crypto/sig.h>
+#include <crypto/internal/sig.h>
+
+#include "cmh_sys.h"
+#include "cmh_qse_abi.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+#include "cmh_pqc.h"
+
+struct cmh_mldsa_tfm_ctx {
+ struct cmh_key_ctx key; /* private key (raw only) */
+ u8 *pub_key;
+ u32 pub_key_len;
+ u32 mode; /* ML_DSA_MODE_44/65/87 */
+ int mode_idx; /* index into size tables */
+};
+
+static inline struct cmh_mldsa_tfm_ctx *cmh_mldsa_ctx(struct crypto_sig *tfm)
+{
+ return crypto_sig_ctx(tfm);
+}
+
+/*
+ * ML-DSA sign (synchronous sig_alg)
+ *
+ * @src: message bytes
+ * @slen: message length
+ * @dst: signature output buffer
+ * @dlen: output buffer length
+ *
+ * Returns signature length on success, negative errno on failure.
+ */
+static int cmh_mldsa_sign(struct crypto_sig *tfm,
+ const void *src, unsigned int slen,
+ void *dst, unsigned int dlen)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+ int mi = ctx->mode_idx;
+ u32 sig_size = ml_dsa_sig_size[mi];
+ u32 sk_size = ml_dsa_sk_size[mi];
+ struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN];
+ struct core_dispatch dd;
+ u8 *m_buf = NULL, *sig_buf = NULL, *sk_buf = NULL;
+ dma_addr_t m_dma = DMA_MAPPING_ERROR;
+ dma_addr_t sig_dma = DMA_MAPPING_ERROR;
+ dma_addr_t sk_dma = DMA_MAPPING_ERROR;
+ int ret, idx;
+
+ if (ctx->key.mode != CMH_KEY_RAW)
+ return -EINVAL;
+ if (dlen < sig_size)
+ return -EINVAL;
+ if (!slen || slen > ML_DSA_MAX_MLEN)
+ return -EINVAL;
+
+ m_buf = kmemdup(src, slen, GFP_KERNEL);
+ sig_buf = kzalloc(sig_size, GFP_KERNEL);
+ if (!m_buf || !sig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (ctx->key.raw.len != sk_size) {
+ ret = -EINVAL;
+ goto out_free;
+ }
+
+ sk_buf = kmemdup(ctx->key.raw.data, ctx->key.raw.len, GFP_KERNEL);
+ if (!sk_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ m_dma = cmh_dma_map_single(m_buf, slen, DMA_TO_DEVICE);
+ sig_dma = cmh_dma_map_single(sig_buf, sig_size, DMA_FROM_DEVICE);
+ sk_dma = cmh_dma_map_single(sk_buf, sk_size, DMA_TO_DEVICE);
+
+ if (cmh_dma_map_error(m_dma) || cmh_dma_map_error(sig_dma) ||
+ cmh_dma_map_error(sk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ dd = cmh_core_select_instance(CMH_CORE_QSE);
+
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ idx = 1;
+ vcq_add_qse_ml_dsa_sign(&vcq[idx++], dd.core_id, ctx->mode,
+ QSE_FLAG_USE_RNG,
+ 0, m_dma, sk_dma, sig_dma, slen, false);
+ vcq_add_qse_flush(&vcq[idx++], dd.core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, 1,
+ dd.mbx_idx);
+ if (!ret) {
+ /* Sync bounce buffer so CPU sees the DMA-written signature */
+ cmh_dma_sync_for_cpu(sig_dma, sig_size, DMA_FROM_DEVICE);
+ memcpy(dst, sig_buf, sig_size);
+ ret = sig_size;
+ }
+
+out_unmap:
+ if (!cmh_dma_map_error(sk_dma))
+ cmh_dma_unmap_single(sk_dma, sk_size, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_size, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, slen, DMA_TO_DEVICE);
+
+out_free:
+ kfree_sensitive(sk_buf);
+ kfree(sig_buf);
+ kfree(m_buf);
+ return ret;
+}
+
+/*
+ * ML-DSA verify (synchronous sig_alg)
+ *
+ * @src: raw signature
+ * @slen: signature length
+ * @digest: message bytes
+ * @dlen: message length
+ *
+ * Returns 0 on successful verification, negative errno on failure.
+ */
+static int cmh_mldsa_verify(struct crypto_sig *tfm,
+ const void *src, unsigned int slen,
+ const void *digest, unsigned int dlen)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+ int mi = ctx->mode_idx;
+ u32 sig_size = ml_dsa_sig_size[mi];
+ u32 pk_size = ml_dsa_pk_size[mi];
+ struct core_dispatch d = cmh_core_select_instance(CMH_CORE_QSE);
+ struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN];
+ u8 *sig_buf = NULL, *m_buf = NULL, *pk_buf = NULL;
+ dma_addr_t sig_dma = DMA_MAPPING_ERROR;
+ dma_addr_t m_dma = DMA_MAPPING_ERROR;
+ dma_addr_t pk_dma = DMA_MAPPING_ERROR;
+ int ret;
+
+ if (!ctx->pub_key)
+ return -EINVAL;
+ if (slen != sig_size)
+ return -EINVAL;
+ if (!dlen || dlen > ML_DSA_MAX_MLEN)
+ return -EINVAL;
+
+ sig_buf = kmemdup(src, slen, GFP_KERNEL);
+ m_buf = kmemdup(digest, dlen, GFP_KERNEL);
+ pk_buf = kmemdup(ctx->pub_key, pk_size, GFP_KERNEL);
+ if (!sig_buf || !m_buf || !pk_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ sig_dma = cmh_dma_map_single(sig_buf, sig_size, DMA_TO_DEVICE);
+ m_dma = cmh_dma_map_single(m_buf, dlen, DMA_TO_DEVICE);
+ pk_dma = cmh_dma_map_single(pk_buf, pk_size, DMA_TO_DEVICE);
+
+ if (cmh_dma_map_error(sig_dma) || cmh_dma_map_error(m_dma) ||
+ cmh_dma_map_error(pk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ vcq_add_qse_ml_dsa_verify(&vcq[1], d.core_id, ctx->mode, 0,
+ m_dma, pk_dma, sig_dma, dlen);
+ vcq_add_qse_flush(&vcq[2], d.core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, 1, d.mbx_idx);
+
+out_unmap:
+ if (!cmh_dma_map_error(pk_dma))
+ cmh_dma_unmap_single(pk_dma, pk_size, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, dlen, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_size, DMA_TO_DEVICE);
+
+out_free:
+ kfree(pk_buf);
+ kfree(m_buf);
+ kfree(sig_buf);
+ return ret;
+}
+
+static int cmh_mldsa_set_pub_key(struct crypto_sig *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+ u32 expected = ml_dsa_pk_size[ctx->mode_idx];
+
+ if (keylen != expected)
+ return -EINVAL;
+
+ kfree(ctx->pub_key);
+ ctx->pub_key = NULL;
+ ctx->pub_key_len = 0;
+
+ ctx->pub_key = kmemdup(key, keylen, GFP_KERNEL);
+ if (!ctx->pub_key)
+ return -ENOMEM;
+
+ ctx->pub_key_len = keylen;
+ return 0;
+}
+
+static int cmh_mldsa_set_priv_key(struct crypto_sig *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+ u32 expected = ml_dsa_sk_size[ctx->mode_idx];
+
+ if (keylen != expected)
+ return -EINVAL;
+
+ return cmh_key_setkey_raw(&ctx->key, key, keylen, CORE_ID_QSE);
+}
+
+static unsigned int cmh_mldsa_key_size(struct crypto_sig *tfm)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+
+ /* crypto_sig_keysize() returns bits, not bytes */
+ return ml_dsa_pk_size[ctx->mode_idx] * 8;
+}
+
+static unsigned int cmh_mldsa_max_size(struct crypto_sig *tfm)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+
+ return ml_dsa_sig_size[ctx->mode_idx];
+}
+
+static int cmh_mldsa_44_init(struct crypto_sig *tfm)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->mode = ML_DSA_MODE_44;
+ ctx->mode_idx = 0;
+ return 0;
+}
+
+static int cmh_mldsa_65_init(struct crypto_sig *tfm)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->mode = ML_DSA_MODE_65;
+ ctx->mode_idx = 1;
+ return 0;
+}
+
+static int cmh_mldsa_87_init(struct crypto_sig *tfm)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->mode = ML_DSA_MODE_87;
+ ctx->mode_idx = 2;
+ return 0;
+}
+
+static void cmh_mldsa_exit(struct crypto_sig *tfm)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+
+ cmh_key_destroy(&ctx->key);
+ kfree(ctx->pub_key);
+ ctx->pub_key = NULL;
+}
+
+/*
+ * Priority 5001: the kernel's software ML-DSA (crypto/mldsa.c) registers
+ * at priority 5000 but only implements verify -- sign returns -EOPNOTSUPP.
+ * We provide full HW-accelerated sign + verify, so we must override.
+ */
+static struct sig_alg cmh_mldsa_algs[] = {
+ {
+ .sign = cmh_mldsa_sign,
+ .verify = cmh_mldsa_verify,
+ .set_pub_key = cmh_mldsa_set_pub_key,
+ .set_priv_key = cmh_mldsa_set_priv_key,
+ .key_size = cmh_mldsa_key_size,
+ .max_size = cmh_mldsa_max_size,
+ .init = cmh_mldsa_44_init,
+ .exit = cmh_mldsa_exit,
+ .base = {
+ .cra_name = "mldsa44",
+ .cra_driver_name = "cri-cmh-mldsa44",
+ .cra_priority = 5001,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_mldsa_tfm_ctx),
+ },
+ },
+ {
+ .sign = cmh_mldsa_sign,
+ .verify = cmh_mldsa_verify,
+ .set_pub_key = cmh_mldsa_set_pub_key,
+ .set_priv_key = cmh_mldsa_set_priv_key,
+ .key_size = cmh_mldsa_key_size,
+ .max_size = cmh_mldsa_max_size,
+ .init = cmh_mldsa_65_init,
+ .exit = cmh_mldsa_exit,
+ .base = {
+ .cra_name = "mldsa65",
+ .cra_driver_name = "cri-cmh-mldsa65",
+ .cra_priority = 5001,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_mldsa_tfm_ctx),
+ },
+ },
+ {
+ .sign = cmh_mldsa_sign,
+ .verify = cmh_mldsa_verify,
+ .set_pub_key = cmh_mldsa_set_pub_key,
+ .set_priv_key = cmh_mldsa_set_priv_key,
+ .key_size = cmh_mldsa_key_size,
+ .max_size = cmh_mldsa_max_size,
+ .init = cmh_mldsa_87_init,
+ .exit = cmh_mldsa_exit,
+ .base = {
+ .cra_name = "mldsa87",
+ .cra_driver_name = "cri-cmh-mldsa87",
+ .cra_priority = 5001,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_mldsa_tfm_ctx),
+ },
+ },
+};
+
+/**
+ * cmh_pqc_mldsa_register() - Register ML-DSA akcipher algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_pqc_mldsa_register(void)
+{
+ int ret, i;
+
+ for (i = 0; i < ARRAY_SIZE(cmh_mldsa_algs); i++) {
+ ret = crypto_register_sig(&cmh_mldsa_algs[i]);
+ if (ret) {
+ dev_err(cmh_dev(), "cmh: failed to register %s (%d)\n",
+ cmh_mldsa_algs[i].base.cra_name, ret);
+ goto err_unregister;
+ }
+ }
+
+ return 0;
+
+err_unregister:
+ while (i--)
+ crypto_unregister_sig(&cmh_mldsa_algs[i]);
+ return ret;
+}
+
+/**
+ * cmh_pqc_mldsa_unregister() - Unregister ML-DSA akcipher algorithms from the crypto framework
+ */
+void cmh_pqc_mldsa_unregister(void)
+{
+ int i = ARRAY_SIZE(cmh_mldsa_algs);
+
+ while (i--)
+ crypto_unregister_sig(&cmh_mldsa_algs[i]);
+}
diff --git a/drivers/crypto/cmh/cmh_pqc_sizes.c b/drivers/crypto/cmh/cmh_pqc_sizes.c
new file mode 100644
index 000000000000..39e3d56f4312
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pqc_sizes.c
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- PQC Algorithm Size Tables
+ *
+ * Centralised ML-DSA and SLH-DSA parameter-size arrays. Declared
+ * extern in cmh_qse_abi.h / cmh_hcq_abi.h, defined here once to
+ * avoid per-TU duplication.
+ */
+
+#include <linux/build_bug.h>
+#include <linux/kernel.h>
+#include <linux/types.h>
+
+#include "cmh_qse_abi.h"
+#include "cmh_hcq_abi.h"
+
+/* ML-DSA size tables (indexed by ml_dsa_mode_idx()) */
+const u32 ml_dsa_pk_size[3] = { 1312U, 1952U, 2592U };
+const u32 ml_dsa_sk_size[3] = { 2560U, 4032U, 4896U };
+const u32 ml_dsa_sk_size_masked[3] = { 3360U, 5472U, 6368U };
+const u32 ml_dsa_sig_size[3] = { 2420U, 3309U, 4627U };
+
+static_assert(ARRAY_SIZE(ml_dsa_pk_size) == ARRAY_SIZE(ml_dsa_sk_size));
+static_assert(ARRAY_SIZE(ml_dsa_pk_size) == ARRAY_SIZE(ml_dsa_sk_size_masked));
+static_assert(ARRAY_SIZE(ml_dsa_pk_size) == ARRAY_SIZE(ml_dsa_sig_size));
+
+/* SLH-DSA n-values and signature sizes (indexed by param_set - 1) */
+const u32 slhdsa_n[12] = {
+ 16, 16, 24, 24, 32, 32, /* SHAKE 128s/f, 192s/f, 256s/f */
+ 16, 16, 24, 24, 32, 32, /* SHA2 128s/f, 192s/f, 256s/f */
+};
+
+const u32 slhdsa_sig_size[12] = {
+ 7856, 17088, 16224, 35664, 29792, 49856, /* SHAKE */
+ 7856, 17088, 16224, 35664, 29792, 49856, /* SHA2 */
+};
+
+static_assert(ARRAY_SIZE(slhdsa_n) == ARRAY_SIZE(slhdsa_sig_size));
diff --git a/drivers/crypto/cmh/cmh_qse.c b/drivers/crypto/cmh/cmh_qse.c
new file mode 100644
index 000000000000..257dc3ee29a8
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_qse.c
@@ -0,0 +1,211 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- QSE Core VCQ Builders
+ *
+ * VCQ builder functions for ML-KEM and ML-DSA commands (plain and masked).
+ * Each function populates a single vcq_cmd slot. Callers assemble
+ * complete VCQs with header + command(s) + flush, then submit via
+ * cmh_tm_submit_sync().
+ */
+
+#include <linux/string.h>
+
+#include "cmh_sys.h"
+
+/* -- QSE flush -- */
+
+/**
+ * vcq_add_qse_flush() - Build a QSE flush VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ */
+void vcq_add_qse_flush(struct vcq_cmd *slot, u32 core_id)
+{
+ vcq_add_flush(slot, core_id);
+}
+
+/* -- ML-KEM -- */
+
+/**
+ * vcq_add_qse_ml_kem_keygen() - Build an ML-KEM key generation VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @k: ML-KEM security parameter (k = 2, 3, or 4)
+ * @flags: Command flags
+ * @seed: DMA address of seed input buffer
+ * @z: DMA address of implicit rejection value buffer
+ * @ek: DMA address of encapsulation key output buffer
+ * @dk: DMA address of decapsulation key output buffer
+ * @dk_type: Decapsulation key datastore type
+ * @masked: Use masked (side-channel protected) variant
+ */
+void vcq_add_qse_ml_kem_keygen(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags,
+ u64 seed, u64 z, u64 ek, u64 dk, u32 dk_type,
+ bool masked)
+{
+ u32 cmd_id = masked ? QSE_CMD_ML_KEM_KEYGEN_MASKED
+ : QSE_CMD_ML_KEM_KEYGEN;
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id);
+ slot->hwc.qse.cmd_ml_kem_keygen.k = k;
+ slot->hwc.qse.cmd_ml_kem_keygen.flags = flags;
+ slot->hwc.qse.cmd_ml_kem_keygen.seed = seed;
+ slot->hwc.qse.cmd_ml_kem_keygen.z = z;
+ slot->hwc.qse.cmd_ml_kem_keygen.ek = ek;
+ slot->hwc.qse.cmd_ml_kem_keygen.dk = dk;
+ slot->hwc.qse.cmd_ml_kem_keygen.dk_type = dk_type;
+}
+
+/**
+ * vcq_add_qse_ml_kem_enc() - Build an ML-KEM encapsulation VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @k: ML-KEM security parameter (k = 2, 3, or 4)
+ * @flags: Command flags
+ * @coin: DMA address of encapsulation coin/randomness buffer
+ * @ek: DMA address of encapsulation key input buffer
+ * @ct: DMA address of ciphertext output buffer
+ * @ss: DMA address of shared secret output buffer
+ * @ss_type: Shared secret datastore type
+ * @masked: Use masked (side-channel protected) variant
+ */
+void vcq_add_qse_ml_kem_enc(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags,
+ u64 coin, u64 ek, u64 ct, u64 ss, u32 ss_type,
+ bool masked)
+{
+ u32 cmd_id = masked ? QSE_CMD_ML_KEM_ENC_MASKED
+ : QSE_CMD_ML_KEM_ENC;
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id);
+ slot->hwc.qse.cmd_ml_kem_enc.k = k;
+ slot->hwc.qse.cmd_ml_kem_enc.flags = flags;
+ slot->hwc.qse.cmd_ml_kem_enc.coin = coin;
+ slot->hwc.qse.cmd_ml_kem_enc.ek = ek;
+ slot->hwc.qse.cmd_ml_kem_enc.ct = ct;
+ slot->hwc.qse.cmd_ml_kem_enc.ss = ss;
+ slot->hwc.qse.cmd_ml_kem_enc.ss_type = ss_type;
+}
+
+/**
+ * vcq_add_qse_ml_kem_dec() - Build an ML-KEM decapsulation VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @k: ML-KEM security parameter (k = 2, 3, or 4)
+ * @flags: Command flags
+ * @ct: DMA address of ciphertext input buffer
+ * @dk: DMA address of decapsulation key input buffer
+ * @ss: DMA address of shared secret output buffer
+ * @ss_type: Shared secret datastore type
+ * @masked: Use masked (side-channel protected) variant
+ */
+void vcq_add_qse_ml_kem_dec(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags,
+ u64 ct, u64 dk, u64 ss, u32 ss_type,
+ bool masked)
+{
+ u32 cmd_id = masked ? QSE_CMD_ML_KEM_DEC_MASKED
+ : QSE_CMD_ML_KEM_DEC;
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id);
+ slot->hwc.qse.cmd_ml_kem_dec.k = k;
+ slot->hwc.qse.cmd_ml_kem_dec.flags = flags;
+ slot->hwc.qse.cmd_ml_kem_dec.ct = ct;
+ slot->hwc.qse.cmd_ml_kem_dec.dk = dk;
+ slot->hwc.qse.cmd_ml_kem_dec.ss = ss;
+ slot->hwc.qse.cmd_ml_kem_dec.ss_type = ss_type;
+}
+
+/* -- ML-DSA -- */
+
+/**
+ * vcq_add_qse_ml_dsa_keygen() - Build an ML-DSA key generation VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @mode: ML-DSA mode (44, 65, or 87)
+ * @flags: Command flags
+ * @seed: DMA address of seed input buffer
+ * @pk: DMA address of public key output buffer
+ * @sk: DMA address of secret key output buffer
+ * @sk_type: Secret key datastore type
+ * @masked: Use masked (side-channel protected) variant
+ */
+void vcq_add_qse_ml_dsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags,
+ u64 seed, u64 pk, u64 sk, u32 sk_type,
+ bool masked)
+{
+ u32 cmd_id = masked ? QSE_CMD_ML_DSA_KEYGEN_MASKED
+ : QSE_CMD_ML_DSA_KEYGEN;
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id);
+ slot->hwc.qse.cmd_ml_dsa_keygen.mode = mode;
+ slot->hwc.qse.cmd_ml_dsa_keygen.flags = flags;
+ slot->hwc.qse.cmd_ml_dsa_keygen.seed = seed;
+ slot->hwc.qse.cmd_ml_dsa_keygen.pk = pk;
+ slot->hwc.qse.cmd_ml_dsa_keygen.sk = sk;
+ slot->hwc.qse.cmd_ml_dsa_keygen.sk_type = sk_type;
+}
+
+/**
+ * vcq_add_qse_ml_dsa_sign() - Build an ML-DSA signing VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @mode: ML-DSA mode (44, 65, or 87)
+ * @flags: Command flags
+ * @rnd: DMA address of signing randomness buffer
+ * @m: DMA address of message buffer
+ * @sk: DMA address of secret key buffer
+ * @sig: DMA address of signature output buffer
+ * @mlen: Length of message in bytes
+ * @masked: Use masked (side-channel protected) variant
+ */
+void vcq_add_qse_ml_dsa_sign(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags,
+ u64 rnd, u64 m, u64 sk, u64 sig, u32 mlen,
+ bool masked)
+{
+ u32 cmd_id = masked ? QSE_CMD_ML_DSA_SIGN_MASKED
+ : QSE_CMD_ML_DSA_SIGN;
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id);
+ slot->hwc.qse.cmd_ml_dsa_sign.mode = mode;
+ slot->hwc.qse.cmd_ml_dsa_sign.flags = flags;
+ slot->hwc.qse.cmd_ml_dsa_sign.rnd = rnd;
+ slot->hwc.qse.cmd_ml_dsa_sign.m = m;
+ slot->hwc.qse.cmd_ml_dsa_sign.sk = sk;
+ slot->hwc.qse.cmd_ml_dsa_sign.sig = sig;
+ slot->hwc.qse.cmd_ml_dsa_sign.mlen = mlen;
+}
+
+/**
+ * vcq_add_qse_ml_dsa_verify() - Build an ML-DSA signature verify VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @mode: ML-DSA mode (44, 65, or 87)
+ * @flags: Command flags
+ * @m: DMA address of message buffer
+ * @pk: DMA address of public key buffer
+ * @sig: DMA address of signature buffer to verify
+ * @mlen: Length of message in bytes
+ */
+void vcq_add_qse_ml_dsa_verify(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags,
+ u64 m, u64 pk, u64 sig, u32 mlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, QSE_CMD_ML_DSA_VERIFY);
+ slot->hwc.qse.cmd_ml_dsa_verify.mode = mode;
+ slot->hwc.qse.cmd_ml_dsa_verify.flags = flags;
+ slot->hwc.qse.cmd_ml_dsa_verify.m = m;
+ slot->hwc.qse.cmd_ml_dsa_verify.pk = pk;
+ slot->hwc.qse.cmd_ml_dsa_verify.sig = sig;
+ slot->hwc.qse.cmd_ml_dsa_verify.mlen = mlen;
+}
--
2.43.7
^ permalink raw reply related
* [PATCH v2 01/19] dt-bindings: crypto: add Rambus CryptoManager Hub
From: Saravanakrishnan Krishnamoorthy @ 2026-07-09 20:30 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260709203037.1884436-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Add device tree binding schema for the CRI CryptoManager Hub (CMH)
hardware crypto accelerator. The binding covers the parent SoC-level
node with register region, interrupt, DMA properties, and per-core
child nodes identified by compatible string and unit address.
Register the 'cri' vendor prefix for Cryptography Research, Inc.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
.../devicetree/bindings/crypto/cri,cmh.yaml | 222 ++++++++++++++++++
.../devicetree/bindings/vendor-prefixes.yaml | 2 +
2 files changed, 224 insertions(+)
create mode 100644 Documentation/devicetree/bindings/crypto/cri,cmh.yaml
diff --git a/Documentation/devicetree/bindings/crypto/cri,cmh.yaml b/Documentation/devicetree/bindings/crypto/cri,cmh.yaml
new file mode 100644
index 000000000000..db41132e0591
--- /dev/null
+++ b/Documentation/devicetree/bindings/crypto/cri,cmh.yaml
@@ -0,0 +1,222 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/crypto/cri,cmh.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: CRI CryptoManager Hub (CMH) Hardware Crypto Accelerator
+
+maintainers:
+ - Alex Ousherovitch <aousherovitch@rambus.com>
+ - Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
+ - Joel Wittenauer <Joel.Wittenauer@cryptography.com>
+
+description: |
+ The CRI CryptoManager Hub (CMH) is a hardware cryptographic accelerator accessed
+ via a mailbox-based VCQ (Virtual Command Queue) interface. The host
+ writes VCQ command sequences into per-mailbox DMA queue buffers and
+ rings a doorbell; the CMH eSW processes them and signals completion
+ via interrupt.
+
+ Supported algorithm families: SHA-2, SHA-3, SM3, AES, SM4,
+ ChaCha20-Poly1305, RSA, ECDSA, EdDSA, ECDH, SM2, ML-KEM, ML-DSA,
+ SLH-DSA, LMS, XMSS, DRBG.
+
+properties:
+ compatible:
+ const: cri,cmh
+
+ reg:
+ maxItems: 1
+ description:
+ SIC (System Interface Controller) MMIO region. Mailbox instance
+ registers are at offsets N * 0x1000 within this region.
+
+ interrupts:
+ minItems: 1
+ maxItems: 64
+ description:
+ Per-mailbox completion/error interrupts from the CryptoManager Hub,
+ matching the real CMH ch_sys_interrupt_mbx[N-1:0] topology.
+ Entry i corresponds to MBX instance i. The driver maps each
+ configured mailbox (cri,mbx-instances) to its DT interrupt
+ index and registers a separate threaded IRQ handler per MBX.
+
+ interrupt-names:
+ minItems: 1
+ maxItems: 64
+ items:
+ pattern: '^mbx[0-9]+$'
+ description:
+ Names for each mailbox interrupt, matching the interrupts array.
+ Format is "mbxN" where N is the mailbox instance index.
+
+ cri,mbx-instances:
+ $ref: /schemas/types.yaml#/definitions/uint32-array
+ minItems: 1
+ maxItems: 64
+ description:
+ Array of 0-based mailbox instance indices to configure.
+ Each index N maps to register offset N * 0x1000 within the
+ SIC region. If absent, defaults to instances 0 and 1.
+
+ cri,mbx-slots-log2:
+ $ref: /schemas/types.yaml#/definitions/uint32-array
+ minItems: 1
+ maxItems: 64
+ description:
+ Per-mailbox slot count as log2. Valid range 1..15.
+ Array length must match cri,mbx-instances.
+ Default is 5 (32 slots).
+
+ cri,mbx-strides-log2:
+ $ref: /schemas/types.yaml#/definitions/uint32-array
+ minItems: 1
+ maxItems: 64
+ description:
+ Per-mailbox stride (bytes per slot) as log2. Valid range 7..10.
+ Array length must match cri,mbx-instances.
+ Default is 7 (128 bytes per slot).
+
+ "#address-cells":
+ const: 1
+
+ "#size-cells":
+ const: 0
+
+patternProperties:
+ "^(hc|aes|sm4|sm3|hcq|qse|pke|drbg|ccp)@[0-9a-f]+$":
+ type: object
+ description:
+ Per-core-type child nodes. Each child represents one crypto core
+ instance available in the hardware. The driver enumerates these at
+ probe to discover which algorithm families are present.
+
+ properties:
+ reg:
+ maxItems: 1
+ description:
+ Hardware core ID for this core type (e.g. 0x02 for HC, 0x03 for AES).
+ Must match the CORE_ID_* values defined by the CMH hardware.
+
+ cri,mbx:
+ $ref: /schemas/types.yaml#/definitions/uint32
+ description:
+ Pin this core instance to a specific mailbox instance index.
+ Multiple child nodes of the same core type may each specify a
+ different cri,mbx value to spread instances across mailboxes.
+ When absent, the driver auto-assigns a mailbox via round-robin
+ across the instances listed in cri,mbx-instances.
+
+ required:
+ - reg
+
+ additionalProperties: false
+
+required:
+ - compatible
+ - reg
+ - interrupts
+ - "#address-cells"
+ - "#size-cells"
+
+additionalProperties: false
+
+examples:
+ - |
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+
+ crypto@a4800000 {
+ compatible = "cri,cmh";
+ reg = <0x0 0xa4800000 0x0 0x41000>;
+ interrupts = <1 2>;
+ interrupt-names = "mbx0", "mbx1";
+ cri,mbx-instances = <0 1>;
+ cri,mbx-slots-log2 = <5 5>;
+ cri,mbx-strides-log2 = <7 7>;
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ hc@2 {
+ reg = <0x02>;
+ };
+
+ aes@3 {
+ reg = <0x03>;
+ };
+
+ sm4@4 {
+ reg = <0x04>;
+ };
+
+ sm3@5 {
+ reg = <0x05>;
+ };
+
+ hcq@8 {
+ reg = <0x08>;
+ };
+
+ qse@9 {
+ reg = <0x09>;
+ };
+
+ pke@a {
+ reg = <0x0a>;
+ cri,mbx = <1>;
+ };
+
+ drbg@f {
+ reg = <0x0f>;
+ };
+
+ ccp@18 {
+ reg = <0x18>;
+ };
+ };
+ };
+
+ - |
+ /* Multi-instance: two AES cores on separate MBXes (future eSW support) */
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+
+ crypto@a4800000 {
+ compatible = "cri,cmh";
+ reg = <0x0 0xa4800000 0x0 0x41000>;
+ interrupts = <1 2>;
+ interrupt-names = "mbx0", "mbx1";
+ cri,mbx-instances = <0 1>;
+ cri,mbx-slots-log2 = <5 5>;
+ cri,mbx-strides-log2 = <7 7>;
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ hc@2 {
+ reg = <0x02>;
+ };
+
+ aes@3 {
+ reg = <0x03>;
+ cri,mbx = <0>;
+ };
+
+ /* Second AES instance at core ID 0x06, pinned to MBX 1 */
+ aes@6 {
+ reg = <0x06>;
+ cri,mbx = <1>;
+ };
+
+ pke@a {
+ reg = <0x0a>;
+ cri,mbx = <1>;
+ };
+
+ drbg@f {
+ reg = <0x0f>;
+ };
+ };
+ };
diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml
index 396044f368e7..8b7187ea0194 100644
--- a/Documentation/devicetree/bindings/vendor-prefixes.yaml
+++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml
@@ -375,6 +375,8 @@ patternProperties:
description: Crane Connectivity Solutions
"^creative,.*":
description: Creative Technology Ltd
+ "^cri,.*":
+ description: Cryptography Research, Inc.
"^crystalfontz,.*":
description: Crystalfontz America, Inc.
"^csky,.*":
--
2.43.7
^ permalink raw reply related
* [PATCH v2 05/19] crypto: cmh - add HMAC ahash
From: Saravanakrishnan Krishnamoorthy @ 2026-07-09 20:30 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260709203037.1884436-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register ahash algorithms for HMAC-SHA-224, HMAC-SHA-256,
HMAC-SHA-384, HMAC-SHA-512, HMAC-SHA3-224, HMAC-SHA3-256,
HMAC-SHA3-384, and HMAC-SHA3-512 using the CMH hash core.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 3 +-
drivers/crypto/cmh/cmh_hmac.c | 684 ++++++++++++++++++++++++++
drivers/crypto/cmh/cmh_main.c | 9 +
drivers/crypto/cmh/include/cmh_hmac.h | 16 +
4 files changed, 711 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_hmac.c
create mode 100644 drivers/crypto/cmh/include/cmh_hmac.h
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index c0531f416229..1f760c0214ef 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -15,7 +15,8 @@ cmh-y := \
cmh_sysfs.o \
cmh_key.o \
cmh_sys.o \
- cmh_hash.o
+ cmh_hash.o \
+ cmh_hmac.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_hmac.c b/drivers/crypto/cmh/cmh_hmac.c
new file mode 100644
index 000000000000..1f536088eabf
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_hmac.c
@@ -0,0 +1,684 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API HMAC Driver
+ *
+ * Registers HMAC ahash algorithms with the Linux crypto subsystem.
+ * Supports HMAC-SHA-2 (224/256/384/512) and HMAC-SHA-3 (224/256/384/512)
+ * using the CMH Hash Core (HC) via HC_CMD_HMAC.
+ *
+ * Uses the same self-contained transaction model as cmh_hash.c:
+ * .setkey() -> store raw key bytes
+ * .init() -> software-only: initialize per-request context
+ * .update() -> software-only: copy SG data into per-call chunk
+ * .final() -> [SYS_CMD_WRITE] + HC_CMD_HMAC + [GATHER] + FINAL + FLUSH
+ *
+ * Raw-key atomicity: SYS_CMD_WRITE to SYS_REF_TEMP is packed into
+ * the same VCQ as HC_CMD_HMAC (see cmh_key.h for details).
+ *
+ * ahash .export()/.import() (state cloning): supported at the
+ * software accumulation level only. The HW hash core does NOT
+ * support save/restore of intermediate HMAC state (SHA3 sponge
+ * invertibility, SHA2 blocked for consistency). Since this driver
+ * accumulates all input data in kernel memory before submitting
+ * atomically in .final(), export/import simply serializes the
+ * input queue -- no keying material or HW state is exposed.
+ *
+ * All HMAC data is accumulated in kernel memory and capped at
+ * HMAC_MAX_DATA (64 KB).
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/hash.h>
+#include <crypto/hash.h>
+#include <linux/scatterlist.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "cmh_hmac.h"
+#include "cmh_vcq.h"
+#include "cmh_hc_abi.h"
+#include "cmh_sys_abi.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+/*
+ * Maximum data that can be accumulated across .update() calls.
+ * HMAC save/restore is intentionally unsupported (see file header),
+ * so all data must be buffered in kernel memory and submitted
+ * atomically in .final(). This cap prevents unbounded allocation.
+ */
+#define HMAC_MAX_DATA (64 * 1024)
+
+/* Algorithm Table */
+
+struct cmh_hmac_alg_info {
+ u32 hc_algo; /* HC_ALGO_* */
+ u32 digest_size; /* bytes */
+ u32 block_size; /* cra_blocksize */
+ const char *alg_name; /* Linux crypto name: "hmac(sha256)" */
+ const char *drv_name; /* driver name: "cri-cmh-hmac-sha256" */
+};
+
+static const struct cmh_hmac_alg_info cmh_hmac_algs_info[] = {
+ /* HMAC-SHA-2 family */
+ {
+ .hc_algo = HC_ALGO_SHA2_224,
+ .digest_size = CMH_SHA224_DIGEST_SIZE,
+ .block_size = 64,
+ .alg_name = "hmac(sha224)",
+ .drv_name = "cri-cmh-hmac-sha224",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA2_256,
+ .digest_size = CMH_SHA256_DIGEST_SIZE,
+ .block_size = 64,
+ .alg_name = "hmac(sha256)",
+ .drv_name = "cri-cmh-hmac-sha256",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA2_384,
+ .digest_size = CMH_SHA384_DIGEST_SIZE,
+ .block_size = 128,
+ .alg_name = "hmac(sha384)",
+ .drv_name = "cri-cmh-hmac-sha384",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA2_512,
+ .digest_size = CMH_SHA512_DIGEST_SIZE,
+ .block_size = 128,
+ .alg_name = "hmac(sha512)",
+ .drv_name = "cri-cmh-hmac-sha512",
+ },
+ /* HMAC-SHA-3 family */
+ {
+ .hc_algo = HC_ALGO_SHA3_224,
+ .digest_size = CMH_SHA3_224_DIGEST_SIZE,
+ .block_size = 144,
+ .alg_name = "hmac(sha3-224)",
+ .drv_name = "cri-cmh-hmac-sha3-224",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA3_256,
+ .digest_size = CMH_SHA3_256_DIGEST_SIZE,
+ .block_size = 136,
+ .alg_name = "hmac(sha3-256)",
+ .drv_name = "cri-cmh-hmac-sha3-256",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA3_384,
+ .digest_size = CMH_SHA3_384_DIGEST_SIZE,
+ .block_size = 104,
+ .alg_name = "hmac(sha3-384)",
+ .drv_name = "cri-cmh-hmac-sha3-384",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA3_512,
+ .digest_size = CMH_SHA3_512_DIGEST_SIZE,
+ .block_size = 72,
+ .alg_name = "hmac(sha3-512)",
+ .drv_name = "cri-cmh-hmac-sha3-512",
+ },
+};
+
+#define CMH_HMAC_ALG_COUNT ARRAY_SIZE(cmh_hmac_algs_info)
+
+/* Per-Request State */
+
+struct cmh_hmac_chunk {
+ struct list_head list;
+ struct list_head tfm_node; /* per-tfm orphan tracking */
+ u32 len;
+ u8 data[];
+};
+
+/*
+ * Maximum payload commands any HMAC transaction can produce:
+ * [SYS_CMD_WRITE] + HC_CMD_HMAC + [GATHER] + FINAL + FLUSH = 5
+ * Worst-case packed output (stride=7, 1 payload per VCQ):
+ * 5 VCQs x 2 entries = 10
+ */
+#define CMH_HMAC_MAX_PAYLOAD 5
+#define CMH_HMAC_MAX_PACKED (CMH_HMAC_MAX_PAYLOAD * 2)
+
+struct cmh_hmac_reqctx {
+ const struct cmh_hmac_alg_info *info;
+ int error;
+ struct list_head chunks;
+ u32 num_chunks;
+ u32 total_len;
+ /* DMA state for async final */
+ dma_addr_t digest_dma;
+ dma_addr_t key_dma;
+ u8 *digest_buf;
+ struct cmh_sg_map *sgm;
+ u32 keylen;
+ struct vcq_cmd packed[CMH_HMAC_MAX_PACKED];
+};
+
+/* Flat state for export/import -- holds accumulated input data only */
+struct cmh_hmac_export_state {
+ u32 total_len;
+ u8 data[];
+};
+
+/*
+ * Flat state buffer for export/import. The CMH hash core does not
+ * support save/restore of intermediate HMAC state, so this driver
+ * accumulates input in SW and serialises the buffer on export.
+ *
+ * PAGE_SIZE (4096) caps the exportable accumulated-data window.
+ * Full-range export (up to HMAC_MAX_DATA = 64 KB) is not feasible
+ * because the crypto subsystem pre-allocates statesize bytes per
+ * request. Export returns -EINVAL if the caller has accumulated
+ * more than CMH_HMAC_EXPORT_MAX.
+ */
+#define CMH_HMAC_STATE_SIZE 4096
+#define CMH_HMAC_EXPORT_MAX (CMH_HMAC_STATE_SIZE - sizeof(struct cmh_hmac_export_state))
+
+/* Per-Transform State (carries key across requests) */
+
+struct cmh_hmac_tfm_ctx {
+ struct cmh_key_ctx key;
+ spinlock_t chunk_lock; /* protects all_chunks */
+ struct list_head all_chunks; /* orphan-safe chunk tracking */
+};
+
+/* VCQ Builders (HMAC-specific; shared builders in cmh_hc_abi.h / cmh_vcq.h) */
+
+/* Add an HC_CMD_HMAC entry */
+static void vcq_add_hc_hmac(struct vcq_cmd *slot, u32 core_id, u64 key_ref,
+ u32 keylen, u32 algo)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_HMAC);
+ slot->hwc.hc.cmd_hmac.key = key_ref;
+ slot->hwc.hc.cmd_hmac.keylen = keylen;
+ slot->hwc.hc.cmd_hmac.algo = algo;
+}
+
+/* Request Context Cleanup */
+
+static void cmh_hmac_free_chunks(struct cmh_hmac_reqctx *rctx,
+ struct cmh_hmac_tfm_ctx *tctx)
+{
+ struct cmh_hmac_chunk *chunk, *tmp;
+
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(chunk, tmp, &rctx->chunks, list) {
+ list_del(&chunk->list);
+ list_del(&chunk->tfm_node);
+ kfree_sensitive(chunk);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->num_chunks = 0;
+ rctx->total_len = 0;
+}
+
+/*
+ * Build a DMA-mapped CMH eSW scatter-gather chain from accumulated chunks.
+ */
+static struct cmh_sg_map *
+cmh_hmac_build_sg(struct cmh_hmac_reqctx *rctx, gfp_t gfp)
+{
+ struct cmh_dma_buf *bufs;
+ struct cmh_hmac_chunk *chunk;
+ struct cmh_sg_map *sgm;
+ u32 i;
+
+ bufs = kcalloc(rctx->num_chunks, sizeof(*bufs), gfp);
+ if (!bufs)
+ return NULL;
+
+ i = 0;
+ list_for_each_entry(chunk, &rctx->chunks, list) {
+ bufs[i].data = chunk->data;
+ bufs[i].len = chunk->len;
+ i++;
+ }
+
+ sgm = cmh_dma_build_sg(bufs, rctx->num_chunks, gfp);
+ kfree(bufs);
+ return sgm;
+}
+
+/* VCQ Packing + Submit */
+
+/* ahash Operations */
+
+struct cmh_hmac_alg_drv {
+ struct ahash_alg alg;
+ const struct cmh_hmac_alg_info *info;
+};
+
+static const struct cmh_hmac_alg_info *
+cmh_hmac_get_info(struct crypto_ahash *tfm)
+{
+ struct ahash_alg *alg = crypto_ahash_alg(tfm);
+
+ return container_of(alg, struct cmh_hmac_alg_drv, alg)->info;
+}
+
+static int cmh_hmac_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_hmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+
+ return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_HC);
+}
+
+static int cmh_hmac_init(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_hmac_reqctx *rctx = ahash_request_ctx(req);
+
+ rctx->info = cmh_hmac_get_info(tfm);
+ rctx->error = 0;
+ INIT_LIST_HEAD(&rctx->chunks);
+ rctx->num_chunks = 0;
+ rctx->total_len = 0;
+
+ return 0;
+}
+
+static int cmh_hmac_update(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_hmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_hmac_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_hmac_chunk *chunk;
+ int nents;
+
+ if (rctx->error)
+ return rctx->error;
+
+ if (!req->nbytes)
+ return 0;
+
+ if (req->nbytes > HMAC_MAX_DATA - rctx->total_len) {
+ rctx->error = -EINVAL;
+ goto err_free_chunks;
+ }
+
+ chunk = kmalloc(sizeof(*chunk) + req->nbytes,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC);
+ if (!chunk) {
+ rctx->error = -ENOMEM;
+ goto err_free_chunks;
+ }
+
+ chunk->len = req->nbytes;
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) {
+ memcpy(chunk->data, req->svirt, req->nbytes);
+ } else {
+ nents = sg_nents_for_len(req->src, req->nbytes);
+ if (nents < 0 ||
+ sg_copy_to_buffer(req->src, nents,
+ chunk->data, req->nbytes) != req->nbytes) {
+ kfree_sensitive(chunk);
+ rctx->error = -EINVAL;
+ goto err_free_chunks;
+ }
+ }
+
+ list_add_tail(&chunk->list, &rctx->chunks);
+ spin_lock_bh(&tctx->chunk_lock);
+ list_add_tail(&chunk->tfm_node, &tctx->all_chunks);
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->num_chunks++;
+ rctx->total_len += req->nbytes;
+
+ return 0;
+
+err_free_chunks:
+ /*
+ * Terminal error -- free all previously accumulated chunks.
+ * The crypto API hash path does not call .final()
+ * on error, and hash_sock_destruct has no per-request
+ * destructor, so chunks would be orphaned otherwise.
+ */
+ cmh_hmac_free_chunks(rctx, tctx);
+ return rctx->error;
+}
+
+static void cmh_hmac_complete(void *data, int error)
+{
+ struct ahash_request *req = data;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_hmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_hmac_reqctx *rctx = ahash_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ cmh_dma_unmap_single(rctx->digest_dma, rctx->info->digest_size,
+ DMA_FROM_DEVICE);
+
+ if (!error)
+ memcpy(req->result, rctx->digest_buf,
+ rctx->info->digest_size);
+
+ kfree(rctx->digest_buf);
+ rctx->digest_buf = NULL;
+ cmh_dma_free_sg(rctx->sgm);
+ rctx->sgm = NULL;
+ cmh_hmac_free_chunks(rctx, tctx);
+ cmh_complete(&req->base, error);
+}
+
+static int cmh_hmac_final(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_hmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_hmac_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_hmac_alg_info *info = rctx->info;
+ struct vcq_cmd cmds[CMH_HMAC_MAX_PAYLOAD];
+ struct cmh_sg_map *sgm = NULL;
+ dma_addr_t digest_dma = DMA_MAPPING_ERROR, key_dma = DMA_MAPPING_ERROR;
+ u8 *digest_buf;
+ u64 key_ref;
+ u32 keylen;
+ struct core_dispatch d;
+ s32 target_mbx;
+ u32 core_id;
+ u32 idx;
+ int ret;
+ gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ if (rctx->error) {
+ ret = rctx->error;
+ goto out_free;
+ }
+
+ if (tctx->key.mode == CMH_KEY_NONE) {
+ ret = -ENOKEY;
+ goto out_free;
+ }
+
+ if (rctx->num_chunks > 0) {
+ sgm = cmh_hmac_build_sg(rctx, gfp);
+ if (!sgm) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ }
+
+ digest_buf = kzalloc(info->digest_size, gfp);
+ if (!digest_buf) {
+ ret = -ENOMEM;
+ goto out_free_sg;
+ }
+ digest_dma = cmh_dma_map_single(digest_buf, info->digest_size,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(digest_dma)) {
+ ret = -ENOMEM;
+ goto out_free_digest;
+ }
+
+ /* Resolve key reference */
+ idx = 0;
+
+ /*
+ * Raw key: pack SYS_CMD_WRITE(SYS_REF_TEMP) into the
+ * same VCQ so the key write + HMAC are atomic.
+ */
+ key_dma = tctx->key.raw.dma;
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, (u64)key_dma,
+ SYS_REF_NONE, tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ keylen = tctx->key.raw.len;
+ d = cmh_core_select_instance(CMH_CORE_HC);
+
+ target_mbx = d.mbx_idx;
+
+ core_id = d.core_id;
+
+ vcq_add_hc_hmac(&cmds[idx++], core_id, key_ref, keylen, info->hc_algo);
+
+ if (sgm)
+ vcq_add_hc_gather(&cmds[idx++], core_id, (u64)sgm->items_dma,
+ HC_CMD_UPDATE);
+
+ vcq_add_hc_final(&cmds[idx++], core_id, (u64)digest_dma, info->digest_size);
+ vcq_add_flush(&cmds[idx++], core_id);
+
+ rctx->digest_buf = digest_buf;
+ rctx->digest_dma = digest_dma;
+ rctx->sgm = sgm;
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_HMAC_MAX_PACKED,
+ target_mbx,
+ cmh_hmac_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_cleanup_all;
+
+ return -EINPROGRESS;
+
+out_cleanup_all:
+ cmh_dma_unmap_single(digest_dma, info->digest_size,
+ DMA_FROM_DEVICE);
+out_free_digest:
+ kfree(digest_buf);
+
+out_free_sg:
+ cmh_dma_free_sg(sgm);
+
+out_free:
+ cmh_hmac_free_chunks(rctx, tctx);
+ return ret;
+}
+
+static int cmh_hmac_finup(struct ahash_request *req)
+{
+ int ret;
+
+ ret = cmh_hmac_update(req);
+ if (ret)
+ return ret;
+
+ return cmh_hmac_final(req);
+}
+
+static int cmh_hmac_digest(struct ahash_request *req)
+{
+ int ret;
+
+ ret = cmh_hmac_init(req);
+ if (ret)
+ return ret;
+
+ return cmh_hmac_finup(req);
+}
+
+/*
+ * ahash .export()/.import(): serialize/deserialize the software
+ * accumulation buffer. No HW state is involved.
+ */
+
+static int cmh_hmac_export(struct ahash_request *req, void *out)
+{
+ struct cmh_hmac_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_hmac_export_state *state = out;
+ struct cmh_hmac_chunk *chunk;
+ u32 offset = 0;
+
+ if (rctx->total_len > CMH_HMAC_EXPORT_MAX)
+ return -ENOSPC;
+
+ state->total_len = rctx->total_len;
+ list_for_each_entry(chunk, &rctx->chunks, list) {
+ memcpy(state->data + offset, chunk->data, chunk->len);
+ offset += chunk->len;
+ }
+ return 0;
+}
+
+static int cmh_hmac_import(struct ahash_request *req, const void *in)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_hmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_hmac_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_hmac_export_state *state = in;
+ struct cmh_hmac_chunk *chunk;
+
+ /*
+ * Do NOT call free_chunks() here: the crypto API does not
+ * guarantee the request context is in a valid state before
+ * import(), so the list pointers may be stale or invalid.
+ * Re-initialize from scratch instead. Any pre-existing chunks
+ * are tracked on tctx->all_chunks and freed in cra_exit.
+ */
+ rctx->info = cmh_hmac_get_info(tfm);
+ rctx->error = 0;
+ INIT_LIST_HEAD(&rctx->chunks);
+ rctx->num_chunks = 0;
+ rctx->total_len = 0;
+
+ if (state->total_len > CMH_HMAC_EXPORT_MAX)
+ return -EINVAL;
+
+ if (state->total_len) {
+ chunk = kmalloc(sizeof(*chunk) + state->total_len, GFP_KERNEL);
+ if (!chunk)
+ return -ENOMEM;
+ chunk->len = state->total_len;
+ memcpy(chunk->data, state->data, state->total_len);
+ list_add_tail(&chunk->list, &rctx->chunks);
+ spin_lock_bh(&tctx->chunk_lock);
+ list_add_tail(&chunk->tfm_node, &tctx->all_chunks);
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->num_chunks = 1;
+ rctx->total_len = state->total_len;
+ }
+ return 0;
+}
+
+/* Transform init/exit (cra_init/cra_exit) */
+
+static int cmh_hmac_cra_init(struct crypto_tfm *tfm)
+{
+ struct cmh_hmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
+
+ memset(tctx, 0, sizeof(*tctx));
+ tctx->key.mode = CMH_KEY_NONE;
+ spin_lock_init(&tctx->chunk_lock);
+ INIT_LIST_HEAD(&tctx->all_chunks);
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct cmh_hmac_reqctx));
+ return 0;
+}
+
+static void cmh_hmac_cra_exit(struct crypto_tfm *tfm)
+{
+ struct cmh_hmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
+ struct cmh_hmac_chunk *chunk, *tmp;
+
+ /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(chunk, tmp, &tctx->all_chunks, tfm_node) {
+ list_del(&chunk->tfm_node);
+ kfree_sensitive(chunk);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+
+ cmh_key_destroy(&tctx->key);
+}
+
+/* Registration */
+
+static struct cmh_hmac_alg_drv cmh_hmac_drvs[CMH_HMAC_ALG_COUNT];
+
+/**
+ * cmh_hmac_register() - Register HMAC-SHA hash algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_hmac_register(void)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < CMH_HMAC_ALG_COUNT; i++) {
+ const struct cmh_hmac_alg_info *info = &cmh_hmac_algs_info[i];
+ struct cmh_hmac_alg_drv *drv = &cmh_hmac_drvs[i];
+ struct ahash_alg *alg = &drv->alg;
+
+ drv->info = info;
+
+ alg->init = cmh_hmac_init;
+ alg->update = cmh_hmac_update;
+ alg->final = cmh_hmac_final;
+ alg->finup = cmh_hmac_finup;
+ alg->digest = cmh_hmac_digest;
+ alg->export = cmh_hmac_export;
+ alg->import = cmh_hmac_import;
+ alg->setkey = cmh_hmac_setkey;
+
+ alg->halg.digestsize = info->digest_size;
+ alg->halg.statesize = CMH_HMAC_STATE_SIZE;
+
+ strscpy(alg->halg.base.cra_name, info->alg_name,
+ CRYPTO_MAX_ALG_NAME);
+ strscpy(alg->halg.base.cra_driver_name, info->drv_name,
+ CRYPTO_MAX_ALG_NAME);
+ alg->halg.base.cra_priority = 300;
+ alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NO_FALLBACK |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_REQ_VIRT;
+ alg->halg.base.cra_blocksize = info->block_size;
+ alg->halg.base.cra_ctxsize = sizeof(struct cmh_hmac_tfm_ctx);
+ alg->halg.base.cra_init = cmh_hmac_cra_init;
+ alg->halg.base.cra_exit = cmh_hmac_cra_exit;
+ alg->halg.base.cra_module = THIS_MODULE;
+
+ ret = crypto_register_ahash(alg);
+ if (ret) {
+ dev_err(cmh_dev(), "hmac: failed to register %s (rc=%d)\n",
+ info->drv_name, ret);
+ while (i--)
+ crypto_unregister_ahash(&cmh_hmac_drvs[i].alg);
+ return ret;
+ }
+
+ dev_dbg(cmh_dev(), "hmac: registered %s (priority 300)\n",
+ info->drv_name);
+ }
+
+ dev_info(cmh_dev(), "hmac: %zu algorithm(s) registered\n",
+ CMH_HMAC_ALG_COUNT);
+ return 0;
+}
+
+/**
+ * cmh_hmac_unregister() - Unregister HMAC-SHA hash algorithms from the crypto framework
+ */
+void cmh_hmac_unregister(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < CMH_HMAC_ALG_COUNT; i++) {
+ crypto_unregister_ahash(&cmh_hmac_drvs[i].alg);
+ dev_dbg(cmh_dev(), "hmac: unregistered %s\n",
+ cmh_hmac_algs_info[i].drv_name);
+ }
+
+ dev_info(cmh_dev(), "hmac: cleaned up\n");
+}
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index dea019346567..618c981e3997 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -30,6 +30,7 @@
#include "cmh_txn.h"
#include "cmh_rh.h"
#include "cmh_hash.h"
+#include "cmh_hmac.h"
#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
@@ -203,6 +204,11 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_hash_register;
+ /* Register HMAC hash algorithms */
+ ret = cmh_hmac_register();
+ if (ret)
+ goto err_hmac_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -215,6 +221,8 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_hmac_unregister();
+err_hmac_register:
cmh_hash_unregister();
err_hash_register:
cmh_rh_cleanup(cfg);
@@ -243,6 +251,7 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_hmac_unregister();
cmh_hash_unregister();
cmh_rh_cleanup(cfg);
cmh_tm_cleanup();
diff --git a/drivers/crypto/cmh/include/cmh_hmac.h b/drivers/crypto/cmh/include/cmh_hmac.h
new file mode 100644
index 000000000000..fb1a11fb76eb
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_hmac.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API HMAC Driver
+ *
+ * Registers HMAC ahash algorithms (HMAC-SHA-2, HMAC-SHA-3) with the
+ * Linux crypto subsystem using HC_CMD_HMAC.
+ */
+
+#ifndef CMH_HMAC_H
+#define CMH_HMAC_H
+
+int cmh_hmac_register(void);
+void cmh_hmac_unregister(void);
+
+#endif /* CMH_HMAC_H */
--
2.43.7
^ permalink raw reply related
* [PATCH v2 03/19] crypto: cmh - add key provisioning and management
From: Saravanakrishnan Krishnamoorthy @ 2026-07-09 20:30 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260709203037.1884436-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Add the CMH key management subsystem:
- Key provisioning: create, import, derive, and destroy hardware keys
stored in the CMH datastore
- System object management: allocate and free CMH system objects
- Management ioctl interface (/dev/cmh_mgmt): ioctl commands
covering key lifecycle, KIC key derivation, PKE operations (RSA,
ECDSA, ECDH, EdDSA), PQC operations (ML-KEM, ML-DSA, SLH-DSA),
SM2, EAC, and DRBG reseeding
- SM2 ioctl handlers: SM2 encrypt, decrypt, sign, and key exchange
-- operations that require multi-step protocol flows not
expressible through the standard crypto API sig interface
- UAPI header: cmh_mgmt_ioctl.h (ioctl definitions and structures)
The misc device requires CAP_SYS_ADMIN for open().
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
Documentation/ABI/testing/cmh-mgmt | 136 ++
drivers/crypto/cmh/Kconfig | 19 +
drivers/crypto/cmh/Makefile | 11 +-
drivers/crypto/cmh/cmh_key.c | 164 +++
drivers/crypto/cmh/cmh_main.c | 9 +
drivers/crypto/cmh/cmh_mgmt.c | 1607 ++++++++++++++++++++++
drivers/crypto/cmh/cmh_mgmt_pke.c | 1100 +++++++++++++++
drivers/crypto/cmh/cmh_mgmt_pqc.c | 1279 +++++++++++++++++
drivers/crypto/cmh/cmh_pke_sm2.c | 827 +++++++++++
drivers/crypto/cmh/cmh_sys.c | 376 +++++
drivers/crypto/cmh/include/cmh_key.h | 82 ++
drivers/crypto/cmh/include/cmh_mgmt.h | 62 +
drivers/crypto/cmh/include/cmh_pke.h | 245 ++++
drivers/crypto/cmh/include/cmh_pke_sm2.h | 30 +
drivers/crypto/cmh/include/cmh_pqc.h | 25 +
drivers/crypto/cmh/include/cmh_sys.h | 111 ++
include/uapi/linux/cmh_mgmt_ioctl.h | 895 ++++++++++++
17 files changed, 6977 insertions(+), 1 deletion(-)
create mode 100644 Documentation/ABI/testing/cmh-mgmt
create mode 100644 drivers/crypto/cmh/cmh_key.c
create mode 100644 drivers/crypto/cmh/cmh_mgmt.c
create mode 100644 drivers/crypto/cmh/cmh_mgmt_pke.c
create mode 100644 drivers/crypto/cmh/cmh_mgmt_pqc.c
create mode 100644 drivers/crypto/cmh/cmh_pke_sm2.c
create mode 100644 drivers/crypto/cmh/cmh_sys.c
create mode 100644 drivers/crypto/cmh/include/cmh_key.h
create mode 100644 drivers/crypto/cmh/include/cmh_mgmt.h
create mode 100644 drivers/crypto/cmh/include/cmh_pke.h
create mode 100644 drivers/crypto/cmh/include/cmh_pke_sm2.h
create mode 100644 drivers/crypto/cmh/include/cmh_pqc.h
create mode 100644 drivers/crypto/cmh/include/cmh_sys.h
create mode 100644 include/uapi/linux/cmh_mgmt_ioctl.h
diff --git a/Documentation/ABI/testing/cmh-mgmt b/Documentation/ABI/testing/cmh-mgmt
new file mode 100644
index 000000000000..2c6fce7ae009
--- /dev/null
+++ b/Documentation/ABI/testing/cmh-mgmt
@@ -0,0 +1,136 @@
+What: /dev/cmh_mgmt
+Date: June 2026
+KernelVersion: 7.1
+Contact: linux-crypto@vger.kernel.org
+Description:
+ Character device (misc) providing a management and
+ key-operations ioctl interface to the CRI CryptoManager Hub
+ hardware crypto accelerator. Used for operations that
+ cannot be represented through the standard in-kernel
+ crypto API: datastore key CRUD, key derivation,
+ asymmetric crypto (EdDSA, SM2), and post-quantum crypto
+ (ML-KEM, ML-DSA, SLH-DSA).
+
+ The ioctl magic is 'J'. All struct arguments are
+ versioned via a leading __u32 version field set to
+ CMH_MGMT_V1 (1).
+
+ Ioctl commands are grouped by function:
+
+ **Key Management (0x01-0x0E):**
+
+ - CMH_IOCTL_KEY_NEW (0x01): Allocate a new datastore slot.
+ Accepts ds_type (CMH_DS_* constant), key length, flags,
+ and caller ID. Returns a 64-bit key reference.
+
+ - CMH_IOCTL_KEY_WRITE (0x02): Write key material into
+ a previously allocated datastore slot. Supports
+ plaintext or wrapped key import via a wrapping key ref.
+
+ - CMH_IOCTL_KEY_READ (0x03): Read key material from
+ a datastore slot, optionally wrapped. Returns data
+ plus a 16-byte SYS header for wrapped reads.
+
+ - CMH_IOCTL_KEY_FIND (0x04): Look up a key reference
+ by caller ID (CID).
+
+ - CMH_IOCTL_KEY_GRANT (0x05): Grant access to a key.
+
+ - CMH_IOCTL_KEY_DELETE (0x06): Delete a datastore slot.
+
+ - CMH_IOCTL_DS_EXPORT (0x07): Export the entire datastore
+ as an encrypted blob.
+
+ - CMH_IOCTL_DS_IMPORT (0x08): Import a previously exported
+ datastore blob.
+
+ - CMH_IOCTL_KEY_NEW_RANDOM (0x0B): Allocate a datastore
+ slot and fill it with hardware-generated random data.
+
+ - CMH_IOCTL_KEY_LIST (0x0E): List active datastore entries,
+ returning CIDs, types, lengths, and flags.
+
+ **Key Derivation -- KIC (0x09-0x0D):**
+
+ - CMH_IOCTL_KIC_HKDF1 (0x09): HKDF-Extract step.
+ - CMH_IOCTL_KIC_HKDF2 (0x0A): HKDF-Expand step.
+ - CMH_IOCTL_KIC_AES_CMAC_KDF (0x0C): AES-CMAC KDF.
+ - CMH_IOCTL_KIC_DKEK_DERIVE (0x0D): DKEK derivation.
+
+ **EAC -- Error and Alarm (0x0F):**
+
+ - CMH_IOCTL_EAC_READ (0x0F): Read and clear hardware
+ error, alarm, and safety notification registers.
+
+ **PKE -- Public Key Engine (0x10-0x1C):**
+
+ - CMH_IOCTL_PKE_RSA_ENC (0x10): RSA public-key encrypt.
+ - CMH_IOCTL_PKE_RSA_DEC (0x11): RSA private-key decrypt.
+ - CMH_IOCTL_PKE_RSA_CRT_DEC (0x12): RSA-CRT decrypt.
+ - CMH_IOCTL_PKE_RSA_KEYGEN (0x13): RSA key pair generation.
+ - CMH_IOCTL_PKE_ECDSA_SIGN (0x14): ECDSA sign.
+ - CMH_IOCTL_PKE_ECDH (0x16): ECDH shared secret.
+ - CMH_IOCTL_PKE_ECDH_KEYGEN (0x17): ECDH key pair generation.
+ - CMH_IOCTL_PKE_EDDSA_SIGN (0x18): EdDSA sign (Ed25519/Ed448).
+ - CMH_IOCTL_PKE_EDDSA_VERIFY (0x19): EdDSA verify.
+ - CMH_IOCTL_PKE_EC_KEYGEN (0x1A): EC key pair generation.
+ - CMH_IOCTL_PKE_EC_PUBGEN (0x1B): EC public key derivation.
+ - CMH_IOCTL_PKE_EDDSA_KEYGEN_SCA (0x1C): EdDSA SCA-protected
+ key generation.
+
+ **PQC -- Post-Quantum Crypto (0x20-0x2D):**
+
+ - CMH_IOCTL_ML_KEM_KEYGEN (0x20): ML-KEM key pair generation
+ (modes 512/768/1024).
+ - CMH_IOCTL_ML_KEM_ENC (0x21): ML-KEM encapsulation.
+ - CMH_IOCTL_ML_KEM_DEC (0x22): ML-KEM decapsulation.
+ - CMH_IOCTL_ML_DSA_KEYGEN (0x23): ML-DSA key pair generation
+ (modes 44/65/87).
+ - CMH_IOCTL_ML_DSA_SIGN (0x24): ML-DSA sign.
+ - CMH_IOCTL_SLHDSA_KEYGEN (0x28): SLH-DSA key pair generation
+ (12 parameter sets).
+ - CMH_IOCTL_SLHDSA_SIGN (0x29): SLH-DSA sign.
+ - CMH_IOCTL_SLHDSA_SIGN_PREHASH (0x2D): SLH-DSA prehash sign.
+
+ **SM2 Operations (0x30-0x37):**
+
+ - CMH_IOCTL_SM2_ECDH_KEYGEN (0x30): SM2 ephemeral key gen.
+ - CMH_IOCTL_SM2_ECDH (0x31): SM2 key exchange.
+ - CMH_IOCTL_SM2_DEC_POINT (0x32): SM2 decrypt (point step).
+ - CMH_IOCTL_SM2_ENC_POINT (0x33): SM2 encrypt (point step).
+ - CMH_IOCTL_SM2_ID_DIGEST (0x34): SM2 ID digest (ZA).
+ - CMH_IOCTL_SM2_ECDH_HASH (0x35): SM2 key exchange hash step.
+ - CMH_IOCTL_SM2_DEC_HASH (0x36): SM2 decrypt (hash step).
+ - CMH_IOCTL_SM2_ENC_HASH (0x37): SM2 encrypt (hash step).
+
+ The SM2 encrypt/decrypt hash-step ioctls accept payloads
+ of at most 32 bytes. The underlying hardware KDF emits a
+ single 32-byte SM3 block, so longer messages cannot be
+ processed in a single command and are rejected with
+ -EINVAL.
+
+ **DRBG Management (0x40):**
+
+ - CMH_IOCTL_DRBG_CONFIG (0x40): Configure the hardware
+ DRBG entropy ratio and security strength. Normally
+ called once at system start-up before hwrng reads.
+
+ All structs contain ``__reserved`` fields that must be
+ zero; the driver returns ``-EINVAL`` if any reserved field
+ is non-zero. This ensures forward compatibility when
+ reserved fields gain meaning in future versions.
+
+ All ioctls return 0 on success or a negative errno on
+ failure. Common errors:
+
+ - EINVAL: Invalid version, parameter, key type, or
+ non-zero reserved field.
+ - ENOENT: Key reference not found in datastore.
+ - ENOMEM: DMA allocation failure.
+ - EBUSY: Hardware mailbox full.
+ - ETIMEDOUT: VCQ operation timed out.
+ - EFAULT: Bad user-space pointer.
+
+ The ioctl UAPI header is <linux/cmh_mgmt_ioctl.h>.
+ All structures, constants, and type definitions are
+ documented in that header file.
diff --git a/drivers/crypto/cmh/Kconfig b/drivers/crypto/cmh/Kconfig
index fa5adeca2512..c607014f8fbc 100644
--- a/drivers/crypto/cmh/Kconfig
+++ b/drivers/crypto/cmh/Kconfig
@@ -44,3 +44,22 @@ config CRYPTO_DEV_CMH_DEBUG
Useful for bringup, validation, and performance analysis.
Not recommended for production.
+
+config CRYPTO_DEV_CMH_MGMT
+ bool "CMH management ioctl device (/dev/cmh_mgmt)"
+ depends on CRYPTO_DEV_CMH
+ default n
+ help
+ Expose /dev/cmh_mgmt, a misc device providing ioctl commands
+ for operations that have no kernel crypto API binding: hardware
+ key lifecycle (create, import, derive, destroy), KIC key
+ derivation, PQC keygen/encaps/decaps (ML-KEM, ML-DSA, SLH-DSA),
+ EdDSA sign/verify, SM2 key exchange, and DRBG
+ configuration.
+
+ The device requires CAP_SYS_ADMIN. Disabling this option
+ removes the ioctl interface but all kernel crypto API
+ algorithms (consumed by in-kernel users and validated by the
+ crypto test manager) remain fully functional.
+
+ If unsure, say N.
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index 0a4591c9fd86..1492e575598c 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -12,7 +12,16 @@ cmh-y := \
cmh_txn.o \
cmh_rh.o \
cmh_dma.o \
- cmh_sysfs.o
+ cmh_sysfs.o \
+ cmh_key.o \
+ cmh_sys.o
+
+# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
+cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
+ cmh_mgmt.o \
+ cmh_mgmt_pke.o \
+ cmh_mgmt_pqc.o \
+ cmh_pke_sm2.o
ccflags-y += -I$(src)/include
diff --git a/drivers/crypto/cmh/cmh_key.c b/drivers/crypto/cmh/cmh_key.c
new file mode 100644
index 000000000000..fde8be50b25c
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_key.c
@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Dual Key Path Implementation
+ *
+ * Two key provisioning paths are supported:
+ *
+ * Raw key: key bytes -> stored in tfm context ->
+ * SYS_CMD_WRITE(SYS_REF_TEMP) packed into every crypto VCQ.
+ * The raw key buffer is DMA-mapped once at setkey time and remains
+ * mapped for the lifetime of the transform (unmapped in destroy).
+ *
+ * Raw key DMA lifetime rationale
+ * ------------------------------
+ * Raw keys are DMA-mapped at setkey time and the mapping persists
+ * until the transform is destroyed (cmh_key_destroy). This is a
+ * deliberate design choice, consistent with upstream HW crypto
+ * drivers (CAAM, ccree, CCP) that also map keys at setkey for
+ * transform-lifetime reuse:
+ *
+ * - The Linux crypto framework expects setkey to prepare the
+ * transform for repeated encrypt/decrypt calls. Remapping the
+ * same key on every request would add DMA API overhead per crypto
+ * operation with no security benefit.
+ * - On destroy, kfree_sensitive() scrubs the key buffer and the
+ * DMA mapping is released. For key-by-ID (persistent), the
+ * per-MBX ref cache is zeroed with memzero_explicit().
+ * - No key material is ever logged; dev_dbg() messages only show
+ * CIDs (content identifiers), not key bytes.
+ *
+ * Hardware-required behaviors (not driver policy)
+ * ------------------------------------------------
+ * - SYS_REF_TEMP lifetime: the eSW firmware reclaims temporary
+ * datastore objects when the mailbox slot is reused. This is a
+ * hardware constraint; the driver packs SYS_CMD_WRITE into every
+ * VCQ to re-provision the raw key for each operation.
+ * - Mailbox flush (SYS_CMD_FLUSH): reclaims temp-stack space on the
+ * target MBX. Required by HW to prevent temp-stack exhaustion
+ * across multi-VCQ operations.
+ */
+
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "cmh_key.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_sys_abi.h"
+#include <uapi/linux/cmh_mgmt_ioctl.h>
+
+/**
+ * cmh_ds_type_to_core_id() - Map a datastore type to a logical core ID
+ * @ds_type: Datastore type constant (e.g. CMH_DS_AES_KEY, CMH_DS_SM4_KEY)
+ *
+ * Returns the algorithm-family identity (e.g. CORE_ID_AES = 0x03), NOT the
+ * VCQ dispatch core_id. With multi-instance, a second AES engine dispatches
+ * at CORE_ID_AES2 (0x06) but keys are still tagged with CORE_ID_AES (0x03)
+ * -- the eSW validates against the logical identity, not the dispatch ID.
+ *
+ * Return: Logical core ID on success, CORE_ID_NUM for unknown @ds_type.
+ */
+u32 cmh_ds_type_to_core_id(u32 ds_type)
+{
+ switch (ds_type) {
+ case CMH_DS_AES_KEY:
+ case CMH_DS_AES_XTS_KEY:
+ return CORE_ID_AES;
+ case CMH_DS_SM4_KEY:
+ return CORE_ID_SM4;
+ case CMH_DS_HMAC_KEY:
+ case CMH_DS_KMAC_KEY:
+ return CORE_ID_HC;
+ case CMH_DS_CHACHA20_KEY:
+ return CORE_ID_CCP;
+ case CMH_DS_RSA_PRIV_KEY:
+ case CMH_DS_RSA_PUB_KEY:
+ case CMH_DS_RSA_CRT_KEY:
+ case CMH_DS_ECDSA_PRIV_KEY:
+ case CMH_DS_ECDSA_PUB_KEY:
+ case CMH_DS_ECDH_PRIV_KEY:
+ case CMH_DS_EDDSA_PRIV_KEY:
+ case CMH_DS_SHARED_SECRET:
+ case CMH_DS_SM2_PRIV_KEY:
+ return CORE_ID_PKE;
+ case CMH_DS_ML_KEM_DK:
+ case CMH_DS_ML_DSA_SK:
+ return CORE_ID_QSE;
+ case CMH_DS_SLHDSA_SK:
+ return CORE_ID_HCQ;
+ default:
+ return CORE_ID_NUM;
+ }
+}
+
+/**
+ * cmh_key_setkey_raw() - Store a raw key in the key context
+ * @ctx: Key context to populate
+ * @key: Pointer to the raw key bytes
+ * @keylen: Length of @key in bytes
+ * @core_id: Logical core ID for SYS_TYPE tagging
+ *
+ * Duplicates the raw key, DMA-maps the copy for the lifetime of the
+ * transform, and stores the mapping in @ctx. Any previously held key
+ * is destroyed first.
+ *
+ * The DMA mapping persists until cmh_key_destroy() is called (typically
+ * from the algorithm .exit_tfm callback). This avoids per-request DMA
+ * mapping overhead and matches the setkey-to-destroy lifetime model used
+ * by other upstream HW crypto drivers (CAAM, ccree, CCP). The key
+ * buffer is freed via kfree_sensitive() on destroy.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_key_setkey_raw(struct cmh_key_ctx *ctx, const u8 *key,
+ u32 keylen, u32 core_id)
+{
+ dma_addr_t dma;
+ u8 *copy;
+
+ if (!keylen || !key)
+ return -EINVAL;
+
+ copy = kmemdup(key, keylen, GFP_KERNEL);
+ if (!copy)
+ return -ENOMEM;
+
+ /* Pre-map for the lifetime of the transform */
+ dma = cmh_dma_map_single(copy, keylen, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(dma)) {
+ kfree_sensitive(copy);
+ return -ENOMEM;
+ }
+
+ /* Clean up any previous key */
+ cmh_key_destroy(ctx);
+
+ ctx->mode = CMH_KEY_RAW;
+ ctx->raw.data = copy;
+ ctx->raw.len = keylen;
+ ctx->raw.dma = dma;
+ ctx->raw.sys_type = SYS_TYPE_SET(SYS_TYPE_FLAG_PT, core_id);
+
+ return 0;
+}
+
+/**
+ * cmh_key_destroy() - Destroy and zero-fill a key context
+ * @ctx: Key context to destroy
+ *
+ * For raw keys, unmaps the DMA buffer and securely frees the key material.
+ * Resets the key mode to CMH_KEY_NONE.
+ */
+void cmh_key_destroy(struct cmh_key_ctx *ctx)
+{
+ if (ctx->mode == CMH_KEY_RAW && ctx->raw.data) {
+ cmh_dma_unmap_single(ctx->raw.dma, ctx->raw.len,
+ DMA_TO_DEVICE);
+ kfree_sensitive(ctx->raw.data);
+ memzero_explicit(&ctx->raw, sizeof(ctx->raw));
+ }
+ ctx->mode = CMH_KEY_NONE;
+}
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index de065a425a2c..7673ed3e5861 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -29,6 +29,7 @@
#include "cmh_mqi.h"
#include "cmh_txn.h"
#include "cmh_rh.h"
+#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
#include "cmh_sysfs.h"
@@ -196,12 +197,19 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_rh_init;
+ /* Register key management device (/dev/cmh_mgmt) */
+ ret = cmh_mgmt_register();
+ if (ret)
+ goto err_mgmt_register;
+
g_cmh_dev = dev;
platform_set_drvdata(pdev, dev);
dev_info(cmh_dev(), "initialized successfully\n");
return 0;
+err_mgmt_register:
+ cmh_rh_cleanup(cfg);
err_rh_init:
cmh_tm_cleanup();
err_tm_init:
@@ -226,6 +234,7 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
+ cmh_mgmt_unregister();
cmh_rh_cleanup(cfg);
cmh_tm_cleanup();
cmh_mqi_cleanup(cfg);
diff --git a/drivers/crypto/cmh/cmh_mgmt.c b/drivers/crypto/cmh/cmh_mgmt.c
new file mode 100644
index 000000000000..d228213f7850
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_mgmt.c
@@ -0,0 +1,1607 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Key Management misc_device (/dev/cmh_mgmt)
+ *
+ * Provides ioctl interface for key provisioning (NEW, NEW_RANDOM, WRITE, READ,
+ * FIND, GRANT, DELETE) and datastore lifecycle (EXPORT, IMPORT).
+ *
+ * Each ioctl handler: copy_from_user -> validate -> DMA alloc ->
+ * build VCQ -> cmh_tm_submit_sync -> copy_to_user -> DMA free.
+ *
+ * Access requires CAP_SYS_ADMIN (checked in open()). The device node
+ * is mode 0660; DAC further limits access to owner/group.
+ * CMH eSW enforces per-MBX access control on top of this.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/miscdevice.h>
+#include <linux/uaccess.h>
+#include <linux/slab.h>
+#include <linux/fs.h>
+#include <linux/capability.h>
+#include <linux/overflow.h>
+
+#include "cmh_mgmt.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_key.h"
+#include "cmh_dma.h"
+#include "cmh_config.h"
+#include "cmh_sys_abi.h"
+#include "cmh_pke.h"
+#include "cmh_pke_sm2.h"
+#include "cmh_qse_abi.h"
+#include "cmh_hcq_abi.h"
+#include <uapi/linux/cmh_mgmt_ioctl.h>
+
+#include <crypto/utils.h>
+
+/*
+ * Pin all mgmt ioctls to a single management mailbox (MBX 0).
+ *
+ * This is a deliberate, structural choice -- not a performance default.
+ * The /dev/cmh_mgmt path is *stateful* with respect to the eSW datastore,
+ * and that state is per-mailbox, so every step of a key's lifecycle must
+ * land on the same mailbox:
+ *
+ * 1. Datastore access control is per-mailbox AND opaque to the driver.
+ * SYS_CMD_NEW grants the creating mailbox a (1 << mbx_id) access mask
+ * (read/write/execute). Crucially, the returned 64-bit ref encodes a
+ * randomised offset -- NOT the owning mailbox -- so given only a ref
+ * (as KEY_GRANT/READ/DELETE/DS_EXPORT receive), the driver cannot
+ * recover which mailbox owns the object. A fixed management mailbox
+ * is therefore the only way to guarantee that NEW, WRITE, GRANT, READ
+ * and the subsequent hardware-held-key compute ops all share the
+ * mailbox that holds the access rights, without exposing mailbox
+ * identity in the UABI. (User space may still widen access to other
+ * mailboxes explicitly via KEY_GRANT.)
+ *
+ * 2. The eSW SYS_REF_TEMP scratch store is per-mailbox and persists
+ * across ioctl calls. A derivation that writes SYS_REF_TEMP (e.g. a
+ * KIC_* derive) must be consumed by a later ioctl on the *same*
+ * mailbox (e.g. DS_EXPORT with wrap_key=SYS_REF_TEMP).
+ *
+ * Device-tree per-core ``cri,mbx`` affinity applies to the *stateless*
+ * registered crypto API path (cmh_core_select_instance()), which carries
+ * no datastore state across calls and is free to balance across mailboxes.
+ *
+ * Note: MBX 0 is NOT reserved exclusively for mgmt -- registered crypto
+ * operations may also land here via TM round-robin (target_mbx = -1).
+ * This is safe because those ops do not allocate from the temp store.
+ */
+
+/* VCQ layout: header + command + flush = 3 entries */
+#define MGMT_VCQ_CMDS 3
+
+/*
+ * Tracks whether any operation has left residual state in the device's
+ * per-mailbox temporary key store since the last flush. The device
+ * reclaims temp storage only on a full mailbox flush (MBX_COMMAND_FLUSH),
+ * which also terminates any executing command queue with -EPIPE.
+ *
+ * To avoid killing concurrent in-flight operations, the flush in
+ * cmh_mgmt_ioctl() is conditional: it fires only when this flag is set.
+ * Operations that allocate temp storage (currently: KIC derivations
+ * targeting SYS_REF_TEMP) set this flag on success.
+ */
+static atomic_t mgmt_temp_dirty = ATOMIC_INIT(0);
+
+/* -- KEY_NEW -------------------------- */
+
+static int cmh_mgmt_key_new(void __user *argp)
+{
+ struct cmh_ioctl_key_new req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ u64 *ref_buf;
+ dma_addr_t ref_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.len)
+ return -EINVAL;
+
+ /* DMA buffer for CMH eSW to write back the ref */
+ ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+
+ *ref_buf = 0;
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.len);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+
+ /*
+ * Unmap before CPU read: single-phase operation (no re-use of
+ * the DMA mapping), so unmap transfers ownership back to the
+ * CPU. On SWIOTLB systems the unmap copies the bounce buffer
+ * to the original allocation. This is the correct pattern for
+ * single-shot sync submits where the buffer is not re-mapped.
+ */
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf), DMA_FROM_DEVICE);
+
+ if (ret) {
+ kfree(ref_buf);
+ return ret;
+ }
+
+ req.ref = *ref_buf;
+ kfree(ref_buf);
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ dev_dbg(cmh_dev(), "mgmt: KEY_NEW cid=0x%llx len=%u -> ref=0x%llx\n",
+ req.cid, req.len, req.ref);
+ return 0;
+}
+
+/* -- KEY_WRITE ------------------------- */
+
+static int cmh_mgmt_key_write(void __user *argp)
+{
+ struct cmh_ioctl_key_write req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ void *dmabuf;
+ dma_addr_t dma_addr;
+ u32 core_id, sys_type;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.len || req.len > CMH_MGMT_MAX_DATA_LEN)
+ return -EINVAL;
+
+ core_id = cmh_ds_type_to_core_id(req.ds_type);
+ if (core_id == CORE_ID_NUM)
+ return -EINVAL;
+ sys_type = SYS_TYPE_SET(req.flags, core_id);
+
+ dmabuf = kmalloc(req.len, GFP_KERNEL);
+ if (!dmabuf)
+ return -ENOMEM;
+
+ if (copy_from_user(dmabuf, u64_to_user_ptr(req.data),
+ req.len)) {
+ kfree_sensitive(dmabuf);
+ return -EFAULT;
+ }
+
+ dma_addr = cmh_dma_map_single(dmabuf, req.len, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(dma_addr)) {
+ kfree_sensitive(dmabuf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_write(&vcq[1], req.ref, dma_addr, req.wrap_key,
+ req.len, sys_type);
+ /*
+ * PKE keys on Weierstrass curves and RSA keys must be byte-swapped
+ * when stored in the DS so they match the internal big-endian
+ * representation used by the PKE sidecar. Edwards curve keys
+ * (EdDSA) use native byte order and must NOT be swapped.
+ */
+ switch (req.ds_type) {
+ case CMH_DS_RSA_PRIV_KEY:
+ case CMH_DS_RSA_PUB_KEY:
+ case CMH_DS_RSA_CRT_KEY:
+ case CMH_DS_ECDSA_PRIV_KEY:
+ case CMH_DS_ECDSA_PUB_KEY:
+ case CMH_DS_ECDH_PRIV_KEY:
+ case CMH_DS_SHARED_SECRET:
+ case CMH_DS_SM2_PRIV_KEY:
+ vcq[1].id |= PKE_SWAP_FLAGS;
+ break;
+ default:
+ /* EdDSA, symmetric keys -- no swap */
+ break;
+ }
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(dma_addr, req.len, DMA_TO_DEVICE);
+ kfree_sensitive(dmabuf);
+
+ if (ret)
+ return ret;
+
+ dev_dbg(cmh_dev(), "mgmt: KEY_WRITE ref=0x%llx len=%u type=0x%x\n",
+ req.ref, req.len, sys_type);
+ return 0;
+}
+
+/* -- KEY_READ -------------------------- */
+
+static int cmh_mgmt_key_read(void __user *argp)
+{
+ struct cmh_ioctl_key_read req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ void *dmabuf;
+ dma_addr_t dma_addr;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (!req.len || req.len > CMH_MGMT_MAX_DATA_LEN)
+ return -EINVAL;
+
+ dmabuf = kzalloc(req.len, GFP_KERNEL);
+ if (!dmabuf)
+ return -ENOMEM;
+
+ dma_addr = cmh_dma_map_single(dmabuf, req.len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(dma_addr)) {
+ kfree(dmabuf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_read(&vcq[1], req.ref, dma_addr, req.wrap_key, req.len);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(dma_addr, req.len, DMA_FROM_DEVICE);
+
+ if (ret) {
+ kfree_sensitive(dmabuf);
+ return ret;
+ }
+
+ if (copy_to_user(u64_to_user_ptr(req.data),
+ dmabuf, req.len)) {
+ kfree_sensitive(dmabuf);
+ return -EFAULT;
+ }
+
+ req.out_len = req.len;
+ kfree_sensitive(dmabuf);
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ dev_dbg(cmh_dev(), "mgmt: KEY_READ ref=0x%llx len=%u\n",
+ req.ref, req.out_len);
+ return 0;
+}
+
+/* -- KEY_FIND -------------------------- */
+
+static int cmh_mgmt_key_find(void __user *argp)
+{
+ struct cmh_ioctl_key_find req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ struct sys_list_item *item;
+ dma_addr_t item_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+
+ item = kzalloc_obj(*item, GFP_KERNEL);
+ if (!item)
+ return -ENOMEM;
+
+ item_dma = cmh_dma_map_single(item, sizeof(*item), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(item_dma)) {
+ kfree(item);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_find(&vcq[1], req.cid, item_dma, sizeof(*item));
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(item_dma, sizeof(*item), DMA_FROM_DEVICE);
+
+ if (ret) {
+ kfree(item);
+ return ret;
+ }
+
+ req.ref = item->ref;
+ req.len = item->len;
+ req.type = item->type;
+ kfree(item);
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ dev_dbg(cmh_dev(), "mgmt: KEY_FIND cid=0x%llx -> ref=0x%llx\n",
+ req.cid, req.ref);
+ return 0;
+}
+
+/* -- KEY_LIST ------------------------- */
+
+static int cmh_mgmt_key_list(void __user *argp)
+{
+ struct cmh_ioctl_key_list req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ struct sys_list_item *item;
+ dma_addr_t item_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+
+ if (req.__reserved)
+ return -EINVAL;
+
+ item = kzalloc_obj(*item, GFP_KERNEL);
+ if (!item)
+ return -ENOMEM;
+
+ item_dma = cmh_dma_map_single(item, sizeof(*item), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(item_dma)) {
+ kfree(item);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_list(&vcq[1], req.start_ref, item_dma, sizeof(*item));
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(item_dma, sizeof(*item), DMA_FROM_DEVICE);
+
+ if (ret) {
+ kfree(item);
+ return ret;
+ }
+
+ req.ref = item->ref;
+ req.cid = item->cid;
+ req.len = item->len;
+ req.type = item->type;
+ kfree(item);
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ return 0;
+}
+
+/* -- KEY_GRANT / KEY_DELETE --------------------- */
+
+static int cmh_mgmt_key_grant(void __user *argp, bool is_delete)
+{
+ struct cmh_ioctl_key_grant req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+
+ /* DELETE = GRANT with all permissions zeroed */
+ if (is_delete) {
+ req.read = 0;
+ req.write = 0;
+ req.execute = 0;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_grant(&vcq[1], req.ref, req.read, req.write, req.execute);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+ if (ret)
+ return ret;
+
+ dev_dbg(cmh_dev(), "mgmt: KEY_%s ref=0x%llx r=0x%llx w=0x%llx x=0x%llx\n",
+ is_delete ? "DELETE" : "GRANT",
+ req.ref, req.read, req.write, req.execute);
+ return 0;
+}
+
+/* -- DS_EXPORT ------------------------- */
+
+static int cmh_mgmt_ds_export(void __user *argp)
+{
+ struct cmh_ioctl_ds_export req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ void *dmabuf;
+ dma_addr_t dma_addr;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (!req.len || req.len > CMH_MGMT_MAX_DATA_LEN)
+ return -EINVAL;
+
+ /*
+ * req.len is the exact DMA buffer size given to the eSW.
+ * Userspace must size it to at least the export blob:
+ *
+ * wrapped: sizeof(sys_wrap_hdr) + 2*AES_BLOCK_SIZE + obj_len
+ * = 16 + 32 + obj_len = 48 + obj_len
+ * plaintext: sizeof(sys_wrap_hdr) + obj_len
+ * = 16 + obj_len
+ *
+ * obj_len is known from KEY_NEW or KEY_FIND. If req.len is
+ * too small, the eSW rejects the command and we return -EIO.
+ */
+ dmabuf = kzalloc(req.len, GFP_KERNEL);
+ if (!dmabuf)
+ return -ENOMEM;
+
+ dma_addr = cmh_dma_map_single(dmabuf, req.len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(dma_addr)) {
+ kfree(dmabuf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_export(&vcq[1], req.cid, dma_addr, req.wrap_key, req.len);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(dma_addr, req.len, DMA_FROM_DEVICE);
+
+ if (ret) {
+ kfree_sensitive(dmabuf);
+ return ret;
+ }
+
+ /* Parse actual blob size from the eSW-written header */
+ {
+ struct sys_wrap_hdr *hdr = (struct sys_wrap_hdr *)dmabuf;
+ u64 actual;
+
+ if (check_add_overflow((u64)sizeof(*hdr), (u64)hdr->wrap,
+ &actual) ||
+ check_add_overflow(actual, (u64)hdr->len, &actual) ||
+ actual > req.len) {
+ kfree_sensitive(dmabuf);
+ return -EIO;
+ }
+ req.out_len = (u32)actual;
+ }
+
+ if (copy_to_user(u64_to_user_ptr(req.data),
+ dmabuf, req.out_len)) {
+ kfree_sensitive(dmabuf);
+ return -EFAULT;
+ }
+
+ kfree_sensitive(dmabuf);
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ dev_dbg(cmh_dev(), "mgmt: DS_EXPORT wrap_key=0x%llx len=%u\n",
+ req.wrap_key, req.out_len);
+ return 0;
+}
+
+/* -- DS_IMPORT ------------------------- */
+
+static int cmh_mgmt_ds_import(void __user *argp)
+{
+ struct cmh_ioctl_ds_import req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ void *dmabuf;
+ dma_addr_t dma_addr;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.len || req.len > CMH_MGMT_MAX_DATA_LEN)
+ return -EINVAL;
+
+ dmabuf = kmalloc(req.len, GFP_KERNEL);
+ if (!dmabuf)
+ return -ENOMEM;
+
+ if (copy_from_user(dmabuf, u64_to_user_ptr(req.data),
+ req.len)) {
+ kfree_sensitive(dmabuf);
+ return -EFAULT;
+ }
+
+ dma_addr = cmh_dma_map_single(dmabuf, req.len, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(dma_addr)) {
+ kfree_sensitive(dmabuf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_import(&vcq[1], dma_addr, req.wrap_key, req.len);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(dma_addr, req.len, DMA_TO_DEVICE);
+ kfree_sensitive(dmabuf);
+
+ if (ret)
+ return ret;
+
+ dev_dbg(cmh_dev(), "mgmt: DS_IMPORT wrap_key=0x%llx len=%u\n",
+ req.wrap_key, req.len);
+ return 0;
+}
+
+/* -- KIC key derivation ioctls --------
+ *
+ * All four KIC derivation handlers (HKDF1, HKDF2, AES-CMAC-KDF,
+ * DKEK-derive) share the same two-mode structure and temp-flush pattern.
+ *
+ * Temp-storage flush rationale:
+ *
+ * The device maintains a small per-mailbox temporary key store
+ * (~960 bytes, LIFO). A derivation targeting SYS_REF_TEMP allocates
+ * from this store; the allocation persists across command-queue
+ * boundaries until either (a) a subsequent command consumes it or
+ * (b) a mailbox flush resets the store.
+ *
+ * Our single-derivation ioctls produce a temp key with no consumer
+ * in the same queue -- the key is consumed by a *later* ioctl
+ * (e.g. DS_EXPORT with wrap_key=SYS_REF_TEMP). If no consumer
+ * follows, the allocation persists. Sequential temp derivations
+ * accumulate allocations until the store is exhausted (3--8 calls
+ * depending on key size), after which the device returns ENOMEM.
+ *
+ * A mailbox flush (cmh_tm_flush_mbx / MBX_COMMAND_FLUSH) resets the
+ * temp store. It does NOT destroy persistent keys, datastore
+ * objects, or DRBG state -- only the command queue and temp store.
+ *
+ * Safe for cross-ioctl temp flows (e.g. export-to-file:
+ * HKDF1->TEMP in ioctl 1, then DS_EXPORT with wrap_key=TEMP in
+ * ioctl 2): the flush only happens in derivation handlers and in
+ * the pre-PKE dispatch path, not in DS_EXPORT/DS_IMPORT, so the
+ * temp key survives until consumed.
+ *
+ * The ioctl dispatch also flushes before PKE/SM2/PQC ioctls to
+ * protect them from temp residue left by earlier derivations on the
+ * same mailbox. The per-handler flushes here remain necessary
+ * because sequential temp derivations (without an intervening
+ * PKE/SM2/PQC ioctl) would still exhaust the store.
+ */
+
+/* -- KIC_HKDF1 ------------------------- */
+
+/*
+ * Derive a key from a KIC base key via one-step HKDF.
+ *
+ * Two modes controlled by CMH_KIC_FLAG_TEMP:
+ *
+ * TEMP (flag set) -- 3-command VCQ:
+ * [0] SYS header
+ * [1] KIC_CMD_HKDF1 (dst=SYS_REF_TEMP)
+ * [2] flush
+ * Returns SYS_REF_TEMP as ref. No DS entry created.
+ *
+ * Persistent (flag clear) -- 4-command VCQ:
+ * [0] SYS header
+ * [1] SYS_CMD_NEW (allocate DS slot, CMH eSW writes ref)
+ * [2] KIC_CMD_HKDF1 (dst=SYS_REF_LAST = just-allocated slot)
+ * [3] flush
+ * Returns the new DS reference.
+ */
+#define KDF_VCQ_MAX 4
+#define KDF_MAX_KEY_LEN 64
+#define KDF_MAX_LABEL_LEN 56
+
+static int cmh_mgmt_kic_hkdf1(void __user *argp)
+{
+ struct cmh_ioctl_kic_hkdf1 req;
+ struct vcq_cmd vcq[KDF_VCQ_MAX];
+ bool temp;
+ u64 *ref_buf = NULL;
+ void *label_buf = NULL;
+ dma_addr_t ref_dma = DMA_MAPPING_ERROR, label_dma = DMA_MAPPING_ERROR;
+ unsigned int n_cmds;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.key_len || req.key_len > KDF_MAX_KEY_LEN)
+ return -EINVAL;
+ if (req.label_len > KDF_MAX_LABEL_LEN)
+ return -EINVAL;
+
+ temp = !!(req.flags & CMH_KIC_FLAG_TEMP);
+
+ /*
+ * Persistent path: need DMA buffer for CMH eSW to write the
+ * newly-allocated DS reference.
+ */
+ if (!temp) {
+ ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+ *ref_buf = 0;
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+ }
+
+ /* DMA buffer for label data (CMH eSW DMA-reads it) */
+ if (req.label_len > 0) {
+ label_buf = kzalloc(req.label_len, GFP_KERNEL);
+ if (!label_buf) {
+ ret = -ENOMEM;
+ goto out_ref;
+ }
+ if (copy_from_user(label_buf,
+ u64_to_user_ptr(req.label),
+ req.label_len)) {
+ ret = -EFAULT;
+ goto out_label;
+ }
+ label_dma = cmh_dma_map_single(label_buf, req.label_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(label_dma)) {
+ ret = -ENOMEM;
+ goto out_label;
+ }
+ }
+
+ /* Build VCQ */
+ memset(vcq, 0, sizeof(vcq));
+
+ if (temp) {
+ /* Flush MBX to reset temp stack -- see KIC section comment */
+ ret = cmh_tm_flush_mbx(MGMT_MBX);
+ if (ret)
+ goto out_unmap_label;
+
+ n_cmds = 3;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_kic_hkdf1(&vcq[1], SYS_REF_TEMP, req.base_key,
+ label_dma, req.key_len, req.label_len,
+ SYS_TYPE_SET(0, CORE_ID_AES));
+ vcq_add_sys_flush(&vcq[2]);
+ } else {
+ n_cmds = 4;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.key_len);
+ vcq_add_kic_hkdf1(&vcq[2], SYS_REF_LAST, req.base_key,
+ label_dma, req.key_len, req.label_len,
+ SYS_TYPE_SET(0, CORE_ID_AES));
+ vcq_add_sys_flush(&vcq[3]);
+ }
+
+ ret = cmh_tm_submit_sync_mbx(vcq, n_cmds, 1, MGMT_MBX);
+
+ /* Cleanup label DMA */
+ if (label_buf) {
+ cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE);
+ kfree(label_buf);
+ label_buf = NULL;
+ }
+
+ if (ret)
+ goto out_ref;
+
+ if (temp) {
+ req.ref = SYS_REF_TEMP;
+ atomic_set(&mgmt_temp_dirty, 1);
+ } else {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ req.ref = *ref_buf;
+ kfree(ref_buf);
+ ref_buf = NULL;
+ }
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ dev_dbg(cmh_dev(),
+ "mgmt: KIC_HKDF1 base=0x%llx len=%u flags=0x%x -> ref=0x%llx\n",
+ req.base_key, req.key_len, req.flags, req.ref);
+ return 0;
+
+out_unmap_label:
+ if (label_buf && !cmh_dma_map_error(label_dma) && label_dma)
+ cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE);
+out_label:
+ kfree(label_buf);
+out_ref:
+ if (ref_buf) {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ kfree(ref_buf);
+ }
+ return ret;
+}
+
+/* -- KIC_HKDF2 ------------------------- */
+
+/*
+ * Two-step HKDF key derivation. Same as HKDF1 but adds a salt key
+ * reference: Step 1: HMAC(salt, base) -> PRK; Step 2: HMAC(PRK, label) -> key.
+ */
+
+static int cmh_mgmt_kic_hkdf2(void __user *argp)
+{
+ struct cmh_ioctl_kic_hkdf2 req;
+ struct vcq_cmd vcq[KDF_VCQ_MAX];
+ bool temp;
+ u64 *ref_buf = NULL;
+ void *label_buf = NULL;
+ dma_addr_t ref_dma = DMA_MAPPING_ERROR, label_dma = DMA_MAPPING_ERROR;
+ unsigned int n_cmds;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.key_len || req.key_len > KDF_MAX_KEY_LEN)
+ return -EINVAL;
+ if (req.label_len > KDF_MAX_LABEL_LEN)
+ return -EINVAL;
+
+ temp = !!(req.flags & CMH_KIC_FLAG_TEMP);
+
+ if (!temp) {
+ ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+ *ref_buf = 0;
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+ }
+
+ if (req.label_len > 0) {
+ label_buf = kzalloc(req.label_len, GFP_KERNEL);
+ if (!label_buf) {
+ ret = -ENOMEM;
+ goto out_ref2;
+ }
+ if (copy_from_user(label_buf,
+ u64_to_user_ptr(req.label),
+ req.label_len)) {
+ ret = -EFAULT;
+ goto out_label2;
+ }
+ label_dma = cmh_dma_map_single(label_buf, req.label_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(label_dma)) {
+ ret = -ENOMEM;
+ goto out_label2;
+ }
+ }
+
+ memset(vcq, 0, sizeof(vcq));
+
+ if (temp) {
+ /* Flush MBX to reset temp stack -- see KIC section comment */
+ ret = cmh_tm_flush_mbx(MGMT_MBX);
+ if (ret)
+ goto out_unmap_label2;
+
+ n_cmds = 3;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_kic_hkdf2(&vcq[1], SYS_REF_TEMP, req.base_key,
+ req.salt_key, label_dma,
+ req.key_len, req.label_len,
+ SYS_TYPE_SET(0, CORE_ID_AES));
+ vcq_add_sys_flush(&vcq[2]);
+ } else {
+ n_cmds = 4;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.key_len);
+ vcq_add_kic_hkdf2(&vcq[2], SYS_REF_LAST, req.base_key,
+ req.salt_key, label_dma,
+ req.key_len, req.label_len,
+ SYS_TYPE_SET(0, CORE_ID_AES));
+ vcq_add_sys_flush(&vcq[3]);
+ }
+
+ ret = cmh_tm_submit_sync_mbx(vcq, n_cmds, 1, MGMT_MBX);
+
+ if (label_buf) {
+ cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE);
+ kfree(label_buf);
+ label_buf = NULL;
+ }
+
+ if (ret)
+ goto out_ref2;
+
+ if (temp) {
+ req.ref = SYS_REF_TEMP;
+ atomic_set(&mgmt_temp_dirty, 1);
+ } else {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ req.ref = *ref_buf;
+ kfree(ref_buf);
+ ref_buf = NULL;
+ }
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ dev_dbg(cmh_dev(),
+ "mgmt: KIC_HKDF2 base=0x%llx salt=0x%llx len=%u flags=0x%x -> ref=0x%llx\n",
+ req.base_key, req.salt_key, req.key_len, req.flags, req.ref);
+ return 0;
+
+out_unmap_label2:
+ if (label_buf && !cmh_dma_map_error(label_dma) && label_dma)
+ cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE);
+out_label2:
+ kfree(label_buf);
+out_ref2:
+ if (ref_buf) {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ kfree(ref_buf);
+ }
+ return ret;
+}
+
+/* -- KIC_AES_CMAC_KDF ------------------ */
+
+/*
+ * Derive a key using AES-CMAC-based KDF (NIST SP800-108 style).
+ * Base key must be 32 bytes. Output is always non-PT (the hub driver
+ * rejects SYS_TYPE_FLAG_PT).
+ *
+ * VCQ layout matches HKDF: TEMP mode uses 3 commands, persistent uses 4.
+ */
+#define CMAC_KDF_KEY_LEN 32
+
+static int cmh_mgmt_kic_aes_cmac_kdf(void __user *argp)
+{
+ struct cmh_ioctl_kic_aes_cmac_kdf req;
+ struct vcq_cmd vcq[KDF_VCQ_MAX];
+ bool temp;
+ u64 *ref_buf = NULL;
+ void *label_buf = NULL;
+ dma_addr_t ref_dma = DMA_MAPPING_ERROR, label_dma = DMA_MAPPING_ERROR;
+ unsigned int n_cmds;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.key_len != CMAC_KDF_KEY_LEN)
+ return -EINVAL;
+ if (req.label_len > KDF_MAX_LABEL_LEN)
+ return -EINVAL;
+
+ temp = !!(req.flags & CMH_KIC_FLAG_TEMP);
+
+ if (!temp) {
+ ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+ *ref_buf = 0;
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+ }
+
+ if (req.label_len > 0) {
+ label_buf = kzalloc(req.label_len, GFP_KERNEL);
+ if (!label_buf) {
+ ret = -ENOMEM;
+ goto out_ref_cmac;
+ }
+ if (copy_from_user(label_buf,
+ u64_to_user_ptr(req.label),
+ req.label_len)) {
+ ret = -EFAULT;
+ goto out_label_cmac;
+ }
+ label_dma = cmh_dma_map_single(label_buf, req.label_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(label_dma)) {
+ ret = -ENOMEM;
+ goto out_label_cmac;
+ }
+ }
+
+ memset(vcq, 0, sizeof(vcq));
+
+ if (temp) {
+ /* Flush MBX to reset temp stack -- see KIC section comment */
+ ret = cmh_tm_flush_mbx(MGMT_MBX);
+ if (ret)
+ goto out_unmap_label_cmac;
+
+ n_cmds = 3;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_kic_aes_cmac_kdf(&vcq[1], SYS_REF_TEMP,
+ req.base_key, label_dma,
+ req.key_len, req.label_len,
+ SYS_TYPE_SET(0, CORE_ID_AES));
+ vcq_add_sys_flush(&vcq[2]);
+ } else {
+ n_cmds = 4;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.key_len);
+ vcq_add_kic_aes_cmac_kdf(&vcq[2], SYS_REF_LAST,
+ req.base_key, label_dma,
+ req.key_len, req.label_len,
+ SYS_TYPE_SET(0, CORE_ID_AES));
+ vcq_add_sys_flush(&vcq[3]);
+ }
+
+ ret = cmh_tm_submit_sync_mbx(vcq, n_cmds, 1, MGMT_MBX);
+
+ if (label_buf) {
+ cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE);
+ kfree(label_buf);
+ label_buf = NULL;
+ }
+
+ if (ret)
+ goto out_ref_cmac;
+
+ if (temp) {
+ req.ref = SYS_REF_TEMP;
+ atomic_set(&mgmt_temp_dirty, 1);
+ } else {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ req.ref = *ref_buf;
+ kfree(ref_buf);
+ ref_buf = NULL;
+ }
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ dev_dbg(cmh_dev(),
+ "mgmt: KIC_AES_CMAC_KDF base=0x%llx len=%u flags=0x%x -> ref=0x%llx\n",
+ req.base_key, req.key_len, req.flags, req.ref);
+ return 0;
+
+out_unmap_label_cmac:
+ if (label_buf && !cmh_dma_map_error(label_dma) && label_dma)
+ cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE);
+out_label_cmac:
+ kfree(label_buf);
+out_ref_cmac:
+ if (ref_buf) {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ kfree(ref_buf);
+ }
+ return ret;
+}
+
+/* -- KIC_DKEK_DERIVE ------------------- */
+
+/*
+ * Derive a Key Encryption Key (KEK) from a KIC base key.
+ * Output is tagged CORE_ID_KIC (usable for further derivation only).
+ * host_id=0 means the caller's own host; non-zero requires management
+ * host privilege (eSW enforces this).
+ */
+#define DKEK_VCQ_MAX 4
+
+static int cmh_mgmt_kic_dkek_derive(void __user *argp)
+{
+ struct cmh_ioctl_kic_dkek_derive req;
+ struct vcq_cmd vcq[DKEK_VCQ_MAX];
+ bool temp;
+ u64 *ref_buf = NULL;
+ void *meta_buf = NULL;
+ dma_addr_t ref_dma = DMA_MAPPING_ERROR, meta_dma = DMA_MAPPING_ERROR;
+ unsigned int n_cmds;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.metadata_len > KIC_DKEK_MAX_METADATA)
+ return -EINVAL;
+
+ temp = !!(req.flags & CMH_KIC_FLAG_TEMP);
+
+ if (!temp) {
+ ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+ *ref_buf = 0;
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+ }
+
+ if (req.metadata_len > 0) {
+ meta_buf = kzalloc(req.metadata_len, GFP_KERNEL);
+ if (!meta_buf) {
+ ret = -ENOMEM;
+ goto out_ref_dkek;
+ }
+ if (copy_from_user(meta_buf,
+ u64_to_user_ptr(req.metadata),
+ req.metadata_len)) {
+ ret = -EFAULT;
+ goto out_meta;
+ }
+ meta_dma = cmh_dma_map_single(meta_buf, req.metadata_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(meta_dma)) {
+ ret = -ENOMEM;
+ goto out_meta;
+ }
+ }
+
+ memset(vcq, 0, sizeof(vcq));
+
+ if (temp) {
+ /* Flush MBX to reset temp stack -- see KIC section comment */
+ ret = cmh_tm_flush_mbx(MGMT_MBX);
+ if (ret)
+ goto out_unmap_meta;
+
+ n_cmds = 3;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_kic_dkek_derive(&vcq[1], SYS_REF_TEMP,
+ req.base_key, req.host_id,
+ meta_dma, req.metadata_len);
+ vcq_add_sys_flush(&vcq[2]);
+ } else {
+ n_cmds = 4;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, KIC_KEY_SIZE);
+ vcq_add_kic_dkek_derive(&vcq[2], SYS_REF_LAST,
+ req.base_key, req.host_id,
+ meta_dma, req.metadata_len);
+ vcq_add_sys_flush(&vcq[3]);
+ }
+
+ ret = cmh_tm_submit_sync_mbx(vcq, n_cmds, 1, MGMT_MBX);
+
+ if (meta_buf) {
+ cmh_dma_unmap_single(meta_dma, req.metadata_len,
+ DMA_TO_DEVICE);
+ kfree(meta_buf);
+ meta_buf = NULL;
+ }
+
+ if (ret)
+ goto out_ref_dkek;
+
+ if (temp) {
+ req.ref = SYS_REF_TEMP;
+ atomic_set(&mgmt_temp_dirty, 1);
+ } else {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ req.ref = *ref_buf;
+ kfree(ref_buf);
+ ref_buf = NULL;
+ }
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ dev_dbg(cmh_dev(),
+ "mgmt: KIC_DKEK_DERIVE base=0x%llx host=%u meta_len=%u flags=0x%x -> ref=0x%llx\n",
+ req.base_key, req.host_id, req.metadata_len, req.flags,
+ req.ref);
+ return 0;
+
+out_unmap_meta:
+ if (meta_buf && !cmh_dma_map_error(meta_dma) && meta_dma)
+ cmh_dma_unmap_single(meta_dma, req.metadata_len, DMA_TO_DEVICE);
+out_meta:
+ kfree(meta_buf);
+out_ref_dkek:
+ if (ref_buf) {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ kfree(ref_buf);
+ }
+ return ret;
+}
+
+/* -- KEY_NEW_RANDOM -- DRBG-backed key generation --- */
+
+/*
+ * Allocate a new datastore slot and fill it with DRBG-generated
+ * random key material in a single atomic VCQ submission:
+ *
+ * [0] SYS header(5)
+ * [1] SYS_CMD_NEW -- allocate DS slot (CMH eSW writes ref)
+ * [2] DRBG_CMD_DATASTORE(SYS_REF_LAST) -- fill with random data
+ * [3] DRBG flush -- release DRBG core ownership
+ * [4] SYS flush
+ *
+ * The DRBG must be configured before this ioctl is used.
+ * Reuses struct cmh_ioctl_key_new (ds_type, flags, cid, len, ref).
+ */
+#define DRBG_KEYGEN_VCQ_CMDS 5
+
+static int cmh_mgmt_key_new_random(void __user *argp)
+{
+ struct cmh_ioctl_key_new req;
+ struct vcq_cmd vcq[DRBG_KEYGEN_VCQ_CMDS];
+ u64 *ref_buf;
+ dma_addr_t ref_dma;
+ u32 core_id, sys_type;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.len)
+ return -EINVAL;
+
+ core_id = cmh_ds_type_to_core_id(req.ds_type);
+ if (core_id == CORE_ID_NUM)
+ return -EINVAL;
+ sys_type = SYS_TYPE_SET(req.flags, core_id);
+
+ ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+
+ *ref_buf = 0;
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], DRBG_KEYGEN_VCQ_CMDS);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.len);
+ vcq_add_drbg_datastore(&vcq[2], SYS_REF_LAST, req.len, sys_type);
+ vcq_add_flush(&vcq[3], CORE_ID_DRBG);
+ vcq_add_sys_flush(&vcq[4]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, DRBG_KEYGEN_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf), DMA_FROM_DEVICE);
+
+ if (ret) {
+ kfree(ref_buf);
+ return ret;
+ }
+
+ req.ref = *ref_buf;
+ kfree(ref_buf);
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ dev_dbg(cmh_dev(),
+ "mgmt: KEY_NEW_RANDOM cid=0x%llx len=%u type=0x%x -> ref=0x%llx\n",
+ req.cid, req.len, sys_type, req.ref);
+ return 0;
+}
+
+#define EAC_VCQ_CMDS 3 /* header + EAC_READ + flush */
+
+static long cmh_mgmt_eac_read(void __user *argp)
+{
+ struct cmh_ioctl_eac_read req;
+ struct eac_read_rsp *rsp;
+ struct vcq_cmd vcq[EAC_VCQ_CMDS];
+ dma_addr_t rsp_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved != 0)
+ return -EINVAL;
+ if (req.__pad != 0)
+ return -EINVAL;
+
+ rsp = kmalloc_obj(*rsp, GFP_KERNEL);
+ if (!rsp)
+ return -ENOMEM;
+
+ rsp_dma = cmh_dma_map_single(rsp, sizeof(*rsp), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rsp_dma)) {
+ kfree(rsp);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], EAC_VCQ_CMDS);
+ vcq_add_eac_read(&vcq[1], rsp_dma, sizeof(*rsp));
+ vcq_add_flush(&vcq[2], CORE_ID_EAC);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, EAC_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(rsp_dma, sizeof(*rsp), DMA_FROM_DEVICE);
+
+ if (ret) {
+ kfree(rsp);
+ return ret;
+ }
+
+ /* Copy response fields into ioctl struct */
+ req.mailbox_notification = rsp->mailbox_notification;
+ req.hw_error = rsp->hw_error;
+ req.hw_nmi = rsp->hw_nmi;
+ req.hw_panic = rsp->hw_panic;
+ req.safety_fatal = rsp->safety_fatal;
+ req.safety_notification = rsp->safety_notification;
+ req.sw_info0 = rsp->sw_info0;
+ req.sw_info1 = rsp->sw_info1;
+ memcpy(req.sram_bank_errors, rsp->sram_bank_errors,
+ sizeof(req.sram_bank_errors));
+ req.__pad = 0;
+
+ kfree(rsp);
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ return 0;
+}
+
+/* -- DRBG CONFIG (management) ------------ */
+
+#define DRBG_CONFIG_VCQ_CMDS 4 /* header + RESET + CONFIG + flush */
+
+static long cmh_mgmt_drbg_config(void __user *argp)
+{
+ struct cmh_ioctl_drbg_config req;
+ struct vcq_cmd vcq[DRBG_CONFIG_VCQ_CMDS];
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved != 0)
+ return -EINVAL;
+ if (req.entropy_ratio > 3)
+ return -EINVAL;
+ if (req.security_strength != CMH_DRBG_STRENGTH_128 &&
+ req.security_strength != CMH_DRBG_STRENGTH_256)
+ return -EINVAL;
+
+ vcq_set_header(&vcq[0], DRBG_CONFIG_VCQ_CMDS);
+ vcq_add_drbg_reset(&vcq[1]);
+ vcq_add_drbg_config(&vcq[2], req.entropy_ratio,
+ req.security_strength);
+ vcq_add_flush(&vcq[3], CORE_ID_DRBG);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, DRBG_CONFIG_VCQ_CMDS, 1, MGMT_MBX);
+ if (ret)
+ dev_warn(cmh_dev(), "mgmt: DRBG CONFIG failed (rc=%d)\n", ret);
+ else
+ dev_info(cmh_dev(), "mgmt: DRBG configured (ratio=%u strength=0x%x)\n",
+ req.entropy_ratio, req.security_strength);
+
+ return ret;
+}
+
+/* -- ioctl dispatch ------------------------ */
+
+/*
+ * PKE, SM2, and PQC ioctls use device-internal temporary storage for
+ * intermediate results. Residual allocations in the per-mailbox temp
+ * store (left by prior operations that targeted SYS_REF_TEMP) reduce
+ * the space available and can cause the device to return ENOMEM.
+ *
+ * Flush the mailbox before these operations to reset the temp store,
+ * but ONLY when the store is actually dirty (mgmt_temp_dirty flag).
+ * Unconditional flushing would kill in-flight command queues from
+ * concurrent callers on the same mailbox -- MBX_COMMAND_FLUSH
+ * terminates any executing queue with -EPIPE and discards all queued
+ * submissions.
+ *
+ * The conditional flush is safe: PKE/SM2/PQC ioctls do not consume
+ * SYS_REF_TEMP from a prior ioctl (unlike DS_EXPORT/DS_IMPORT which
+ * may reference a temp key produced by a preceding derivation), so
+ * clearing the temp store before them loses no needed state.
+ */
+static inline bool cmh_mgmt_needs_temp_flush(unsigned int cmd)
+{
+ unsigned int nr = _IOC_NR(cmd);
+
+ /*
+ * Range invariant: all PKE/SM2/PQC ioctls must have consecutive
+ * NR values between PKE_RSA_ENC (0x10) and SM2_ENC_HASH (0x37).
+ * If a new ioctl is added outside this range, update the bounds
+ * and adjust these assertions.
+ */
+ BUILD_BUG_ON(_IOC_NR(CMH_IOCTL_PKE_RSA_ENC) != 0x10);
+ BUILD_BUG_ON(_IOC_NR(CMH_IOCTL_SM2_ENC_HASH) != 0x37);
+
+ return nr >= _IOC_NR(CMH_IOCTL_PKE_RSA_ENC) &&
+ nr <= _IOC_NR(CMH_IOCTL_SM2_ENC_HASH);
+}
+
+static long cmh_mgmt_ioctl(struct file *file, unsigned int cmd,
+ unsigned long arg)
+{
+ void __user *argp = (void __user *)arg;
+ int ret;
+
+ if (cmh_mgmt_needs_temp_flush(cmd) &&
+ atomic_xchg(&mgmt_temp_dirty, 0)) {
+ ret = cmh_tm_flush_mbx(MGMT_MBX);
+ if (ret)
+ return ret;
+ }
+
+ switch (cmd) {
+ case CMH_IOCTL_KEY_NEW:
+ return cmh_mgmt_key_new(argp);
+ case CMH_IOCTL_KEY_WRITE:
+ return cmh_mgmt_key_write(argp);
+ case CMH_IOCTL_KEY_READ:
+ return cmh_mgmt_key_read(argp);
+ case CMH_IOCTL_KEY_FIND:
+ return cmh_mgmt_key_find(argp);
+ case CMH_IOCTL_KEY_GRANT:
+ return cmh_mgmt_key_grant(argp, false);
+ case CMH_IOCTL_KEY_DELETE:
+ return cmh_mgmt_key_grant(argp, true);
+ case CMH_IOCTL_DS_EXPORT:
+ return cmh_mgmt_ds_export(argp);
+ case CMH_IOCTL_DS_IMPORT:
+ return cmh_mgmt_ds_import(argp);
+ case CMH_IOCTL_KIC_HKDF1:
+ return cmh_mgmt_kic_hkdf1(argp);
+ case CMH_IOCTL_KIC_HKDF2:
+ return cmh_mgmt_kic_hkdf2(argp);
+ case CMH_IOCTL_KEY_NEW_RANDOM:
+ return cmh_mgmt_key_new_random(argp);
+ case CMH_IOCTL_KIC_AES_CMAC_KDF:
+ return cmh_mgmt_kic_aes_cmac_kdf(argp);
+ case CMH_IOCTL_KIC_DKEK_DERIVE:
+ return cmh_mgmt_kic_dkek_derive(argp);
+ case CMH_IOCTL_KEY_LIST:
+ return cmh_mgmt_key_list(argp);
+ case CMH_IOCTL_EAC_READ:
+ return cmh_mgmt_eac_read(argp);
+ /* PKE operations */
+ case CMH_IOCTL_PKE_RSA_ENC:
+ return cmh_mgmt_pke_rsa_enc(argp);
+ case CMH_IOCTL_PKE_RSA_DEC:
+ return cmh_mgmt_pke_rsa_dec(argp);
+ case CMH_IOCTL_PKE_RSA_CRT_DEC:
+ return cmh_mgmt_pke_rsa_crt_dec(argp);
+ case CMH_IOCTL_PKE_RSA_KEYGEN:
+ return cmh_mgmt_pke_rsa_keygen(argp);
+ case CMH_IOCTL_PKE_ECDSA_SIGN:
+ return cmh_mgmt_pke_ecdsa_sign(argp);
+ case CMH_IOCTL_PKE_ECDH:
+ return cmh_mgmt_pke_ecdh(argp);
+ case CMH_IOCTL_PKE_ECDH_KEYGEN:
+ return cmh_mgmt_pke_ecdh_keygen(argp);
+ case CMH_IOCTL_PKE_EDDSA_SIGN:
+ return cmh_mgmt_pke_eddsa_sign(argp);
+ case CMH_IOCTL_PKE_EDDSA_VERIFY:
+ return cmh_mgmt_pke_eddsa_verify(argp);
+ case CMH_IOCTL_PKE_EC_KEYGEN:
+ return cmh_mgmt_pke_ec_keygen(argp);
+ case CMH_IOCTL_PKE_EC_PUBGEN:
+ return cmh_mgmt_pke_ec_pubgen(argp);
+ case CMH_IOCTL_PKE_EDDSA_KEYGEN_SCA:
+ return cmh_mgmt_pke_eddsa_keygen_sca(argp);
+ /* SM2 operations */
+ case CMH_IOCTL_SM2_ECDH_KEYGEN:
+ return cmh_mgmt_sm2_ecdh_keygen(argp);
+ case CMH_IOCTL_SM2_ECDH:
+ return cmh_mgmt_sm2_ecdh(argp);
+ case CMH_IOCTL_SM2_DEC_POINT:
+ return cmh_mgmt_sm2_dec_point(argp);
+ case CMH_IOCTL_SM2_ENC_POINT:
+ return cmh_mgmt_sm2_enc_point(argp);
+ case CMH_IOCTL_SM2_ID_DIGEST:
+ return cmh_mgmt_sm2_id_digest(argp);
+ case CMH_IOCTL_SM2_ECDH_HASH:
+ return cmh_mgmt_sm2_ecdh_hash(argp);
+ case CMH_IOCTL_SM2_DEC_HASH:
+ return cmh_mgmt_sm2_dec_hash(argp);
+ case CMH_IOCTL_SM2_ENC_HASH:
+ return cmh_mgmt_sm2_enc_hash(argp);
+ /* PQC operations */
+ case CMH_IOCTL_ML_KEM_KEYGEN:
+ return cmh_mgmt_ml_kem_keygen(argp);
+ case CMH_IOCTL_ML_KEM_ENC:
+ return cmh_mgmt_ml_kem_enc(argp);
+ case CMH_IOCTL_ML_KEM_DEC:
+ return cmh_mgmt_ml_kem_dec(argp);
+ case CMH_IOCTL_ML_DSA_KEYGEN:
+ return cmh_mgmt_ml_dsa_keygen(argp);
+ case CMH_IOCTL_ML_DSA_SIGN:
+ return cmh_mgmt_ml_dsa_sign(argp);
+ case CMH_IOCTL_SLHDSA_KEYGEN:
+ return cmh_mgmt_slhdsa_keygen(argp);
+ case CMH_IOCTL_SLHDSA_SIGN:
+ return cmh_mgmt_slhdsa_sign(argp);
+ case CMH_IOCTL_SLHDSA_SIGN_PREHASH:
+ return cmh_mgmt_slhdsa_sign_prehash(argp);
+ /* DRBG management */
+ case CMH_IOCTL_DRBG_CONFIG:
+ return cmh_mgmt_drbg_config(argp);
+ default:
+ return -ENOTTY;
+ }
+}
+
+/* -- File operations ----------------------- */
+
+/*
+ * Capability is checked once at open time. A privileged process may
+ * pass the resulting fd to an unprivileged helper -- this delegation
+ * model is intentional and mirrors /dev/kvm, /dev/loop-control, etc.
+ */
+static int cmh_mgmt_open(struct inode *inode, struct file *file)
+{
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
+
+ return 0;
+}
+
+static const struct file_operations cmh_mgmt_fops = {
+ .owner = THIS_MODULE,
+ .open = cmh_mgmt_open,
+ .unlocked_ioctl = cmh_mgmt_ioctl,
+ .compat_ioctl = compat_ptr_ioctl,
+};
+
+static struct miscdevice cmh_mgmt_dev = {
+ .minor = MISC_DYNAMIC_MINOR,
+ .name = "cmh_mgmt",
+ .fops = &cmh_mgmt_fops,
+ .mode = 0660,
+};
+
+static bool cmh_mgmt_registered;
+
+/**
+ * cmh_mgmt_register() - Register the /dev/cmh_mgmt misc device
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_register(void)
+{
+ int ret;
+
+ /*
+ * ABI size guards -- catch silent layout changes at compile time.
+ * All ioctl structs use only __u32 and __u64 with explicit padding,
+ * guaranteeing identical layout on 32-bit and 64-bit (compat_ptr_ioctl).
+ */
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_new) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_write) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_read) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_find) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_list) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_grant) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_ds_export) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_ds_import) != 24);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_kic_hkdf1) != 48);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_kic_hkdf2) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_kic_aes_cmac_kdf) != 48);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_kic_dkek_derive) != 48);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_rsa_enc) != 48);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_rsa_dec) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_rsa_crt_dec) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_rsa_keygen) != 64);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ecdsa_sign) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ecdh) != 48);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ecdh_keygen) != 24);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_eddsa_sign) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_eddsa_verify) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ec_keygen) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ec_pubgen) != 24);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_eddsa_keygen_sca) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_kem_keygen) != 64);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_kem_enc) != 64);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_kem_dec) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_dsa_keygen) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_dsa_sign) != 48);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_slhdsa_keygen) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_slhdsa_sign) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_slhdsa_sign_prehash) != 64);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_ecdh_keygen) != 24);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_ecdh) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_dec_point) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_enc_point) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_id_digest) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_ecdh_hash) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_dec_hash) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_enc_hash) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_eac_read) != 64);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_drbg_config) != 16);
+
+ ret = misc_register(&cmh_mgmt_dev);
+ if (ret) {
+ dev_err(cmh_dev(), "mgmt: misc_register failed (rc=%d)\n", ret);
+ return ret;
+ }
+
+ cmh_mgmt_registered = true;
+ dev_info(cmh_dev(), "mgmt: registered /dev/cmh_mgmt\n");
+ return 0;
+}
+
+/**
+ * cmh_mgmt_unregister() - Unregister the /dev/cmh_mgmt misc device
+ */
+void cmh_mgmt_unregister(void)
+{
+ if (!cmh_mgmt_registered)
+ return;
+
+ misc_deregister(&cmh_mgmt_dev);
+ cmh_mgmt_registered = false;
+ dev_info(cmh_dev(), "mgmt: unregistered /dev/cmh_mgmt\n");
+}
diff --git a/drivers/crypto/cmh/cmh_mgmt_pke.c b/drivers/crypto/cmh/cmh_mgmt_pke.c
new file mode 100644
index 000000000000..6954832fa8ac
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_mgmt_pke.c
@@ -0,0 +1,1100 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH -- PKE ioctl handlers for /dev/cmh_mgmt
+ *
+ * RSA encrypt/decrypt/CRT/keygen, ECDSA sign, ECDH/keygen,
+ * EdDSA sign/verify, EC keygen/pubgen.
+ *
+ * Split from cmh_mgmt.c for maintainability.
+ */
+
+#include <linux/kernel.h>
+#include <linux/uaccess.h>
+#include <linux/slab.h>
+#include <linux/overflow.h>
+
+#include "cmh_mgmt.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_key.h"
+#include "cmh_dma.h"
+#include "cmh_config.h"
+#include "cmh_pke.h"
+#include "cmh_pke_abi.h"
+#include "cmh_sys_abi.h"
+#include <uapi/linux/cmh_mgmt_ioctl.h>
+
+#include <crypto/utils.h>
+
+/* -- PKE ioctl helpers ------------------- */
+
+/*
+ * Maximum PKE operand size: 512 bytes (RSA 4096-bit),
+ * or 2 * 68 = 136 bytes (P-521 coordinate pair).
+ */
+#define PKE_MAX_OPERAND 512
+
+/* Validate curve ID and return coordinate length; 0 = invalid */
+static u32 cmh_pke_validate_curve(u32 curve)
+{
+ return pke_curve_clen(curve);
+}
+
+/**
+ * cmh_mgmt_pke_rsa_enc() - Handle CMH_MGMT_IOC_PKE_RSA_ENC ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_rsa_enc(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_rsa_enc req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 n_len, e_padded;
+ u8 *e_buf, *n_buf, *m_buf, *c_buf;
+ dma_addr_t e_dma, n_dma, m_dma, c_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (req.bits < PKE_RSA_MIN_BITS || req.bits > PKE_RSA_MAX_BITS)
+ return -EINVAL;
+ if (!req.e_len || req.e_len > PKE_MAX_OPERAND)
+ return -EINVAL;
+
+ n_len = req.bits / 8;
+ e_padded = ALIGN(req.e_len, 4);
+
+ e_buf = kzalloc(e_padded, GFP_KERNEL);
+ n_buf = kmalloc(n_len, GFP_KERNEL);
+ m_buf = kmalloc(n_len, GFP_KERNEL);
+ c_buf = kzalloc(n_len, GFP_KERNEL);
+ if (!e_buf || !n_buf || !m_buf || !c_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ /* Right-align exponent in zero-padded buffer for DMA alignment */
+ if (copy_from_user(e_buf + e_padded - req.e_len,
+ u64_to_user_ptr(req.e), req.e_len) ||
+ copy_from_user(n_buf, u64_to_user_ptr(req.n), n_len) ||
+ copy_from_user(m_buf, u64_to_user_ptr(req.input), n_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ e_dma = cmh_dma_map_single(e_buf, e_padded, DMA_TO_DEVICE);
+ n_dma = cmh_dma_map_single(n_buf, n_len, DMA_TO_DEVICE);
+ m_dma = cmh_dma_map_single(m_buf, n_len, DMA_TO_DEVICE);
+ c_dma = cmh_dma_map_single(c_buf, n_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(e_dma) || cmh_dma_map_error(n_dma) ||
+ cmh_dma_map_error(m_dma) || cmh_dma_map_error(c_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_rsa_enc(&vcq[1], pke_cid, req.bits, e_padded,
+ e_dma, n_dma, m_dma, c_dma, PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(c_dma))
+ cmh_dma_unmap_single(c_dma, n_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, n_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(n_dma))
+ cmh_dma_unmap_single(n_dma, n_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(e_dma))
+ cmh_dma_unmap_single(e_dma, e_padded, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.output), c_buf, n_len))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(c_buf);
+ kfree_sensitive(m_buf);
+ kfree(n_buf);
+ kfree(e_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_rsa_dec() - Handle CMH_MGMT_IOC_PKE_RSA_DEC ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_rsa_dec(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_rsa_dec req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 n_len, e_padded;
+ u8 *e_buf, *n_buf, *c_buf, *m_buf;
+ dma_addr_t e_dma, n_dma, c_dma, m_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (req.bits < PKE_RSA_MIN_BITS || req.bits > PKE_RSA_MAX_BITS)
+ return -EINVAL;
+ if (!req.e_len || req.e_len > PKE_MAX_OPERAND)
+ return -EINVAL;
+
+ n_len = req.bits / 8;
+ e_padded = ALIGN(req.e_len, 4);
+
+ e_buf = kzalloc(e_padded, GFP_KERNEL);
+ n_buf = kmalloc(n_len, GFP_KERNEL);
+ c_buf = kmalloc(n_len, GFP_KERNEL);
+ m_buf = kzalloc(n_len, GFP_KERNEL);
+ if (!e_buf || !n_buf || !c_buf || !m_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ /* Right-align exponent in zero-padded buffer for DMA alignment */
+ if (copy_from_user(e_buf + e_padded - req.e_len,
+ u64_to_user_ptr(req.e), req.e_len) ||
+ copy_from_user(n_buf, u64_to_user_ptr(req.n), n_len) ||
+ copy_from_user(c_buf, u64_to_user_ptr(req.input), n_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ e_dma = cmh_dma_map_single(e_buf, e_padded, DMA_TO_DEVICE);
+ n_dma = cmh_dma_map_single(n_buf, n_len, DMA_TO_DEVICE);
+ c_dma = cmh_dma_map_single(c_buf, n_len, DMA_TO_DEVICE);
+ m_dma = cmh_dma_map_single(m_buf, n_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(e_dma) || cmh_dma_map_error(n_dma) ||
+ cmh_dma_map_error(c_dma) || cmh_dma_map_error(m_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_rsa_dec(&vcq[1], pke_cid, req.bits, e_padded,
+ e_dma, n_dma, c_dma, m_dma, req.key_ref,
+ PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, n_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(c_dma))
+ cmh_dma_unmap_single(c_dma, n_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(n_dma))
+ cmh_dma_unmap_single(n_dma, n_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(e_dma))
+ cmh_dma_unmap_single(e_dma, e_padded, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.output), m_buf, n_len))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(m_buf);
+ kfree(c_buf);
+ kfree(n_buf);
+ kfree(e_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_rsa_crt_dec() - Handle CMH_MGMT_IOC_PKE_RSA_CRT_DEC ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_rsa_crt_dec(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_rsa_crt_dec req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 n_len, e_padded;
+ u8 *e_buf, *n_buf, *c_buf, *m_buf;
+ dma_addr_t e_dma, n_dma, c_dma, m_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (req.bits < PKE_RSA_MIN_BITS || req.bits > PKE_RSA_MAX_BITS)
+ return -EINVAL;
+ if (!req.e_len || req.e_len > PKE_MAX_OPERAND)
+ return -EINVAL;
+
+ n_len = req.bits / 8;
+ e_padded = ALIGN(req.e_len, 4);
+
+ e_buf = kzalloc(e_padded, GFP_KERNEL);
+ n_buf = kmalloc(n_len, GFP_KERNEL);
+ c_buf = kmalloc(n_len, GFP_KERNEL);
+ m_buf = kzalloc(n_len, GFP_KERNEL);
+ if (!e_buf || !n_buf || !c_buf || !m_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ /* Right-align exponent in zero-padded buffer for DMA alignment */
+ if (copy_from_user(e_buf + e_padded - req.e_len,
+ u64_to_user_ptr(req.e), req.e_len) ||
+ copy_from_user(n_buf, u64_to_user_ptr(req.n), n_len) ||
+ copy_from_user(c_buf, u64_to_user_ptr(req.input), n_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ e_dma = cmh_dma_map_single(e_buf, e_padded, DMA_TO_DEVICE);
+ n_dma = cmh_dma_map_single(n_buf, n_len, DMA_TO_DEVICE);
+ c_dma = cmh_dma_map_single(c_buf, n_len, DMA_TO_DEVICE);
+ m_dma = cmh_dma_map_single(m_buf, n_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(e_dma) || cmh_dma_map_error(n_dma) ||
+ cmh_dma_map_error(c_dma) || cmh_dma_map_error(m_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_rsa_crt_dec(&vcq[1], pke_cid, req.bits, e_padded,
+ e_dma, n_dma, c_dma, m_dma, req.crt_ref,
+ PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, n_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(c_dma))
+ cmh_dma_unmap_single(c_dma, n_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(n_dma))
+ cmh_dma_unmap_single(n_dma, n_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(e_dma))
+ cmh_dma_unmap_single(e_dma, e_padded, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.output), m_buf, n_len))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(m_buf);
+ kfree(c_buf);
+ kfree(n_buf);
+ kfree(e_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_rsa_keygen() - Handle CMH_MGMT_IOC_PKE_RSA_KEYGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_rsa_keygen(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_rsa_keygen req;
+ /*
+ * When has_crt, we use a two-VCQ approach (CRI pattern):
+ * VCQ #1: header + SYS_NEW(d) + SYS_NEW(crt) + SYS_FLUSH (4 slots)
+ * VCQ #2: header + RSA_KEYGEN + PKE_FLUSH + SYS_FLUSH (4 slots)
+ * Without CRT, single VCQ:
+ * header + SYS_NEW(d) + RSA_KEYGEN + PKE_FLUSH + SYS_FLUSH (5 slots)
+ */
+ struct vcq_cmd vcq[5];
+ u32 n_len, e_padded, key_flags, d_ds_len, crt_ds_len;
+ u8 *e_buf, *n_buf;
+ u64 *d_ref_buf, *crt_ref_buf;
+ dma_addr_t e_dma, n_dma, d_ref_dma, crt_ref_dma;
+ int idx, ret;
+ bool has_crt, is_sca;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.bits < PKE_RSA_MIN_BITS || req.bits > PKE_RSA_MAX_BITS)
+ return -EINVAL;
+ if (!req.e_len || req.e_len > PKE_MAX_OPERAND)
+ return -EINVAL;
+ if (req.flags & ~CMH_FLAG_MASK)
+ return -EINVAL;
+
+ n_len = req.bits / 8;
+ has_crt = (req.crt_cid != 0);
+ e_padded = ALIGN(req.e_len, 4);
+ key_flags = req.flags & CMH_FLAG_MASK;
+ is_sca = !!(req.flags & CMH_FLAG_SCA);
+
+ /*
+ * SCA keys are stored in 2 shares -- DS allocation must be enlarged.
+ * CRI reference formulas: cmh_pke_rsa_private_key_size().
+ */
+ if (is_sca) {
+ d_ds_len = n_len * 2;
+ crt_ds_len = (7 + n_len / 2) * 4;
+ } else {
+ d_ds_len = n_len;
+ crt_ds_len = 5 * (n_len / 2);
+ }
+
+ e_buf = kzalloc(e_padded, GFP_KERNEL);
+ n_buf = kzalloc(n_len, GFP_KERNEL);
+ d_ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ crt_ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ if (!e_buf || !n_buf || !d_ref_buf || !crt_ref_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(e_buf + e_padded - req.e_len,
+ u64_to_user_ptr(req.e), req.e_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ e_dma = cmh_dma_map_single(e_buf, e_padded, DMA_TO_DEVICE);
+ n_dma = cmh_dma_map_single(n_buf, n_len, DMA_FROM_DEVICE);
+ d_ref_dma = cmh_dma_map_single(d_ref_buf, sizeof(u64), DMA_FROM_DEVICE);
+ crt_ref_dma = cmh_dma_map_single(crt_ref_buf, sizeof(u64),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(e_dma) || cmh_dma_map_error(n_dma) ||
+ cmh_dma_map_error(d_ref_dma) || cmh_dma_map_error(crt_ref_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ if (has_crt) {
+ /*
+ * Two-VCQ approach (CRI pattern): SYS_REF_LAST can only
+ * refer to the most recently created DS object. When we
+ * need both d and crt refs, we must first allocate DS
+ * objects, read back the opaque refs, then pass them by
+ * value in the keygen VCQ.
+ *
+ * VCQ #1: allocate both DS objects.
+ */
+ idx = 0;
+ vcq_set_header(&vcq[idx++], 4);
+ vcq_add_sys_new(&vcq[idx++], req.d_cid, d_ref_dma, d_ds_len);
+ vcq_add_sys_new(&vcq[idx++], req.crt_cid, crt_ref_dma,
+ crt_ds_len);
+ vcq_add_sys_flush(&vcq[idx++]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 4, 1, MGMT_MBX);
+ if (ret)
+ goto out_unmap;
+
+ /* Sync DMA so we can read back the opaque refs */
+ cmh_dma_unmap_single(d_ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ cmh_dma_unmap_single(crt_ref_dma, sizeof(u64),
+ DMA_FROM_DEVICE);
+ d_ref_dma = 0;
+ crt_ref_dma = 0;
+
+ /*
+ * VCQ #2: keygen with resolved refs.
+ */
+ idx = 0;
+ memset(vcq, 0, sizeof(vcq));
+ vcq_set_header(&vcq[idx++], 4);
+
+ vcq[idx].magic = VCQ_CMD_MAGIC;
+ vcq[idx].id = VCQ_CMD_ID(pke_cid, PKE_SWAP_FLAGS, 1,
+ PKE_CMD_RSA_KEYGEN);
+ vcq[idx].hwc.pke.cmd_rsa_keygen.bits = req.bits;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.e = e_dma;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.n = n_dma;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.d = *d_ref_buf;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.d_type =
+ SYS_TYPE_SET(key_flags, CORE_ID_PKE);
+ vcq[idx].hwc.pke.cmd_rsa_keygen.crt = *crt_ref_buf;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.crt_type =
+ SYS_TYPE_SET(key_flags, CORE_ID_PKE);
+ idx++;
+
+ vcq_add_pke_flush(&vcq[idx++], pke_cid);
+ vcq_add_sys_flush(&vcq[idx++]);
+
+ ret = cmh_tm_submit_sync_tmo(vcq, 4, 1, MGMT_MBX,
+ cmh_tm_slow_op_timeout_jiffies());
+ } else {
+ /*
+ * Single-VCQ: only d, so SYS_REF_LAST is unambiguous.
+ */
+ idx = 0;
+ vcq_set_header(&vcq[idx++], 5);
+ vcq_add_sys_new(&vcq[idx++], req.d_cid, d_ref_dma, d_ds_len);
+
+ vcq[idx].magic = VCQ_CMD_MAGIC;
+ vcq[idx].id = VCQ_CMD_ID(pke_cid, PKE_SWAP_FLAGS, 1,
+ PKE_CMD_RSA_KEYGEN);
+ vcq[idx].hwc.pke.cmd_rsa_keygen.bits = req.bits;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.e = e_dma;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.n = n_dma;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.d = SYS_REF_LAST;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.d_type =
+ SYS_TYPE_SET(key_flags, CORE_ID_PKE);
+ vcq[idx].hwc.pke.cmd_rsa_keygen.crt = SYS_REF_NONE;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.crt_type = 0;
+ idx++;
+
+ vcq_add_pke_flush(&vcq[idx++], pke_cid);
+ vcq_add_sys_flush(&vcq[idx++]);
+
+ ret = cmh_tm_submit_sync_tmo(vcq, 5, 1, MGMT_MBX,
+ cmh_tm_slow_op_timeout_jiffies());
+ }
+
+out_unmap:
+ if (crt_ref_dma && !cmh_dma_map_error(crt_ref_dma))
+ cmh_dma_unmap_single(crt_ref_dma, sizeof(u64),
+ DMA_FROM_DEVICE);
+ if (d_ref_dma && !cmh_dma_map_error(d_ref_dma))
+ cmh_dma_unmap_single(d_ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(n_dma))
+ cmh_dma_unmap_single(n_dma, n_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(e_dma))
+ cmh_dma_unmap_single(e_dma, e_padded, DMA_TO_DEVICE);
+
+ if (!ret) {
+ /* Copy generated modulus and refs back */
+ if (copy_to_user(u64_to_user_ptr(req.n), n_buf, n_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ req.d_ref = *d_ref_buf;
+ req.crt_ref = has_crt ? *crt_ref_buf : 0;
+ if (copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(crt_ref_buf);
+ kfree(d_ref_buf);
+ kfree(n_buf);
+ kfree(e_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_ecdsa_sign() - Handle CMH_MGMT_IOC_PKE_ECDSA_SIGN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_ecdsa_sign(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_ecdsa_sign req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 clen, sig_len, dig_map_len;
+ u8 *dig_buf, *sig_buf;
+ dma_addr_t dig_dma, sig_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ clen = cmh_pke_validate_curve(req.curve);
+ if (!clen || !req.digest_len ||
+ req.digest_len > CMH_MGMT_MAX_DATA_LEN)
+ return -EINVAL;
+
+ sig_len = 2 * clen;
+
+ /*
+ * eSW requires digest_len >= clen. Zero-pad shorter hashes.
+ */
+ dig_map_len = max_t(u32, req.digest_len, clen);
+
+ dig_buf = kzalloc(dig_map_len, GFP_KERNEL);
+ sig_buf = kzalloc(sig_len, GFP_KERNEL);
+ if (!dig_buf || !sig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(dig_buf, u64_to_user_ptr(req.digest),
+ req.digest_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ dig_dma = cmh_dma_map_single(dig_buf, dig_map_len, DMA_TO_DEVICE);
+ sig_dma = cmh_dma_map_single(sig_buf, sig_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(dig_dma) || cmh_dma_map_error(sig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_ecdsa_sign(&vcq[1], pke_cid, req.curve, clen,
+ dig_dma, sig_dma, req.key_ref,
+ dig_map_len, pke_swap_flags(req.curve));
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(dig_dma))
+ cmh_dma_unmap_single(dig_dma, dig_map_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.signature),
+ sig_buf, sig_len))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(sig_buf);
+ kfree(dig_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_ecdh() - Handle CMH_MGMT_IOC_PKE_ECDH ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_ecdh(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_ecdh req;
+ /* Phase 1: hdr + sys_new + pke_ecdh + pke_flush; reused for Phase 2 */
+ struct vcq_cmd vcq[4];
+ u32 clen, swap, ss_type;
+ u8 *peer_buf, *ss_buf;
+ u64 *ref_buf;
+ dma_addr_t peer_dma, ss_dma, ref_dma;
+ int ret, idx;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ clen = cmh_pke_validate_curve(req.curve);
+ if (!clen)
+ return -EINVAL;
+
+ swap = PKE_SWAP_FLAGS;
+ ss_type = SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_PKE);
+
+ peer_buf = kmalloc(clen, GFP_KERNEL);
+ ss_buf = kzalloc(clen, GFP_KERNEL);
+ ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ if (!peer_buf || !ss_buf || !ref_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(peer_buf, u64_to_user_ptr(req.peer_key_x), clen)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ peer_dma = cmh_dma_map_single(peer_buf, clen, DMA_TO_DEVICE);
+ ss_dma = cmh_dma_map_single(ss_buf, clen, DMA_FROM_DEVICE);
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(peer_dma) || cmh_dma_map_error(ss_dma) ||
+ cmh_dma_map_error(ref_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ idx = 0;
+ vcq_set_header(&vcq[idx++], 4);
+ vcq_add_sys_new(&vcq[idx++], 0, ref_dma, clen);
+ vcq_add_pke_ecdh(&vcq[idx++], pke_cid, req.curve, clen, clen,
+ ss_type, peer_dma, req.key_ref,
+ SYS_REF_LAST, swap);
+ vcq_add_pke_flush(&vcq[idx++], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 4, 1, MGMT_MBX);
+ if (ret)
+ goto out_unmap;
+
+ /* Sync bounce buffer so CPU sees the DMA-written ref */
+ cmh_dma_sync_for_cpu(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+
+ /* Phase 2: extract shared secret from DS via actual ref */
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_data(&vcq[1], *ref_buf, ss_dma, clen);
+ vcq[1].id |= pke_swap_flags(req.curve);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(ref_dma))
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ss_dma))
+ cmh_dma_unmap_single(ss_dma, clen, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(peer_dma))
+ cmh_dma_unmap_single(peer_dma, clen, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.output), ss_buf, clen))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(ref_buf);
+ kfree_sensitive(ss_buf);
+ kfree(peer_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_ecdh_keygen() - Handle CMH_MGMT_IOC_PKE_ECDH_KEYGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_ecdh_keygen(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_ecdh_keygen req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 clen, out_len;
+ u8 *pkx_buf;
+ dma_addr_t pkx_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ clen = cmh_pke_validate_curve(req.curve);
+ if (!clen)
+ return -EINVAL;
+
+ /*
+ * ECDH_KEYGEN always outputs both X and Y coordinates
+ * (2 * clen bytes total) even though only X is useful for
+ * the ECDH exchange. Allocate the full output size to avoid
+ * a DMA buffer overflow, but copy only X back to userspace.
+ */
+ out_len = 2 * clen;
+
+ pkx_buf = kzalloc(out_len, GFP_KERNEL);
+ if (!pkx_buf)
+ return -ENOMEM;
+
+ pkx_dma = cmh_dma_map_single(pkx_buf, out_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(pkx_dma)) {
+ kfree(pkx_buf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_ecdh_keygen(&vcq[1], pke_cid, req.curve, clen,
+ pkx_dma, req.key_ref,
+ PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(pkx_dma, out_len, DMA_FROM_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.public_key_x),
+ pkx_buf, clen))
+ ret = -EFAULT;
+ }
+
+ kfree(pkx_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_eddsa_sign() - Handle CMH_MGMT_IOC_PKE_EDDSA_SIGN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_eddsa_sign(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_eddsa_sign req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 klen, sig_len;
+ u8 *msg_buf, *sig_buf;
+ dma_addr_t msg_dma, sig_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (!cmh_pke_validate_curve(req.curve) || !req.digest_len ||
+ req.digest_len > CMH_MGMT_MAX_DATA_LEN)
+ return -EINVAL;
+ if (!pke_curve_is_edwards(req.curve))
+ return -EINVAL;
+
+ klen = pke_eddsa_key_len(req.curve);
+ sig_len = 2 * klen;
+
+ msg_buf = kmalloc(req.digest_len, GFP_KERNEL);
+ sig_buf = kzalloc(sig_len, GFP_KERNEL);
+ if (!msg_buf || !sig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(msg_buf, u64_to_user_ptr(req.digest),
+ req.digest_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ msg_dma = cmh_dma_map_single(msg_buf, req.digest_len, DMA_TO_DEVICE);
+ sig_dma = cmh_dma_map_single(sig_buf, sig_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(msg_dma) || cmh_dma_map_error(sig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_eddsa_sign(&vcq[1], pke_cid, req.curve, klen,
+ msg_dma, sig_dma, req.key_ref,
+ req.digest_len, pke_swap_flags(req.curve));
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(msg_dma))
+ cmh_dma_unmap_single(msg_dma, req.digest_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.signature),
+ sig_buf, sig_len))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(sig_buf);
+ kfree(msg_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_eddsa_verify() - Handle CMH_MGMT_IOC_PKE_EDDSA_VERIFY ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_eddsa_verify(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_eddsa_verify req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 clen, klen, sig_len;
+ u8 *msg_buf, *sig_buf, *pky_buf, *rp_buf;
+ dma_addr_t msg_dma, sig_dma, pky_dma, rp_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ clen = cmh_pke_validate_curve(req.curve);
+ if (!clen || !req.digest_len ||
+ req.digest_len > CMH_MGMT_MAX_DATA_LEN)
+ return -EINVAL;
+ if (!pke_curve_is_edwards(req.curve))
+ return -EINVAL;
+
+ klen = pke_eddsa_key_len(req.curve);
+ sig_len = 2 * klen;
+
+ msg_buf = kmalloc(req.digest_len, GFP_KERNEL);
+ sig_buf = kmalloc(sig_len, GFP_KERNEL);
+ pky_buf = kmalloc(klen, GFP_KERNEL);
+ rp_buf = kzalloc(clen, GFP_KERNEL);
+ if (!msg_buf || !sig_buf || !pky_buf || !rp_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(msg_buf, u64_to_user_ptr(req.digest),
+ req.digest_len) ||
+ copy_from_user(sig_buf, u64_to_user_ptr(req.signature),
+ sig_len) ||
+ copy_from_user(pky_buf, u64_to_user_ptr(req.public_key_y),
+ klen)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ msg_dma = cmh_dma_map_single(msg_buf, req.digest_len, DMA_TO_DEVICE);
+ sig_dma = cmh_dma_map_single(sig_buf, sig_len, DMA_TO_DEVICE);
+ pky_dma = cmh_dma_map_single(pky_buf, klen, DMA_TO_DEVICE);
+ rp_dma = cmh_dma_map_single(rp_buf, clen, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(msg_dma) || cmh_dma_map_error(sig_dma) ||
+ cmh_dma_map_error(pky_dma) || cmh_dma_map_error(rp_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_eddsa_verify(&vcq[1], pke_cid, req.curve, req.digest_len,
+ pky_dma, msg_dma, sig_dma, rp_dma,
+ pke_swap_flags(req.curve));
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(rp_dma))
+ cmh_dma_unmap_single(rp_dma, clen, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(pky_dma))
+ cmh_dma_unmap_single(pky_dma, klen, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(msg_dma))
+ cmh_dma_unmap_single(msg_dma, req.digest_len, DMA_TO_DEVICE);
+
+out_free:
+ kfree(rp_buf);
+ kfree(pky_buf);
+ kfree(sig_buf);
+ kfree(msg_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_ec_keygen() - Handle CMH_MGMT_IOC_PKE_EC_KEYGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_ec_keygen(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_ec_keygen req;
+ /* header + SYS_NEW + ECDSA_KEYGEN + flush_pke + flush_sys */
+ struct vcq_cmd vcq[5];
+ u32 clen, key_flags, ds_len;
+ u64 *ref_buf;
+ dma_addr_t ref_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (req.flags & ~CMH_FLAG_MASK)
+ return -EINVAL;
+ clen = cmh_pke_validate_curve(req.curve);
+ if (!clen)
+ return -EINVAL;
+
+ key_flags = req.flags & CMH_FLAG_MASK;
+ /* SCA keys are stored in 2 shares -- allocate double the curve length */
+ ds_len = (req.flags & CMH_FLAG_SCA) ? clen * 2 : clen;
+
+ ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], 5);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, ds_len);
+ vcq_add_pke_ecdsa_keygen(&vcq[2], pke_cid, req.curve, clen,
+ SYS_REF_LAST,
+ SYS_TYPE_SET(key_flags, CORE_ID_PKE),
+ pke_swap_flags(req.curve));
+ vcq_add_pke_flush(&vcq[3], pke_cid);
+ vcq_add_sys_flush(&vcq[4]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 5, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+
+ if (!ret) {
+ req.ref = *ref_buf;
+ if (copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ }
+
+ kfree(ref_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_ec_pubgen() - Handle CMH_MGMT_IOC_PKE_EC_PUBGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_ec_pubgen(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_ec_pubgen req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 clen, pk_len;
+ u8 *pk_buf;
+ dma_addr_t pk_dma;
+ bool is_ed;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ clen = cmh_pke_validate_curve(req.curve);
+ if (!clen)
+ return -EINVAL;
+
+ is_ed = pke_curve_is_edwards(req.curve);
+ pk_len = is_ed ? pke_eddsa_key_len(req.curve) : 2 * clen;
+
+ pk_buf = kzalloc(pk_len, GFP_KERNEL);
+ if (!pk_buf)
+ return -ENOMEM;
+
+ pk_dma = cmh_dma_map_single(pk_buf, pk_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(pk_dma)) {
+ kfree(pk_buf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ if (is_ed)
+ vcq_add_pke_eddsa_pubgen(&vcq[1], pke_cid, req.curve,
+ pke_eddsa_key_len(req.curve),
+ pk_dma, req.key_ref,
+ pke_swap_flags(req.curve));
+ else
+ vcq_add_pke_ecdsa_pubgen(&vcq[1], pke_cid, req.curve, clen,
+ pk_dma, req.key_ref,
+ pke_swap_flags(req.curve));
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(pk_dma, pk_len, DMA_FROM_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.public_key),
+ pk_buf, pk_len))
+ ret = -EFAULT;
+ }
+
+ kfree(pk_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_eddsa_keygen_sca() - Handle CMH_MGMT_IOC_PKE_EDDSA_KEYGEN_SCA ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_eddsa_keygen_sca(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_eddsa_keygen_sca req;
+ /* header + SYS_NEW + EDDSA_KEYGEN_SCA + flush_pke + flush_sys */
+ struct vcq_cmd vcq[5];
+ u64 *ref_buf;
+ dma_addr_t ref_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ /* EdDSA SCA keygen is only supported for Ed448 */
+ if (req.curve != PKE_CURVE_448)
+ return -EINVAL;
+
+ ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], 5);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, PKE_ED448_SK_SCA_LEN);
+ vcq_add_pke_eddsa_keygen_sca(&vcq[2], pke_cid, req.curve, req.key_ref,
+ SYS_REF_LAST);
+ vcq_add_pke_flush(&vcq[3], pke_cid);
+ vcq_add_sys_flush(&vcq[4]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 5, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+
+ if (!ret) {
+ req.sca_ref = *ref_buf;
+ if (copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ }
+
+ kfree(ref_buf);
+ return ret;
+}
diff --git a/drivers/crypto/cmh/cmh_mgmt_pqc.c b/drivers/crypto/cmh/cmh_mgmt_pqc.c
new file mode 100644
index 000000000000..db479e80326b
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_mgmt_pqc.c
@@ -0,0 +1,1279 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH -- PQC ioctl handlers for /dev/cmh_mgmt
+ *
+ * ML-KEM keygen/encapsulate/decapsulate, ML-DSA keygen/sign,
+ * SLH-DSA keygen/sign (pure + prehash).
+ *
+ * Split from cmh_mgmt.c for maintainability.
+ */
+
+#include <linux/kernel.h>
+#include <linux/uaccess.h>
+#include <linux/slab.h>
+#include <linux/overflow.h>
+
+#include "cmh_mgmt.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_key.h"
+#include "cmh_dma.h"
+#include "cmh_config.h"
+#include "cmh_pqc.h"
+#include "cmh_qse_abi.h"
+#include "cmh_sys_abi.h"
+#include <uapi/linux/cmh_mgmt_ioctl.h>
+
+#include <crypto/utils.h>
+
+/* -- PQC -- ML-KEM -- */
+
+/**
+ * cmh_mgmt_ml_kem_keygen() - Handle CMH_MGMT_IOC_ML_KEM_KEYGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_ml_kem_keygen(void __user *argp)
+{
+ u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE);
+
+ struct cmh_ioctl_ml_kem_keygen req;
+ struct vcq_cmd vcq[QSE_VCQ_CMDS_MAX];
+ u32 ek_len, dk_len, seed_len, key_flags;
+ u32 qse_flags = 0;
+ bool masked, ds_ref, hw_rng;
+ u8 *seed_buf = NULL, *z_buf = NULL, *ek_buf, *dk_buf = NULL;
+ u64 *ref_buf = NULL;
+ dma_addr_t seed_dma = DMA_MAPPING_ERROR, z_dma = DMA_MAPPING_ERROR;
+ dma_addr_t ek_dma, dk_dma = DMA_MAPPING_ERROR, ref_dma = DMA_MAPPING_ERROR;
+ int ret, idx;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (ml_kem_k_idx(req.k) < 0)
+ return -EINVAL;
+ if (req.flags & ~(CMH_QSE_FLAG_MASK | CMH_FLAG_MASK))
+ return -EINVAL;
+
+ masked = !!(req.flags & CMH_QSE_FLAG_MASKED);
+ ds_ref = !!(req.flags & CMH_QSE_FLAG_DS_REF);
+ hw_rng = !!(req.flags & CMH_QSE_FLAG_HW_RNG);
+
+ /*
+ * QSE keys only support PT storage -- the eSW dec/sign paths
+ * hardcode SYS_TYPE_FLAG_PT when reading the key back.
+ * QSE SCA protection uses masking (CMH_QSE_FLAG_MASKED),
+ * not the 2-share mechanism (CMH_FLAG_SCA).
+ */
+ key_flags = req.flags & CMH_FLAG_MASK;
+ if (key_flags && key_flags != CMH_FLAG_PT)
+ return -EINVAL;
+ key_flags = CMH_FLAG_PT;
+
+ /* Masked keygen must store dk in DS -- polynomial unmasking not supported */
+ if (masked && !ds_ref)
+ return -EINVAL;
+
+ ek_len = ML_KEM_EK_SIZE(req.k);
+ dk_len = masked ? ML_KEM_DK_SIZE_MASKED(req.k)
+ : ML_KEM_DK_SIZE(req.k);
+ seed_len = masked ? QSE_SEED_LEN_MASKED : QSE_SEED_LEN;
+
+ if (hw_rng)
+ qse_flags |= QSE_FLAG_USE_RNG;
+ if (ds_ref)
+ qse_flags |= QSE_FLAG_USE_REF;
+
+ ek_buf = kzalloc(ek_len, GFP_KERNEL);
+ if (!ek_buf)
+ return -ENOMEM;
+
+ if (!hw_rng && req.seed && req.z) {
+ seed_buf = kmalloc(seed_len, GFP_KERNEL);
+ z_buf = kmalloc(seed_len, GFP_KERNEL);
+ if (!seed_buf || !z_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(seed_buf, u64_to_user_ptr(req.seed),
+ seed_len) ||
+ copy_from_user(z_buf, u64_to_user_ptr(req.z), seed_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ if (ds_ref) {
+ ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ if (!ref_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ } else {
+ dk_buf = kzalloc(dk_len, GFP_KERNEL);
+ if (!dk_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ }
+
+ /* DMA map */
+ ek_dma = cmh_dma_map_single(ek_buf, ek_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ek_dma)) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (seed_buf) {
+ seed_dma = cmh_dma_map_single(seed_buf, seed_len,
+ DMA_TO_DEVICE);
+ z_dma = cmh_dma_map_single(z_buf, seed_len, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(seed_dma) || cmh_dma_map_error(z_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ if (ds_ref) {
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ } else {
+ dk_dma = cmh_dma_map_single(dk_buf, dk_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(dk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ idx = 0;
+ if (ds_ref) {
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MAX);
+ idx++;
+ vcq_add_sys_new(&vcq[idx++], req.dk_cid, ref_dma, dk_len);
+ vcq_add_qse_ml_kem_keygen(&vcq[idx++], qse_cid, req.k, qse_flags,
+ seed_dma, z_dma,
+ ek_dma, SYS_REF_LAST,
+ SYS_TYPE_SET(key_flags,
+ CORE_ID_QSE),
+ masked);
+ vcq_add_qse_flush(&vcq[idx++], qse_cid);
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MAX,
+ 1, MGMT_MBX);
+ } else {
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ idx++;
+ vcq_add_qse_ml_kem_keygen(&vcq[idx++], qse_cid, req.k, qse_flags,
+ seed_dma, z_dma,
+ ek_dma, dk_dma, 0, masked);
+ vcq_add_qse_flush(&vcq[idx++], qse_cid);
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN,
+ 1, MGMT_MBX);
+ }
+
+out_unmap:
+ if (ds_ref && !cmh_dma_map_error(ref_dma))
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ if (!ds_ref && dk_buf && !cmh_dma_map_error(dk_dma))
+ cmh_dma_unmap_single(dk_dma, dk_len, DMA_FROM_DEVICE);
+ if (z_buf && !cmh_dma_map_error(z_dma))
+ cmh_dma_unmap_single(z_dma, seed_len, DMA_TO_DEVICE);
+ if (seed_buf && !cmh_dma_map_error(seed_dma))
+ cmh_dma_unmap_single(seed_dma, seed_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(ek_dma))
+ cmh_dma_unmap_single(ek_dma, ek_len, DMA_FROM_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.ek), ek_buf, ek_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ if (ds_ref) {
+ req.dk_ref = *ref_buf;
+ } else {
+ if (copy_to_user(u64_to_user_ptr(req.dk),
+ dk_buf, dk_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+ if (copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(dk_buf);
+ kfree(ref_buf);
+ kfree_sensitive(z_buf);
+ kfree_sensitive(seed_buf);
+ kfree(ek_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_ml_kem_enc() - Handle CMH_MGMT_IOC_ML_KEM_ENC ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_ml_kem_enc(void __user *argp)
+{
+ u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE);
+
+ struct cmh_ioctl_ml_kem_enc req;
+ struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN];
+ u32 ek_len, ct_len, ss_out_len;
+ u32 qse_flags = 0;
+ bool masked, hw_rng;
+ u8 *ek_buf, *coin_buf = NULL, *ct_buf, *ss_buf;
+ dma_addr_t ek_dma, coin_dma = DMA_MAPPING_ERROR, ct_dma, ss_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved || req.__reserved2[0] || req.__reserved2[1])
+ return -EINVAL;
+ if (ml_kem_k_idx(req.k) < 0)
+ return -EINVAL;
+
+ masked = !!(req.flags & CMH_QSE_FLAG_MASKED);
+ hw_rng = !!(req.flags & CMH_QSE_FLAG_HW_RNG);
+
+ ek_len = ML_KEM_EK_SIZE(req.k);
+ ct_len = ML_KEM_CT_SIZE(req.k);
+ ss_out_len = masked ? ML_KEM_SS_LEN_MASKED : ML_KEM_SS_LEN;
+
+ if (hw_rng)
+ qse_flags |= QSE_FLAG_USE_RNG;
+
+ ek_buf = kmalloc(ek_len, GFP_KERNEL);
+ ct_buf = kzalloc(ct_len, GFP_KERNEL);
+ ss_buf = kzalloc(ss_out_len, GFP_KERNEL);
+ if (!ek_buf || !ct_buf || !ss_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(ek_buf, u64_to_user_ptr(req.ek), ek_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ if (!hw_rng && req.coin) {
+ u32 coin_len = masked ? QSE_SEED_LEN_MASKED : QSE_SEED_LEN;
+
+ coin_buf = kmalloc(coin_len, GFP_KERNEL);
+ if (!coin_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(coin_buf, u64_to_user_ptr(req.coin),
+ coin_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ coin_dma = cmh_dma_map_single(coin_buf, coin_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(coin_dma)) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ }
+
+ ek_dma = cmh_dma_map_single(ek_buf, ek_len, DMA_TO_DEVICE);
+ ct_dma = cmh_dma_map_single(ct_buf, ct_len, DMA_FROM_DEVICE);
+ ss_dma = cmh_dma_map_single(ss_buf, ss_out_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ek_dma) || cmh_dma_map_error(ct_dma) ||
+ cmh_dma_map_error(ss_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ vcq_add_qse_ml_kem_enc(&vcq[1], qse_cid, req.k, qse_flags,
+ coin_dma, ek_dma, ct_dma, ss_dma, 0, masked);
+ vcq_add_qse_flush(&vcq[2], qse_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(ss_dma))
+ cmh_dma_unmap_single(ss_dma, ss_out_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ct_dma))
+ cmh_dma_unmap_single(ct_dma, ct_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ek_dma))
+ cmh_dma_unmap_single(ek_dma, ek_len, DMA_TO_DEVICE);
+ if (coin_buf && !cmh_dma_map_error(coin_dma))
+ cmh_dma_unmap_single(coin_dma,
+ masked ? QSE_SEED_LEN_MASKED
+ : QSE_SEED_LEN,
+ DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.ct), ct_buf, ct_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ /* Unmask ss if masked: ss = share0 ^ share1 */
+ if (masked) {
+ crypto_xor(ss_buf, ss_buf + ML_KEM_SS_LEN,
+ ML_KEM_SS_LEN);
+ }
+ if (copy_to_user(u64_to_user_ptr(req.ss), ss_buf,
+ ML_KEM_SS_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ if (copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(ss_buf);
+ kfree(ct_buf);
+ kfree(coin_buf);
+ kfree(ek_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_ml_kem_dec() - Handle CMH_MGMT_IOC_ML_KEM_DEC ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_ml_kem_dec(void __user *argp)
+{
+ u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE);
+
+ struct cmh_ioctl_ml_kem_dec req;
+ struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN];
+ u32 ct_len, dk_len, ss_out_len;
+ u32 qse_flags = 0;
+ bool masked, ds_ref;
+ u8 *ct_buf, *dk_buf = NULL, *ss_buf;
+ dma_addr_t ct_dma, dk_dma = DMA_MAPPING_ERROR, ss_dma;
+ u64 dk_ref;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved || req.__reserved2[0] || req.__reserved2[1])
+ return -EINVAL;
+ if (ml_kem_k_idx(req.k) < 0)
+ return -EINVAL;
+
+ masked = !!(req.flags & CMH_QSE_FLAG_MASKED);
+ ds_ref = !!(req.flags & CMH_QSE_FLAG_DS_REF);
+
+ ct_len = ML_KEM_CT_SIZE(req.k);
+ dk_len = masked ? ML_KEM_DK_SIZE_MASKED(req.k)
+ : ML_KEM_DK_SIZE(req.k);
+ ss_out_len = masked ? ML_KEM_SS_LEN_MASKED : ML_KEM_SS_LEN;
+
+ ct_buf = kmalloc(ct_len, GFP_KERNEL);
+ ss_buf = kzalloc(ss_out_len, GFP_KERNEL);
+ if (!ct_buf || !ss_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(ct_buf, u64_to_user_ptr(req.ct), ct_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ ct_dma = cmh_dma_map_single(ct_buf, ct_len, DMA_TO_DEVICE);
+ ss_dma = cmh_dma_map_single(ss_buf, ss_out_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ct_dma) || cmh_dma_map_error(ss_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ /*
+ * dk: if DS_REF flag is set, req.dk is a DS reference.
+ * Otherwise, copy raw dk from user-space and use extmem DMA.
+ * Masked decaps requires DS ref (polynomial unmasking not supported).
+ */
+ if (ds_ref) {
+ dk_ref = req.dk;
+ qse_flags |= QSE_FLAG_USE_REF;
+ } else {
+ if (masked) {
+ ret = -EINVAL;
+ goto out_unmap;
+ }
+ dk_buf = kmalloc(dk_len, GFP_KERNEL);
+ if (!dk_buf) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ if (copy_from_user(dk_buf, u64_to_user_ptr(req.dk), dk_len)) {
+ ret = -EFAULT;
+ goto out_unmap;
+ }
+ dk_dma = cmh_dma_map_single(dk_buf, dk_len, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(dk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ dk_ref = dk_dma;
+ }
+
+ if (ds_ref) {
+ /*
+ * DS_REF decaps: CMH eSW resolves both dk and ss from DS.
+ * Phase 1: dec stores ss into SYS_REF_TEMP.
+ * Phase 2: sys_data reads ss from SYS_REF_TEMP to DMA.
+ */
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ vcq_add_qse_ml_kem_dec(&vcq[1], qse_cid, req.k, qse_flags,
+ ct_dma, dk_ref, SYS_REF_TEMP,
+ SYS_TYPE_SET(SYS_TYPE_FLAG_PT,
+ CORE_ID_QSE),
+ masked);
+ vcq_add_qse_flush(&vcq[2], qse_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN,
+ 1, MGMT_MBX);
+ if (ret)
+ goto out_unmap;
+
+ /* Phase 2: extract ss from SYS_REF_TEMP */
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ vcq_add_sys_data(&vcq[1], SYS_REF_TEMP, ss_dma,
+ ss_out_len);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN,
+ 1, MGMT_MBX);
+ } else {
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ vcq_add_qse_ml_kem_dec(&vcq[1], qse_cid, req.k, qse_flags,
+ ct_dma, dk_ref, ss_dma, 0, masked);
+ vcq_add_qse_flush(&vcq[2], qse_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN,
+ 1, MGMT_MBX);
+ }
+
+out_unmap:
+ if (dk_buf && !cmh_dma_map_error(dk_dma))
+ cmh_dma_unmap_single(dk_dma, dk_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(ss_dma))
+ cmh_dma_unmap_single(ss_dma, ss_out_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ct_dma))
+ cmh_dma_unmap_single(ct_dma, ct_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (masked) {
+ crypto_xor(ss_buf, ss_buf + ML_KEM_SS_LEN,
+ ML_KEM_SS_LEN);
+ }
+ if (copy_to_user(u64_to_user_ptr(req.ss), ss_buf,
+ ML_KEM_SS_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ if (copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(dk_buf);
+ kfree_sensitive(ss_buf);
+ kfree(ct_buf);
+ return ret;
+}
+
+/* -- PQC -- ML-DSA -- */
+
+/**
+ * cmh_mgmt_ml_dsa_keygen() - Handle CMH_MGMT_IOC_ML_DSA_KEYGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_ml_dsa_keygen(void __user *argp)
+{
+ u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE);
+
+ struct cmh_ioctl_ml_dsa_keygen req;
+ struct vcq_cmd vcq[QSE_VCQ_CMDS_MAX];
+ u32 pk_size, sk_size, seed_len, key_flags;
+ u32 qse_flags = 0;
+ bool masked, ds_ref, hw_rng;
+ u8 *seed_buf = NULL, *pk_buf, *sk_buf = NULL;
+ u64 *ref_buf = NULL;
+ dma_addr_t seed_dma = DMA_MAPPING_ERROR, pk_dma;
+ dma_addr_t sk_dma = DMA_MAPPING_ERROR, ref_dma = DMA_MAPPING_ERROR;
+ int ret, idx, mi;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ mi = ml_dsa_mode_idx(req.mode);
+ if (mi < 0)
+ return -EINVAL;
+ if (req.flags & ~(CMH_QSE_FLAG_MASK | CMH_FLAG_MASK))
+ return -EINVAL;
+
+ masked = !!(req.flags & CMH_QSE_FLAG_MASKED);
+ ds_ref = !!(req.flags & CMH_QSE_FLAG_DS_REF);
+ hw_rng = !!(req.flags & CMH_QSE_FLAG_HW_RNG);
+
+ /*
+ * QSE keys only support PT storage -- the eSW sign path
+ * hardcodes SYS_TYPE_FLAG_PT when reading the key back.
+ * QSE SCA protection uses masking (CMH_QSE_FLAG_MASKED),
+ * not the 2-share mechanism (CMH_FLAG_SCA).
+ */
+ key_flags = req.flags & CMH_FLAG_MASK;
+ if (key_flags && key_flags != CMH_FLAG_PT)
+ return -EINVAL;
+ key_flags = CMH_FLAG_PT;
+
+ if (masked && !ds_ref)
+ return -EINVAL;
+
+ pk_size = ml_dsa_pk_size[mi];
+ sk_size = masked ? ml_dsa_sk_size_masked[mi] : ml_dsa_sk_size[mi];
+ seed_len = masked ? QSE_SEED_LEN_MASKED : QSE_SEED_LEN;
+
+ if (hw_rng)
+ qse_flags |= QSE_FLAG_USE_RNG;
+ if (ds_ref)
+ qse_flags |= QSE_FLAG_USE_REF;
+
+ pk_buf = kzalloc(pk_size, GFP_KERNEL);
+ if (!pk_buf)
+ return -ENOMEM;
+
+ if (!hw_rng && req.seed) {
+ seed_buf = kmalloc(seed_len, GFP_KERNEL);
+ if (!seed_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(seed_buf, u64_to_user_ptr(req.seed),
+ seed_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ if (ds_ref) {
+ ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ if (!ref_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ } else {
+ sk_buf = kzalloc(sk_size, GFP_KERNEL);
+ if (!sk_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ }
+
+ pk_dma = cmh_dma_map_single(pk_buf, pk_size, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(pk_dma)) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (seed_buf) {
+ seed_dma = cmh_dma_map_single(seed_buf, seed_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(seed_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ if (ds_ref) {
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ } else {
+ sk_dma = cmh_dma_map_single(sk_buf, sk_size, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(sk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ idx = 0;
+ if (ds_ref) {
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MAX);
+ idx++;
+ vcq_add_sys_new(&vcq[idx++], req.sk_cid, ref_dma, sk_size);
+ vcq_add_qse_ml_dsa_keygen(&vcq[idx++], qse_cid, req.mode, qse_flags,
+ seed_dma, pk_dma,
+ SYS_REF_LAST,
+ SYS_TYPE_SET(key_flags,
+ CORE_ID_QSE),
+ masked);
+ vcq_add_qse_flush(&vcq[idx++], qse_cid);
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MAX,
+ 1, MGMT_MBX);
+ } else {
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ idx++;
+ vcq_add_qse_ml_dsa_keygen(&vcq[idx++], qse_cid, req.mode, qse_flags,
+ seed_dma, pk_dma,
+ sk_dma, 0, masked);
+ vcq_add_qse_flush(&vcq[idx++], qse_cid);
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN,
+ 1, MGMT_MBX);
+ }
+
+out_unmap:
+ if (ds_ref && !cmh_dma_map_error(ref_dma))
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ if (!ds_ref && sk_buf && !cmh_dma_map_error(sk_dma))
+ cmh_dma_unmap_single(sk_dma, sk_size, DMA_FROM_DEVICE);
+ if (seed_buf && !cmh_dma_map_error(seed_dma))
+ cmh_dma_unmap_single(seed_dma, seed_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(pk_dma))
+ cmh_dma_unmap_single(pk_dma, pk_size, DMA_FROM_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.pk), pk_buf, pk_size)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ if (ds_ref) {
+ req.sk_ref = *ref_buf;
+ } else {
+ if (copy_to_user(u64_to_user_ptr(req.sk),
+ sk_buf, sk_size)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+ if (copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(sk_buf);
+ kfree(ref_buf);
+ kfree_sensitive(seed_buf);
+ kfree(pk_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_ml_dsa_sign() - Handle CMH_MGMT_IOC_ML_DSA_SIGN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_ml_dsa_sign(void __user *argp)
+{
+ u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE);
+
+ struct cmh_ioctl_ml_dsa_sign req;
+ struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN];
+ u32 sig_size, copy_len, rnd_len;
+ u32 qse_flags = 0;
+ bool masked;
+ u8 *m_buf, *sig_buf, *sk_buf = NULL, *rnd_buf = NULL;
+ dma_addr_t m_dma = DMA_MAPPING_ERROR, sig_dma;
+ dma_addr_t sk_dma = DMA_MAPPING_ERROR, rnd_dma = DMA_MAPPING_ERROR;
+ u64 sk_ref;
+ int mi, ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ mi = ml_dsa_mode_idx(req.mode);
+ if (mi < 0)
+ return -EINVAL;
+ if (req.mlen > ML_DSA_MAX_MLEN && req.mlen != ML_DSA_MLEN_EXTERNAL_MU)
+ return -EINVAL;
+
+ masked = !!(req.flags & CMH_QSE_FLAG_MASKED);
+ rnd_len = masked ? QSE_SEED_LEN_MASKED : QSE_SEED_LEN;
+ sig_size = ml_dsa_sig_size[mi];
+ copy_len = (req.mlen == ML_DSA_MLEN_EXTERNAL_MU)
+ ? ML_DSA_EXTMU_LEN : req.mlen;
+
+ /*
+ * sk: if DS_REF, req.sk is a DS reference (masked sk lives in DS).
+ * Otherwise, copy raw sk from user-space.
+ * Masked sign requires DS ref (polynomial unmasking not supported).
+ */
+ if (req.flags & CMH_QSE_FLAG_DS_REF) {
+ sk_ref = req.sk;
+ qse_flags |= QSE_FLAG_USE_REF;
+ } else {
+ u32 sk_size;
+
+ if (masked)
+ return -EINVAL;
+ sk_size = ml_dsa_sk_size[mi];
+ sk_buf = kmalloc(sk_size, GFP_KERNEL);
+ if (!sk_buf)
+ return -ENOMEM;
+ if (copy_from_user(sk_buf, u64_to_user_ptr(req.sk), sk_size)) {
+ kfree_sensitive(sk_buf);
+ return -EFAULT;
+ }
+ }
+
+ m_buf = kmalloc(max_t(u32, copy_len, 1), GFP_KERNEL);
+ sig_buf = kzalloc(sig_size, GFP_KERNEL);
+ if (!m_buf || !sig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_len > 0 &&
+ copy_from_user(m_buf, u64_to_user_ptr(req.m), copy_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ if (req.rnd) {
+ rnd_buf = kmalloc(rnd_len, GFP_KERNEL);
+ if (!rnd_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(rnd_buf, u64_to_user_ptr(req.rnd),
+ rnd_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ if (copy_len > 0) {
+ m_dma = cmh_dma_map_single(m_buf, copy_len, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(m_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+ sig_dma = cmh_dma_map_single(sig_buf, sig_size, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(sig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ if (sk_buf) {
+ sk_dma = cmh_dma_map_single(sk_buf, ml_dsa_sk_size[mi],
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(sk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ sk_ref = sk_dma;
+ }
+
+ if (rnd_buf) {
+ rnd_dma = cmh_dma_map_single(rnd_buf, rnd_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rnd_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ vcq_add_qse_ml_dsa_sign(&vcq[1], qse_cid, req.mode, qse_flags,
+ rnd_dma, m_dma, sk_ref, sig_dma,
+ req.mlen, masked);
+ vcq_add_qse_flush(&vcq[2], qse_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (rnd_buf && !cmh_dma_map_error(rnd_dma))
+ cmh_dma_unmap_single(rnd_dma, rnd_len, DMA_TO_DEVICE);
+ if (sk_buf && !cmh_dma_map_error(sk_dma))
+ cmh_dma_unmap_single(sk_dma, ml_dsa_sk_size[mi],
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_size, DMA_FROM_DEVICE);
+ if (copy_len > 0 && !cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, copy_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.sig), sig_buf, sig_size))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(rnd_buf);
+ kfree(sig_buf);
+ kfree(m_buf);
+ kfree_sensitive(sk_buf);
+ return ret;
+}
+
+/* -- PQC -- SLH-DSA -- */
+
+/**
+ * cmh_mgmt_slhdsa_keygen() - Handle CMH_MGMT_IOC_SLHDSA_KEYGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_slhdsa_keygen(void __user *argp)
+{
+ u32 hcq_cid = cmh_core_default_id(CMH_CORE_HCQ);
+
+ struct cmh_ioctl_slhdsa_keygen req;
+ struct vcq_cmd vcq[HCQ_VCQ_CMDS_MAX];
+ u32 pk_sz, sk_sz, seed_sz, sk_alloc, vcq_cnt, key_flags;
+ bool ds_ref;
+ u8 *seed_buf, *pk_buf, *sk_buf = NULL;
+ u64 *ref_buf = NULL;
+ dma_addr_t seed_dma, pk_dma, sk_dma = DMA_MAPPING_ERROR, ref_dma = DMA_MAPPING_ERROR;
+ int ret, idx;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (req.parameter_set < 1 || req.parameter_set > HCQ_SLHDSA_PARAM_MAX)
+ return -EINVAL;
+ if (req.flags & ~(CMH_QSE_FLAG_DS_REF | CMH_FLAG_MASK))
+ return -EINVAL;
+
+ ds_ref = !!(req.flags & CMH_QSE_FLAG_DS_REF);
+
+ /*
+ * QSE keys only support PT storage -- the eSW sign path
+ * hardcodes SYS_TYPE_FLAG_PT when reading the key back.
+ * HCQ core sets key type internally during keygen.
+ */
+ key_flags = req.flags & CMH_FLAG_MASK;
+ if (key_flags && key_flags != CMH_FLAG_PT)
+ return -EINVAL;
+ (void)key_flags;
+
+ pk_sz = slhdsa_pk_size(req.parameter_set);
+ sk_sz = slhdsa_sk_size(req.parameter_set);
+ seed_sz = slhdsa_seed_size(req.parameter_set);
+
+ seed_buf = kmalloc(seed_sz, GFP_KERNEL);
+ pk_buf = kzalloc(pk_sz, GFP_KERNEL);
+ if (!seed_buf || !pk_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(seed_buf, u64_to_user_ptr(req.seed), seed_sz)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ /*
+ * Both paths need ref_buf for sys_new output. Non-ds_ref also
+ * needs sk_buf (+16 for SYS header) to read back via sys_read.
+ */
+ ref_buf = kzalloc(sizeof(u64), GFP_KERNEL);
+ if (!ref_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (!ds_ref) {
+ sk_alloc = sk_sz + SYS_WRAP_HDR_SIZE;
+ sk_buf = kzalloc(sk_alloc, GFP_KERNEL);
+ if (!sk_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ }
+
+ seed_dma = cmh_dma_map_single(seed_buf, seed_sz, DMA_TO_DEVICE);
+ pk_dma = cmh_dma_map_single(pk_buf, pk_sz, DMA_FROM_DEVICE);
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(seed_dma) || cmh_dma_map_error(pk_dma) ||
+ cmh_dma_map_error(ref_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ if (!ds_ref) {
+ sk_dma = cmh_dma_map_single(sk_buf, sk_alloc,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(sk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ /*
+ * SLH-DSA keygen requires seed and sk as DS references.
+ * VCQ: hdr + sys_new(sk) + sys_write(seed->TEMP) + keygen + [sys_read] + flush
+ */
+ idx = 0;
+ if (ds_ref) {
+ vcq_cnt = HCQ_VCQ_CMDS_MAX - 1; /* hdr+new+write+keygen+flush */
+ vcq_set_header(&vcq[idx++], vcq_cnt);
+ vcq_add_sys_new(&vcq[idx++], req.sk_cid, ref_dma,
+ sk_sz);
+ } else {
+ vcq_cnt = HCQ_VCQ_CMDS_MAX; /* hdr+new+write+keygen+read+flush */
+ vcq_set_header(&vcq[idx++], vcq_cnt);
+ vcq_add_sys_new(&vcq[idx++], SYS_CID_NONE, ref_dma,
+ sk_sz);
+ }
+ vcq_add_sys_write(&vcq[idx++], SYS_REF_TEMP, seed_dma, 0,
+ seed_sz,
+ SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_HCQ));
+ vcq_add_hcq_slhdsa_keygen(&vcq[idx++], hcq_cid, req.parameter_set,
+ seed_sz, pk_sz, sk_sz,
+ SYS_REF_TEMP, pk_dma, SYS_REF_LAST);
+ if (!ds_ref)
+ vcq_add_sys_read(&vcq[idx++], SYS_REF_LAST, sk_dma,
+ 0, sk_sz + SYS_WRAP_HDR_SIZE);
+ vcq_add_hcq_flush(&vcq[idx++], hcq_cid);
+
+ ret = cmh_tm_submit_sync_tmo(vcq, vcq_cnt, 1, MGMT_MBX,
+ cmh_tm_slow_op_timeout_jiffies());
+
+out_unmap:
+ if (!ds_ref && sk_buf && !cmh_dma_map_error(sk_dma))
+ cmh_dma_unmap_single(sk_dma, sk_alloc, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ref_dma))
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(pk_dma))
+ cmh_dma_unmap_single(pk_dma, pk_sz, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(seed_dma))
+ cmh_dma_unmap_single(seed_dma, seed_sz, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.pk), pk_buf, pk_sz)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ if (ds_ref) {
+ req.sk_ref = *ref_buf;
+ } else {
+ if (copy_to_user(u64_to_user_ptr(req.sk),
+ sk_buf + SYS_WRAP_HDR_SIZE,
+ sk_sz)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+ if (copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(sk_buf);
+ kfree(ref_buf);
+ kfree(pk_buf);
+ kfree_sensitive(seed_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_slhdsa_sign() - Handle CMH_MGMT_IOC_SLHDSA_SIGN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_slhdsa_sign(void __user *argp)
+{
+ u32 hcq_cid = cmh_core_default_id(CMH_CORE_HCQ);
+
+ struct cmh_ioctl_slhdsa_sign req;
+ struct vcq_cmd vcq[HCQ_VCQ_CMDS_MIN];
+ u32 sig_sz, n_val;
+ u8 *msg_buf, *ctx_buf = NULL, *sig_buf, *rnd_buf = NULL;
+ dma_addr_t msg_dma = DMA_MAPPING_ERROR, ctx_dma = DMA_MAPPING_ERROR;
+ dma_addr_t sig_dma, rnd_dma = DMA_MAPPING_ERROR;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.parameter_set < 1 || req.parameter_set > HCQ_SLHDSA_PARAM_MAX)
+ return -EINVAL;
+ if (req.msg_len > SLHDSA_MAX_MSG_LEN)
+ return -EINVAL;
+ if (req.ctx_len > SLHDSA_MAX_CTX_LEN)
+ return -EINVAL;
+
+ sig_sz = slhdsa_get_sig_size(req.parameter_set);
+ n_val = slhdsa_n[req.parameter_set - 1];
+
+ msg_buf = kmalloc(max_t(u32, req.msg_len, 1), GFP_KERNEL);
+ sig_buf = kzalloc(sig_sz, GFP_KERNEL);
+ if (!msg_buf || !sig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (req.msg_len > 0 &&
+ copy_from_user(msg_buf, u64_to_user_ptr(req.msg), req.msg_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ if (req.ctx_len > 0 && req.ctx) {
+ ctx_buf = kmalloc(req.ctx_len, GFP_KERNEL);
+ if (!ctx_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(ctx_buf, u64_to_user_ptr(req.ctx),
+ req.ctx_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ if (req.add_random) {
+ rnd_buf = kmalloc(n_val, GFP_KERNEL);
+ if (!rnd_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(rnd_buf, u64_to_user_ptr(req.add_random),
+ n_val)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ sig_dma = cmh_dma_map_single(sig_buf, sig_sz, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(sig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ if (req.msg_len > 0) {
+ msg_dma = cmh_dma_map_single(msg_buf, req.msg_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(msg_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ if (ctx_buf) {
+ ctx_dma = cmh_dma_map_single(ctx_buf, req.ctx_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(ctx_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ if (rnd_buf) {
+ rnd_dma = cmh_dma_map_single(rnd_buf, n_val, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rnd_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ vcq_set_header(&vcq[0], HCQ_VCQ_CMDS_MIN);
+ vcq_add_hcq_slhdsa_sign(&vcq[1], hcq_cid, req.parameter_set,
+ req.msg_len, req.ctx_len,
+ rnd_dma, msg_dma, ctx_dma,
+ req.sk, sig_dma);
+ vcq_add_hcq_flush(&vcq[2], hcq_cid);
+
+ ret = cmh_tm_submit_sync_tmo(vcq, HCQ_VCQ_CMDS_MIN, 1, MGMT_MBX,
+ cmh_tm_slow_op_timeout_jiffies());
+
+out_unmap:
+ if (rnd_buf && !cmh_dma_map_error(rnd_dma))
+ cmh_dma_unmap_single(rnd_dma, n_val, DMA_TO_DEVICE);
+ if (ctx_buf && !cmh_dma_map_error(ctx_dma))
+ cmh_dma_unmap_single(ctx_dma, req.ctx_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_sz, DMA_FROM_DEVICE);
+ if (req.msg_len > 0 && !cmh_dma_map_error(msg_dma))
+ cmh_dma_unmap_single(msg_dma, req.msg_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.sig), sig_buf, sig_sz))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(rnd_buf);
+ kfree(ctx_buf);
+ kfree(sig_buf);
+ kfree(msg_buf);
+ return ret;
+}
+
+/* -- PQC -- SLH-DSA prehash -- */
+
+/**
+ * cmh_mgmt_slhdsa_sign_prehash() - Handle CMH_MGMT_IOC_SLHDSA_SIGN_PREHASH ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_slhdsa_sign_prehash(void __user *argp)
+{
+ u32 hcq_cid = cmh_core_default_id(CMH_CORE_HCQ);
+
+ struct cmh_ioctl_slhdsa_sign_prehash req;
+ struct vcq_cmd vcq[HCQ_VCQ_CMDS_MIN];
+ u32 sig_sz, n_val, hcq_cmd;
+ u8 *msg_buf, *ctx_buf = NULL, *sig_buf, *rnd_buf = NULL;
+ dma_addr_t msg_dma = DMA_MAPPING_ERROR, ctx_dma = DMA_MAPPING_ERROR;
+ dma_addr_t sig_dma, rnd_dma = DMA_MAPPING_ERROR;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.parameter_set < 1 || req.parameter_set > HCQ_SLHDSA_PARAM_MAX)
+ return -EINVAL;
+ if (req.prehash_algo < 1 || req.prehash_algo > HCQ_SLHDSA_PREHASH_SHAKE256)
+ return -EINVAL;
+ if (req.msg_len > SLHDSA_MAX_MSG_LEN)
+ return -EINVAL;
+ if (req.ctx_len > SLHDSA_MAX_CTX_LEN)
+ return -EINVAL;
+
+ hcq_cmd = req.digest ? HCQ_CMD_SLHDSA_SIGN_PREHASH_DIGEST
+ : HCQ_CMD_SLHDSA_SIGN_PREHASH;
+
+ sig_sz = slhdsa_get_sig_size(req.parameter_set);
+ n_val = slhdsa_n[req.parameter_set - 1];
+
+ msg_buf = kmalloc(max_t(u32, req.msg_len, 1), GFP_KERNEL);
+ sig_buf = kzalloc(sig_sz, GFP_KERNEL);
+ if (!msg_buf || !sig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (req.msg_len > 0 &&
+ copy_from_user(msg_buf, u64_to_user_ptr(req.msg), req.msg_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ if (req.ctx_len > 0 && req.ctx) {
+ ctx_buf = kmalloc(req.ctx_len, GFP_KERNEL);
+ if (!ctx_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(ctx_buf, u64_to_user_ptr(req.ctx),
+ req.ctx_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ if (req.add_random) {
+ rnd_buf = kmalloc(n_val, GFP_KERNEL);
+ if (!rnd_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(rnd_buf, u64_to_user_ptr(req.add_random),
+ n_val)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ sig_dma = cmh_dma_map_single(sig_buf, sig_sz, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(sig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ if (req.msg_len > 0) {
+ msg_dma = cmh_dma_map_single(msg_buf, req.msg_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(msg_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ if (ctx_buf) {
+ ctx_dma = cmh_dma_map_single(ctx_buf, req.ctx_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(ctx_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ if (rnd_buf) {
+ rnd_dma = cmh_dma_map_single(rnd_buf, n_val, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rnd_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ vcq_set_header(&vcq[0], HCQ_VCQ_CMDS_MIN);
+ vcq_add_hcq_slhdsa_sign_prehash(&vcq[1], hcq_cid,
+ hcq_cmd, req.parameter_set,
+ req.prehash_algo,
+ req.msg_len, req.ctx_len,
+ rnd_dma, msg_dma, ctx_dma,
+ req.sk, sig_dma);
+ vcq_add_hcq_flush(&vcq[2], hcq_cid);
+
+ ret = cmh_tm_submit_sync_tmo(vcq, HCQ_VCQ_CMDS_MIN, 1, MGMT_MBX,
+ cmh_tm_slow_op_timeout_jiffies());
+
+out_unmap:
+ if (rnd_buf && !cmh_dma_map_error(rnd_dma))
+ cmh_dma_unmap_single(rnd_dma, n_val, DMA_TO_DEVICE);
+ if (ctx_buf && !cmh_dma_map_error(ctx_dma))
+ cmh_dma_unmap_single(ctx_dma, req.ctx_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_sz, DMA_FROM_DEVICE);
+ if (req.msg_len > 0 && !cmh_dma_map_error(msg_dma))
+ cmh_dma_unmap_single(msg_dma, req.msg_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.sig), sig_buf, sig_sz))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(rnd_buf);
+ kfree(ctx_buf);
+ kfree(sig_buf);
+ kfree(msg_buf);
+ return ret;
+}
+
+/* -- EAC (Error and Alarm Controller) ---- */
+
diff --git a/drivers/crypto/cmh/cmh_pke_sm2.c b/drivers/crypto/cmh/cmh_pke_sm2.c
new file mode 100644
index 000000000000..9a6e30c7f5e5
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pke_sm2.c
@@ -0,0 +1,827 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- SM2 PKE Ioctl Handlers
+ *
+ * SM2 (GM/T 0003) is the Chinese national public-key standard over the
+ * sm2p256v1 curve (256-bit). It defines three protocols:
+ *
+ * - Signature: reuses ECDSA sign/verify with SM2_CURVE (0x18), handled
+ * by the existing cmh_mgmt_pke_ecdsa_{sign,verify}() paths.
+ * - Encryption: two-step (ENC_POINT + ENC_HASH / DEC_POINT + DEC_HASH).
+ * - Key Exchange: four-step (ECDH_KEYGEN + ID_DIGEST + ECDH + ECDH_HASH).
+ *
+ * This file implements the 8 SM2-specific ioctl handlers (0x16--0x1D).
+ * Sign/verify/keygen/pubgen use the existing ECDSA/EC paths unchanged.
+ *
+ * VCQ flag convention (from eSW API):
+ * - Most SM2 commands use flags=0 (no swap).
+ * - SM2_DEC_POINT and SM2_ECDH_HASH use PKE_SWAP_FLAGS on the
+ * PKE command itself.
+ * - SM2_ECDH and SM2_ECDH_HASH also apply PKE_SWAP_FLAGS on
+ * their sys_new/sys_data VCQ phases (Weierstrass DS format).
+ */
+
+#include <linux/uaccess.h>
+#include <linux/slab.h>
+
+#include "cmh_pke.h"
+#include "cmh_pke_sm2.h"
+#include "cmh_sys.h"
+#include "cmh_dma.h"
+#include "cmh_txn.h"
+#include "cmh_mgmt.h"
+#include "cmh_sys_abi.h"
+#include <uapi/linux/cmh_mgmt_ioctl.h>
+
+/* SM2 fixed sizes (sm2p256v1: 256-bit curve) */
+#define SM2_CLEN 32U /* coordinate length */
+#define SM2_POINT_LEN 64U /* uncompressed EC point (x||y) */
+#define SM2_SHARED_KEY_LEN 16U /* ECDH shared key output */
+#define SM2_DIGEST_LEN 32U /* SM3 ZA digest */
+#define SM2_NONCE_LEN 32U /* nonce (when caller-provided) */
+/*
+ * SM2 enc_hash/dec_hash payload limit.
+ *
+ * The eSW PKE driver expands the GM/T 0003.4 KDF by issuing a single SM3
+ * invocation per command (one 32-byte block of key stream). Messages
+ * longer than 32 bytes would require ceil(msg_len / 32) SM3 invocations
+ * with an incremented counter, which the eSW does not perform; longer
+ * inputs would silently produce incorrect ciphertext / plaintext.
+ *
+ * The eSW PKE SRAM can physically hold up to 4000 bytes of payload, but
+ * that capacity is unusable until a future eSW change implements the full
+ * KDF expansion. Until then we cap the LKM at the 32-byte limit
+ * documented in Documentation/ABI/testing/cmh-mgmt.
+ */
+#define SM2_MAX_MSG_LEN 32U /* max plaintext for encrypt/decrypt */
+#define SM2_MAX_ID_LEN 32U /* max identity string */
+#define SM2_CT_OVERHEAD 96U /* C1(64) + C3(32) */
+#define SM2_MAX_CT_LEN (SM2_CT_OVERHEAD + SM2_MAX_MSG_LEN) /* 128 */
+
+/* -- SM2_ECDH_KEYGEN ------------------- */
+
+/**
+ * cmh_mgmt_sm2_ecdh_keygen() - Handle CMH_MGMT_IOC_SM2_ECDH_KEYGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_ecdh_keygen(void __user *argp)
+{
+ struct cmh_ioctl_sm2_ecdh_keygen req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 core_id = cmh_core_default_id(CMH_CORE_PKE);
+ u8 *nonce_buf, *sk_buf;
+ dma_addr_t nonce_dma, sk_dma;
+ int nonce_dir;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.nonce_len != 0 && req.nonce_len != SM2_NONCE_LEN)
+ return -EINVAL;
+
+ sk_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL);
+ nonce_buf = kzalloc(SM2_NONCE_LEN, GFP_KERNEL);
+ if (!sk_buf || !nonce_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ /*
+ * nonce_len=32: caller provides ephemeral scalar r (DMA_TO_DEVICE).
+ * nonce_len=0: HW generates r and writes it back (DMA_FROM_DEVICE).
+ * The caller MUST supply a valid nonce pointer in both cases.
+ */
+ if (req.nonce_len) {
+ if (copy_from_user(nonce_buf, u64_to_user_ptr(req.nonce),
+ SM2_NONCE_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ nonce_dir = DMA_TO_DEVICE;
+ } else {
+ nonce_dir = DMA_FROM_DEVICE;
+ }
+
+ sk_dma = cmh_dma_map_single(sk_buf, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ nonce_dma = cmh_dma_map_single(nonce_buf, SM2_NONCE_LEN, nonce_dir);
+ if (cmh_dma_map_error(sk_dma) || cmh_dma_map_error(nonce_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_sm2_ecdh_keygen(&vcq[1], core_id, nonce_dma, sk_dma,
+ req.nonce_len, 0);
+ vcq_add_pke_flush(&vcq[2], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(nonce_dma))
+ cmh_dma_unmap_single(nonce_dma, SM2_NONCE_LEN, nonce_dir);
+ if (!cmh_dma_map_error(sk_dma))
+ cmh_dma_unmap_single(sk_dma, SM2_POINT_LEN, DMA_FROM_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.session_key),
+ sk_buf, SM2_POINT_LEN))
+ ret = -EFAULT;
+ /* Write back HW-generated nonce when nonce_len=0 */
+ if (!ret && !req.nonce_len) {
+ if (copy_to_user(u64_to_user_ptr(req.nonce),
+ nonce_buf, SM2_NONCE_LEN))
+ ret = -EFAULT;
+ }
+ }
+
+out_free:
+ kfree_sensitive(nonce_buf);
+ kfree_sensitive(sk_buf);
+ return ret;
+}
+
+/* -- SM2_ECDH -------------------------- */
+
+/**
+ * cmh_mgmt_sm2_ecdh() - Handle CMH_MGMT_IOC_SM2_ECDH ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_ecdh(void __user *argp)
+{
+ struct cmh_ioctl_sm2_ecdh req;
+ /* Phase 1: hdr + sys_new + sm2_ecdh + pke_flush */
+ struct vcq_cmd vcq[4];
+ u32 sp_type, core_id;
+ u8 *nonce_buf, *peer_pk_buf, *peer_sk_buf, *sp_buf;
+ u64 *ref_buf;
+ dma_addr_t nonce_dma, peer_pk_dma, peer_sk_dma, sp_dma, ref_dma;
+ int nonce_dir, ret, idx;
+ bool keep_ds;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.nonce_len != 0 && req.nonce_len != SM2_NONCE_LEN)
+ return -EINVAL;
+
+ keep_ds = (req.shared_point_ref != 0);
+ sp_type = SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_PKE);
+ core_id = cmh_core_default_id(CMH_CORE_PKE);
+
+ peer_pk_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL);
+ peer_sk_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL);
+ sp_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL);
+ ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ nonce_buf = kzalloc(SM2_NONCE_LEN, GFP_KERNEL);
+ if (!peer_pk_buf || !peer_sk_buf || !sp_buf || !ref_buf ||
+ !nonce_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(peer_pk_buf, u64_to_user_ptr(req.peer_public_key),
+ SM2_POINT_LEN) ||
+ copy_from_user(peer_sk_buf, u64_to_user_ptr(req.peer_session_key),
+ SM2_POINT_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ if (req.nonce_len) {
+ if (copy_from_user(nonce_buf, u64_to_user_ptr(req.nonce),
+ SM2_NONCE_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ nonce_dir = DMA_TO_DEVICE;
+ } else {
+ nonce_dir = DMA_FROM_DEVICE;
+ }
+
+ peer_pk_dma = cmh_dma_map_single(peer_pk_buf, SM2_POINT_LEN,
+ DMA_TO_DEVICE);
+ peer_sk_dma = cmh_dma_map_single(peer_sk_buf, SM2_POINT_LEN,
+ DMA_TO_DEVICE);
+ sp_dma = cmh_dma_map_single(sp_buf, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE);
+ nonce_dma = cmh_dma_map_single(nonce_buf, SM2_NONCE_LEN, nonce_dir);
+
+ if (cmh_dma_map_error(peer_pk_dma) || cmh_dma_map_error(peer_sk_dma) ||
+ cmh_dma_map_error(sp_dma) || cmh_dma_map_error(ref_dma) ||
+ cmh_dma_map_error(nonce_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ /* Phase 1: sys_new(shared_point_ref) + SM2_ECDH(->SYS_REF_LAST) */
+ idx = 0;
+ vcq_set_header(&vcq[idx++], 4);
+ vcq_add_sys_new(&vcq[idx], 0, ref_dma, SM2_POINT_LEN);
+ vcq[idx++].id |= PKE_SWAP_FLAGS;
+ vcq_add_pke_sm2_ecdh(&vcq[idx++], core_id, req.nonce_len, SM2_CLEN,
+ nonce_dma, peer_pk_dma, peer_sk_dma,
+ req.key_ref, SYS_REF_LAST, sp_type, 0);
+ vcq_add_pke_flush(&vcq[idx++], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 4, 1, MGMT_MBX);
+ if (ret)
+ goto out_unmap;
+
+ if (!keep_ds) {
+ /* Sync bounce buffer so CPU sees the DMA-written ref */
+ cmh_dma_sync_for_cpu(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+
+ /* Phase 2: read shared point from DS -> DMA, consuming the slot */
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_data(&vcq[1], *ref_buf, sp_dma, SM2_POINT_LEN);
+ vcq[1].id |= PKE_SWAP_FLAGS;
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+ }
+
+out_unmap:
+ if (!cmh_dma_map_error(nonce_dma))
+ cmh_dma_unmap_single(nonce_dma, SM2_NONCE_LEN, nonce_dir);
+ if (!cmh_dma_map_error(ref_dma))
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(sp_dma))
+ cmh_dma_unmap_single(sp_dma, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(peer_sk_dma))
+ cmh_dma_unmap_single(peer_sk_dma, SM2_POINT_LEN,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(peer_pk_dma))
+ cmh_dma_unmap_single(peer_pk_dma, SM2_POINT_LEN,
+ DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (!keep_ds) {
+ if (copy_to_user(u64_to_user_ptr(req.shared_point),
+ sp_buf, SM2_POINT_LEN))
+ ret = -EFAULT;
+ } else {
+ /* Return DS ref for ECDH_HASH to consume */
+ u64 __user *sp_refp = (__u64 __user *)
+ u64_to_user_ptr(req.shared_point_ref);
+
+ if (put_user(*ref_buf, sp_refp)) {
+ /*
+ * Failed to deliver the DS ref to
+ * userspace. Logically delete the
+ * orphaned slot so it does not leak.
+ */
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_grant(&vcq[1], *ref_buf,
+ 0, 0, 0);
+ vcq_add_sys_flush(&vcq[2]);
+ cmh_tm_submit_sync_mbx(vcq, 3, 1,
+ MGMT_MBX);
+ dev_warn(cmh_dev(), "SM2 ECDH put_user failed, DS slot cleaned up\n");
+ ret = -EFAULT;
+ }
+ }
+ /* Write back HW-generated nonce when nonce_len=0 */
+ if (!ret && !req.nonce_len) {
+ if (copy_to_user(u64_to_user_ptr(req.nonce),
+ nonce_buf, SM2_NONCE_LEN))
+ ret = -EFAULT;
+ }
+ }
+
+out_free:
+ kfree_sensitive(nonce_buf);
+ kfree(ref_buf);
+ kfree_sensitive(sp_buf);
+ kfree(peer_sk_buf);
+ kfree(peer_pk_buf);
+ return ret;
+}
+
+/* -- SM2_DEC_POINT --------------------- */
+
+/**
+ * cmh_mgmt_sm2_dec_point() - Handle CMH_MGMT_IOC_SM2_DEC_POINT ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_dec_point(void __user *argp)
+{
+ struct cmh_ioctl_sm2_dec_point req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 core_id = cmh_core_default_id(CMH_CORE_PKE);
+ u8 *ct_buf, *dp_buf;
+ dma_addr_t ct_dma, dp_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.ciphertext_len <= SM2_CT_OVERHEAD ||
+ req.ciphertext_len > SM2_MAX_CT_LEN)
+ return -EINVAL;
+
+ /* Only need C1 (first 64 bytes) for the sidecar */
+ ct_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL);
+ dp_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL);
+ if (!ct_buf || !dp_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(ct_buf, u64_to_user_ptr(req.ciphertext),
+ SM2_POINT_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ ct_dma = cmh_dma_map_single(ct_buf, SM2_POINT_LEN, DMA_TO_DEVICE);
+ dp_dma = cmh_dma_map_single(dp_buf, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ct_dma) || cmh_dma_map_error(dp_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_sm2_dec_point(&vcq[1], core_id, req.ciphertext_len, SM2_CLEN,
+ ct_dma, dp_dma, req.key_ref,
+ PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[2], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(dp_dma))
+ cmh_dma_unmap_single(dp_dma, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ct_dma))
+ cmh_dma_unmap_single(ct_dma, SM2_POINT_LEN, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.dec_point),
+ dp_buf, SM2_POINT_LEN))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(dp_buf);
+ kfree(ct_buf);
+ return ret;
+}
+
+/* -- SM2_ENC_POINT --------------------- */
+
+/**
+ * cmh_mgmt_sm2_enc_point() - Handle CMH_MGMT_IOC_SM2_ENC_POINT ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_enc_point(void __user *argp)
+{
+ struct cmh_ioctl_sm2_enc_point req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 core_id = cmh_core_default_id(CMH_CORE_PKE);
+ u8 *nonce_buf = NULL, *pk_buf, *ct_buf, *ep_buf;
+ dma_addr_t nonce_dma = DMA_MAPPING_ERROR, pk_dma, ct_dma, ep_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.nonce_len != 0 && req.nonce_len != SM2_NONCE_LEN)
+ return -EINVAL;
+
+ pk_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL);
+ ct_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL);
+ ep_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL);
+ if (!pk_buf || !ct_buf || !ep_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(pk_buf, u64_to_user_ptr(req.public_key),
+ SM2_POINT_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ if (req.nonce_len) {
+ nonce_buf = kmalloc(SM2_NONCE_LEN, GFP_KERNEL);
+ if (!nonce_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(nonce_buf, u64_to_user_ptr(req.nonce),
+ SM2_NONCE_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ pk_dma = cmh_dma_map_single(pk_buf, SM2_POINT_LEN, DMA_TO_DEVICE);
+ ct_dma = cmh_dma_map_single(ct_buf, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ ep_dma = cmh_dma_map_single(ep_buf, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ if (nonce_buf)
+ nonce_dma = cmh_dma_map_single(nonce_buf, SM2_NONCE_LEN,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(pk_dma) || cmh_dma_map_error(ct_dma) ||
+ cmh_dma_map_error(ep_dma) ||
+ (nonce_buf && cmh_dma_map_error(nonce_dma))) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_sm2_enc_point(&vcq[1], core_id, nonce_dma, pk_dma, ct_dma,
+ ep_dma, req.nonce_len, 0);
+ vcq_add_pke_flush(&vcq[2], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (nonce_buf && !cmh_dma_map_error(nonce_dma))
+ cmh_dma_unmap_single(nonce_dma, SM2_NONCE_LEN, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(ep_dma))
+ cmh_dma_unmap_single(ep_dma, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ct_dma))
+ cmh_dma_unmap_single(ct_dma, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(pk_dma))
+ cmh_dma_unmap_single(pk_dma, SM2_POINT_LEN, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.ciphertext),
+ ct_buf, SM2_POINT_LEN) ||
+ copy_to_user(u64_to_user_ptr(req.enc_point),
+ ep_buf, SM2_POINT_LEN))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(nonce_buf);
+ kfree(ep_buf);
+ kfree(ct_buf);
+ kfree(pk_buf);
+ return ret;
+}
+
+/* -- SM2_ID_DIGEST --------------------- */
+
+/**
+ * cmh_mgmt_sm2_id_digest() - Handle CMH_MGMT_IOC_SM2_ID_DIGEST ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_id_digest(void __user *argp)
+{
+ struct cmh_ioctl_sm2_id_digest req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 core_id = cmh_core_default_id(CMH_CORE_PKE);
+ u8 *id_buf, *pk_buf, *dig_buf;
+ dma_addr_t id_dma, pk_dma, dig_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.id_len || req.id_len > SM2_MAX_ID_LEN)
+ return -EINVAL;
+
+ id_buf = kmalloc(req.id_len, GFP_KERNEL);
+ pk_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL);
+ dig_buf = kzalloc(SM2_DIGEST_LEN, GFP_KERNEL);
+ if (!id_buf || !pk_buf || !dig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(id_buf, u64_to_user_ptr(req.id), req.id_len) ||
+ copy_from_user(pk_buf, u64_to_user_ptr(req.public_key),
+ SM2_POINT_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ id_dma = cmh_dma_map_single(id_buf, req.id_len, DMA_TO_DEVICE);
+ pk_dma = cmh_dma_map_single(pk_buf, SM2_POINT_LEN, DMA_TO_DEVICE);
+ dig_dma = cmh_dma_map_single(dig_buf, SM2_DIGEST_LEN,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(id_dma) || cmh_dma_map_error(pk_dma) ||
+ cmh_dma_map_error(dig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_sm2_id_digest(&vcq[1], core_id, id_dma, pk_dma, dig_dma,
+ req.id_len, 0);
+ vcq_add_pke_flush(&vcq[2], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(dig_dma))
+ cmh_dma_unmap_single(dig_dma, SM2_DIGEST_LEN,
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(pk_dma))
+ cmh_dma_unmap_single(pk_dma, SM2_POINT_LEN, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(id_dma))
+ cmh_dma_unmap_single(id_dma, req.id_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.digest),
+ dig_buf, SM2_DIGEST_LEN))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(dig_buf);
+ kfree(pk_buf);
+ kfree(id_buf);
+ return ret;
+}
+
+/* -- SM2_ECDH_HASH --------------------- */
+
+/**
+ * cmh_mgmt_sm2_ecdh_hash() - Handle CMH_MGMT_IOC_SM2_ECDH_HASH ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_ecdh_hash(void __user *argp)
+{
+ struct cmh_ioctl_sm2_ecdh_hash req;
+ /* Phase 1: hdr + sys_new + sm2_ecdh_hash + pke_flush; reused for Phase 2 */
+ struct vcq_cmd vcq[4];
+ u32 sk_type, core_id;
+ u8 *peer_dig_buf, *dig_buf, *sk_buf;
+ u64 *ref_buf;
+ dma_addr_t peer_dig_dma, dig_dma, sk_dma, ref_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+
+ sk_type = SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_PKE);
+ core_id = cmh_core_default_id(CMH_CORE_PKE);
+
+ peer_dig_buf = kmalloc(SM2_DIGEST_LEN, GFP_KERNEL);
+ dig_buf = kmalloc(SM2_DIGEST_LEN, GFP_KERNEL);
+ sk_buf = kzalloc(SM2_SHARED_KEY_LEN, GFP_KERNEL);
+ ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ if (!peer_dig_buf || !dig_buf || !sk_buf || !ref_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(peer_dig_buf, u64_to_user_ptr(req.peer_id_digest),
+ SM2_DIGEST_LEN) ||
+ copy_from_user(dig_buf, u64_to_user_ptr(req.id_digest),
+ SM2_DIGEST_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ peer_dig_dma = cmh_dma_map_single(peer_dig_buf, SM2_DIGEST_LEN,
+ DMA_TO_DEVICE);
+ dig_dma = cmh_dma_map_single(dig_buf, SM2_DIGEST_LEN, DMA_TO_DEVICE);
+ sk_dma = cmh_dma_map_single(sk_buf, SM2_SHARED_KEY_LEN,
+ DMA_FROM_DEVICE);
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(peer_dig_dma) || cmh_dma_map_error(dig_dma) ||
+ cmh_dma_map_error(sk_dma) || cmh_dma_map_error(ref_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ /*
+ * Phase 1: sys_new(shared_key_ref) + SM2_ECDH_HASH
+ * The shared_point_ref from the ECDH step is passed directly
+ * as a DS reference -- the eSW hub reads it from DS.
+ */
+ vcq_set_header(&vcq[0], 4);
+ vcq_add_sys_new(&vcq[1], 0, ref_dma, SM2_SHARED_KEY_LEN);
+ vcq[1].id |= PKE_SWAP_FLAGS;
+ vcq_add_pke_sm2_ecdh_hash(&vcq[2], core_id, peer_dig_dma, dig_dma,
+ req.shared_point_ref, SYS_REF_LAST,
+ sk_type, PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[3], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 4, 1, MGMT_MBX);
+ if (ret)
+ goto out_unmap;
+
+ /* Sync bounce buffer so CPU sees the DMA-written ref */
+ cmh_dma_sync_for_cpu(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+
+ /* Phase 2: read shared key from DS -> DMA */
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_data(&vcq[1], *ref_buf, sk_dma, SM2_SHARED_KEY_LEN);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(ref_dma))
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(sk_dma))
+ cmh_dma_unmap_single(sk_dma, SM2_SHARED_KEY_LEN,
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(dig_dma))
+ cmh_dma_unmap_single(dig_dma, SM2_DIGEST_LEN, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(peer_dig_dma))
+ cmh_dma_unmap_single(peer_dig_dma, SM2_DIGEST_LEN,
+ DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.shared_key),
+ sk_buf, SM2_SHARED_KEY_LEN))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(ref_buf);
+ kfree_sensitive(sk_buf);
+ kfree(dig_buf);
+ kfree(peer_dig_buf);
+ return ret;
+}
+
+/* -- SM2_DEC_HASH ---------------------- */
+
+/**
+ * cmh_mgmt_sm2_dec_hash() - Handle CMH_MGMT_IOC_SM2_DEC_HASH ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_dec_hash(void __user *argp)
+{
+ struct cmh_ioctl_sm2_dec_hash req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 msg_len, core_id;
+ u8 *ct_buf, *dp_buf, *pt_buf;
+ dma_addr_t ct_dma, dp_dma, pt_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.ciphertext_len <= SM2_CT_OVERHEAD ||
+ req.ciphertext_len > SM2_MAX_CT_LEN)
+ return -EINVAL;
+
+ msg_len = req.ciphertext_len - SM2_CT_OVERHEAD;
+ core_id = cmh_core_default_id(CMH_CORE_PKE);
+
+ ct_buf = kmalloc(req.ciphertext_len, GFP_KERNEL);
+ dp_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL);
+ pt_buf = kzalloc(msg_len, GFP_KERNEL);
+ if (!ct_buf || !dp_buf || !pt_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(ct_buf, u64_to_user_ptr(req.ciphertext),
+ req.ciphertext_len) ||
+ copy_from_user(dp_buf, u64_to_user_ptr(req.dec_point),
+ SM2_POINT_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ ct_dma = cmh_dma_map_single(ct_buf, req.ciphertext_len,
+ DMA_TO_DEVICE);
+ dp_dma = cmh_dma_map_single(dp_buf, SM2_POINT_LEN, DMA_TO_DEVICE);
+ pt_dma = cmh_dma_map_single(pt_buf, msg_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ct_dma) || cmh_dma_map_error(dp_dma) ||
+ cmh_dma_map_error(pt_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_sm2_dec_hash(&vcq[1], core_id, ct_dma, dp_dma, pt_dma,
+ req.ciphertext_len, 0);
+ vcq_add_pke_flush(&vcq[2], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(pt_dma))
+ cmh_dma_unmap_single(pt_dma, msg_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(dp_dma))
+ cmh_dma_unmap_single(dp_dma, SM2_POINT_LEN, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(ct_dma))
+ cmh_dma_unmap_single(ct_dma, req.ciphertext_len,
+ DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.plaintext),
+ pt_buf, msg_len))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(pt_buf);
+ kfree_sensitive(dp_buf);
+ kfree(ct_buf);
+ return ret;
+}
+
+/* -- SM2_ENC_HASH ---------------------- */
+
+/**
+ * cmh_mgmt_sm2_enc_hash() - Handle CMH_MGMT_IOC_SM2_ENC_HASH ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_enc_hash(void __user *argp)
+{
+ struct cmh_ioctl_sm2_enc_hash req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 ct_len, core_id;
+ u8 *msg_buf, *ep_buf, *ct_buf;
+ dma_addr_t msg_dma, ep_dma, ct_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.message_len || req.message_len > SM2_MAX_MSG_LEN)
+ return -EINVAL;
+
+ ct_len = SM2_CT_OVERHEAD + req.message_len;
+ core_id = cmh_core_default_id(CMH_CORE_PKE);
+
+ msg_buf = kmalloc(req.message_len, GFP_KERNEL);
+ ep_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL);
+ ct_buf = kzalloc(ct_len, GFP_KERNEL);
+ if (!msg_buf || !ep_buf || !ct_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(msg_buf, u64_to_user_ptr(req.message),
+ req.message_len) ||
+ copy_from_user(ep_buf, u64_to_user_ptr(req.enc_point),
+ SM2_POINT_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ msg_dma = cmh_dma_map_single(msg_buf, req.message_len, DMA_TO_DEVICE);
+ ep_dma = cmh_dma_map_single(ep_buf, SM2_POINT_LEN, DMA_TO_DEVICE);
+ ct_dma = cmh_dma_map_single(ct_buf, ct_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(msg_dma) || cmh_dma_map_error(ep_dma) ||
+ cmh_dma_map_error(ct_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_sm2_enc_hash(&vcq[1], core_id, msg_dma, ep_dma, ct_dma,
+ req.message_len, 0);
+ vcq_add_pke_flush(&vcq[2], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(ct_dma))
+ cmh_dma_unmap_single(ct_dma, ct_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ep_dma))
+ cmh_dma_unmap_single(ep_dma, SM2_POINT_LEN, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(msg_dma))
+ cmh_dma_unmap_single(msg_dma, req.message_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.ciphertext),
+ ct_buf, ct_len))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(ct_buf);
+ kfree(ep_buf);
+ kfree_sensitive(msg_buf);
+ return ret;
+}
diff --git a/drivers/crypto/cmh/cmh_sys.c b/drivers/crypto/cmh/cmh_sys.c
new file mode 100644
index 000000000000..b01d058e6d89
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_sys.c
@@ -0,0 +1,376 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- SYS Core VCQ Builders
+ *
+ * VCQ builder functions for SYS core datastore commands. Each function
+ * populates a single vcq_cmd slot. Callers (cmh_mgmt.c, cmh_key.c)
+ * assemble complete VCQs by combining header + command(s) + flush,
+ * then submit via cmh_tm_submit_sync().
+ *
+ * Hardware-required datastore semantics
+ * --------------------------------------
+ * The commands below (NEW, WRITE, DATA, FIND, DELETE, FLUSH) are
+ * direct mappings of the eSW firmware SYS core command set. The
+ * eSW maintains per-mailbox datastore namespaces with two object
+ * classes:
+ *
+ * SYS_REF_TEMP -- Temporary objects. Lifetime is scoped to the
+ * current mailbox slot; reclaimed automatically
+ * when the slot is reused or on explicit FLUSH.
+ * Used for raw-key provisioning on every VCQ.
+ *
+ * SYS_REF_PERSIST -- Persistent objects. Survive across slots;
+ * require explicit DELETE to reclaim. Identified
+ * by a 64-bit Content ID (CID) and resolved to
+ * a per-MBX ref via SYS_CMD_FIND.
+ *
+ * These semantics are hardware requirements, not driver policy.
+ * The per-MBX temp-stack and per-MBX ref namespace are eSW firmware
+ * design constraints that cannot be changed by the kernel driver.
+ */
+
+#include <linux/string.h>
+
+#include "cmh_sys.h"
+
+/**
+ * vcq_add_sys_flush() - Build a SYS_FLUSH VCQ command
+ * @slot: VCQ command slot to populate
+ */
+void vcq_add_sys_flush(struct vcq_cmd *slot)
+{
+ vcq_add_flush(slot, CORE_ID_SYS);
+}
+
+/**
+ * vcq_add_sys_new() - Build a SYS_NEW VCQ command
+ * @slot: VCQ command slot to populate
+ * @cid: Content identifier for the new datastore object
+ * @ref_dma: DMA address of the object reference buffer
+ * @len: Length of the object data in bytes
+ */
+void vcq_add_sys_new(struct vcq_cmd *slot, u64 cid, u64 ref_dma, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_NEW);
+ slot->hwc.sys.cmd_new.cid = cid;
+ slot->hwc.sys.cmd_new.ref = ref_dma;
+ slot->hwc.sys.cmd_new.len = len;
+}
+
+/**
+ * vcq_add_sys_write() - Build a SYS_WRITE VCQ command
+ * @slot: VCQ command slot to populate
+ * @ref: Datastore object reference handle
+ * @src_dma: DMA address of source data buffer
+ * @wrap_key: Wrapping key reference (0 if none)
+ * @len: Length of data to write in bytes
+ * @sys_type: Datastore object type identifier
+ */
+void vcq_add_sys_write(struct vcq_cmd *slot, u64 ref, u64 src_dma,
+ u64 wrap_key, u32 len, u32 sys_type)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_WRITE);
+ slot->hwc.sys.cmd_write.ref = ref;
+ slot->hwc.sys.cmd_write.src = src_dma;
+ slot->hwc.sys.cmd_write.key = wrap_key;
+ slot->hwc.sys.cmd_write.len = len;
+ slot->hwc.sys.cmd_write.type = sys_type;
+}
+
+/**
+ * vcq_add_sys_read() - Build a SYS_READ VCQ command
+ * @slot: VCQ command slot to populate
+ * @ref: Datastore object reference handle
+ * @dst_dma: DMA address of destination buffer
+ * @wrap_key: Wrapping key reference (0 if none)
+ * @len: Length of data to read in bytes
+ */
+void vcq_add_sys_read(struct vcq_cmd *slot, u64 ref, u64 dst_dma,
+ u64 wrap_key, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_READ);
+ slot->hwc.sys.cmd_read.ref = ref;
+ slot->hwc.sys.cmd_read.dst = dst_dma;
+ slot->hwc.sys.cmd_read.key = wrap_key;
+ slot->hwc.sys.cmd_read.len = len;
+}
+
+/**
+ * vcq_add_sys_data() - Build a SYS_DATA VCQ command
+ * @slot: VCQ command slot to populate
+ * @ref: Datastore object reference handle
+ * @dst_dma: DMA address of destination buffer
+ * @len: Length of data section to read in bytes
+ */
+void vcq_add_sys_data(struct vcq_cmd *slot, u64 ref, u64 dst_dma, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_DATA);
+ slot->hwc.sys.cmd_data.ref = ref;
+ slot->hwc.sys.cmd_data.dst = dst_dma;
+ slot->hwc.sys.cmd_data.len = len;
+}
+
+/**
+ * vcq_add_sys_find() - Build a SYS_FIND VCQ command
+ * @slot: VCQ command slot to populate
+ * @cid: Content identifier to search for
+ * @dst_dma: DMA address of destination buffer for result
+ * @len: Length of destination buffer in bytes
+ */
+void vcq_add_sys_find(struct vcq_cmd *slot, u64 cid, u64 dst_dma, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_FIND);
+ slot->hwc.sys.cmd_find.cid = cid;
+ slot->hwc.sys.cmd_find.dst = dst_dma;
+ slot->hwc.sys.cmd_find.len = len;
+}
+
+/**
+ * vcq_add_sys_list() - Build a SYS_LIST VCQ command
+ * @slot: VCQ command slot to populate
+ * @ref: Datastore object reference for enumeration start
+ * @dst_dma: DMA address of destination buffer for list
+ * @len: Length of destination buffer in bytes
+ */
+void vcq_add_sys_list(struct vcq_cmd *slot, u64 ref, u64 dst_dma, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_LIST);
+ slot->hwc.sys.cmd_list.ref = ref;
+ slot->hwc.sys.cmd_list.dst = dst_dma;
+ slot->hwc.sys.cmd_list.len = len;
+}
+
+/**
+ * vcq_add_sys_grant() - Build a SYS_GRANT VCQ command
+ * @slot: VCQ command slot to populate
+ * @ref: Datastore object reference handle
+ * @read: Read permission bitmask
+ * @write: Write permission bitmask
+ * @execute: Execute permission bitmask
+ */
+void vcq_add_sys_grant(struct vcq_cmd *slot, u64 ref, u64 read,
+ u64 write, u64 execute)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_GRANT);
+ slot->hwc.sys.cmd_grant.ref = ref;
+ slot->hwc.sys.cmd_grant.read = read;
+ slot->hwc.sys.cmd_grant.write = write;
+ slot->hwc.sys.cmd_grant.execute = execute;
+}
+
+/**
+ * vcq_add_sys_export() - Build a SYS_EXPORT VCQ command
+ * @slot: VCQ command slot to populate
+ * @cid: Content identifier of object to export
+ * @dst_dma: DMA address of destination buffer for wrapped blob
+ * @wrap_key: Wrapping key reference for export
+ * @len: Length of destination buffer in bytes
+ */
+void vcq_add_sys_export(struct vcq_cmd *slot, u64 cid, u64 dst_dma,
+ u64 wrap_key, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_EXPORT);
+ slot->hwc.sys.cmd_export.cid = cid;
+ slot->hwc.sys.cmd_export.dst = dst_dma;
+ slot->hwc.sys.cmd_export.key = wrap_key;
+ slot->hwc.sys.cmd_export.len = len;
+}
+
+/**
+ * vcq_add_sys_import() - Build a SYS_IMPORT VCQ command
+ * @slot: VCQ command slot to populate
+ * @src_dma: DMA address of wrapped datastore blob to import
+ * @wrap_key: Wrapping key reference for unwrapping
+ * @len: Length of wrapped blob in bytes
+ */
+void vcq_add_sys_import(struct vcq_cmd *slot, u64 src_dma,
+ u64 wrap_key, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_IMPORT);
+ slot->hwc.sys.cmd_import.src = src_dma;
+ slot->hwc.sys.cmd_import.key = wrap_key;
+ slot->hwc.sys.cmd_import.len = len;
+}
+
+/* -- KIC Core VCQ Builders --------------------- */
+
+/**
+ * vcq_add_kic_hkdf1() - Build a KIC HKDF-Expand VCQ command
+ * @slot: VCQ command slot to populate
+ * @dst: Datastore reference for derived key output
+ * @base: Datastore reference for base key input
+ * @label_dma: DMA address of HKDF label/info buffer
+ * @key_len: Derived key length in bytes
+ * @label_len: Length of label buffer in bytes
+ * @type: Derived key datastore type
+ */
+void vcq_add_kic_hkdf1(struct vcq_cmd *slot, u64 dst, u64 base,
+ u64 label_dma, u32 key_len, u32 label_len, u32 type)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_KIC, 0, 1, KIC_CMD_HKDF1);
+ slot->hwc.kic.cmd_hkdf1.dst = dst;
+ slot->hwc.kic.cmd_hkdf1.base = base;
+ slot->hwc.kic.cmd_hkdf1.label = label_dma;
+ slot->hwc.kic.cmd_hkdf1.llen = label_len;
+ slot->hwc.kic.cmd_hkdf1.len = key_len;
+ slot->hwc.kic.cmd_hkdf1.type = type;
+}
+
+/**
+ * vcq_add_kic_hkdf2() - Build a KIC HKDF-with-salt VCQ command
+ * @slot: VCQ command slot to populate
+ * @dst: Datastore reference for derived key output
+ * @base: Datastore reference for base key input
+ * @salt: Datastore reference for HKDF salt key
+ * @label_dma: DMA address of HKDF label/info buffer
+ * @key_len: Derived key length in bytes
+ * @label_len: Length of label buffer in bytes
+ * @type: Derived key datastore type
+ */
+void vcq_add_kic_hkdf2(struct vcq_cmd *slot, u64 dst, u64 base, u64 salt,
+ u64 label_dma, u32 key_len, u32 label_len, u32 type)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_KIC, 0, 1, KIC_CMD_HKDF2);
+ slot->hwc.kic.cmd_hkdf2.dst = dst;
+ slot->hwc.kic.cmd_hkdf2.base = base;
+ slot->hwc.kic.cmd_hkdf2.salt = salt;
+ slot->hwc.kic.cmd_hkdf2.label = label_dma;
+ slot->hwc.kic.cmd_hkdf2.llen = label_len;
+ slot->hwc.kic.cmd_hkdf2.len = key_len;
+ slot->hwc.kic.cmd_hkdf2.type = type;
+}
+
+/**
+ * vcq_add_kic_aes_cmac_kdf() - Build a KIC AES-CMAC KDF VCQ command
+ * @slot: VCQ command slot to populate
+ * @out_key: Datastore reference for derived key output
+ * @base_key: Datastore reference for base key input
+ * @label_dma: DMA address of KDF label buffer
+ * @key_len: Derived key length in bytes
+ * @label_len: Length of label buffer in bytes
+ * @type: Derived key datastore type
+ */
+void vcq_add_kic_aes_cmac_kdf(struct vcq_cmd *slot, u64 out_key, u64 base_key,
+ u64 label_dma, u32 key_len, u32 label_len,
+ u32 type)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_KIC, 0, 1, KIC_CMD_AES_CMAC_KDF);
+ slot->hwc.kic.cmd_aes_cmac_kdf.base_key = base_key;
+ slot->hwc.kic.cmd_aes_cmac_kdf.out_key = out_key;
+ slot->hwc.kic.cmd_aes_cmac_kdf.label = label_dma;
+ slot->hwc.kic.cmd_aes_cmac_kdf.key_len = key_len;
+ slot->hwc.kic.cmd_aes_cmac_kdf.label_len = label_len;
+ slot->hwc.kic.cmd_aes_cmac_kdf.type = type;
+}
+
+/**
+ * vcq_add_kic_dkek_derive() - Build a KIC DKEK derivation VCQ command
+ * @slot: VCQ command slot to populate
+ * @out_key: Datastore reference for derived DKEK output
+ * @base_key: Datastore reference for base key input
+ * @host_id: Host identifier for key binding
+ * @metadata_dma: DMA address of derivation metadata buffer
+ * @metadata_len: Length of metadata buffer in bytes
+ */
+void vcq_add_kic_dkek_derive(struct vcq_cmd *slot, u64 out_key, u64 base_key,
+ u32 host_id, u64 metadata_dma, u32 metadata_len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_KIC, 0, 1, KIC_CMD_DKEK_DERIVE);
+ slot->hwc.kic.cmd_dkek_derive.base_key = base_key;
+ slot->hwc.kic.cmd_dkek_derive.out_key = out_key;
+ slot->hwc.kic.cmd_dkek_derive.host_id = host_id;
+ slot->hwc.kic.cmd_dkek_derive.metadata = metadata_dma;
+ slot->hwc.kic.cmd_dkek_derive.metadata_len = metadata_len;
+}
+
+/* -- DRBG Core VCQ Builders -------------------- */
+
+/**
+ * vcq_add_drbg_reset() - Build a DRBG reset VCQ command
+ * @slot: VCQ command slot to populate
+ *
+ * Issues DRBG_CMD_RESET which clears the instantiated state, allowing
+ * a subsequent CONFIG to proceed without a double-instantiate error.
+ */
+void vcq_add_drbg_reset(struct vcq_cmd *slot)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_DRBG, 0, 1, DRBG_CMD_RESET);
+}
+
+/**
+ * vcq_add_drbg_config() - Build a DRBG configuration VCQ command
+ * @slot: VCQ command slot to populate
+ * @ratio: Entropy-to-output ratio
+ * @strength: Security strength in bits
+ */
+void vcq_add_drbg_config(struct vcq_cmd *slot, u32 ratio, u32 strength)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_DRBG, 0, 1, DRBG_CMD_CONFIG);
+ slot->hwc.drbg.cmd_config.entropy_ratio = ratio;
+ slot->hwc.drbg.cmd_config.security_strength = strength;
+}
+
+/**
+ * vcq_add_drbg_datastore() - Build a DRBG datastore setup VCQ command
+ * @slot: VCQ command slot to populate
+ * @ref: Datastore object reference handle
+ * @len: Length of datastore allocation in bytes
+ * @type: Datastore object type
+ */
+void vcq_add_drbg_datastore(struct vcq_cmd *slot, u64 ref, u32 len, u32 type)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_DRBG, 0, 1, DRBG_CMD_DATASTORE);
+ slot->hwc.drbg.cmd_datastore.ref = ref;
+ slot->hwc.drbg.cmd_datastore.len = len;
+ slot->hwc.drbg.cmd_datastore.type = type;
+}
+
+/* -- EAC Core VCQ Builder ---------------------- */
+
+/**
+ * vcq_add_eac_read() - Build an EAC read VCQ command
+ * @slot: VCQ command slot to populate
+ * @dst_dma: DMA address of destination buffer
+ * @len: Length of data to read in bytes
+ */
+void vcq_add_eac_read(struct vcq_cmd *slot, u64 dst_dma, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_EAC, 0, 1, EAC_CMD_READ);
+ slot->hwc.eac.cmd_read.dst = dst_dma;
+ slot->hwc.eac.cmd_read.len = len;
+}
diff --git a/drivers/crypto/cmh/include/cmh_key.h b/drivers/crypto/cmh/include/cmh_key.h
new file mode 100644
index 000000000000..bad69c92b892
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_key.h
@@ -0,0 +1,82 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Per-transform key context
+ *
+ * Per-transform key context used by all keyed crypto algorithms (AES,
+ * SM4, CCP, HMAC, KMAC). Stores raw key bytes supplied via the crypto
+ * API .setkey() callback: the key is DMA-mapped once at setkey time and
+ * written to SYS_REF_TEMP in every VCQ.
+ *
+ * Each keyed algorithm driver embeds a struct cmh_key_ctx in its
+ * per-transform context and calls cmh_key_setkey_raw() from its
+ * .setkey() callback.
+ *
+ * Raw-key atomicity (SYS_REF_TEMP)
+ * ---------------------------------
+ * SYS_CMD_WRITE to SYS_REF_TEMP is packed into the same VCQ as the
+ * algorithm commands (AES_CMD_INIT, HC_CMD_HMAC, etc.). SYS_REF_TEMP
+ * is per-MBX -- the CMH eSW allocates it in the tail of each mailbox's
+ * own VCQ buffer (mbx_alloc_temp), so concurrent raw-key requests on
+ * different MBXes do not interfere.
+ */
+
+#ifndef CMH_KEY_H
+#define CMH_KEY_H
+
+#include <linux/types.h>
+#include "cmh_config.h"
+#include "cmh_vcq.h"
+
+/* Key context mode */
+enum cmh_key_mode {
+ CMH_KEY_NONE = 0, /* no key set yet */
+ CMH_KEY_RAW, /* raw key bytes in memory */
+};
+
+/* Per-transform key context */
+struct cmh_key_ctx {
+ enum cmh_key_mode mode;
+ struct {
+ u8 *data; /* kmemdup'd raw key bytes */
+ u32 len; /* key length in bytes */
+ u32 sys_type; /* SYS_TYPE_SET(flags, core_id) */
+ dma_addr_t dma; /* pre-mapped DMA addr (DMA_TO_DEVICE) */
+ } raw;
+};
+
+/**
+ * cmh_key_setkey_raw() - Store raw key bytes in the transform context
+ * @ctx: Per-transform key context
+ * @key: Raw key bytes
+ * @keylen: Key length in bytes
+ * @core_id: Target algorithm core (e.g. CORE_ID_AES)
+ *
+ * SYS_TYPE_FLAG_PT is set so the written temp key
+ * can be read back as plaintext if needed. The actual SYS_CMD_WRITE
+ * to SYS_REF_TEMP is deferred to each encrypt/decrypt VCQ, where it
+ * is packed inline for atomicity.
+ *
+ * Return: 0 on success, -ENOMEM on allocation failure.
+ */
+int cmh_key_setkey_raw(struct cmh_key_ctx *ctx, const u8 *key,
+ u32 keylen, u32 core_id);
+
+/**
+ * cmh_key_destroy() - Free key resources
+ * @ctx: Per-transform key context
+ *
+ * Zeroises and frees the raw key buffer.
+ */
+void cmh_key_destroy(struct cmh_key_ctx *ctx);
+
+/**
+ * cmh_ds_type_to_core_id() - Map datastore key type to core ID
+ * @ds_type: CMH_DS_* key type constant
+ *
+ * Return: Corresponding CORE_ID_*, or CORE_ID_NUM (0x1F) on
+ * unrecognised type (caller should return -EINVAL).
+ */
+u32 cmh_ds_type_to_core_id(u32 ds_type);
+
+#endif /* CMH_KEY_H */
diff --git a/drivers/crypto/cmh/include/cmh_mgmt.h b/drivers/crypto/cmh/include/cmh_mgmt.h
new file mode 100644
index 000000000000..b211014bd71d
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_mgmt.h
@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH -- Key Management misc_device (/dev/cmh_mgmt)
+ *
+ * ioctl interface for key CRUD + datastore export/import,
+ * PKE operations (RSA, ECDSA, ECDH, EdDSA),
+ * and PQC operations (ML-KEM, ML-DSA, SLH-DSA).
+ *
+ * Registered alongside crypto algorithms in module_init,
+ * unregistered before them in module_exit.
+ */
+
+#ifndef CMH_MGMT_H
+#define CMH_MGMT_H
+
+#ifdef CONFIG_CRYPTO_DEV_CMH_MGMT
+
+/*
+ * Pin all mgmt ioctls to MBX 0 for DS ownership and SYS_REF_TEMP scope.
+ * Shared by cmh_mgmt.c, cmh_mgmt_pke.c, cmh_mgmt_pqc.c, cmh_pke_sm2.c.
+ */
+#define MGMT_MBX 0
+
+/* Maximum DMA buffer size for key data / datastore blobs */
+#define CMH_MGMT_MAX_DATA_LEN (256 * 1024) /* 256 KB */
+
+int cmh_mgmt_register(void);
+void cmh_mgmt_unregister(void);
+
+/* -- PKE ioctl handlers (cmh_mgmt_pke.c) -- */
+int cmh_mgmt_pke_rsa_enc(void __user *argp);
+int cmh_mgmt_pke_rsa_dec(void __user *argp);
+int cmh_mgmt_pke_rsa_crt_dec(void __user *argp);
+int cmh_mgmt_pke_rsa_keygen(void __user *argp);
+int cmh_mgmt_pke_ecdsa_sign(void __user *argp);
+int cmh_mgmt_pke_ecdh(void __user *argp);
+int cmh_mgmt_pke_ecdh_keygen(void __user *argp);
+int cmh_mgmt_pke_eddsa_sign(void __user *argp);
+int cmh_mgmt_pke_eddsa_verify(void __user *argp);
+int cmh_mgmt_pke_ec_keygen(void __user *argp);
+int cmh_mgmt_pke_ec_pubgen(void __user *argp);
+int cmh_mgmt_pke_eddsa_keygen_sca(void __user *argp);
+
+/* -- PQC ioctl handlers (cmh_mgmt_pqc.c) -- */
+int cmh_mgmt_ml_kem_keygen(void __user *argp);
+int cmh_mgmt_ml_kem_enc(void __user *argp);
+int cmh_mgmt_ml_kem_dec(void __user *argp);
+int cmh_mgmt_ml_dsa_keygen(void __user *argp);
+int cmh_mgmt_ml_dsa_sign(void __user *argp);
+int cmh_mgmt_slhdsa_keygen(void __user *argp);
+int cmh_mgmt_slhdsa_sign(void __user *argp);
+int cmh_mgmt_slhdsa_sign_prehash(void __user *argp);
+
+#else /* !CONFIG_CRYPTO_DEV_CMH_MGMT */
+
+static inline int cmh_mgmt_register(void) { return 0; }
+static inline void cmh_mgmt_unregister(void) { }
+
+#endif /* CONFIG_CRYPTO_DEV_CMH_MGMT */
+
+#endif /* CMH_MGMT_H */
diff --git a/drivers/crypto/cmh/include/cmh_pke.h b/drivers/crypto/cmh/include/cmh_pke.h
new file mode 100644
index 000000000000..dcfdb3fc3cd6
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_pke.h
@@ -0,0 +1,245 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- PKE Common Types and Helpers
+ *
+ * Shared definitions for RSA, ECDSA, ECDH, EdDSA, and SM2 drivers.
+ * Curve -> coordinate-length mapping, VCQ byte-swap flags, and
+ * common VCQ builder prototypes.
+ */
+
+#ifndef CMH_PKE_H
+#define CMH_PKE_H
+
+#include <linux/types.h>
+#include "cmh_vcq.h"
+#include "cmh_pke_abi.h"
+
+/* VCQ byte-swap flags for DMA transfers (per CMH VCQ ABI) */
+#define VCQ_FLAG_SWAP_BYTES 0x400000U
+#define VCQ_FLAG_SWAP_WORDS 0x200000U
+
+/* VCQ byte-swap flags for PKE -- big-endian data on LE bus */
+#define PKE_SWAP_FLAGS (VCQ_FLAG_SWAP_BYTES | VCQ_FLAG_SWAP_WORDS)
+
+/* VCQ layout: header + [SYS_WRITE] + PKE_CMD + flush */
+#define PKE_VCQ_CMDS_MIN 3 /* header + cmd + flush */
+#define PKE_VCQ_CMDS_MAX 4 /* header + SYS_WRITE + cmd + flush */
+
+/* Max RSA key size in bytes (4096 bits) */
+#define PKE_RSA_MAX_BYTES 512
+#define PKE_RSA_MIN_BITS 1024
+#define PKE_RSA_MAX_BITS 4096
+
+/* EdDSA SCA: Ed448 blinded private key length (bytes) */
+#define PKE_ED448_SK_SCA_LEN 226
+
+/**
+ * pke_curve_clen() - Get EC curve coordinate length in bytes
+ * @curve: PKE curve identifier (PKE_CURVE_*)
+ *
+ * Return: Coordinate length in bytes, or 0 for unknown curves.
+ */
+static inline u32 pke_curve_clen(u32 curve)
+{
+ switch (curve) {
+ case PKE_CURVE_P192:
+ case PKE_CURVE_BP192R1:
+ return 24;
+ case PKE_CURVE_P224:
+ case PKE_CURVE_BP224R1:
+ return 28;
+ case PKE_CURVE_P256:
+ case PKE_CURVE_SECP256K1:
+ case PKE_CURVE_BP256R1:
+ case PKE_CURVE_ANSSI_FRP256V1:
+ case PKE_CURVE_SM2:
+ case PKE_CURVE_25519:
+ return 32;
+ case PKE_CURVE_BP320R1:
+ return 40;
+ case PKE_CURVE_P384:
+ case PKE_CURVE_BP384R1:
+ return 48;
+ case PKE_CURVE_BP512R1:
+ return 64;
+ case PKE_CURVE_P521:
+ return 68; /* ceil(521/8) = 66, ABI uses ALIGN(66, 4) = 68 */
+ case PKE_CURVE_448:
+ return 56;
+ default:
+ return 0;
+ }
+}
+
+/**
+ * pke_curve_bits() - Get EC curve size in bits
+ * @curve: PKE curve identifier (PKE_CURVE_*)
+ *
+ * Return: Curve size in bits, or 0 for unknown curves.
+ */
+static inline u32 pke_curve_bits(u32 curve)
+{
+ switch (curve) {
+ case PKE_CURVE_P192:
+ case PKE_CURVE_BP192R1:
+ return 192;
+ case PKE_CURVE_P224:
+ case PKE_CURVE_BP224R1:
+ return 224;
+ case PKE_CURVE_P256:
+ case PKE_CURVE_SECP256K1:
+ case PKE_CURVE_BP256R1:
+ case PKE_CURVE_ANSSI_FRP256V1:
+ case PKE_CURVE_SM2:
+ case PKE_CURVE_25519:
+ return 256;
+ case PKE_CURVE_BP320R1:
+ return 320;
+ case PKE_CURVE_P384:
+ case PKE_CURVE_BP384R1:
+ return 384;
+ case PKE_CURVE_BP512R1:
+ return 512;
+ case PKE_CURVE_P521:
+ return 521;
+ case PKE_CURVE_448:
+ return 448;
+ default:
+ return 0;
+ }
+}
+
+/**
+ * pke_eddsa_key_len() - Get EdDSA key/pubkey length
+ * @curve: PKE curve identifier (PKE_CURVE_25519 or PKE_CURVE_448)
+ *
+ * Ed25519 uses 32 bytes (== clen), Ed448 uses 57 bytes (clen + 1
+ * flag byte per RFC 8032). Signature length is 2 * pke_eddsa_key_len().
+ *
+ * Return: Key length in bytes.
+ */
+static inline u32 pke_eddsa_key_len(u32 curve)
+{
+ u32 clen = pke_curve_clen(curve);
+
+ return (curve == PKE_CURVE_448) ? clen + 1 : clen;
+}
+
+/**
+ * pke_curve_is_edwards() - Check if curve uses Edwards form
+ * @curve: PKE curve identifier (PKE_CURVE_*)
+ *
+ * Return: true for Curve25519 and Curve448, false otherwise.
+ */
+static inline bool pke_curve_is_edwards(u32 curve)
+{
+ return curve == PKE_CURVE_25519 || curve == PKE_CURVE_448;
+}
+
+/**
+ * pke_swap_flags() - Get VCQ byte-swap flags for a given curve
+ * @curve: PKE curve identifier (PKE_CURVE_*)
+ *
+ * Weierstrass curves need byte+word swap; Edwards curves do not.
+ *
+ * Return: VCQ swap flags to OR into the command ID.
+ */
+static inline u32 pke_swap_flags(u32 curve)
+{
+ return pke_curve_is_edwards(curve) ? 0 : PKE_SWAP_FLAGS;
+}
+
+/* Common VCQ builder prototypes */
+
+void vcq_add_pke_flush(struct vcq_cmd *slot, u32 core_id);
+
+void vcq_add_pke_rsa_enc(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len,
+ u64 e_dma, u64 n_dma, u64 m_dma, u64 c_dma,
+ u32 flags);
+
+void vcq_add_pke_rsa_dec(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len,
+ u64 e_dma, u64 n_dma, u64 c_dma, u64 m_dma,
+ u64 d_ref, u32 flags);
+
+void vcq_add_pke_rsa_crt_dec(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len,
+ u64 e_dma, u64 n_dma, u64 c_dma, u64 m_dma,
+ u64 crt_ref, u32 flags);
+
+void vcq_add_pke_ecdsa_verify(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 dlen,
+ u64 pk_dma, u64 dig_dma, u64 sig_dma,
+ u64 rp_dma, u32 flags);
+
+void vcq_add_pke_ecdsa_sign(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 dig_dma, u64 sig_dma, u64 sk_ref,
+ u32 dlen, u32 flags);
+
+void vcq_add_pke_ecdsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 pk_dma, u64 sk_ref, u32 flags);
+
+void vcq_add_pke_ecdsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 sk_ref, u32 sk_type, u32 flags);
+
+void vcq_add_pke_ecdh_keygen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 pkx_dma, u64 sk_ref, u32 flags);
+
+void vcq_add_pke_ecdh(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u32 sslen, u32 ss_type, u64 peer_dma, u64 sk_ref,
+ u64 ss_ref, u32 flags);
+
+void vcq_add_pke_eddsa_verify(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 dlen,
+ u64 pky_dma, u64 dig_dma, u64 sig_dma,
+ u64 rp_dma, u32 flags);
+
+void vcq_add_pke_eddsa_sign(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 dig_dma, u64 sig_dma, u64 sk_ref,
+ u32 dlen, u32 flags);
+
+void vcq_add_pke_eddsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 pky_dma, u64 sk_ref, u32 flags);
+
+void vcq_add_pke_eddsa_keygen_sca(struct vcq_cmd *slot, u32 core_id, u32 curve,
+ u64 sk_ref, u64 sca_sk_ref);
+
+/* SM2 VCQ builders */
+
+void vcq_add_pke_sm2_ecdh_keygen(struct vcq_cmd *slot, u32 core_id, u64 nonce_dma,
+ u64 session_key_dma, u32 nonce_len, u32 flags);
+
+void vcq_add_pke_sm2_ecdh(struct vcq_cmd *slot, u32 core_id, u32 nonce_len,
+ u32 private_key_len, u64 nonce_dma,
+ u64 peer_pk_dma, u64 peer_sk_dma,
+ u64 priv_ref, u64 sp_ref, u32 sp_type, u32 flags);
+
+void vcq_add_pke_sm2_dec_point(struct vcq_cmd *slot, u32 core_id, u32 ct_len,
+ u32 pk_len, u64 ct_dma, u64 dp_dma,
+ u64 priv_ref, u32 flags);
+
+void vcq_add_pke_sm2_enc_point(struct vcq_cmd *slot, u32 core_id, u64 nonce_dma,
+ u64 pk_dma, u64 ct_dma, u64 ep_dma,
+ u32 nonce_len, u32 flags);
+
+void vcq_add_pke_sm2_id_digest(struct vcq_cmd *slot, u32 core_id, u64 id_dma,
+ u64 pk_dma, u64 dig_dma, u32 id_len,
+ u32 flags);
+
+void vcq_add_pke_sm2_ecdh_hash(struct vcq_cmd *slot, u32 core_id, u64 peer_dig_dma,
+ u64 dig_dma, u64 sp_ref, u64 sk_ref,
+ u32 sk_type, u32 flags);
+
+void vcq_add_pke_sm2_dec_hash(struct vcq_cmd *slot, u32 core_id, u64 ct_dma,
+ u64 dp_dma, u64 pt_dma, u32 ct_len, u32 flags);
+
+void vcq_add_pke_sm2_enc_hash(struct vcq_cmd *slot, u32 core_id, u64 msg_dma,
+ u64 ep_dma, u64 ct_dma, u32 msg_len, u32 flags);
+
+/* Registration */
+
+int cmh_pke_rsa_register(void);
+void cmh_pke_rsa_unregister(void);
+int cmh_pke_ecdsa_register(void);
+void cmh_pke_ecdsa_unregister(void);
+int cmh_pke_ecdh_register(void);
+void cmh_pke_ecdh_unregister(void);
+
+#endif /* CMH_PKE_H */
diff --git a/drivers/crypto/cmh/include/cmh_pke_sm2.h b/drivers/crypto/cmh/include/cmh_pke_sm2.h
new file mode 100644
index 000000000000..a2c7164b8d49
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_pke_sm2.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- SM2 PKE Ioctl Handler Declarations
+ *
+ * SM2 signature (GM/T 0003.2) requires the caller to compute
+ * ZA = SM3(ENTLA || IDA || a || b || xG || yG || xA || yA)
+ * and pass SM3(ZA || M) as the digest to the sign/verify path.
+ * The CMH eSW does NOT compute ZA internally; the full
+ * identity pre-hash is the caller's responsibility.
+ *
+ * For the in-kernel akcipher "sm2" algorithm this means the
+ * caller (e.g. asymmetric_key subsystem) must pre-hash with ZA
+ * before invoking verify. The SM2_ID_DIGEST ioctl below can
+ * compute ZA for userspace callers of the misc-device path.
+ */
+
+#ifndef CMH_PKE_SM2_H
+#define CMH_PKE_SM2_H
+
+int cmh_mgmt_sm2_ecdh_keygen(void __user *argp);
+int cmh_mgmt_sm2_ecdh(void __user *argp);
+int cmh_mgmt_sm2_dec_point(void __user *argp);
+int cmh_mgmt_sm2_enc_point(void __user *argp);
+int cmh_mgmt_sm2_id_digest(void __user *argp);
+int cmh_mgmt_sm2_ecdh_hash(void __user *argp);
+int cmh_mgmt_sm2_dec_hash(void __user *argp);
+int cmh_mgmt_sm2_enc_hash(void __user *argp);
+
+#endif /* CMH_PKE_SM2_H */
diff --git a/drivers/crypto/cmh/include/cmh_pqc.h b/drivers/crypto/cmh/include/cmh_pqc.h
new file mode 100644
index 000000000000..cd4761a0ce5c
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_pqc.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- PQC Algorithm Registration
+ *
+ * Registration/unregistration functions for PQC akcipher algorithms:
+ * ML-DSA, SLH-DSA, LMS, XMSS.
+ */
+
+#ifndef CMH_PQC_H
+#define CMH_PQC_H
+
+int cmh_pqc_mldsa_register(void);
+void cmh_pqc_mldsa_unregister(void);
+
+int cmh_pqc_slhdsa_register(void);
+void cmh_pqc_slhdsa_unregister(void);
+
+int cmh_pqc_lms_register(void);
+void cmh_pqc_lms_unregister(void);
+
+int cmh_pqc_xmss_register(void);
+void cmh_pqc_xmss_unregister(void);
+
+#endif /* CMH_PQC_H */
diff --git a/drivers/crypto/cmh/include/cmh_sys.h b/drivers/crypto/cmh/include/cmh_sys.h
new file mode 100644
index 000000000000..dd336b67bd65
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_sys.h
@@ -0,0 +1,111 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- SYS Core VCQ Builders
+ *
+ * VCQ builder functions for SYS core commands (NEW, WRITE, READ,
+ * FIND, GRANT, DATA, EXPORT, IMPORT). Each builder populates one
+ * vcq_cmd slot with the appropriate magic, command ID, and payload.
+ *
+ * Callers combine these with vcq_set_header() + vcq_add_flush()
+ * and submit via cmh_tm_submit_sync().
+ */
+
+#ifndef CMH_SYS_H
+#define CMH_SYS_H
+
+#include "cmh_vcq.h"
+
+void vcq_add_sys_new(struct vcq_cmd *slot, u64 cid, u64 ref_dma, u32 len);
+void vcq_add_sys_write(struct vcq_cmd *slot, u64 ref, u64 src_dma,
+ u64 wrap_key, u32 len, u32 sys_type);
+void vcq_add_sys_read(struct vcq_cmd *slot, u64 ref, u64 dst_dma,
+ u64 wrap_key, u32 len);
+void vcq_add_sys_data(struct vcq_cmd *slot, u64 ref, u64 dst_dma, u32 len);
+void vcq_add_sys_find(struct vcq_cmd *slot, u64 cid, u64 dst_dma, u32 len);
+void vcq_add_sys_list(struct vcq_cmd *slot, u64 ref, u64 dst_dma, u32 len);
+void vcq_add_sys_grant(struct vcq_cmd *slot, u64 ref, u64 read,
+ u64 write, u64 execute);
+void vcq_add_sys_export(struct vcq_cmd *slot, u64 cid, u64 dst_dma,
+ u64 wrap_key, u32 len);
+void vcq_add_sys_import(struct vcq_cmd *slot, u64 src_dma,
+ u64 wrap_key, u32 len);
+
+/* KIC core VCQ builders */
+void vcq_add_kic_hkdf1(struct vcq_cmd *slot, u64 dst, u64 base,
+ u64 label_dma, u32 key_len, u32 label_len, u32 type);
+void vcq_add_kic_hkdf2(struct vcq_cmd *slot, u64 dst, u64 base, u64 salt,
+ u64 label_dma, u32 key_len, u32 label_len, u32 type);
+void vcq_add_kic_aes_cmac_kdf(struct vcq_cmd *slot, u64 out_key, u64 base_key,
+ u64 label_dma, u32 key_len, u32 label_len,
+ u32 type);
+void vcq_add_kic_dkek_derive(struct vcq_cmd *slot, u64 out_key, u64 base_key,
+ u32 host_id, u64 metadata_dma, u32 metadata_len);
+
+/* DRBG core VCQ builders */
+void vcq_add_drbg_reset(struct vcq_cmd *slot);
+void vcq_add_drbg_config(struct vcq_cmd *slot, u32 ratio, u32 strength);
+void vcq_add_drbg_datastore(struct vcq_cmd *slot, u64 ref, u32 len, u32 type);
+
+/* QSE core VCQ builders */
+void vcq_add_qse_flush(struct vcq_cmd *slot, u32 core_id);
+void vcq_add_qse_ml_kem_keygen(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags,
+ u64 seed, u64 z, u64 ek, u64 dk, u32 dk_type,
+ bool masked);
+void vcq_add_qse_ml_kem_enc(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags,
+ u64 coin, u64 ek, u64 ct, u64 ss, u32 ss_type,
+ bool masked);
+void vcq_add_qse_ml_kem_dec(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags,
+ u64 ct, u64 dk, u64 ss, u32 ss_type,
+ bool masked);
+void vcq_add_qse_ml_dsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags,
+ u64 seed, u64 pk, u64 sk, u32 sk_type,
+ bool masked);
+void vcq_add_qse_ml_dsa_sign(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags,
+ u64 rnd, u64 m, u64 sk, u64 sig, u32 mlen,
+ bool masked);
+void vcq_add_qse_ml_dsa_verify(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags,
+ u64 m, u64 pk, u64 sig, u32 mlen);
+
+/* HCQ core VCQ builders */
+void vcq_add_hcq_flush(struct vcq_cmd *slot, u32 core_id);
+void vcq_add_hcq_slhdsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 seed_len, u32 pk_len, u32 sk_len,
+ u64 seed, u64 pk, u64 sk);
+void vcq_add_hcq_slhdsa_sign(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 msg_len, u32 ctx_len,
+ u64 add_random, u64 msg, u64 ctx,
+ u64 sk, u64 sig);
+void vcq_add_hcq_slhdsa_sign_internal(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 msg_len, u64 add_random,
+ u64 msg, u64 sk, u64 sig);
+void vcq_add_hcq_slhdsa_verify(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 msg_len, u32 ctx_len,
+ u64 msg, u64 ctx, u64 pk, u64 sig);
+void vcq_add_hcq_slhdsa_sign_prehash(struct vcq_cmd *slot, u32 core_id,
+ u32 cmd, u32 param_set, u32 prehash_algo,
+ u32 msg_len, u32 ctx_len,
+ u64 add_random, u64 msg, u64 ctx,
+ u64 sk, u64 sig);
+void vcq_add_hcq_slhdsa_verify_prehash(struct vcq_cmd *slot, u32 core_id,
+ u32 cmd, u32 param_set, u32 prehash_algo,
+ u32 msg_len, u32 ctx_len,
+ u64 msg, u64 ctx, u64 pk, u64 sig);
+void vcq_add_hcq_slhdsa_verify_internal(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 msg_len, u64 msg, u64 pk, u64 sig);
+void vcq_add_hcq_slhdsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 sk_len, u64 sk, u64 pk);
+void vcq_add_hcq_lms_verify(struct vcq_cmd *slot, u32 core_id, u32 lms_hss,
+ u32 pk_len, u32 sig_len, u32 dig_len,
+ u64 pk, u64 sig, u64 dig);
+void vcq_add_hcq_xmss_verify(struct vcq_cmd *slot, u32 core_id, u32 xmss_mt,
+ u32 pk_len, u32 sig_len, u32 dig_len,
+ u64 pk, u64 sig, u64 dig);
+
+/* SYS core flush */
+void vcq_add_sys_flush(struct vcq_cmd *slot);
+
+/* EAC core VCQ builder */
+void vcq_add_eac_read(struct vcq_cmd *slot, u64 dst_dma, u32 len);
+
+#endif /* CMH_SYS_H */
diff --git a/include/uapi/linux/cmh_mgmt_ioctl.h b/include/uapi/linux/cmh_mgmt_ioctl.h
new file mode 100644
index 000000000000..a690454fae69
--- /dev/null
+++ b/include/uapi/linux/cmh_mgmt_ioctl.h
@@ -0,0 +1,895 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Key Management ioctl Interface (User-Space API)
+ *
+ * ioctl commands for /dev/cmh_mgmt -- key CRUD, datastore
+ * export/import, KIC key derivation, PKE, SM2, and PQC operations.
+ *
+ * Relationship to the in-kernel crypto API
+ * -----------------------------------------
+ * Most commands here have no crypto API representation (no transform
+ * type or verb exists): keystore CRUD, key generation, KIC key
+ * derivation, ML-KEM encapsulate/decapsulate, SM2 multi-step
+ * encrypt/decrypt/key-exchange, EdDSA, EAC, and DRBG configuration.
+ * For these the character device is the only available UAPI.
+ *
+ * A bounded subset names primitives the driver ALSO registers with
+ * the crypto API, and the overlap is intentional:
+ * - Hardware-held-key operations (RSA decrypt, ECDSA/ML-DSA/SLH-DSA
+ * sign, ECDH) reference a private key by datastore handle. The
+ * crypto API set_priv_key()/set_secret() take only raw key bytes
+ * and cannot name a key that never leaves the hardware; these
+ * ioctls keep the key hardware-resident. The registered
+ * transforms serve raw-key in-kernel users -- the paths are
+ * complementary.
+ *
+ * Multi-step protocol flows are documented above the PKE and SM2
+ * struct sections. Single-command ioctls are self-documenting.
+ *
+ * Versioned structs: user space sets .version = CMH_MGMT_V1 so the
+ * driver can extend structs in the future without breaking ABI.
+ */
+
+#ifndef _UAPI_CMH_MGMT_IOCTL_H
+#define _UAPI_CMH_MGMT_IOCTL_H
+
+#include <linux/types.h>
+#include <linux/ioctl.h>
+#include <linux/const.h>
+
+#define CMH_MGMT_V1 1
+
+/* Special reference values */
+#define CMH_REF_NONE 0x0000000000000000ULL /* no key (plaintext) */
+
+/* Flags for cmh_ioctl_key_new.flags / cmh_ioctl_key_write.flags */
+#define CMH_FLAG_PT _BITUL(16) /* key can be read as plaintext */
+#define CMH_FLAG_XC _BITUL(17) /* key can be exported over XC bus */
+#define CMH_FLAG_SCA _BITUL(18) /* SCA key stored in 2 shares */
+#define CMH_FLAG_MASK (CMH_FLAG_PT | CMH_FLAG_XC | CMH_FLAG_SCA)
+
+/*
+ * Datastore key types -- the LKM maps these to core IDs internally.
+ * User space passes these in cmh_ioctl_key_new.ds_type.
+ */
+#define CMH_DS_RAW_VALUE 1
+#define CMH_DS_AES_KEY 2
+#define CMH_DS_AES_XTS_KEY 3
+#define CMH_DS_HMAC_KEY 4
+#define CMH_DS_KMAC_KEY 5
+#define CMH_DS_SM4_KEY 6
+#define CMH_DS_CHACHA20_KEY 7
+
+/* PKE key types -- all map to CORE_ID_PKE (0x0A) */
+#define CMH_DS_RSA_PRIV_KEY 10
+#define CMH_DS_RSA_PUB_KEY 11
+#define CMH_DS_RSA_CRT_KEY 12
+#define CMH_DS_ECDSA_PRIV_KEY 13
+#define CMH_DS_ECDSA_PUB_KEY 14
+#define CMH_DS_ECDH_PRIV_KEY 15
+#define CMH_DS_EDDSA_PRIV_KEY 16
+#define CMH_DS_SHARED_SECRET 17
+#define CMH_DS_SM2_PRIV_KEY 18
+
+/* QSE key types -- map to CORE_ID_QSE (0x09) */
+#define CMH_DS_ML_KEM_DK 20
+#define CMH_DS_ML_DSA_SK 21
+
+/* HCQ key types -- map to CORE_ID_HCQ (0x08) */
+#define CMH_DS_SLHDSA_SK 25
+
+/* ioctl argument structures */
+
+struct cmh_ioctl_key_new {
+ __u32 version; /* must be CMH_MGMT_V1 */
+ __u32 ds_type; /* CMH_DS_* key type */
+ __u32 len; /* key length in bytes */
+ __u32 flags; /* CMH_FLAG_* (e.g. CMH_FLAG_PT) */
+ __u64 cid; /* caller ID (name) for the key */
+ __u64 ref; /* [out] CMH eSW returns key_ref here */
+};
+
+struct cmh_ioctl_key_write {
+ __u32 version;
+ __u32 len; /* key data length */
+ __u32 ds_type; /* CMH_DS_* key type */
+ __u32 flags; /* CMH_FLAG_* (e.g. CMH_FLAG_PT) */
+ __u64 ref; /* key reference from KEY_NEW */
+ __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */
+ __u64 data; /* user-space pointer to key material */
+};
+
+struct cmh_ioctl_key_read {
+ __u32 version;
+ __u32 len; /* buffer length */
+ __u64 ref; /* key reference */
+ __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */
+ __u64 data; /* user-space pointer to output buffer */
+ __u32 out_len; /* [out] actual bytes written */
+ __u32 __reserved;
+};
+
+struct cmh_ioctl_key_find {
+ __u32 version;
+ __u32 __reserved;
+ __u64 cid; /* caller ID to search for */
+ __u64 ref; /* [out] resolved key reference */
+ __u32 len; /* [out] key length */
+ __u32 type; /* [out] key type */
+};
+
+/*
+ * KEY_LIST -- iterate datastore objects.
+ *
+ * Pass start_ref=0 to begin from the first accessible object.
+ * On return, ref/cid/len/type describe that object. Pass the
+ * returned ref as start_ref in the next call to advance. Iteration
+ * ends when ref == 0 (no more objects).
+ */
+struct cmh_ioctl_key_list {
+ __u32 version;
+ __u32 __reserved;
+ __u64 start_ref; /* starting DS reference (0 = first) */
+ __u64 ref; /* [out] object reference */
+ __u64 cid; /* [out] caller ID */
+ __u32 len; /* [out] object length */
+ __u32 type; /* [out] object type */
+};
+
+struct cmh_ioctl_key_grant {
+ __u32 version;
+ __u32 __reserved;
+ __u64 ref; /* key reference */
+ __u64 read; /* per-MBX read permission bitfield */
+ __u64 write; /* per-MBX write permission bitfield */
+ __u64 execute; /* per-MBX execute permission bitfield */
+};
+
+/* Export blob overhead beyond the raw object data (bytes) */
+#define CMH_DS_EXPORT_OVERHEAD_WRAPPED 48 /* 16B hdr + 16B nonce + 16B tag */
+#define CMH_DS_EXPORT_OVERHEAD_PLAIN 16 /* 16B hdr only */
+
+/**
+ * struct cmh_ioctl_ds_export - Export a datastore object to a wrapped blob
+ * @version: protocol version (CMH_MGMT_V1)
+ * @len: DMA buffer size; must be >= export blob size:
+ * wrapped: CMH_DS_EXPORT_OVERHEAD_WRAPPED + object_len
+ * plaintext: CMH_DS_EXPORT_OVERHEAD_PLAIN + object_len
+ * object_len is known from KEY_NEW or KEY_FIND.
+ * If too small, the eSW rejects the command (-EIO).
+ * @cid: caller ID of the object to export
+ * @wrap_key: wrapping key ref (CMH_REF_NONE = plaintext export)
+ * @data: user-space pointer to output buffer (at least @len bytes)
+ * @out_len: [out] actual blob bytes written on success
+ * @__reserved: must be zero
+ */
+struct cmh_ioctl_ds_export {
+ __u32 version;
+ __u32 len; /* buffer length (see sizing rule above) */
+ __u64 cid; /* caller ID for response tagging */
+ __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */
+ __u64 data; /* user-space pointer to output buffer */
+ __u32 out_len; /* [out] actual bytes written */
+ __u32 __reserved;
+};
+
+struct cmh_ioctl_ds_import {
+ __u32 version;
+ __u32 len; /* blob length */
+ __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */
+ __u64 data; /* user-space pointer to import blob */
+};
+
+/* Flags for cmh_ioctl_kic_hkdf1.flags / cmh_ioctl_kic_hkdf2.flags */
+#define CMH_KIC_FLAG_TEMP 0x01 /* store result in TEMP (not persistent DS) */
+
+/*
+ * KIC hardware base key references.
+ *
+ * Each CMH device has up to 8 hardware base keys provisioned in OTP/fuses.
+ * These values are passed in the base_key field of KIC ioctl structs.
+ * The key valid bitmask is visible via R_KIC_KEY_VALID (MMIO 0x100).
+ */
+#define CMH_KIC_KEY1 0x0000000100000001ULL
+#define CMH_KIC_KEY2 0x0000000200000002ULL
+#define CMH_KIC_KEY3 0x0000000300000003ULL
+#define CMH_KIC_KEY4 0x0000000400000004ULL
+#define CMH_KIC_KEY5 0x0000000500000005ULL
+#define CMH_KIC_KEY6 0x0000000600000006ULL
+#define CMH_KIC_KEY7 0x0000000700000007ULL
+#define CMH_KIC_KEY8 0x0000000800000008ULL
+
+struct cmh_ioctl_kic_hkdf1 {
+ __u32 version;
+ __u32 key_len; /* output key length (e.g., 32) */
+ __u64 base_key; /* KIC base key reference */
+ __u64 cid; /* CID for the new DS entry (ignored if TEMP) */
+ __u64 label; /* user-space pointer to label data */
+ __u32 label_len; /* label length in bytes */
+ __u32 flags; /* CMH_KIC_FLAG_* */
+ __u64 ref; /* [out] derived key reference */
+};
+
+struct cmh_ioctl_kic_hkdf2 {
+ __u32 version;
+ __u32 key_len; /* output key length (e.g., 32) */
+ __u64 base_key; /* KIC base key reference */
+ __u64 salt_key; /* salt key reference (CMH_REF_NONE = no salt) */
+ __u64 cid; /* CID for the new DS entry (ignored if TEMP) */
+ __u64 label; /* user-space pointer to label data */
+ __u32 label_len; /* label length in bytes */
+ __u32 flags; /* CMH_KIC_FLAG_* */
+ __u64 ref; /* [out] derived key reference */
+};
+
+struct cmh_ioctl_kic_aes_cmac_kdf {
+ __u32 version;
+ __u32 key_len; /* base & output key length (must be 32) */
+ __u64 base_key; /* KIC base key or DS reference */
+ __u64 cid; /* CID for the new DS entry (ignored if TEMP) */
+ __u64 label; /* user-space pointer to label data */
+ __u32 label_len; /* label length in bytes */
+ __u32 flags; /* CMH_KIC_FLAG_* */
+ __u64 ref; /* [out] derived key reference */
+};
+
+#define KIC_DKEK_MAX_METADATA 64 /* max metadata length for DKEK */
+
+struct cmh_ioctl_kic_dkek_derive {
+ __u32 version;
+ __u32 host_id; /* target host ID (0 = caller's own) */
+ __u64 base_key; /* KIC base key reference */
+ __u64 cid; /* CID for the new DS entry (ignored if TEMP) */
+ __u64 metadata; /* user-space pointer to metadata */
+ __u32 metadata_len; /* metadata length in bytes */
+ __u32 flags; /* CMH_KIC_FLAG_* */
+ __u64 ref; /* [out] derived KEK reference */
+};
+
+/* -- PKE ioctl argument structures ----------- */
+
+/*
+ * PKE multi-step protocol flows
+ *
+ * RSA encrypt/decrypt:
+ * 1. KEY_NEW(CMH_DS_RSA_PRIV_KEY) + KEY_WRITE -> priv_ref (or RSA_KEYGEN -> priv_ref)
+ * 2. RSA_ENC(e, n, plaintext) -> ciphertext (public key = raw e,n)
+ * 3. RSA_DEC(e, n, ciphertext, priv_ref) -> plaintext (or RSA_CRT_DEC)
+ *
+ * ECDSA sign:
+ * 1. EC_KEYGEN(curve) -> priv_ref (or KEY_NEW + KEY_WRITE)
+ * 2. EC_PUBGEN(priv_ref) -> public_key (raw x||y returned)
+ * 3. ECDSA_SIGN(digest, priv_ref) -> signature
+ * SM2 sign uses the same path with curve=CMH_CURVE_SM2.
+ *
+ * ECDH shared secret:
+ * 1. EC_KEYGEN(curve) -> priv_ref (or KEY_NEW + KEY_WRITE)
+ * 2. ECDH_KEYGEN(priv_ref) -> public_key_x (derive pub from priv)
+ * 3. Exchange public keys with peer
+ * 4. ECDH(peer_key_x, priv_ref) -> shared_secret (raw or DS ref via FLAG_DS_RESULT)
+ *
+ * EdDSA sign/verify:
+ * 1. EC_KEYGEN(CURVE_25519 or CURVE_448) -> priv_ref
+ * 2. EC_PUBGEN(priv_ref) -> public_key
+ * 3. EDDSA_SIGN(message, priv_ref) -> signature (pure EdDSA, not prehash)
+ * 4. EDDSA_VERIFY(message, signature, public_key_y)
+ * For Ed448 SCA: EDDSA_KEYGEN_SCA(priv_ref) -> sca_ref (2-share blinded key)
+ *
+ * SM2 encryption (GM/T 0003.4):
+ * 1. EC_KEYGEN(CMH_CURVE_SM2) -> priv_ref (or KEY_NEW + KEY_WRITE)
+ * 2. EC_PUBGEN(priv_ref) -> public_key
+ * 3. SM2_ENC_POINT(public_key) -> C1, enc_point (nonce_len=0: HW ephemeral)
+ * 4. SM2_ENC_HASH(enc_point, message) -> ciphertext (C1||C3||C2)
+ * Decrypt:
+ * 5. SM2_DEC_POINT(C1, priv_ref) -> dec_point
+ * 6. SM2_DEC_HASH(ciphertext, dec_point) -> plaintext
+ * enc_point and dec_point are raw DMA buffers (64B each), not DS refs.
+ *
+ * SM2 key exchange (GM/T 0003.3):
+ * 1. EC_KEYGEN(CMH_CURVE_SM2) -> priv_ref (long-lived, persistent DS)
+ * 2. EC_PUBGEN(priv_ref) -> public_key
+ * 3. SM2_ID_DIGEST(id, public_key) -> ZA (SM3-based identity digest)
+ * 4. SM2_ECDH_KEYGEN(nonce) -> session_key, r (ephemeral scalar r)
+ * - nonce_len=32: caller supplies r (deterministic)
+ * - nonce_len=0: HW generates r, writes it back to .nonce
+ * Exchange session_key with peer.
+ * 5. SM2_ECDH(r, priv_ref, peer_pub, peer_sess) -> shared_point
+ * - Must pass the same r from step 4 (nonce_len=32)
+ * - shared_point_ref=0: reads back raw shared_point, destroys DS slot
+ * - shared_point_ref=&ref: keeps DS slot alive, writes ref for ECDH_HASH
+ * 6. SM2_ECDH_HASH(shared_point_ref, ZA_self, ZA_peer) -> shared_key (16B)
+ * - shared_point_ref is a persistent DS reference from step 5
+ * - The DS slot is consumed by the hub; caller should delete it afterward
+ * The nonce r is a raw 32-byte scalar in userspace memory between steps 4-5.
+ * The shared_point is a persistent DS ref between steps 5-6.
+ * The long-lived private key (priv_ref) persists independently.
+ */
+
+/* PKE operation flags */
+#define CMH_PKE_FLAG_DS_RESULT _BITUL(0) /* store result in DS, return ref */
+
+struct cmh_ioctl_pke_rsa_enc {
+ __u32 version;
+ __u32 bits; /* RSA key size in bits (512-4096) */
+ __u64 e; /* user-space pointer to public exponent */
+ __u32 e_len; /* exponent length in bytes */
+ __u32 __reserved;
+ __u64 n; /* user-space pointer to modulus */
+ __u64 input; /* user-space pointer to input data */
+ __u64 output; /* user-space pointer to output buffer */
+};
+
+struct cmh_ioctl_pke_rsa_dec {
+ __u32 version;
+ __u32 bits;
+ __u64 e; /* public exponent */
+ __u32 e_len;
+ __u32 __reserved;
+ __u64 n; /* modulus */
+ __u64 input; /* ciphertext */
+ __u64 output; /* plaintext output */
+ __u64 key_ref; /* private key DS reference */
+};
+
+struct cmh_ioctl_pke_rsa_crt_dec {
+ __u32 version;
+ __u32 bits;
+ __u64 e;
+ __u32 e_len;
+ __u32 __reserved;
+ __u64 n;
+ __u64 input;
+ __u64 output;
+ __u64 crt_ref; /* CRT key DS reference */
+};
+
+struct cmh_ioctl_pke_rsa_keygen {
+ __u32 version;
+ __u32 bits; /* key size in bits */
+ __u64 e; /* user-space pointer to public exponent */
+ __u32 e_len;
+ __u32 flags; /* CMH_FLAG_* */
+ __u64 n; /* [out] user-space pointer to modulus buffer */
+ __u64 d_cid; /* CID for private key DS entry */
+ __u64 d_ref; /* [out] private key reference */
+ __u64 crt_cid; /* CID for CRT key DS entry (0 = skip CRT) */
+ __u64 crt_ref; /* [out] CRT key reference */
+};
+
+struct cmh_ioctl_pke_ecdsa_sign {
+ __u32 version;
+ __u32 curve; /* ABI curve ID (e.g. 0x03 = P-256) */
+ __u64 digest; /* user-space pointer to hash digest */
+ __u32 digest_len; /* digest length in bytes */
+ __u32 __reserved;
+ __u64 signature; /* [out] user-space pointer to (r,s) */
+ __u64 key_ref; /* private key DS reference */
+};
+
+struct cmh_ioctl_pke_ecdh {
+ __u32 version;
+ __u32 curve;
+ __u64 peer_key_x; /* user-space pointer to peer public key X */
+ __u64 key_ref; /* private key DS reference */
+ __u32 flags; /* CMH_PKE_FLAG_DS_RESULT */
+ __u32 __reserved;
+ __u64 result_cid; /* CID for DS result (if FLAG_DS_RESULT) */
+ __u64 output; /* [out] raw shared secret or DS ref */
+};
+
+struct cmh_ioctl_pke_ecdh_keygen {
+ __u32 version;
+ __u32 curve;
+ __u64 key_ref; /* private key DS reference */
+ __u64 public_key_x; /* [out] user-space pointer to public key X */
+};
+
+struct cmh_ioctl_pke_eddsa_sign {
+ __u32 version;
+ __u32 curve; /* CURVE_25519 or CURVE_448 */
+ __u64 digest; /* user-space ptr to message (not digest) */
+ __u32 digest_len;
+ __u32 __reserved;
+ __u64 signature; /* [out] user-space pointer to signature */
+ __u64 key_ref; /* private key DS reference */
+};
+
+struct cmh_ioctl_pke_eddsa_verify {
+ __u32 version;
+ __u32 curve;
+ __u64 digest;
+ __u32 digest_len;
+ __u32 __reserved;
+ __u64 signature;
+ __u64 public_key_y; /* user-space pointer to public key Y */
+};
+
+struct cmh_ioctl_pke_ec_keygen {
+ __u32 version;
+ __u32 curve;
+ __u32 flags; /* CMH_FLAG_* */
+ __u32 __reserved;
+ __u64 cid; /* CID for the new key DS entry */
+ __u64 ref; /* [out] private key reference */
+};
+
+struct cmh_ioctl_pke_ec_pubgen {
+ __u32 version;
+ __u32 curve;
+ __u64 key_ref; /* private key DS reference */
+ __u64 public_key; /* [out] user-space pointer to public key */
+};
+
+struct cmh_ioctl_pke_eddsa_keygen_sca {
+ __u32 version;
+ __u32 curve; /* must be CURVE_448 */
+ __u64 key_ref; /* input: normal Ed448 private key DS ref */
+ __u64 cid; /* CID for the new SCA key DS entry */
+ __u64 sca_ref; /* [out] SCA private key reference */
+};
+
+/*
+ * ioctl numbers -- type 'J', sequential.
+ * 'C' conflicts with OSS sound, CAPI/ISDN, and COSA WAN drivers;
+ * 'J' is unregistered in Documentation/userspace-api/ioctl/ioctl-number.rst.
+ */
+
+#define CMH_MGMT_IOC_MAGIC 'J'
+
+#define CMH_IOCTL_KEY_NEW _IOWR(CMH_MGMT_IOC_MAGIC, 0x01, struct cmh_ioctl_key_new)
+#define CMH_IOCTL_KEY_WRITE _IOW(CMH_MGMT_IOC_MAGIC, 0x02, struct cmh_ioctl_key_write)
+#define CMH_IOCTL_KEY_READ _IOWR(CMH_MGMT_IOC_MAGIC, 0x03, struct cmh_ioctl_key_read)
+#define CMH_IOCTL_KEY_FIND _IOWR(CMH_MGMT_IOC_MAGIC, 0x04, struct cmh_ioctl_key_find)
+#define CMH_IOCTL_KEY_GRANT _IOW(CMH_MGMT_IOC_MAGIC, 0x05, struct cmh_ioctl_key_grant)
+#define CMH_IOCTL_KEY_DELETE _IOW(CMH_MGMT_IOC_MAGIC, 0x06, struct cmh_ioctl_key_grant)
+#define CMH_IOCTL_DS_EXPORT _IOWR(CMH_MGMT_IOC_MAGIC, 0x07, struct cmh_ioctl_ds_export)
+#define CMH_IOCTL_DS_IMPORT _IOW(CMH_MGMT_IOC_MAGIC, 0x08, struct cmh_ioctl_ds_import)
+#define CMH_IOCTL_KIC_HKDF1 _IOWR(CMH_MGMT_IOC_MAGIC, 0x09, struct cmh_ioctl_kic_hkdf1)
+#define CMH_IOCTL_KIC_HKDF2 _IOWR(CMH_MGMT_IOC_MAGIC, 0x0A, struct cmh_ioctl_kic_hkdf2)
+#define CMH_IOCTL_KEY_NEW_RANDOM _IOWR(CMH_MGMT_IOC_MAGIC, 0x0B, struct cmh_ioctl_key_new)
+#define CMH_IOCTL_KIC_AES_CMAC_KDF _IOWR(CMH_MGMT_IOC_MAGIC, 0x0C, \
+ struct cmh_ioctl_kic_aes_cmac_kdf)
+#define CMH_IOCTL_KIC_DKEK_DERIVE _IOWR(CMH_MGMT_IOC_MAGIC, 0x0D, \
+ struct cmh_ioctl_kic_dkek_derive)
+#define CMH_IOCTL_KEY_LIST _IOWR(CMH_MGMT_IOC_MAGIC, 0x0E, struct cmh_ioctl_key_list)
+
+/* PKE operation ioctls */
+#define CMH_IOCTL_PKE_RSA_ENC _IOWR(CMH_MGMT_IOC_MAGIC, 0x10, \
+ struct cmh_ioctl_pke_rsa_enc)
+#define CMH_IOCTL_PKE_RSA_DEC _IOWR(CMH_MGMT_IOC_MAGIC, 0x11, \
+ struct cmh_ioctl_pke_rsa_dec)
+#define CMH_IOCTL_PKE_RSA_CRT_DEC _IOWR(CMH_MGMT_IOC_MAGIC, 0x12, \
+ struct cmh_ioctl_pke_rsa_crt_dec)
+#define CMH_IOCTL_PKE_RSA_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x13, \
+ struct cmh_ioctl_pke_rsa_keygen)
+#define CMH_IOCTL_PKE_ECDSA_SIGN _IOWR(CMH_MGMT_IOC_MAGIC, 0x14, \
+ struct cmh_ioctl_pke_ecdsa_sign)
+#define CMH_IOCTL_PKE_ECDH _IOWR(CMH_MGMT_IOC_MAGIC, 0x16, \
+ struct cmh_ioctl_pke_ecdh)
+#define CMH_IOCTL_PKE_ECDH_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x17, \
+ struct cmh_ioctl_pke_ecdh_keygen)
+#define CMH_IOCTL_PKE_EDDSA_SIGN _IOWR(CMH_MGMT_IOC_MAGIC, 0x18, \
+ struct cmh_ioctl_pke_eddsa_sign)
+#define CMH_IOCTL_PKE_EDDSA_VERIFY _IOW(CMH_MGMT_IOC_MAGIC, 0x19, \
+ struct cmh_ioctl_pke_eddsa_verify)
+#define CMH_IOCTL_PKE_EC_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x1A, \
+ struct cmh_ioctl_pke_ec_keygen)
+#define CMH_IOCTL_PKE_EC_PUBGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x1B, \
+ struct cmh_ioctl_pke_ec_pubgen)
+#define CMH_IOCTL_PKE_EDDSA_KEYGEN_SCA _IOWR(CMH_MGMT_IOC_MAGIC, 0x1C, \
+ struct cmh_ioctl_pke_eddsa_keygen_sca)
+
+/* -- PQC ioctl argument structures ----------- */
+
+/*
+ * PQC operation flags (bits [2:0]).
+ * PQC keygen ioctls accept CMH_FLAG_PT in bits [18:16] to explicitly
+ * set the DS key storage attribute when CMH_QSE_FLAG_DS_REF is set.
+ * CMH_FLAG_SCA and CMH_FLAG_XC are rejected -- QSE SCA protection uses
+ * polynomial masking (CMH_QSE_FLAG_MASKED), not 2-share storage,
+ * and the eSW dec/sign paths hardcode SYS_TYPE_FLAG_PT.
+ * If no CMH_FLAG_* bits are set, DS keys default to CMH_FLAG_PT.
+ */
+#define CMH_QSE_FLAG_MASKED _BITUL(0) /* use masked (SCA-resistant) HW commands */
+#define CMH_QSE_FLAG_DS_REF _BITUL(1) /* store key output in DS, return ref */
+#define CMH_QSE_FLAG_HW_RNG _BITUL(2) /* use HW RNG for seed/randomness */
+#define CMH_QSE_FLAG_MASK (_BITUL(0) | _BITUL(1) | _BITUL(2))
+
+/* -- SYS wrap header size -------------------- */
+/* sys_read prepends a 16-byte header even for plaintext reads */
+#define CMH_SYS_WRAP_HDR_SIZE 16
+
+/* -- Seed / randomness lengths --------------- */
+
+#define CMH_QSE_SEED_LEN 32 /* ML-KEM/ML-DSA seed size */
+#define CMH_QSE_SEED_LEN_MASKED 64 /* seed size for masked mode */
+
+/* -- ML-DSA ExternalMu sentinel -------------- */
+/* Pass this as mlen to use 64-byte pre-hashed mu instead of raw message */
+#define CMH_ML_DSA_MLEN_EXTERNAL_MU 0xFFFFFFFFU
+
+/* -- ML-KEM size macros ---------------------- */
+
+#define CMH_ML_KEM_EK_SIZE(k) (384U * (k) + 32U)
+#define CMH_ML_KEM_DK_SIZE(k) (768U * (k) + 96U)
+/* CT sizes: k=2 -> 768, k=3 -> 1088, k=4 -> 1568 */
+#define CMH_ML_KEM_CT_SIZE_512 768U
+#define CMH_ML_KEM_CT_SIZE_768 1088U
+#define CMH_ML_KEM_CT_SIZE_1024 1568U
+#define CMH_ML_KEM_SS_LEN 32U
+
+/* -- ML-DSA size macros ---------------------- */
+/* Indexed by mode: [0]=44 (mode=2), [1]=65 (mode=3), [2]=87 (mode=5) */
+
+#define CMH_ML_DSA_44_PK_SIZE 1312U
+#define CMH_ML_DSA_44_SK_SIZE 2560U
+#define CMH_ML_DSA_44_SIG_SIZE 2420U
+#define CMH_ML_DSA_65_PK_SIZE 1952U
+#define CMH_ML_DSA_65_SK_SIZE 4032U
+#define CMH_ML_DSA_65_SIG_SIZE 3309U
+#define CMH_ML_DSA_87_PK_SIZE 2592U
+#define CMH_ML_DSA_87_SK_SIZE 4896U
+#define CMH_ML_DSA_87_SIG_SIZE 4627U
+
+/* -- SLH-DSA parameter set IDs --------------- */
+
+#define CMH_SLHDSA_SHAKE_128S 1U
+#define CMH_SLHDSA_SHAKE_128F 2U
+#define CMH_SLHDSA_SHAKE_192S 3U
+#define CMH_SLHDSA_SHAKE_192F 4U
+#define CMH_SLHDSA_SHAKE_256S 5U
+#define CMH_SLHDSA_SHAKE_256F 6U
+#define CMH_SLHDSA_SHA2_128S 7U
+#define CMH_SLHDSA_SHA2_128F 8U
+#define CMH_SLHDSA_SHA2_192S 9U
+#define CMH_SLHDSA_SHA2_192F 10U
+#define CMH_SLHDSA_SHA2_256S 11U
+#define CMH_SLHDSA_SHA2_256F 12U
+#define CMH_SLHDSA_PARAM_MAX 12U
+
+/* SLH-DSA prehash algorithm IDs */
+#define CMH_SLHDSA_PREHASH_SHA256 1U
+#define CMH_SLHDSA_PREHASH_SHA512 2U
+#define CMH_SLHDSA_PREHASH_SHAKE128 3U
+#define CMH_SLHDSA_PREHASH_SHAKE256 4U
+#define CMH_SLHDSA_PREHASH_MAX 4U
+
+/* SLH-DSA n-value table indexed by (param_set - 1) */
+#define CMH_SLHDSA_N_128 16U
+#define CMH_SLHDSA_N_192 24U
+#define CMH_SLHDSA_N_256 32U
+
+/* SLH-DSA key sizes: pk = 2*n, sk = 4*n, seed = 3*n */
+#define CMH_SLHDSA_PK_SIZE(n) (2U * (n))
+#define CMH_SLHDSA_SK_SIZE(n) (4U * (n))
+#define CMH_SLHDSA_SEED_SIZE(n) (3U * (n))
+
+/* SLH-DSA signature sizes indexed by (param_set - 1) */
+#define CMH_SLHDSA_SIG_SIZE_SHAKE_128S 7856U
+#define CMH_SLHDSA_SIG_SIZE_SHAKE_128F 17088U
+#define CMH_SLHDSA_SIG_SIZE_SHAKE_192S 16224U
+#define CMH_SLHDSA_SIG_SIZE_SHAKE_192F 35664U
+#define CMH_SLHDSA_SIG_SIZE_SHAKE_256S 29792U
+#define CMH_SLHDSA_SIG_SIZE_SHAKE_256F 49856U
+#define CMH_SLHDSA_SIG_SIZE_SHA2_128S 7856U
+#define CMH_SLHDSA_SIG_SIZE_SHA2_128F 17088U
+#define CMH_SLHDSA_SIG_SIZE_SHA2_192S 16224U
+#define CMH_SLHDSA_SIG_SIZE_SHA2_192F 35664U
+#define CMH_SLHDSA_SIG_SIZE_SHA2_256S 29792U
+#define CMH_SLHDSA_SIG_SIZE_SHA2_256F 49856U
+
+/* -- PKE curve IDs -------------- */
+
+#define CMH_CURVE_P192 0x01U
+#define CMH_CURVE_P224 0x02U
+#define CMH_CURVE_P256 0x03U
+#define CMH_CURVE_P384 0x04U
+#define CMH_CURVE_P521 0x05U
+#define CMH_CURVE_SECP256K1 0x07U
+#define CMH_CURVE_BP192R1 0x11U
+#define CMH_CURVE_BP224R1 0x12U
+#define CMH_CURVE_BP256R1 0x13U
+#define CMH_CURVE_BP320R1 0x14U
+#define CMH_CURVE_BP384R1 0x15U
+#define CMH_CURVE_BP512R1 0x16U
+#define CMH_CURVE_SM2 0x18U
+#define CMH_CURVE_25519 0x21U
+#define CMH_CURVE_448 0x22U
+
+/* ML-KEM */
+
+struct cmh_ioctl_ml_kem_keygen {
+ __u32 version;
+ __u32 k; /* security parameter: 2/3/4 */
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 __reserved;
+ __u64 seed; /* user-space pointer to seed (or 0 for HW RNG) */
+ __u64 z; /* user-space pointer to z (or 0 for HW RNG) */
+ __u64 ek; /* [out] user-space pointer to encapsulation key */
+ __u64 dk; /* [out] user-space pointer to decapsulation key
+ * or [out] DS ref if CMH_QSE_FLAG_DS_REF
+ */
+ __u64 dk_cid; /* CID for DS entry (if DS_REF) */
+ __u64 dk_ref; /* [out] dk DS reference (if DS_REF) */
+};
+
+struct cmh_ioctl_ml_kem_enc {
+ __u32 version;
+ __u32 k;
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 __reserved;
+ __u64 coin; /* user-space pointer to random coin (or 0) */
+ __u64 ek; /* user-space pointer to encapsulation key */
+ __u64 ct; /* [out] user-space pointer to ciphertext */
+ __u64 ss; /* [out] user-space pointer to shared secret */
+ __u64 __reserved2[2]; /* reserved for future use */
+};
+
+struct cmh_ioctl_ml_kem_dec {
+ __u32 version;
+ __u32 k;
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 __reserved;
+ __u64 ct; /* user-space pointer to ciphertext */
+ __u64 dk; /* user-space pointer to dk or DS ref */
+ __u64 ss; /* [out] user-space pointer to shared secret */
+ __u64 __reserved2[2]; /* reserved for future use */
+};
+
+/* ML-DSA */
+
+struct cmh_ioctl_ml_dsa_keygen {
+ __u32 version;
+ __u32 mode; /* security parameter: 2/3/5 */
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 __reserved;
+ __u64 seed; /* user-space pointer to seed (or 0 for HW RNG) */
+ __u64 pk; /* [out] user-space pointer to public key */
+ __u64 sk; /* [out] user-space pointer to secret key
+ * or [out] DS ref if CMH_QSE_FLAG_DS_REF
+ */
+ __u64 sk_cid; /* CID for DS entry (if DS_REF) */
+ __u64 sk_ref; /* [out] sk DS reference (if DS_REF) */
+};
+
+struct cmh_ioctl_ml_dsa_sign {
+ __u32 version;
+ __u32 mode;
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 mlen; /* message length in bytes */
+ __u64 m; /* user-space pointer to message */
+ __u64 sk; /* user-space pointer to sk or DS ref */
+ __u64 sig; /* [out] user-space pointer to signature */
+ __u64 rnd; /* user-space pointer to randomness (or 0) */
+};
+
+/* SLH-DSA */
+
+struct cmh_ioctl_slhdsa_keygen {
+ __u32 version;
+ __u32 parameter_set; /* HCQ_SLHDSA_SHAKE_128S .. SHA2_256F */
+ __u32 flags; /* CMH_QSE_FLAG_DS_REF */
+ __u32 __reserved;
+ __u64 seed; /* user-space pointer to seed */
+ __u64 pk; /* [out] user-space pointer to public key */
+ __u64 sk; /* [out] user-space pointer to secret key
+ * or [out] DS ref if CMH_QSE_FLAG_DS_REF
+ */
+ __u64 sk_cid; /* CID for DS entry (if DS_REF) */
+ __u64 sk_ref; /* [out] sk DS reference (if DS_REF) */
+};
+
+struct cmh_ioctl_slhdsa_sign {
+ __u32 version;
+ __u32 parameter_set;
+ __u32 msg_len;
+ __u32 ctx_len;
+ __u64 msg; /* user-space pointer to message */
+ __u64 ctx; /* user-space pointer to context (or 0) */
+ __u64 sk; /* DS ref for secret key */
+ __u64 sig; /* [out] user-space pointer to signature */
+ __u64 add_random; /* user-space pointer to addl. randomness (or 0) */
+};
+
+struct cmh_ioctl_slhdsa_sign_prehash {
+ __u32 version;
+ __u32 parameter_set;
+ __u32 prehash_algo; /* CMH_SLHDSA_PREHASH_* */
+ __u32 digest; /* 0 = raw msg (eSW hashes), 1 = pre-computed digest */
+ __u32 msg_len;
+ __u32 ctx_len;
+ __u64 msg; /* user-space pointer to message/digest */
+ __u64 ctx; /* user-space pointer to context (or 0) */
+ __u64 sk; /* DS ref for secret key */
+ __u64 sig; /* [out] user-space pointer to signature */
+ __u64 add_random; /* user-space pointer to addl. randomness (or 0) */
+};
+
+/* -- SM2 ioctl argument structures ----------- */
+
+/* SM2 fixed key sizes (sm2p256v1 curve, 256-bit) */
+#define CMH_SM2_CLEN 32U /* coordinate length */
+#define CMH_SM2_PUBKEY_LEN 64U /* uncompressed (x||y) */
+#define CMH_SM2_POINT_LEN 64U /* EC point (x||y) */
+#define CMH_SM2_SHARED_KEY_LEN 16U /* ECDH shared key */
+#define CMH_SM2_DIGEST_LEN 32U /* SM3 digest (ZA) */
+/*
+ * SM2 enc_hash/dec_hash payload limit.
+ *
+ * The eSW PKE driver implements only a single-block GM/T 0003.4 KDF
+ * (one SM3 invocation, 32 bytes of key stream). Longer messages would
+ * silently produce incorrect ciphertext / plaintext, so the driver caps
+ * the payload at 32 bytes. See Documentation/ABI/testing/cmh-mgmt.
+ */
+#define CMH_SM2_MAX_MSG_LEN 32U /* encrypt/decrypt */
+#define CMH_SM2_MAX_ID_LEN 32U /* identity string */
+#define CMH_SM2_CT_OVERHEAD 96U /* C1(64) + C3(32) */
+#define CMH_SM2_MAX_CT_LEN 128U /* 96 + max_msg = 128 */
+
+struct cmh_ioctl_sm2_ecdh_keygen {
+ __u32 version;
+ __u32 nonce_len; /* 0 = HW generates r (written back), 32 = caller provides r */
+ __u64 nonce; /* [in/out] user-space pointer to nonce buffer (32B) */
+ __u64 session_key; /* [out] user-space pointer to session key R=r*G (64B) */
+};
+
+struct cmh_ioctl_sm2_ecdh {
+ __u32 version;
+ __u32 nonce_len; /* 0 = HW generates (written back), 32 = caller provides */
+ __u64 nonce; /* [in/out] user-space pointer to nonce r (32B) */
+ __u64 peer_public_key; /* user-space pointer to peer pub key (64B) */
+ __u64 peer_session_key; /* user-space pointer to peer session key (64B) */
+ __u64 key_ref; /* private key DS reference */
+ __u64 shared_point; /* [out] user-space pointer to shared point (64B) */
+ __u64 shared_point_ref; /* [in/out] 0 = read-back mode; &ref = keep DS, write ref */
+};
+
+struct cmh_ioctl_sm2_dec_point {
+ __u32 version;
+ __u32 ciphertext_len; /* total ciphertext length (97..128) */
+ __u64 ciphertext; /* user-space pointer to ciphertext (64B: C1) */
+ __u64 dec_point; /* [out] user-space pointer to dec point (64B) */
+ __u64 key_ref; /* private key DS reference */
+};
+
+struct cmh_ioctl_sm2_enc_point {
+ __u32 version;
+ __u32 nonce_len; /* 0 = HW generates, 32 = caller provides */
+ __u64 nonce; /* user-space pointer to nonce (or 0) */
+ __u64 public_key; /* user-space pointer to public key (64B) */
+ __u64 ciphertext; /* [out] user-space pointer to C1 (64B) */
+ __u64 enc_point; /* [out] user-space pointer to enc point (64B) */
+};
+
+struct cmh_ioctl_sm2_id_digest {
+ __u32 version;
+ __u32 id_len; /* identity length in bytes (<=32) */
+ __u64 id; /* user-space pointer to identity string */
+ __u64 public_key; /* user-space pointer to public key (64B) */
+ __u64 digest; /* [out] user-space pointer to ZA digest (32B) */
+};
+
+/*
+ * SM2 ECDH_HASH -- derive shared key from shared point + ZA digests.
+ *
+ * IMPORTANT: The digest fields use ABSOLUTE ordering per GM/T 0003.3,
+ * NOT relative own/peer ordering. Both parties must pass:
+ * peer_id_digest = Z_A (initiator's digest) -- hashed FIRST
+ * id_digest = Z_B (responder's digest) -- hashed SECOND
+ * The eSW computes: KDF(shared_point || peer_id_digest || id_digest).
+ */
+struct cmh_ioctl_sm2_ecdh_hash {
+ __u32 version;
+ __u32 __reserved;
+ __u64 peer_id_digest; /* ptr to Z_A -- initiator's digest (32B) */
+ __u64 id_digest; /* ptr to Z_B -- responder's digest (32B) */
+ __u64 shared_point_ref; /* DS reference from SM2_ECDH */
+ __u64 shared_key; /* [out] ptr to shared key (16B) */
+};
+
+struct cmh_ioctl_sm2_dec_hash {
+ __u32 version;
+ __u32 ciphertext_len; /* ciphertext length (97..128) */
+ __u64 ciphertext; /* user-space pointer to full ciphertext */
+ __u64 dec_point; /* user-space pointer to dec point (64B) */
+ __u64 plaintext; /* [out] user-space pointer to plaintext */
+};
+
+struct cmh_ioctl_sm2_enc_hash {
+ __u32 version;
+ __u32 message_len; /* message length (1..32) */
+ __u64 message; /* user-space pointer to plaintext */
+ __u64 enc_point; /* user-space pointer to enc point (64B) */
+ __u64 ciphertext; /* [out] user-space pointer to ciphertext */
+};
+
+/* PQC ioctl numbers */
+#define CMH_IOCTL_ML_KEM_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x20, \
+ struct cmh_ioctl_ml_kem_keygen)
+#define CMH_IOCTL_ML_KEM_ENC _IOWR(CMH_MGMT_IOC_MAGIC, 0x21, \
+ struct cmh_ioctl_ml_kem_enc)
+#define CMH_IOCTL_ML_KEM_DEC _IOWR(CMH_MGMT_IOC_MAGIC, 0x22, \
+ struct cmh_ioctl_ml_kem_dec)
+#define CMH_IOCTL_ML_DSA_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x23, \
+ struct cmh_ioctl_ml_dsa_keygen)
+#define CMH_IOCTL_ML_DSA_SIGN _IOWR(CMH_MGMT_IOC_MAGIC, 0x24, \
+ struct cmh_ioctl_ml_dsa_sign)
+#define CMH_IOCTL_SLHDSA_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x28, \
+ struct cmh_ioctl_slhdsa_keygen)
+#define CMH_IOCTL_SLHDSA_SIGN _IOWR(CMH_MGMT_IOC_MAGIC, 0x29, \
+ struct cmh_ioctl_slhdsa_sign)
+#define CMH_IOCTL_SLHDSA_SIGN_PREHASH _IOWR(CMH_MGMT_IOC_MAGIC, 0x2D, \
+ struct cmh_ioctl_slhdsa_sign_prehash)
+
+/* SM2 operation ioctls */
+#define CMH_IOCTL_SM2_ECDH_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x30, \
+ struct cmh_ioctl_sm2_ecdh_keygen)
+#define CMH_IOCTL_SM2_ECDH _IOWR(CMH_MGMT_IOC_MAGIC, 0x31, \
+ struct cmh_ioctl_sm2_ecdh)
+#define CMH_IOCTL_SM2_DEC_POINT _IOWR(CMH_MGMT_IOC_MAGIC, 0x32, \
+ struct cmh_ioctl_sm2_dec_point)
+#define CMH_IOCTL_SM2_ENC_POINT _IOWR(CMH_MGMT_IOC_MAGIC, 0x33, \
+ struct cmh_ioctl_sm2_enc_point)
+#define CMH_IOCTL_SM2_ID_DIGEST _IOWR(CMH_MGMT_IOC_MAGIC, 0x34, \
+ struct cmh_ioctl_sm2_id_digest)
+#define CMH_IOCTL_SM2_ECDH_HASH _IOWR(CMH_MGMT_IOC_MAGIC, 0x35, \
+ struct cmh_ioctl_sm2_ecdh_hash)
+#define CMH_IOCTL_SM2_DEC_HASH _IOWR(CMH_MGMT_IOC_MAGIC, 0x36, \
+ struct cmh_ioctl_sm2_dec_hash)
+#define CMH_IOCTL_SM2_ENC_HASH _IOWR(CMH_MGMT_IOC_MAGIC, 0x37, \
+ struct cmh_ioctl_sm2_enc_hash)
+
+/*
+ * EAC (Error and Alarm Controller) -- read and clear error registers.
+ *
+ * Returns a snapshot of all hardware error/safety/notification registers.
+ * The eSW atomically reads and clears the registers on each call, so
+ * successive reads show only new events.
+ */
+struct cmh_ioctl_eac_read {
+ __u32 version; /* must be CMH_MGMT_V1 */
+ __u32 __reserved;
+ __u64 mailbox_notification; /* [out] MBX safety notification bitmask */
+ __u32 hw_error; /* [out] HWC error bitmask */
+ __u32 hw_nmi; /* [out] HWC NMI bitmask */
+ __u32 hw_panic; /* [out] HWC panic bitmask */
+ __u32 safety_fatal; /* [out] HWC fatal safety bitmask */
+ __u32 safety_notification; /* [out] HWC safety notification bitmask */
+ __u32 sw_info0; /* [out] eSW tracing info */
+ __u32 sw_info1; /* [out] eSW tracing info */
+ __u32 sram_bank_errors[4]; /* [out] correctable ECC error counts */
+ __u32 __pad; /* explicit tail padding (prevent info leak) */
+};
+
+/*
+ * DRBG CONFIG -- configure the hardware DRBG before first use.
+ *
+ * This is a management operation normally performed once at system
+ * start-up. Must be called before any hwrng reads or DRBG GENERATE
+ * operations.
+ */
+#define CMH_DRBG_RATIO_ONE 0 /* 1:1 entropy ratio */
+#define CMH_DRBG_RATIO_ONE_HALF 1 /* 1:2 */
+#define CMH_DRBG_RATIO_ONE_THIRD 2 /* 1:3 */
+#define CMH_DRBG_RATIO_ONE_FOURTH 3 /* 1:4 */
+
+#define CMH_DRBG_STRENGTH_128 0x00 /* 128-bit security */
+#define CMH_DRBG_STRENGTH_256 0x10 /* 256-bit security */
+
+struct cmh_ioctl_drbg_config {
+ __u32 version; /* must be CMH_MGMT_V1 */
+ __u32 entropy_ratio; /* CMH_DRBG_RATIO_* */
+ __u32 security_strength; /* CMH_DRBG_STRENGTH_* */
+ __u32 __reserved;
+};
+
+/* EAC ioctl number */
+#define CMH_IOCTL_EAC_READ _IOWR(CMH_MGMT_IOC_MAGIC, 0x0F, \
+ struct cmh_ioctl_eac_read)
+
+/* DRBG management ioctl number */
+#define CMH_IOCTL_DRBG_CONFIG _IOW(CMH_MGMT_IOC_MAGIC, 0x40, \
+ struct cmh_ioctl_drbg_config)
+
+#endif /* _UAPI_CMH_MGMT_IOCTL_H */
--
2.43.7
^ permalink raw reply related
* [PATCH v2 00/19] crypto: cmh - add CRI CryptoManager Hub driver
From: Saravanakrishnan Krishnamoorthy @ 2026-07-09 20:30 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
From: Alex Ousherovitch <aousherovitch@rambus.com>
crypto: cmh - add CRI CryptoManager Hub hardware crypto accelerator
This series adds a driver for the CRI CryptoManager Hub (CMH), a
hardware cryptographic accelerator IP from Cryptography Research at
Rambus Inc. (https://www.rambus.com/cryptographyresearch/).
CMH provides a broad set of symmetric, asymmetric, and post-quantum
cryptographic algorithms accelerated in hardware, accessed via a
mailbox-based Virtual Command Queue (VCQ) interface.
The hardware is a platform device matched via device tree
(compatible = "cri,cmh"). It exposes a single MMIO register region
(SIC) with per-mailbox doorbell, status, and command registers.
Each mailbox has DMA-coherent queue memory for VCQ command
submission and completion.
Driver architecture:
In-kernel users /dev/cmh_mgmt (ioctl)
(dm-crypt, IPsec, kTLS, fscrypt) (key management)
| |
v v
+----------------------------------------------------+
| Kernel Crypto API + hwrng (72 total) |
| ahash | skcipher | aead | akcipher | sig | kpp |
+----------------------------------------------------+
| |
v v
+------------------+ +------------------------+
| Transaction Mgr |--->| Key / Mgmt subsystem |
| (kthread, CMQ) | | (datastore, ioctl ops) |
+------------------+ +------------------------+
|
v
+------------------+ +-------------------+
| MQI (VCQ pack, |---->| Response Handler |
| DMA map, submit)| | (threaded IRQ, |
+------------------+ | watchdog, unmap) |
| +-------------------+
v ^
+-----------+ +-----------+
| Hardware |--- IRQ ----->| Hardware |
| (mailbox) | | (mailbox) |
+-----------+ +-----------+
The transaction manager runs as a dedicated kthread that pulls
requests from a central command queue, packs VCQ entries, maps DMA
buffers, and submits to the least-loaded mailbox. Completion is
handled by per-mailbox threaded IRQs. The driver returns
-EINPROGRESS for async crypto requests and supports the
CRYPTO_TFM_REQ_MAY_BACKLOG flag for queue-full backpressure.
Registered algorithms (72 total):
Type Count Algorithms
--------- ----- --------------------------------------------------
ahash 15 SHA-{224,256,384,512}, SHA3-{224,256,384,512},
SHAKE-{128,256}, cSHAKE-{128,256},
KMAC-{128,256}, SM3
ahash(HMAC) 8 HMAC-SHA-{224,256,384,512},
HMAC-SHA3-{224,256,384,512}
ahash(MAC) 4 CMAC(AES), CMAC(SM4), XCBC(SM4), Poly1305
skcipher 11 AES-{ECB,CBC,CTR,CFB,XTS},
SM4-{ECB,CBC,CTR,CFB,XTS}, ChaCha20
aead 6 AES-{GCM,CCM}, SM4-{GCM,CCM},
rfc7539(chacha20,poly1305),
rfc7539esp(chacha20,poly1305)
akcipher 1 RSA (2048--4096 bit; 512/1024 legacy/test)
sig 23 ECDSA P-{256,384,521}, SM2 (verify-only),
ML-DSA-{44,65,87},
SLH-DSA (12 parameter sets),
LMS, LMS-HSS, XMSS, XMSS-MT
kpp 3 ECDH P-{256,384}, X25519
hwrng 1 DRBG-backed /dev/hwrng
Ioctl-only algorithms (not registered with the crypto API at all):
- EdDSA (Ed25519, Ed448): sign and verify
- ML-KEM (ML-KEM-512/768/1024): no standard kernel KEM API exists
The driver also exposes /dev/cmh_mgmt, a misc device providing 44
ioctl commands. Relative to the in-kernel crypto API these fall into
two groups; the distinction matters because some commands name the
same primitives the driver also registers, and that overlap is
deliberate and bounded:
(1) Operations with no crypto API representation - the large
majority. The crypto API has no transform type or verb for
these, so a character device is the only available UAPI:
- hardware key lifecycle: create, import, export, derive,
destroy, enumerate (keystore CRUD) - no keystore verb
- KIC key derivation (HKDF, AES-CMAC-KDF, DKEK)
- asymmetric key generation (RSA, EC, EdDSA, ML-DSA, SLH-DSA)
and public-key derivation - the crypto API has no keygen verb
- ML-KEM encapsulate/decapsulate - no kernel KEM API exists
- SM2 encrypt/decrypt and key exchange (multi-step GM/T 0003)
- EdDSA sign/verify - not registered with the crypto API
- EAC Chip Authentication and DRBG (re)configuration
(2) Hardware-held-key operations on algorithms that ARE also
registered (RSA decrypt, ECDSA/ML-DSA/SLH-DSA sign, ECDH). These
name the same primitives as the registered akcipher/sig/kpp
transforms, but the crypto API's set_priv_key()/set_secret()
accept only raw key bytes supplied by the caller; they cannot
reference a private key that is generated inside, and never
leaves, the hardware datastore - the central security property of
this device. The ioctl path keeps the private key
hardware-resident, while the registered transforms serve raw-key
in-kernel users. The two paths are complementary, not redundant.
The device requires CAP_SYS_ADMIN.
/dev/cmh_mgmt is built conditionally on CONFIG_CRYPTO_DEV_CMH_MGMT
(default n); when disabled the ioctl interface is absent while all
kernel crypto API algorithms remain registered.
The ML-DSA sig algorithms are registered at priority 5001. The
kernel's crypto/mldsa.c registers at priority 5000 with verify-only
(sign returns -EOPNOTSUPP). Our driver provides full HW-accelerated
sign + verify, so the higher priority ensures the hardware
implementation is preferred when the driver is loaded.
Power management uses DEFINE_SIMPLE_DEV_PM_OPS. On suspend the
transaction manager drains in-flight requests (configurable 10s
timeout, returns -ECANCELED on timeout), stops the kthread, and
masks IRQs. On resume it re-verifies SIC/boot status and restarts
the kthread.
Dependencies:
- Kernel 7.1+ (based on Herbert Xu's cryptodev-2.6 tree, 7.2.0-rc1)
- sig_alg backend (upstream since 6.13)
- CRYPTO_AHASH_REQ_VIRT (native support, no fallback needed)
- CMH eSW loaded independently by hardware before driver probe
The driver registers all algorithms through the standard in-kernel
crypto API; in-kernel users (dm-crypt, fscrypt, IPsec, etc.) consume
them directly. Key provisioning and hardware-held-key operations are
exposed to user space via /dev/cmh_mgmt ioctls.
Public hardware documentation:
Product brief: https://go.rambus.com/ch-7xx-and-cc-7xx-product-brief
No public datasheets are currently available. The driver was
developed against the CRI CryptoManager Hub Hardware Reference
Manual (Rambus Inc. confidential). Detailed hardware reference is
available under NDA from Rambus Inc.; contact the maintainers listed
in MAINTAINERS for access during review.
Tested on RISC-V and ARM64 QEMU emulation with the CMH hardware
model (QEMU TCG, 512 MiB RAM). Also exercised on Xilinx VMK180
FPGA board with real CMH IP.
- testmgr: 41 CMH algorithm registrations matched by upstream
test vectors, all pass; 30 names report "No test for" (PQC
families, KMAC, cSHAKE - no upstream vectors yet).
- kselftest tools/testing/selftests/drivers/crypto/cmh:
6 pass, 0 fail.
checkpatch.pl --strict: 0 errors, 0 warnings, 0 checks on all
files (the only output is the expected per-file "does MAINTAINERS
need updating?" reminder, satisfied by the MAINTAINERS patch).
sparse (C=2): 0 warnings.
W=1 -Werror: clean.
make dt_binding_check: clean (dtschema validates the
cri,cmh.yaml binding).
Tested with the following debug options enabled simultaneously
(submit-checklist "Test your code" item 1):
CONFIG_PROVE_LOCKING, CONFIG_PROVE_RCU, CONFIG_DEBUG_LOCK_ALLOC,
CONFIG_DEBUG_OBJECTS_RCU_HEAD, CONFIG_SLUB_DEBUG,
CONFIG_DEBUG_PAGEALLOC, CONFIG_DEBUG_MUTEXES, CONFIG_DEBUG_SPINLOCK,
CONFIG_DEBUG_PREEMPT, CONFIG_DEBUG_ATOMIC_SLEEP.
Result: no lockdep warnings, no ODEBUG splats, no slab corruption.
Additionally tested (separate passes - mutually exclusive configs):
- CONFIG_KASAN + CONFIG_UBSAN + CONFIG_DEBUG_KMEMLEAK + CONFIG_KFENCE:
no sanitizer findings; KMEMLEAK scan reports 0 unreferenced objects.
- CONFIG_KCSAN (arm64; riscv64 lacks HAVE_ARCH_KCSAN):
0 data-race reports attributed to the driver.
Stack usage: worst-case under 1 KB on both riscv64 and arm64
(scripts/checkstack.pl). Hardware command buffers live in
per-request context (heap-allocated by the crypto framework).
Alex Ousherovitch (19):
dt-bindings: crypto: add Rambus CryptoManager Hub
crypto: cmh - add core platform driver
crypto: cmh - add key provisioning and management
crypto: cmh - add SHA-2/SHA-3/SHAKE ahash
crypto: cmh - add HMAC ahash
crypto: cmh - add CSHAKE/KMAC ahash
crypto: cmh - add SM3 ahash
crypto: cmh - add AES skcipher/aead/cmac
crypto: cmh - add SM4 skcipher/aead/cmac/xcbc
crypto: cmh - add ChaCha20-Poly1305
crypto: cmh - add DRBG hwrng
crypto: cmh - add RSA akcipher
crypto: cmh - add ECDSA/SM2 sig
crypto: cmh - add ECDH/X25519 kpp
crypto: cmh - add ML-KEM/ML-DSA (QSE)
crypto: cmh - add SLH-DSA/LMS/XMSS (HCQ)
Documentation: ioctl: add CMH ioctl documentation and register 'J'
selftests: crypto: cmh - add kselftest for management ioctl
MAINTAINERS: add Rambus CryptoManager Hub (CMH)
Changes in v2:
- crypto: cmh: wait for both eSW readiness bits at probe. The probe
readiness gate polled SW_BOOT_STATUS for MISSION (bit 6, primary
VCQ engine) only, but the sidecar engine signals readiness
separately via MISSION2 (bit 7), asserted asynchronously. Gate on
both bits so an operation offloaded to the sidecar cannot race an
engine that is not yet up.
- crypto: cmh: compute cmac(sm4) of a zero-length message in software.
The SM4 core mishandles an empty CMAC input, so -- exactly like the
existing xcbc(sm4) empty-message path, and per NIST SP 800-38B -- the
driver now derives the K2 subkey and computes the tag with a software
cipher. Surfaced by the testmgr fuzz comparison against the generic
cmac(sm4); no change for non-empty messages.
- Documentation: link cmh_mgmt.rst into the userspace-api/ioctl index
toctree, fixing a "document isn't included in any toctree" htmldocs
warning reported on the v1 posting of patch 17.
- crypto: cmh: carry the XTS block offset (xts_offset) inline in the
AES INIT command, matching the updated CMH hardware ABI that
retired the separate XTS_OFFSET command. Always 0 on the skcipher
path (data units start at block 0); no functional change.
- MAINTAINERS: drop the internal moderated list (sipsupport@rambus.com)
and the T: tree line from the CMH entry, per review. The driver
goes through the crypto tree, so keep only linux-crypto@vger.kernel.org
and no T: line.
- Rebased onto current cryptodev-2.6 (v7.2-rc1); no code changes were
required (the merge window touches none of the crypto APIs the driver
uses).
v1: https://lore.kernel.org/linux-crypto/20260625173328.1140487-1-skrishnamoorthy@rambus.com/
base-commit: e264401ce4776a288524e5b87593d4d864147115
--
2.43.7
^ permalink raw reply
* [PATCH v2 19/19] MAINTAINERS: add Rambus CryptoManager Hub (CMH)
From: Saravanakrishnan Krishnamoorthy @ 2026-07-09 20:30 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260709203037.1884436-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Add MAINTAINERS entry for the CRI CryptoManager Hub (CMH) hardware
crypto accelerator driver under drivers/crypto/cmh/.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
MAINTAINERS | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index d1dbf2f07104..1c153a84f70f 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -6891,6 +6891,23 @@ F: kernel/cred.c
F: rust/kernel/cred.rs
F: Documentation/security/credentials.rst
+CRI CRYPTOMANAGER HUB (CMH) HARDWARE CRYPTO ACCELERATOR
+M: Alex Ousherovitch <aousherovitch@rambus.com>
+M: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
+R: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
+R: Thi Nguyen <thin@rambus.com>
+L: linux-crypto@vger.kernel.org
+S: Maintained
+F: Documentation/ABI/testing/cmh-mgmt
+F: Documentation/ABI/testing/debugfs-driver-cmh
+F: Documentation/ABI/testing/sysfs-driver-cmh
+F: Documentation/crypto/device_drivers/cmh.rst
+F: Documentation/devicetree/bindings/crypto/cri,cmh.yaml
+F: Documentation/userspace-api/ioctl/cmh_mgmt.rst
+F: drivers/crypto/cmh/
+F: include/uapi/linux/cmh_mgmt_ioctl.h
+F: tools/testing/selftests/drivers/crypto/cmh/
+
INTEL CRPS COMMON REDUNDANT PSU DRIVER
M: Ninad Palsule <ninad@linux.ibm.com>
L: linux-hwmon@vger.kernel.org
--
2.43.7
^ permalink raw reply related
* Re: [PATCH] ovl: add ioctls to retrieve layer file descriptors
From: Amir Goldstein @ 2026-07-09 14:32 UTC (permalink / raw)
To: Miklos Szeredi
Cc: Giuseppe Scrivano, linux-unionfs, linux-fsdevel, linux-api,
Al Viro, Christian Brauner, Daan De Meyer
In-Reply-To: <CAJfpegvUEEbFnCa1N9k3gMXpw46YFpWuta=G6brON1=xU36bgw@mail.gmail.com>
On Wed, Jul 8, 2026 at 9:45 PM Miklos Szeredi <miklos@szeredi.hu> wrote:
>
> On Wed, 8 Jul 2026 at 17:55, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
> >
> > Miklos Szeredi <miklos@szeredi.hu> writes:
> >
> > > On Wed, 8 Jul 2026 at 16:32, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
> > >
> > >> Amir suggested to add that functionality when I've asked for some
> > >> feedback before sending the patch here. I am fine to drop it if this is
> > >> the consensus although I see its utility from user space.
> > >
> > > How about a completely different interface:
> > >
> > > int get_fd_opt(const char *name, unsigned int index, unsigned int flags);
> > >
> > > Enumerating layers would be as easy as passing an index stating from
> > > zero and stopping when -ERANGE is received.
> > >
> > > It would work for all filesystems that use files as options. No more
> > > fs specific ioctls.
> >
> > Is a new syscall really justified for such a narrow use case?
>
> That's the reason I advocate pseudo fs based solutions. Let's see, we
> had a proposal to use openat(), something. like:
>
> openat(base_fd, "mount/options/lowerdir+/0", O_ALT | O_PATH);
>
Ack for this, but please don't use mount/options/... it's a bit ugly
because most mount options are not opan-able.
Please stick to something logical like "fs/layers/N" corresponding to
ovl_fs::layers, where layer 0 is reserved for upper.
We could also support opening by aliases fs/layers/upper->0 etc,
but no rush IMO.
What I am contemplating is whether we should implement
introspection of the ovl_entry stack for merge dirs in addition or instead
introspection of fs/layers.
IIRC, the root dir stack does NOT hold references to lowerdata dirs,
so support for "fs/layers/N" is still needed.
For directories we could implement "real/0..N" with aliases
real/upper->0 real/origin->1.
For a regular file we could have constant aliases real/upper->0
real/lower->1 real/lowerdata->2 and/or dynamic aliases
real/data->{0,1,2}, real/metadata->{0,1}, real/origin->{0,1}.
Those could be used to implement "revert to origin".
Specifically, I think Daan asked for it during the last LSFMM.
Although I don't think we currently store the original lowerdata
in ovl_entry when lookup is done post data copy up.
Thanks,
Amir.
^ permalink raw reply
* Re: [PATCH] ovl: add ioctls to retrieve layer file descriptors
From: Miklos Szeredi @ 2026-07-09 6:11 UTC (permalink / raw)
To: Giuseppe Scrivano
Cc: linux-unionfs, linux-fsdevel, linux-api, Amir Goldstein, Al Viro,
Christian Brauner
In-Reply-To: <87fr1t6wyt.fsf@redhat.com>
On Wed, 8 Jul 2026 at 22:35, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
> Would you be OK with the ioctl as a solution that works today, and
> duplicate it when an alternative API materializes?
Let me have a go at the pseudo fs interface first. I can probably do
a quick 'n dirty prototype within a couple of days and see what
reaction it gets...
Thanks,
Miklos
^ permalink raw reply
* Re: [PATCH] ovl: add ioctls to retrieve layer file descriptors
From: Giuseppe Scrivano @ 2026-07-08 20:45 UTC (permalink / raw)
To: Miklos Szeredi; +Cc: Amir Goldstein, linux-unionfs, linux-fsdevel, linux-api
In-Reply-To: <CAJfpegvtd--TEQP9CVjSZuosU4meELdzhTV9oFo6joR1f4Dpgw@mail.gmail.com>
Miklos Szeredi <miklos@szeredi.hu> writes:
> On Wed, 8 Jul 2026 at 15:44, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
>>
>> Amir Goldstein <amir73il@gmail.com> writes:
>>
>> > On Wed, Jul 8, 2026 at 2:31 PM Giuseppe Scrivano <gscrivan@redhat.com> wrote:
>> >>
>> >> Miklos Szeredi <miklos@szeredi.hu> writes:
>> >>
>> >> > On Wed, 8 Jul 2026 at 12:00, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
>> >> >>
>> >> >> Add two ioctls to overlay filesystem to allow userspace to retrieve
>> >> >> information about the overlay layers:
>> >> >>
>> >> >> OVL_IOC_OPEN_LAYER: return an O_PATH fd to the root of a layer.
>> >> >> arg == 0 returns the upper layer (-ENOENT if
>> >> >> no upper is configured), arg >= 1 returns
>> >> >> lower layers (-ENOENT if index is out of
>> >> >> range).
>> >> >
>> >> > We could do this with a plain open() call. Something like the magic
>> >> > symlinks we have under /proc/PID/fd/. Question is where could these
>> >> > live...
>> >>
>> >> is there any existing user of such a mechanism? I don't see any mount
>> >> specific info under /proc or /sys.
>> >>
>> >> >
>> >> >> OVL_IOC_GET_LAYERS_INFO: copy a struct ovl_layers_info to userspace
>> >> >> with numlower, numlowerdata, and has_upper.
>> >> >
>> >> > Isn't this info obtainable via statmount(2) already? If not, it
>> >> > should be there, instead of a specialized ioctl.
>> >>
>> >> no that is not exposed by statmount and I don't see any way to export
>> >> file system specific data through it. Do you've anything in mind?
>> >>
>> >> >> --- a/fs/overlayfs/ovl_entry.h
>> >> >> +++ b/fs/overlayfs/ovl_entry.h
>> >> >> @@ -35,6 +35,8 @@ struct ovl_layer {
>> >> >> struct vfsmount *mnt;
>> >> >> /* Trap in ovl inode cache */
>> >> >> struct inode *trap;
>> >> >> + /* Keeps the original fsmount file alive for OVL_IOC_OPEN_LAYER */
>> >> >> + struct file *origin;
>> >> >
>> >> > Don't need to keep the file open: the only info missing is the
>> >> > original vfsmount, everything else is already there to reconstruct the
>> >> > file.
>> >>
>> >> I didn't manage to get that to work. As soon as the userspace process
>> >> closes the mount fd that was passed to fsconfig, the anonymous mount
>> >> namespace is destroyed and dissolve_on_fput sets mnt->mnt_ns to NULL.
>> >>
>> >> So whenever I try to use this mount again from userspace, it is not
>> >> usable because the mount namespace is empty, causing check_mnt() to
>> >> fail.
>> >>
>> >> Do you have any suggestions on how to solve this problem?
>> >
>> > The suggestion was to store origin->f_path->mnt instead of storing origin file,
>> > because you only end up using the origin vfsmount.
>>
>> the reason I am keeping the file and not just the vfsmount is that the
>> file is what keeps the mount namespace alive (preventing
>> dissolve_on_fput from fire).
>>
>> Should we export open_detached_copy from fs/namespace.c? I've not
>> tested it, but it might work. Are there other ways to solve it that I
>> am not seeing?
>
> Using an anon namespace sounds good to me, that means the original
> vfsmount isn't needed at all.
>
> Not sure if it's okay for the case where the original ns is not anon,
> but we can save the vfsmount in that case if it turns out to be a
> problem.
>
> Thanks,
> Miklos
for this to work, I need something like:
diff --git a/fs/namespace.c b/fs/namespace.c
index 3d5cd5bf3b05..138d15ab37ef 100644
--- a/fs/namespace.c
+++ b/fs/namespace.c
@@ -3091,6 +3091,52 @@ static struct file *open_detached_copy(struct path *path, unsigned int flags)
return file;
}
+struct file *open_detached_copy_internal(struct path *path)
+{
+ struct mnt_namespace *ns, *mnt_ns = current->nsproxy->mnt_ns;
+ struct mount *mnt;
+ struct file *file;
+
+ ns = alloc_mnt_ns(mnt_ns->user_ns, true);
+ if (IS_ERR(ns))
+ return ERR_CAST(ns);
+
+ guard(namespace_excl)();
+
+ mnt = clone_mnt(real_mount(path->mnt), path->dentry,
+ CL_COPY_MNT_NS_FILE);
+ if (IS_ERR(mnt)) {
+ free_mnt_ns(ns);
+ return ERR_CAST(mnt);
+ }
+
+ mnt_add_to_ns(ns, mnt);
+ ns->nr_mounts++;
+ ns->root = mnt;
+
+ mntput(path->mnt);
+ path->mnt = mntget(&mnt->mnt);
+ file = dentry_open(path, O_PATH, current_cred());
+ if (IS_ERR(file))
+ dissolve_on_fput(path->mnt);
+ else
+ file->f_mode |= FMODE_NEED_UNMOUNT;
+ return file;
+}
+EXPORT_SYMBOL_GPL(open_detached_copy_internal);
+
enum mount_copy_flags_t {
MOUNT_COPY_RECURSIVE = (1 << 0),
MOUNT_COPY_NEW = (1 << 1),
is this acceptable?
Regards,
Giuseppe
^ permalink raw reply related
* Re: [PATCH] ovl: add ioctls to retrieve layer file descriptors
From: Giuseppe Scrivano @ 2026-07-08 20:35 UTC (permalink / raw)
To: Miklos Szeredi
Cc: linux-unionfs, linux-fsdevel, linux-api, Amir Goldstein, Al Viro,
Christian Brauner
In-Reply-To: <CAJfpegvUEEbFnCa1N9k3gMXpw46YFpWuta=G6brON1=xU36bgw@mail.gmail.com>
Miklos Szeredi <miklos@szeredi.hu> writes:
> On Wed, 8 Jul 2026 at 17:55, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
>>
>> Miklos Szeredi <miklos@szeredi.hu> writes:
>>
>> > On Wed, 8 Jul 2026 at 16:32, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
>> >
>> >> Amir suggested to add that functionality when I've asked for some
>> >> feedback before sending the patch here. I am fine to drop it if this is
>> >> the consensus although I see its utility from user space.
>> >
>> > How about a completely different interface:
>> >
>> > int get_fd_opt(const char *name, unsigned int index, unsigned int flags);
>> >
>> > Enumerating layers would be as easy as passing an index stating from
>> > zero and stopping when -ERANGE is received.
>> >
>> > It would work for all filesystems that use files as options. No more
>> > fs specific ioctls.
>>
>> Is a new syscall really justified for such a narrow use case?
>
> That's the reason I advocate pseudo fs based solutions. Let's see, we
> had a proposal to use openat(), something. like:
>
> openat(base_fd, "mount/options/lowerdir+/0", O_ALT | O_PATH);
>
> Meaning that O_ALT switches to an alternative/meta namespace that is
> based on the given fd and in that meta namespace the tree under mount/
> represents the attributes of the mount that base_fd is on.
>
> See this post from Linus as well:
>
> https://lore.kernel.org/all/CAHk-=wjzLmMRf=QG-n+1HnxWCx4KTQn9+OhVvUSJ=ZCQd6Y1WA@mail.gmail.com/
>
> The statmount api took a different route, but for getting an O_PATH
> file this would be a very natural interface without added syscalls.
I don't argue against such an API, I only argue that I am probably not
the best person to drive such a significant change :-)
Would you be OK with the ioctl as a solution that works today, and
duplicate it when an alternative API materializes?
Thanks,
Giuseppe
^ permalink raw reply
* Re: [PATCH] ovl: add ioctls to retrieve layer file descriptors
From: Colin Walters @ 2026-07-08 20:16 UTC (permalink / raw)
To: Amir Goldstein, Giuseppe Scrivano
Cc: Miklos Szeredi, linux-unionfs@vger.kernel.org,
linux-fsdevel@vger.kernel.org, linux-api, Al Viro,
Christian Brauner, Gao Xiang, linux-erofs mailing list
In-Reply-To: <CAOQ4uxgbNhdzKN7tvRmFDpt-8CZWh9pVcMLv25HxJzA0_0WfSg@mail.gmail.com>
On Wed, Jul 8, 2026, at 3:01 PM, Amir Goldstein wrote:
> If you want to keep a pool of mounted erofs images, you could do that
> in userspace -
> create a service that indexes mounted erofs images by unique mount point paths.
You're right that it’d be possible to do this in user space.
However, overlayfs is used by many things (it’s a powerful “Swiss army knife”!) and this ability to reliably introspect its components is I think generally useful.
For example: https://github.com/systemd/systemd/issues/35017#issuecomment-2457333218
Various tools want to know the backing filesystem(s) and/or block devices thereof, and if we don’t have something like this in the general case every overlayfs user would need to agree on a scheme to store this data out of band - and manage its lifecycle as mounts change and deal with where that data is stored with respect to mount namespaces etc.
Even if we had this API, would it be better to have a user space cache server for the original use case here? Maybe. I could personally go either way.
But the introspection I think really would be generally useful and there are code bases (in systemd and in composefs at least) that would start using it (with fallback for old kernels) for at least that use case right away.
^ permalink raw reply
* Re: [PATCH] ovl: add ioctls to retrieve layer file descriptors
From: Giuseppe Scrivano @ 2026-07-08 20:18 UTC (permalink / raw)
To: Amir Goldstein
Cc: Miklos Szeredi, linux-unionfs, linux-fsdevel, linux-api, Al Viro,
Christian Brauner, Gao Xiang, linux-erofs mailing list
In-Reply-To: <CAOQ4uxgbNhdzKN7tvRmFDpt-8CZWh9pVcMLv25HxJzA0_0WfSg@mail.gmail.com>
Amir Goldstein <amir73il@gmail.com> writes:
> On Wed, Jul 8, 2026 at 5:55 PM Giuseppe Scrivano <gscrivan@redhat.com> wrote:
>>
>> Miklos Szeredi <miklos@szeredi.hu> writes:
>>
>> > On Wed, 8 Jul 2026 at 16:32, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
>> >
>> >> Amir suggested to add that functionality when I've asked for some
>> >> feedback before sending the patch here. I am fine to drop it if this is
>> >> the consensus although I see its utility from user space.
>
> I was thinking that getting the number of layers or info would be
> a good idea to complement getting a layer fd.
>
> I agree that the same information is probably available via statmount
> by parsing the upperdir/lowerdir/datadir mount options.
>
>> >
>> > How about a completely different interface:
>> >
>> > int get_fd_opt(const char *name, unsigned int index, unsigned int flags);
>> >
>> > Enumerating layers would be as easy as passing an index stating from
>> > zero and stopping when -ERANGE is received.
>> >
>> > It would work for all filesystems that use files as options. No more
>> > fs specific ioctls.
>>
>> Is a new syscall really justified for such a narrow use case?
>>
>
> I feel the same way.
>
> Giuseppe,
>
> Could you add some high level context in this thread on why you need
> this functionality.
> I think it's this composefs-rs work. right?
> https://github.com/giuseppe/composefs-rs/commits/reuse-mounts-and-prevent-gc-overlay/
>
> I must say this seems a bit upside down to me.
>
> If you want to keep a pool of mounted erofs images, you could do that
> in userspace -
> create a service that indexes mounted erofs images by unique mount point paths.
> Then you can introspect the overlayfs mount options referring to those
> mount points.
A first issue is that the mount options won't have this information
anymore, as we use /proc/self/fd/$i paths as lower dirs so we are sure
the fd points exactly to the file we have measured its fs-verity digest
before using it.
I know this can be achieved with a system daemon, but do we really need
one if this information is already known to the kernel?
Combined with listmount/statmount for discovery and fs-verity for
validation, the entire mechanism is stateless from userspace.
More in general we need a way to introspect overlay mounts to know where
they are pointing to since paths can be hidden using /proc/*/fd
symlinks, or files get replaced.
Another similar request:
https://github.com/systemd/systemd/issues/35017#issuecomment-2457333218
> Going through the kernel to get an fd and reuse that fd for a new
> overlayfs mount
> sounds like a strange way of accomplishing this.
>
> If the overlayfs mounter is unprivileged, it would have to go through
> systemd-mountfsd
> to request a mount of erofs trusted image, right?
off-topic but for now we are considering FUSE to deal with mounting
EROFS as it would serve only the metadata anyway in a composefs setup.
Regards,
Giuseppe
> Can't the same service provide the "is_image_mounted" query which provides
> the mount path?
>
> I am not against introspection of overlayfs, but I'd like to understand
> the use cases before finalizing the uapi.
>
> Thanks,
> Amir.
^ permalink raw reply
* Re: [PATCH] ovl: add ioctls to retrieve layer file descriptors
From: Miklos Szeredi @ 2026-07-08 19:45 UTC (permalink / raw)
To: Giuseppe Scrivano
Cc: linux-unionfs, linux-fsdevel, linux-api, Amir Goldstein, Al Viro,
Christian Brauner
In-Reply-To: <87o6gh79yi.fsf@redhat.com>
On Wed, 8 Jul 2026 at 17:55, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
>
> Miklos Szeredi <miklos@szeredi.hu> writes:
>
> > On Wed, 8 Jul 2026 at 16:32, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
> >
> >> Amir suggested to add that functionality when I've asked for some
> >> feedback before sending the patch here. I am fine to drop it if this is
> >> the consensus although I see its utility from user space.
> >
> > How about a completely different interface:
> >
> > int get_fd_opt(const char *name, unsigned int index, unsigned int flags);
> >
> > Enumerating layers would be as easy as passing an index stating from
> > zero and stopping when -ERANGE is received.
> >
> > It would work for all filesystems that use files as options. No more
> > fs specific ioctls.
>
> Is a new syscall really justified for such a narrow use case?
That's the reason I advocate pseudo fs based solutions. Let's see, we
had a proposal to use openat(), something. like:
openat(base_fd, "mount/options/lowerdir+/0", O_ALT | O_PATH);
Meaning that O_ALT switches to an alternative/meta namespace that is
based on the given fd and in that meta namespace the tree under mount/
represents the attributes of the mount that base_fd is on.
See this post from Linus as well:
https://lore.kernel.org/all/CAHk-=wjzLmMRf=QG-n+1HnxWCx4KTQn9+OhVvUSJ=ZCQd6Y1WA@mail.gmail.com/
The statmount api took a different route, but for getting an O_PATH
file this would be a very natural interface without added syscalls.
Thanks,
Miklos
^ permalink raw reply
* Re: [PATCH] ovl: add ioctls to retrieve layer file descriptors
From: Amir Goldstein @ 2026-07-08 19:01 UTC (permalink / raw)
To: Giuseppe Scrivano
Cc: Miklos Szeredi, linux-unionfs, linux-fsdevel, linux-api, Al Viro,
Christian Brauner, Gao Xiang, linux-erofs mailing list
In-Reply-To: <87o6gh79yi.fsf@redhat.com>
On Wed, Jul 8, 2026 at 5:55 PM Giuseppe Scrivano <gscrivan@redhat.com> wrote:
>
> Miklos Szeredi <miklos@szeredi.hu> writes:
>
> > On Wed, 8 Jul 2026 at 16:32, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
> >
> >> Amir suggested to add that functionality when I've asked for some
> >> feedback before sending the patch here. I am fine to drop it if this is
> >> the consensus although I see its utility from user space.
I was thinking that getting the number of layers or info would be
a good idea to complement getting a layer fd.
I agree that the same information is probably available via statmount
by parsing the upperdir/lowerdir/datadir mount options.
> >
> > How about a completely different interface:
> >
> > int get_fd_opt(const char *name, unsigned int index, unsigned int flags);
> >
> > Enumerating layers would be as easy as passing an index stating from
> > zero and stopping when -ERANGE is received.
> >
> > It would work for all filesystems that use files as options. No more
> > fs specific ioctls.
>
> Is a new syscall really justified for such a narrow use case?
>
I feel the same way.
Giuseppe,
Could you add some high level context in this thread on why you need
this functionality.
I think it's this composefs-rs work. right?
https://github.com/giuseppe/composefs-rs/commits/reuse-mounts-and-prevent-gc-overlay/
I must say this seems a bit upside down to me.
If you want to keep a pool of mounted erofs images, you could do that
in userspace -
create a service that indexes mounted erofs images by unique mount point paths.
Then you can introspect the overlayfs mount options referring to those
mount points.
Going through the kernel to get an fd and reuse that fd for a new
overlayfs mount
sounds like a strange way of accomplishing this.
If the overlayfs mounter is unprivileged, it would have to go through
systemd-mountfsd
to request a mount of erofs trusted image, right?
Can't the same service provide the "is_image_mounted" query which provides
the mount path?
I am not against introspection of overlayfs, but I'd like to understand
the use cases before finalizing the uapi.
Thanks,
Amir.
^ permalink raw reply
* Re: [PATCH] ovl: add ioctls to retrieve layer file descriptors
From: Giuseppe Scrivano @ 2026-07-08 15:55 UTC (permalink / raw)
To: Miklos Szeredi
Cc: linux-unionfs, linux-fsdevel, linux-api, Amir Goldstein, Al Viro,
Christian Brauner
In-Reply-To: <CAJfpegtTixwWSh9M-9NbwP0nUbJJ9rh0rxqO7BzgK7Su_RpM+A@mail.gmail.com>
Miklos Szeredi <miklos@szeredi.hu> writes:
> On Wed, 8 Jul 2026 at 16:32, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
>
>> Amir suggested to add that functionality when I've asked for some
>> feedback before sending the patch here. I am fine to drop it if this is
>> the consensus although I see its utility from user space.
>
> How about a completely different interface:
>
> int get_fd_opt(const char *name, unsigned int index, unsigned int flags);
>
> Enumerating layers would be as easy as passing an index stating from
> zero and stopping when -ERANGE is received.
>
> It would work for all filesystems that use files as options. No more
> fs specific ioctls.
Is a new syscall really justified for such a narrow use case?
On the other hand, it would cover both ioctl's I am working on right
now.
Regards,
Giuseppe
^ permalink raw reply
* Re: [PATCH] ovl: add ioctls to retrieve layer file descriptors
From: Miklos Szeredi @ 2026-07-08 15:37 UTC (permalink / raw)
To: Giuseppe Scrivano; +Cc: linux-unionfs, linux-fsdevel, linux-api, Amir Goldstein
In-Reply-To: <87wlv57dt1.fsf@redhat.com>
On Wed, 8 Jul 2026 at 16:32, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
> Amir suggested to add that functionality when I've asked for some
> feedback before sending the patch here. I am fine to drop it if this is
> the consensus although I see its utility from user space.
How about a completely different interface:
int get_fd_opt(const char *name, unsigned int index, unsigned int flags);
Enumerating layers would be as easy as passing an index stating from
zero and stopping when -ERANGE is received.
It would work for all filesystems that use files as options. No more
fs specific ioctls.
Thanks,
Miklos
^ permalink raw reply
* Re: [PATCH 2/2] erofs: add ioctl to retrieve the backing source file descriptor
From: Giuseppe Scrivano @ 2026-07-08 15:20 UTC (permalink / raw)
To: Andy Lutomirski; +Cc: linux-erofs, linux-fsdevel, linux-api
In-Reply-To: <CALCETrVE0g-qKC079K4Ch=26VH6R+q1tXVU7HiN_Og9o1zzpow@mail.gmail.com>
Andy Lutomirski <luto@amacapital.net> writes:
> On Wed, Jul 8, 2026 at 2:36 AM Giuseppe Scrivano <gscrivan@redhat.com> wrote:
>>
>> Add EROFS_IOC_GET_SOURCE_FD ioctl that returns a file descriptor to the
>> backing image file for file-backed erofs mounts.
>
> What’s the use case?
the use case is to reuse EROFS mounts across multiple composefs mounts
without needing a user space daemon to keep the fd alive. The advantage
is that we can share the same superblock across multiple overlay mounts,
instead of limiting the sharing to the same backing file.
> In any event, this seems to have potential security and API-oddity implications.
>
> 1. That capable(CAP_SYS_ADMIN) seems critical — otherwise a task that
> is admin in a userns can get an fd to a backing file *outside* its
> container or to an otherwise inaccessible file. It at least needs a
> comment IMO.
thanks, I'll add that.
> 2. This series appears to fully round-trip the struct file. That
> means that f_cred and mode are preserved. This seems strange and may
> have all kinds of accidental effects. For mode in parallel, if the
> program that mounts the backing store opened it for write, then this
> API gives a writable fd, which seems like an odd choice for a
> filesystem that literally has read only in the name.
> The OVL variant has these issues to a lesser extent due to the fact
> that it returns an O_PATH fd instead of an actual open file.
I didn't think much of the file mode as I assumed requiring
CAP_SYS_ADMIN was enough. I'll change the file mode to O_RDONLY and use
current_cred() to open it. Would that be enough?
Thanks,
Giuseppe
^ permalink raw reply
* Re: [PATCH 2/2] erofs: add ioctl to retrieve the backing source file descriptor
From: Andy Lutomirski @ 2026-07-08 14:55 UTC (permalink / raw)
To: Giuseppe Scrivano; +Cc: linux-erofs, linux-fsdevel, linux-api
In-Reply-To: <20260708093446.3370200-3-gscrivan@redhat.com>
On Wed, Jul 8, 2026 at 2:36 AM Giuseppe Scrivano <gscrivan@redhat.com> wrote:
>
> Add EROFS_IOC_GET_SOURCE_FD ioctl that returns a file descriptor to the
> backing image file for file-backed erofs mounts.
What’s the use case?
In any event, this seems to have potential security and API-oddity implications.
1. That capable(CAP_SYS_ADMIN) seems critical — otherwise a task that
is admin in a userns can get an fd to a backing file *outside* its
container or to an otherwise inaccessible file. It at least needs a
comment IMO.
2. This series appears to fully round-trip the struct file. That
means that f_cred and mode are preserved. This seems strange and may
have all kinds of accidental effects. For mode in parallel, if the
program that mounts the backing store opened it for write, then this
API gives a writable fd, which seems like an odd choice for a
filesystem that literally has read only in the name.
The OVL variant has these issues to a lesser extent due to the fact
that it returns an O_PATH fd instead of an actual open file.
^ permalink raw reply
* Re: [PATCH] ovl: add ioctls to retrieve layer file descriptors
From: Giuseppe Scrivano @ 2026-07-08 14:31 UTC (permalink / raw)
To: Miklos Szeredi; +Cc: linux-unionfs, linux-fsdevel, linux-api, Amir Goldstein
In-Reply-To: <CAJfpegvQ06=2E0V_ADgxwmo7e5weTfOMozmBB-QVNLLWYAm8WQ@mail.gmail.com>
Miklos Szeredi <miklos@szeredi.hu> writes:
> On Wed, 8 Jul 2026 at 14:27, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
>
>> is there any existing user of such a mechanism? I don't see any mount
>> specific info under /proc or /sys.
>
> Not yet, but I think it would be good to have one. It's definitely a
> bigger bite than adding an ioctl(), though.
>
>> >> OVL_IOC_GET_LAYERS_INFO: copy a struct ovl_layers_info to userspace
>> >> with numlower, numlowerdata, and has_upper.
>> >
>> > Isn't this info obtainable via statmount(2) already? If not, it
>> > should be there, instead of a specialized ioctl.
>>
>
>> no that is not exposed by statmount
>
> I mean, you should be able to get that info by counting the lowerdir+,
> datadir+ and upperdir options, no?
Amir suggested to add that functionality when I've asked for some
feedback before sending the patch here. I am fine to drop it if this is
the consensus although I see its utility from user space.
Regards,
Giuseppe
^ permalink raw reply
* Re: [PATCH] ovl: add ioctls to retrieve layer file descriptors
From: Miklos Szeredi @ 2026-07-08 14:08 UTC (permalink / raw)
To: Giuseppe Scrivano; +Cc: linux-unionfs, linux-fsdevel, linux-api
In-Reply-To: <878q7l8y4y.fsf@redhat.com>
On Wed, 8 Jul 2026 at 14:27, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
> is there any existing user of such a mechanism? I don't see any mount
> specific info under /proc or /sys.
Not yet, but I think it would be good to have one. It's definitely a
bigger bite than adding an ioctl(), though.
> >> OVL_IOC_GET_LAYERS_INFO: copy a struct ovl_layers_info to userspace
> >> with numlower, numlowerdata, and has_upper.
> >
> > Isn't this info obtainable via statmount(2) already? If not, it
> > should be there, instead of a specialized ioctl.
>
> no that is not exposed by statmount
I mean, you should be able to get that info by counting the lowerdir+,
datadir+ and upperdir options, no?
Thanks,
Miklos
^ permalink raw reply
* Re: [PATCH] ovl: add ioctls to retrieve layer file descriptors
From: Miklos Szeredi @ 2026-07-08 14:01 UTC (permalink / raw)
To: Giuseppe Scrivano; +Cc: Amir Goldstein, linux-unionfs, linux-fsdevel, linux-api
In-Reply-To: <871pdd8ukx.fsf@redhat.com>
On Wed, 8 Jul 2026 at 15:44, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
>
> Amir Goldstein <amir73il@gmail.com> writes:
>
> > On Wed, Jul 8, 2026 at 2:31 PM Giuseppe Scrivano <gscrivan@redhat.com> wrote:
> >>
> >> Miklos Szeredi <miklos@szeredi.hu> writes:
> >>
> >> > On Wed, 8 Jul 2026 at 12:00, Giuseppe Scrivano <gscrivan@redhat.com> wrote:
> >> >>
> >> >> Add two ioctls to overlay filesystem to allow userspace to retrieve
> >> >> information about the overlay layers:
> >> >>
> >> >> OVL_IOC_OPEN_LAYER: return an O_PATH fd to the root of a layer.
> >> >> arg == 0 returns the upper layer (-ENOENT if
> >> >> no upper is configured), arg >= 1 returns
> >> >> lower layers (-ENOENT if index is out of
> >> >> range).
> >> >
> >> > We could do this with a plain open() call. Something like the magic
> >> > symlinks we have under /proc/PID/fd/. Question is where could these
> >> > live...
> >>
> >> is there any existing user of such a mechanism? I don't see any mount
> >> specific info under /proc or /sys.
> >>
> >> >
> >> >> OVL_IOC_GET_LAYERS_INFO: copy a struct ovl_layers_info to userspace
> >> >> with numlower, numlowerdata, and has_upper.
> >> >
> >> > Isn't this info obtainable via statmount(2) already? If not, it
> >> > should be there, instead of a specialized ioctl.
> >>
> >> no that is not exposed by statmount and I don't see any way to export
> >> file system specific data through it. Do you've anything in mind?
> >>
> >> >> --- a/fs/overlayfs/ovl_entry.h
> >> >> +++ b/fs/overlayfs/ovl_entry.h
> >> >> @@ -35,6 +35,8 @@ struct ovl_layer {
> >> >> struct vfsmount *mnt;
> >> >> /* Trap in ovl inode cache */
> >> >> struct inode *trap;
> >> >> + /* Keeps the original fsmount file alive for OVL_IOC_OPEN_LAYER */
> >> >> + struct file *origin;
> >> >
> >> > Don't need to keep the file open: the only info missing is the
> >> > original vfsmount, everything else is already there to reconstruct the
> >> > file.
> >>
> >> I didn't manage to get that to work. As soon as the userspace process
> >> closes the mount fd that was passed to fsconfig, the anonymous mount
> >> namespace is destroyed and dissolve_on_fput sets mnt->mnt_ns to NULL.
> >>
> >> So whenever I try to use this mount again from userspace, it is not
> >> usable because the mount namespace is empty, causing check_mnt() to
> >> fail.
> >>
> >> Do you have any suggestions on how to solve this problem?
> >
> > The suggestion was to store origin->f_path->mnt instead of storing origin file,
> > because you only end up using the origin vfsmount.
>
> the reason I am keeping the file and not just the vfsmount is that the
> file is what keeps the mount namespace alive (preventing
> dissolve_on_fput from fire).
>
> Should we export open_detached_copy from fs/namespace.c? I've not
> tested it, but it might work. Are there other ways to solve it that I
> am not seeing?
Using an anon namespace sounds good to me, that means the original
vfsmount isn't needed at all.
Not sure if it's okay for the case where the original ns is not anon,
but we can save the vfsmount in that case if it turns out to be a
problem.
Thanks,
Miklos
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox