* [PATCH v3 11/19] crypto: cmh - add DRBG hwrng
From: Saravanakrishnan Krishnamoorthy @ 2026-08-06 19:55 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register the CMH DRBG core (core ID 0x0f) as an hwrng provider.
The hardware implements a NIST SP 800-90A compliant DRBG with
automatic self-seeding.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 3 +-
drivers/crypto/cmh/cmh_main.c | 9 +
drivers/crypto/cmh/cmh_rng.c | 333 ++++++++++++++++++++++++++++++++++
3 files changed, 344 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_rng.c
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index ef24879ba1f1..407fd2870f6d 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -28,7 +28,8 @@ cmh-y := \
cmh_sm4_cmac.o \
cmh_ccp.o \
cmh_ccp_aead.o \
- cmh_ccp_poly.o
+ cmh_ccp_poly.o \
+ cmh_rng.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index 73b25ed7edcb..1e0f5dad8575 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -36,6 +36,7 @@
#include "cmh_cshake.h"
#include "cmh_kmac.h"
#include "cmh_sm3.h"
+#include "cmh_rng.h"
#include "cmh_aes.h"
#include "cmh_sm4.h"
#include "cmh_ccp.h"
@@ -270,6 +271,11 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_sm3_register;
+ /* Register hwrng backed by DRBG core */
+ ret = cmh_rng_register(pdev);
+ if (ret)
+ goto err_rng_register;
+
/* Register AES skcipher algorithms */
ret = cmh_aes_register();
if (ret)
@@ -345,6 +351,8 @@ static int cmh_probe(struct platform_device *pdev)
err_aes_aead_register:
cmh_aes_unregister();
err_aes_register:
+ cmh_rng_unregister();
+err_rng_register:
cmh_sm3_unregister();
err_sm3_register:
cmh_kmac_unregister();
@@ -390,6 +398,7 @@ static void cmh_remove(struct platform_device *pdev)
cmh_aes_cmac_unregister();
cmh_aes_aead_unregister();
cmh_aes_unregister();
+ cmh_rng_unregister();
cmh_sm3_unregister();
cmh_kmac_unregister();
cmh_cshake_unregister();
diff --git a/drivers/crypto/cmh/cmh_rng.c b/drivers/crypto/cmh/cmh_rng.c
new file mode 100644
index 000000000000..9532b8d66b4c
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_rng.c
@@ -0,0 +1,333 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Hardware RNG (DRBG) Driver
+ *
+ * Implements a Linux hwrng backed by the CMH DRBG core. Each .read()
+ * builds a 3-entry VCQ (header + GENERATE + FLUSH) and submits it
+ * synchronously through the Transaction Manager.
+ *
+ * DRBG configuration (CONFIG) is a management-host operation in the
+ * CMH security model. The driver's behaviour is controlled by the
+ * drbg_config setting (debug-only module parameter):
+ *
+ * "auto" (default) -- attempt CONFIG at probe with the hardcoded
+ * ratio/strength defaults. Succeeds in stateless mode (any host may
+ * CONFIG) or when this host is the management host in stateful
+ * mode. On -EPERM the driver logs a notice and continues --
+ * GENERATE will work once the management host configures the DRBG.
+ *
+ * "skip" -- do not issue CONFIG; assume an external management host
+ * will configure the DRBG. hwrng is still registered; .read()
+ * returns -EAGAIN until GENERATE succeeds.
+ *
+ * The management host (or any privileged user-space process) can also
+ * reconfigure the DRBG at runtime via CMH_IOCTL_DRBG_CONFIG.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/hw_random.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+
+#include "cmh_rng.h"
+#include "cmh_vcq.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_sys.h"
+#include "cmh_config.h"
+
+/* VCQ layout for .read(): header + GENERATE + FLUSH = 3 entries. */
+#define DRBG_READ_VCQ_CMDS 3
+
+/* VCQ layout for CONFIG: header + RESET + CONFIG + FLUSH = 4 entries. */
+#define DRBG_CONFIG_VCQ_CMDS 4
+
+/*
+ * Linux hwrng quality is expressed in bits of entropy per 1024 bits of
+ * input. The kernel clamps to this maximum; mirror it here so our
+ * MODULE_PARM_DESC and clamp logic stay in sync.
+ */
+#define CMH_HWRNG_QUALITY_MAX 1024
+
+/* Module parameters */
+
+static int hwrng_quality;
+module_param(hwrng_quality, int, 0444);
+MODULE_PARM_DESC(hwrng_quality,
+ "hwrng entropy quality in bits per 1024 bits of output (1-1024). 0 (default) is left to the hwrng core, which elevates it to full trust (1024); set an explicit value to lower the estimate.");
+
+#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG
+static char *drbg_config = "auto";
+module_param(drbg_config, charp, 0444);
+MODULE_PARM_DESC(drbg_config,
+ "[debug] DRBG config at probe: \"auto\"=attempt CONFIG, \"skip\"=assume external (default: auto)");
+#else
+static const char *drbg_config = "auto";
+#endif
+
+/*
+ * DRBG parameters -- hardcoded to production defaults.
+ * Entropy ratio 0 = 1:1 (full entropy), security strength 0x10 = 256-bit.
+ */
+#define CMH_DRBG_ENTROPY_RATIO 0
+#define CMH_DRBG_SECURITY_STRENGTH 0x10
+
+static unsigned int drbg_timeout_ms = 500;
+
+/* VCQ Builders */
+
+static void vcq_add_drbg_generate(struct vcq_cmd *slot, u64 dst_phys, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_DRBG, 0, 1, DRBG_CMD_GENERATE);
+ slot->hwc.drbg.cmd_generate.dst = dst_phys;
+ slot->hwc.drbg.cmd_generate.len = len;
+}
+
+/*
+ * Maximum bytes per DRBG GENERATE request. The kernel calls .read()
+ * repeatedly to fill larger requests, so capping here is safe.
+ * 32 bytes matches the 256-bit security strength natural output size.
+ */
+#define CMH_DRBG_MAX_GENERATE 32U
+
+/* hwrng .read() callback */
+
+static int cmh_rng_read(struct hwrng *rng, void *data, size_t max, bool wait)
+{
+ struct cmh_dma_orphan *orphan;
+ struct vcq_cmd vcq[DRBG_READ_VCQ_CMDS];
+ dma_addr_t dma_addr;
+ void *dmabuf;
+ size_t nbytes;
+ int ret;
+
+ if (max == 0)
+ return 0;
+
+ /*
+ * Our path uses GFP_KERNEL allocations and synchronous VCQ
+ * submission -- both may sleep. When the caller indicates
+ * non-blocking context (!wait), return 0 ("no data yet") so
+ * the hwrng core retries later.
+ */
+ if (!wait)
+ return 0;
+
+ nbytes = min_t(size_t, max, CMH_DRBG_MAX_GENERATE);
+
+ orphan = kmalloc_obj(*orphan, GFP_KERNEL);
+ if (!orphan)
+ return -ENOMEM;
+
+ dmabuf = kmalloc(nbytes, GFP_KERNEL);
+ if (!dmabuf) {
+ kfree(orphan);
+ return -ENOMEM;
+ }
+
+ dma_addr = cmh_dma_map_single(dmabuf, nbytes, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(dma_addr)) {
+ kfree(dmabuf);
+ kfree(orphan);
+ return -ENOMEM;
+ }
+
+ orphan->buf = dmabuf;
+ orphan->addr = dma_addr;
+ orphan->len = nbytes;
+ orphan->dir = DMA_FROM_DEVICE;
+
+ vcq_set_header(&vcq[0], DRBG_READ_VCQ_CMDS);
+ vcq_add_drbg_generate(&vcq[1], dma_addr, nbytes);
+ vcq_add_flush(&vcq[2], CORE_ID_DRBG);
+
+ /*
+ * Use the noabort variant: if the MBX is occupied by a slow
+ * operation (e.g. SLH-DSA sign at 120 s), we must not issue
+ * MBX_COMMAND_ABORT -- that would kill the unrelated in-flight
+ * VCQ. On timeout with an in-flight VCQ (-EINPROGRESS), the
+ * orphan callback defers DMA cleanup until the RH fires.
+ */
+ ret = cmh_tm_submit_sync_noabort(vcq, DRBG_READ_VCQ_CMDS, 1,
+ msecs_to_jiffies(drbg_timeout_ms),
+ cmh_dma_orphan_free, orphan);
+ if (ret == -EINPROGRESS) {
+ /*
+ * The orphan callback owns dmabuf and frees it on VCQ
+ * completion. Return 0 (not -EAGAIN): .read() only runs with
+ * wait=true (see the !wait early return above), and the hwrng
+ * core forwards a negative errno straight to a blocking read
+ * whereas a 0 return makes it retry.
+ */
+ return 0;
+ }
+
+ /* Normal path or cancelled-from-queue: caller owns DMA */
+ cmh_dma_unmap_single(dma_addr, nbytes, DMA_FROM_DEVICE);
+ kfree(orphan);
+
+ if (ret) {
+ /*
+ * .read() only runs with wait=true (see the !wait early
+ * return above). For known transient conditions return 0 so
+ * the hwrng core retries the blocking read; a negative errno
+ * here would be forwarded to userspace on a blocking fd
+ * (e.g. -EAGAIN violates POSIX). Propagate genuinely
+ * unexpected failures so real faults are not masked into an
+ * indefinite retry loop.
+ */
+ switch (ret) {
+ case -EAGAIN:
+ case -EBUSY:
+ case -ETIMEDOUT:
+ case -EIO:
+ /*
+ * -ENODEV: the TM is not running -- occurs when the
+ * hwrng kthread (PF_NOFREEZE, not frozen during
+ * suspend) calls .read() while the device is suspended.
+ * Treat as transient: the TM restarts on resume.
+ */
+ case -ENODEV:
+ dev_dbg_ratelimited(cmh_dev(),
+ "rng: transient DRBG failure (rc=%d)\n",
+ ret);
+ kfree_sensitive(dmabuf);
+ return 0;
+ default:
+ dev_err_ratelimited(cmh_dev(),
+ "rng: DRBG generate failed (rc=%d)\n",
+ ret);
+ kfree_sensitive(dmabuf);
+ return ret;
+ }
+ }
+
+ memcpy(data, dmabuf, nbytes);
+ kfree_sensitive(dmabuf);
+
+ return nbytes;
+}
+
+/* Registration */
+
+static bool cmh_rng_registered;
+
+static struct hwrng cmh_hwrng = {
+ .name = "rambus-cmh-drbg",
+ .read = cmh_rng_read,
+};
+
+/**
+ * cmh_rng_register() - Register the CMH hardware RNG device
+ * @pdev: Platform device for the CMH accelerator
+ *
+ * Reads hwrng quality from device tree and module parameters, validates
+ * DRBG configuration, optionally sends a DRBG CONFIG VCQ to firmware,
+ * and registers the hwrng device with the kernel hwrng framework.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_rng_register(struct platform_device *pdev)
+{
+ int ret;
+
+ /*
+ * The hwrng core forces a proper quality at registration:
+ * rng->quality = min3(default_quality, 1024, quality ?: 1024).
+ * So quality 0 is NOT "no seeding" -- it is elevated to the core
+ * default (full trust, 1024). Pass the module param through and
+ * clamp to the max; set hwrng_quality explicitly to lower the
+ * entropy estimate.
+ */
+ cmh_hwrng.quality = hwrng_quality;
+
+ if (cmh_hwrng.quality > CMH_HWRNG_QUALITY_MAX)
+ cmh_hwrng.quality = CMH_HWRNG_QUALITY_MAX;
+
+ /*
+ * DRBG CONFIG is a management-host operation. In "auto" mode,
+ * attempt it -- this succeeds in stateless mode (any host) or
+ * when we are the management host in stateful mode. On -EPERM
+ * (not management host) we continue without error -- GENERATE
+ * will work once the management host configures the DRBG.
+ *
+ * In "skip" mode, do not issue CONFIG -- assume the management
+ * host has already configured (or will configure) the DRBG.
+ */
+ if (strcmp(drbg_config, "skip") != 0) {
+ struct vcq_cmd cfg_vcq[DRBG_CONFIG_VCQ_CMDS];
+
+ if (strcmp(drbg_config, "auto") != 0)
+ dev_warn(&pdev->dev,
+ "rng: unrecognized drbg_config=\"%s\", treating as \"auto\"\n",
+ drbg_config);
+
+ vcq_set_header(&cfg_vcq[0], DRBG_CONFIG_VCQ_CMDS);
+ vcq_add_drbg_reset(&cfg_vcq[1]);
+ vcq_add_drbg_config(&cfg_vcq[2], CMH_DRBG_ENTROPY_RATIO,
+ CMH_DRBG_SECURITY_STRENGTH);
+ vcq_add_flush(&cfg_vcq[3], CORE_ID_DRBG);
+ ret = cmh_tm_submit_sync(cfg_vcq, DRBG_CONFIG_VCQ_CMDS, 1);
+ if (ret == -EPERM)
+ dev_notice(&pdev->dev,
+ "rng: DRBG config not permitted (not management host); assuming external configuration\n");
+ else if (ret)
+ dev_warn(&pdev->dev,
+ "rng: DRBG config failed (rc=%d)\n", ret);
+ else
+ dev_info(&pdev->dev,
+ "rng: DRBG configured (ratio=%u strength=0x%02x)\n",
+ CMH_DRBG_ENTROPY_RATIO,
+ CMH_DRBG_SECURITY_STRENGTH);
+ } else {
+ dev_info(&pdev->dev,
+ "rng: DRBG config skipped (drbg_config=skip); assuming external configuration\n");
+ }
+
+ ret = hwrng_register(&cmh_hwrng);
+ if (ret) {
+ dev_err(&pdev->dev, "rng: hwrng_register failed (rc=%d)\n",
+ ret);
+ return ret;
+ }
+
+ dev_info(&pdev->dev,
+ "rng: registered rambus-cmh-drbg (quality=%d timeout=%ums)\n",
+ cmh_hwrng.quality, drbg_timeout_ms);
+
+ cmh_rng_registered = true;
+ return 0;
+}
+
+/**
+ * cmh_rng_unregister() - Unregister the CMH hardware RNG device
+ *
+ * Unregisters the hwrng device from the kernel hwrng framework if it
+ * was previously registered.
+ */
+void cmh_rng_unregister(void)
+{
+ if (!cmh_rng_registered)
+ return;
+ hwrng_unregister(&cmh_hwrng);
+ cmh_rng_registered = false;
+ dev_info(cmh_dev(), "rng: unregistered rambus-cmh-drbg\n");
+}
+
+/* -- debugfs timeout accessor ------------------------------------------ */
+
+#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG
+/**
+ * cmh_rng_timeout_drbg_ptr() - Return pointer to drbg_timeout_ms for debugfs
+ *
+ * Exposes the DRBG operation timeout for runtime tuning via debugfs
+ * config/ directory.
+ *
+ * Return: pointer to the static drbg_timeout_ms variable.
+ */
+unsigned int *cmh_rng_timeout_drbg_ptr(void) { return &drbg_timeout_ms; }
+#endif
--
2.43.7
^ permalink raw reply related
* [PATCH v3 13/19] crypto: cmh - add ECDSA/SM2 sig
From: Saravanakrishnan Krishnamoorthy @ 2026-08-06 19:55 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register ECDSA and SM2 sig algorithms using the CMH PKE core.
Supports P-256, P-384, P-521, and SM2 curves for sign and verify
operations. SM2 is registered as verify-only via the crypto API;
full SM2 operations (encrypt, decrypt, key exchange) are available
through the /dev/cmh_mgmt ioctl interface.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 3 +-
drivers/crypto/cmh/cmh_main.c | 8 +
drivers/crypto/cmh/cmh_pke_ecdsa.c | 602 +++++++++++++++++++++++++++++
3 files changed, 612 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_pke_ecdsa.c
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index cdbcc8cdac5f..ae1f74a93e99 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -31,7 +31,8 @@ cmh-y := \
cmh_ccp_poly.o \
cmh_rng.o \
cmh_pke_common.o \
- cmh_pke_rsa.o
+ cmh_pke_rsa.o \
+ cmh_pke_ecdsa.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index 46738193d354..82b734b42805 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -327,6 +327,11 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_pke_rsa_register;
+ /* Register PKE ECDSA/SM2 sig */
+ ret = cmh_pke_ecdsa_register();
+ if (ret)
+ goto err_pke_ecdsa_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -339,6 +344,8 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_pke_ecdsa_unregister();
+err_pke_ecdsa_register:
cmh_pke_rsa_unregister();
err_pke_rsa_register:
cmh_ccp_poly_unregister();
@@ -397,6 +404,7 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_pke_ecdsa_unregister();
cmh_pke_rsa_unregister();
cmh_ccp_poly_unregister();
cmh_ccp_aead_unregister();
diff --git a/drivers/crypto/cmh/cmh_pke_ecdsa.c b/drivers/crypto/cmh/cmh_pke_ecdsa.c
new file mode 100644
index 000000000000..6d6b8aec6c88
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pke_ecdsa.c
@@ -0,0 +1,602 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- ECDSA / SM2 Signature Driver (sig_alg, synchronous)
+ *
+ * Registers "ecdsa-nist-p256", "ecdsa-nist-p384", and "ecdsa-nist-p521"
+ * sig algorithms with sign, verify, set_pub_key, and set_priv_key callbacks.
+ * Registers "sm2" as verify-only (set_pub_key + verify); SM2 sign is
+ * provided via the cmh_mgmt ioctl path in cmh_pke_sm2.c.
+ *
+ * In-kernel consumers typically use verify-only (module signatures, IMA),
+ * but we provide sign as well for completeness -- matching the CMH eSW
+ * capability.
+ *
+ * Key format: Public key = raw 04 || X || Y (uncompressed).
+ * Signature format: struct ecdsa_raw_sig (two u64[ECC_MAX_DIGITS] arrays
+ * in VLI format -- native byte order, LE digit order) for both sign
+ * output and verify input. This matches the kernel crypto sig API.
+ *
+ * Private key via cmh_key_ctx: raw keys written via SYS_REF_TEMP.
+ * Datastore-referenced keys are only reachable through the ioctl
+ * path (cmh_mgmt.c).
+ *
+ * SM2 note: The SM2 sig entry is verify-only (no sign/set_priv_key).
+ * SM2 signature verification requires the digest to be SM3(ZA || M)
+ * where ZA = SM3(ENTLA || IDA || a || b || xG || yG || xA || yA).
+ * The ZA identity pre-hash is the caller's responsibility; the driver
+ * passes the digest directly to the CMH eSW SM2 verify engine.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <crypto/sha2.h>
+#include <crypto/sig.h>
+#include <crypto/internal/sig.h>
+#include <crypto/internal/ecc.h>
+
+#include "cmh_pke.h"
+#include "cmh_sys.h"
+#include "cmh_sys_abi.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+/*
+ * Number of ECC digits needed for a given coordinate byte length.
+ * P-256: 4, P-384: 6, P-521/SM2(clen=68): 9.
+ */
+static inline unsigned int clen_to_ndigits(u32 clen)
+{
+ return DIV_ROUND_UP(clen, sizeof(u64));
+}
+
+struct cmh_ecdsa_tfm_ctx {
+ struct cmh_key_ctx key; /* private key (raw only) */
+ u8 *pub_key; /* uncompressed (x, y) without 04 prefix */
+ u32 pub_key_len;
+ u32 curve; /* PKE_CURVE_* */
+ u32 clen; /* coordinate length in bytes */
+};
+
+static inline struct cmh_ecdsa_tfm_ctx *cmh_ecdsa_ctx(struct crypto_sig *tfm)
+{
+ return crypto_sig_ctx(tfm);
+}
+
+/*
+ * Convert one VLI component (u64 array, LE digit order, native byte order)
+ * to big-endian byte array of @out_len bytes. The VLI value is right-aligned
+ * in the output (leading zero padding added if ndigits*8 < out_len). If the
+ * VLI is wider than @out_len, the skipped leading bytes must be zero; a
+ * non-zero leading byte is malformed caller input and returns -EBADMSG.
+ *
+ * Return: 0 on success, -EBADMSG if the value does not fit in @out_len.
+ */
+static int ecdsa_vli_to_be(const u64 *vli, unsigned int ndigits,
+ u8 *out, unsigned int out_len)
+{
+ unsigned int full_len = ndigits * sizeof(u64);
+ unsigned int i, skip;
+
+ memset(out, 0, out_len);
+
+ if (full_len <= out_len) {
+ /* VLI fits entirely -- write at right end of out */
+ u8 *dst = out + (out_len - full_len);
+
+ for (i = 0; i < ndigits; i++)
+ put_unaligned_be64(vli[ndigits - 1 - i],
+ &dst[i * sizeof(u64)]);
+ } else {
+ /*
+ * VLI wider than out -- the skipped leading bytes MUST be
+ * zero. This runs on caller-supplied signature data, so a
+ * non-zero high byte is malformed input: reject it with
+ * -EBADMSG rather than WARN_ON_ONCE (which would let an
+ * unprivileged caller trip panic_on_warn).
+ */
+ u8 tmp[ECC_MAX_BYTES];
+
+ for (i = 0; i < ndigits; i++)
+ put_unaligned_be64(vli[ndigits - 1 - i],
+ &tmp[i * sizeof(u64)]);
+ skip = full_len - out_len;
+ if (memchr_inv(tmp, 0, skip))
+ return -EBADMSG;
+ memcpy(out, tmp + skip, out_len);
+ }
+
+ return 0;
+}
+
+/*
+ * Convert big-endian byte array to VLI (u64 array, LE digit order).
+ * Output is zero-filled to @max_digits entries.
+ */
+static void ecdsa_be_to_vli(const u8 *in, unsigned int in_len,
+ u64 *vli, unsigned int max_digits)
+{
+ unsigned int full_len = max_digits * sizeof(u64);
+ u8 tmp[ECC_MAX_BYTES];
+ unsigned int i;
+
+ if (WARN_ON_ONCE(max_digits > ECC_MAX_DIGITS))
+ max_digits = ECC_MAX_DIGITS;
+
+ memset(tmp, 0, full_len);
+ if (in_len <= full_len)
+ memcpy(tmp + (full_len - in_len), in, in_len);
+ else
+ memcpy(tmp, in + (in_len - full_len), full_len);
+
+ for (i = 0; i < max_digits; i++) {
+ unsigned int off = (max_digits - 1 - i) * sizeof(u64);
+
+ vli[i] = get_unaligned_be64(&tmp[off]);
+ }
+}
+
+/*
+ * Extract raw (r || s) big-endian byte arrays from struct ecdsa_raw_sig.
+ * Each component is written as @clen bytes into @raw_rs.
+ */
+static int ecdsa_sig_to_raw(const void *src, unsigned int slen,
+ u8 *raw_rs, u32 clen)
+{
+ const struct ecdsa_raw_sig *sig = src;
+ unsigned int ndigits = clen_to_ndigits(clen);
+ int ret;
+
+ if (slen != sizeof(struct ecdsa_raw_sig))
+ return -EINVAL;
+
+ ret = ecdsa_vli_to_be(sig->r, ndigits, raw_rs, clen);
+ if (ret)
+ return ret;
+ return ecdsa_vli_to_be(sig->s, ndigits, raw_rs + clen, clen);
+}
+
+/*
+ * Encode raw (r || s) big-endian byte arrays into struct ecdsa_raw_sig.
+ * Returns sizeof(struct ecdsa_raw_sig) on success.
+ */
+static int ecdsa_raw_to_sig(const u8 *raw_rs, u32 clen,
+ void *dst, unsigned int dlen)
+{
+ struct ecdsa_raw_sig *sig = dst;
+
+ if (dlen < sizeof(struct ecdsa_raw_sig))
+ return -ENOSPC;
+
+ memset(sig, 0, sizeof(*sig));
+ ecdsa_be_to_vli(raw_rs, clen, sig->r, ECC_MAX_DIGITS);
+ ecdsa_be_to_vli(raw_rs + clen, clen, sig->s, ECC_MAX_DIGITS);
+ return sizeof(struct ecdsa_raw_sig);
+}
+
+/*
+ * ECDSA verify (synchronous sig_alg)
+ *
+ * @src: struct ecdsa_raw_sig (VLI format)
+ * @slen: signature length (must be sizeof(struct ecdsa_raw_sig))
+ * @digest: hash digest
+ * @dlen: digest length
+ *
+ * Returns 0 on successful verification, negative errno on failure.
+ */
+static int cmh_ecdsa_verify(struct crypto_sig *tfm,
+ const void *src, unsigned int slen,
+ const void *digest, unsigned int dlen)
+{
+ struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm);
+ u32 clen = ctx->clen;
+ u32 sig_raw_len = 2 * clen;
+ u32 copy_len = min_t(u32, dlen, clen);
+ struct core_dispatch d = cmh_core_select_instance(CMH_CORE_PKE);
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u8 *sig_raw = NULL, *dig_buf = NULL, *pk_buf = NULL, *rp_buf = NULL;
+ dma_addr_t pk_dma, dig_dma, sig_dma, rp_dma;
+ int ret;
+
+ if (!ctx->pub_key)
+ return -EINVAL;
+
+ sig_raw = kzalloc(sig_raw_len, GFP_KERNEL);
+ dig_buf = kzalloc(clen, GFP_KERNEL);
+ pk_buf = kmemdup(ctx->pub_key, ctx->pub_key_len, GFP_KERNEL);
+ rp_buf = kzalloc(clen, GFP_KERNEL);
+ if (!sig_raw || !dig_buf || !pk_buf || !rp_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ /* Extract raw (r, s) big-endian from VLI signature */
+ ret = ecdsa_sig_to_raw(src, slen, sig_raw, clen);
+ if (ret)
+ goto out_free;
+
+ /*
+ * Truncate or zero-pad digest to clen bytes, right-aligned.
+ * Matches ECDSA bits2int: use leftmost min(dlen, clen) bytes,
+ * zero-pad on the left when dlen < clen.
+ */
+ memcpy(dig_buf + (clen - copy_len), digest, copy_len);
+
+ pk_dma = cmh_dma_map_single(pk_buf, ctx->pub_key_len, DMA_TO_DEVICE);
+ dig_dma = cmh_dma_map_single(dig_buf, clen, DMA_TO_DEVICE);
+ sig_dma = cmh_dma_map_single(sig_raw, sig_raw_len, DMA_TO_DEVICE);
+ rp_dma = cmh_dma_map_single(rp_buf, clen, DMA_FROM_DEVICE);
+
+ if (cmh_dma_map_error(pk_dma) || cmh_dma_map_error(dig_dma) ||
+ cmh_dma_map_error(sig_dma) || cmh_dma_map_error(rp_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_ecdsa_verify(&vcq[1], d.core_id, ctx->curve, clen,
+ pk_dma, dig_dma, sig_dma, rp_dma,
+ pke_swap_flags(ctx->curve));
+ vcq_add_pke_flush(&vcq[2], d.core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, d.mbx_idx);
+
+out_unmap:
+ if (!cmh_dma_map_error(rp_dma))
+ cmh_dma_unmap_single(rp_dma, clen, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_raw_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(dig_dma))
+ cmh_dma_unmap_single(dig_dma, clen, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(pk_dma))
+ cmh_dma_unmap_single(pk_dma, ctx->pub_key_len, DMA_TO_DEVICE);
+
+out_free:
+ kfree(rp_buf);
+ kfree(pk_buf);
+ kfree(sig_raw);
+ kfree(dig_buf);
+ return ret;
+}
+
+/*
+ * ECDSA sign (synchronous sig_alg)
+ *
+ * @src: hash digest
+ * @slen: digest length
+ * @dst: output buffer for struct ecdsa_raw_sig (VLI format)
+ * @dlen: output buffer length
+ *
+ * Returns sizeof(struct ecdsa_raw_sig) on success, negative errno on failure.
+ */
+static int cmh_ecdsa_sign(struct crypto_sig *tfm,
+ const void *src, unsigned int slen,
+ void *dst, unsigned int dlen)
+{
+ struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm);
+ u32 clen = ctx->clen;
+ u32 sig_raw_len = 2 * clen;
+ u32 copy_len = min_t(u32, slen, clen);
+ struct core_dispatch dd;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MAX];
+ u8 *dig_buf = NULL, *sig_buf = NULL;
+ dma_addr_t dig_dma, sig_dma;
+ int ret, idx;
+
+ if (ctx->key.mode != CMH_KEY_RAW)
+ return -EINVAL;
+ if (dlen < sizeof(struct ecdsa_raw_sig))
+ return -EINVAL;
+
+ dig_buf = kzalloc(clen, GFP_KERNEL);
+ sig_buf = kzalloc(sig_raw_len, GFP_KERNEL);
+ if (!dig_buf || !sig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ /*
+ * Truncate or zero-pad digest to clen bytes, right-aligned.
+ * Matches ECDSA bits2int: use leftmost min(slen, clen) bytes,
+ * zero-pad on the left when slen < clen.
+ */
+ memcpy(dig_buf + (clen - copy_len), src, copy_len);
+
+ dig_dma = cmh_dma_map_single(dig_buf, clen, DMA_TO_DEVICE);
+ sig_dma = cmh_dma_map_single(sig_buf, sig_raw_len, DMA_FROM_DEVICE);
+
+ if (cmh_dma_map_error(dig_dma) || cmh_dma_map_error(sig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ dd = cmh_core_select_instance(CMH_CORE_PKE);
+
+ idx = 1;
+ vcq_add_sys_write(&vcq[idx], SYS_REF_TEMP, ctx->key.raw.dma,
+ SYS_REF_NONE, ctx->key.raw.len,
+ ctx->key.raw.sys_type);
+ vcq[idx].id |= pke_swap_flags(ctx->curve);
+ idx++;
+ vcq_add_pke_ecdsa_sign(&vcq[idx++], dd.core_id, ctx->curve, clen,
+ dig_dma, sig_dma, SYS_REF_TEMP,
+ clen, pke_swap_flags(ctx->curve));
+ vcq_add_pke_flush(&vcq[idx++], dd.core_id);
+ vcq_set_header(&vcq[0], idx);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, idx, 1, dd.mbx_idx);
+ if (!ret) {
+ /* Sync bounce buffer so CPU sees the DMA-written signature */
+ cmh_dma_sync_for_cpu(sig_dma, sig_raw_len, DMA_FROM_DEVICE);
+
+ /* Encode raw (r||s) into VLI ecdsa_raw_sig for kernel API */
+ ret = ecdsa_raw_to_sig(sig_buf, clen, dst, dlen);
+ }
+
+out_unmap:
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_raw_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(dig_dma))
+ cmh_dma_unmap_single(dig_dma, clen, DMA_TO_DEVICE);
+
+out_free:
+ kfree(sig_buf);
+ kfree(dig_buf);
+ return ret;
+}
+
+static int cmh_ecdsa_set_pub_key(struct crypto_sig *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm);
+ const u8 *d = key;
+ u32 clen = ctx->clen;
+ u32 raw_clen;
+
+ /* Accept 04 || X || Y (uncompressed point) */
+ if (keylen < 1 || d[0] != 0x04)
+ return -EINVAL;
+ d++;
+ keylen--;
+
+ if (keylen & 1)
+ return -EINVAL;
+ raw_clen = keylen / 2;
+
+ /*
+ * Kernel passes ceil(bits/8) per coordinate (e.g. 66 for P-521),
+ * but our HW ABI uses clen (ALIGN(66,4)=68 for P-521).
+ * Accept raw_clen <= clen and zero-pad on the left.
+ */
+ if (raw_clen > clen || raw_clen == 0)
+ return -EINVAL;
+
+ kfree(ctx->pub_key);
+ ctx->pub_key = NULL;
+ ctx->pub_key_len = 0;
+
+ ctx->pub_key = kzalloc(2 * clen, GFP_KERNEL);
+ if (!ctx->pub_key)
+ return -ENOMEM;
+
+ /* Right-align each coordinate to clen bytes */
+ memcpy(ctx->pub_key + (clen - raw_clen), d, raw_clen);
+ memcpy(ctx->pub_key + clen + (clen - raw_clen), d + raw_clen,
+ raw_clen);
+ ctx->pub_key_len = 2 * clen;
+ return 0;
+}
+
+static int cmh_ecdsa_set_priv_key(struct crypto_sig *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm);
+ u32 clen = ctx->clen;
+ u32 api_len = DIV_ROUND_UP(pke_curve_bits(ctx->curve), 8);
+ u8 padded[ECC_MAX_BYTES];
+ int ret;
+
+ /*
+ * The crypto sig API passes the scalar as ceil(field_bits/8) bytes
+ * (e.g. 66 for P-521), which may be shorter than the HW width clen
+ * (ALIGN(.,4) = 68 for P-521). Accept exactly that width and
+ * left-pad to clen. A strict "keylen == clen" wrongly rejected the
+ * valid 66-byte P-521 key; accepting "keylen <= clen" wrongly
+ * admitted malformed short keys.
+ */
+ if (keylen != api_len)
+ return -EINVAL;
+
+ if (keylen == clen)
+ return cmh_key_setkey_raw(&ctx->key, key, keylen, CORE_ID_PKE);
+
+ memset(padded, 0, clen);
+ memcpy(padded + (clen - keylen), key, keylen);
+ ret = cmh_key_setkey_raw(&ctx->key, padded, clen, CORE_ID_PKE);
+ memzero_explicit(padded, clen);
+ return ret;
+}
+
+static unsigned int cmh_ecdsa_key_size(struct crypto_sig *tfm)
+{
+ struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm);
+
+ /* crypto_sig_keysize() returns bits, not bytes */
+ return pke_curve_bits(ctx->curve);
+}
+
+static unsigned int cmh_ecdsa_max_size(struct crypto_sig *tfm)
+{
+ return sizeof(struct ecdsa_raw_sig);
+}
+
+static unsigned int cmh_ecdsa_digest_size(struct crypto_sig *tfm)
+{
+ /*
+ * Accept digests up to SHA-512 (64 bytes). Digests longer
+ * than the curve order are truncated per ECDSA bits2int.
+ * Matches kernel ecdsa_digest_size().
+ */
+ return SHA512_DIGEST_SIZE;
+}
+
+static int cmh_ecdsa_p256_init(struct crypto_sig *tfm)
+{
+ struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->curve = PKE_CURVE_P256;
+ ctx->clen = pke_curve_clen(PKE_CURVE_P256);
+ return 0;
+}
+
+static int cmh_ecdsa_p384_init(struct crypto_sig *tfm)
+{
+ struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->curve = PKE_CURVE_P384;
+ ctx->clen = pke_curve_clen(PKE_CURVE_P384);
+ return 0;
+}
+
+static int cmh_ecdsa_p521_init(struct crypto_sig *tfm)
+{
+ struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->curve = PKE_CURVE_P521;
+ ctx->clen = pke_curve_clen(PKE_CURVE_P521);
+ return 0;
+}
+
+static int cmh_sm2_init(struct crypto_sig *tfm)
+{
+ struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->curve = PKE_CURVE_SM2;
+ ctx->clen = pke_curve_clen(PKE_CURVE_SM2);
+ return 0;
+}
+
+static void cmh_ecdsa_exit(struct crypto_sig *tfm)
+{
+ struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm);
+
+ cmh_key_destroy(&ctx->key);
+ kfree(ctx->pub_key);
+ ctx->pub_key = NULL;
+}
+
+static struct sig_alg cmh_ecdsa_algs[] = {
+ {
+ .sign = cmh_ecdsa_sign,
+ .verify = cmh_ecdsa_verify,
+ .set_pub_key = cmh_ecdsa_set_pub_key,
+ .set_priv_key = cmh_ecdsa_set_priv_key,
+ .key_size = cmh_ecdsa_key_size,
+ .max_size = cmh_ecdsa_max_size,
+ .digest_size = cmh_ecdsa_digest_size,
+ .init = cmh_ecdsa_p256_init,
+ .exit = cmh_ecdsa_exit,
+ .base = {
+ .cra_name = "ecdsa-nist-p256",
+ .cra_driver_name = "rambus-cmh-ecdsa-nist-p256",
+ .cra_priority = 300,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_ecdsa_tfm_ctx),
+ },
+ },
+ {
+ .sign = cmh_ecdsa_sign,
+ .verify = cmh_ecdsa_verify,
+ .set_pub_key = cmh_ecdsa_set_pub_key,
+ .set_priv_key = cmh_ecdsa_set_priv_key,
+ .key_size = cmh_ecdsa_key_size,
+ .max_size = cmh_ecdsa_max_size,
+ .digest_size = cmh_ecdsa_digest_size,
+ .init = cmh_ecdsa_p384_init,
+ .exit = cmh_ecdsa_exit,
+ .base = {
+ .cra_name = "ecdsa-nist-p384",
+ .cra_driver_name = "rambus-cmh-ecdsa-nist-p384",
+ .cra_priority = 300,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_ecdsa_tfm_ctx),
+ },
+ },
+ {
+ .sign = cmh_ecdsa_sign,
+ .verify = cmh_ecdsa_verify,
+ .set_pub_key = cmh_ecdsa_set_pub_key,
+ .set_priv_key = cmh_ecdsa_set_priv_key,
+ .key_size = cmh_ecdsa_key_size,
+ .max_size = cmh_ecdsa_max_size,
+ .digest_size = cmh_ecdsa_digest_size,
+ .init = cmh_ecdsa_p521_init,
+ .exit = cmh_ecdsa_exit,
+ .base = {
+ .cra_name = "ecdsa-nist-p521",
+ .cra_driver_name = "rambus-cmh-ecdsa-nist-p521",
+ .cra_priority = 300,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_ecdsa_tfm_ctx),
+ },
+ },
+ {
+ .verify = cmh_ecdsa_verify,
+ .set_pub_key = cmh_ecdsa_set_pub_key,
+ .key_size = cmh_ecdsa_key_size,
+ .max_size = cmh_ecdsa_max_size,
+ .digest_size = cmh_ecdsa_digest_size,
+ .init = cmh_sm2_init,
+ .exit = cmh_ecdsa_exit,
+ .base = {
+ .cra_name = "sm2",
+ .cra_driver_name = "rambus-cmh-sm2",
+ .cra_priority = 300,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_ecdsa_tfm_ctx),
+ },
+ },
+};
+
+/**
+ * cmh_pke_ecdsa_register() - Register ECDSA/SM2 sig algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_pke_ecdsa_register(void)
+{
+ int ret, i;
+
+ for (i = 0; i < ARRAY_SIZE(cmh_ecdsa_algs); i++) {
+ ret = crypto_register_sig(&cmh_ecdsa_algs[i]);
+ if (ret) {
+ dev_err(cmh_dev(), "cmh: failed to register %s (%d)\n",
+ cmh_ecdsa_algs[i].base.cra_name, ret);
+ goto err_unregister;
+ }
+ }
+
+ return 0;
+
+err_unregister:
+ while (i--)
+ crypto_unregister_sig(&cmh_ecdsa_algs[i]);
+ return ret;
+}
+
+/**
+ * cmh_pke_ecdsa_unregister() - Unregister ECDSA/SM2 sig algorithms from the crypto framework
+ */
+void cmh_pke_ecdsa_unregister(void)
+{
+ int i = ARRAY_SIZE(cmh_ecdsa_algs);
+
+ while (i--)
+ crypto_unregister_sig(&cmh_ecdsa_algs[i]);
+}
--
2.43.7
^ permalink raw reply related
* [PATCH v3 17/19] Documentation: ioctl: add CMH ioctl documentation and register 'J'
From: Saravanakrishnan Krishnamoorthy @ 2026-08-06 19:55 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Add Documentation/userspace-api/ioctl/cmh_mgmt.rst documenting the
ioctl commands on the /dev/cmh_mgmt misc device for the Rambus
CryptoManager Hub (CMH) hardware crypto accelerator driver. Covers
key management, KIC key derivation, PKE (RSA, ECDSA, ECDH, EdDSA),
PQC (ML-KEM, ML-DSA, SLH-DSA), SM2, EAC, and DRBG. Link the page
into the userspace-api/ioctl index toctree.
Register ioctl magic number 'J' (0x4A) in ioctl-number.rst. The
driver uses ioctls 0x01-0x40.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
.../userspace-api/ioctl/cmh_mgmt.rst | 1295 +++++++++++++++++
Documentation/userspace-api/ioctl/index.rst | 1 +
.../userspace-api/ioctl/ioctl-number.rst | 1 +
3 files changed, 1297 insertions(+)
create mode 100644 Documentation/userspace-api/ioctl/cmh_mgmt.rst
diff --git a/Documentation/userspace-api/ioctl/cmh_mgmt.rst b/Documentation/userspace-api/ioctl/cmh_mgmt.rst
new file mode 100644
index 000000000000..d17a77975df6
--- /dev/null
+++ b/Documentation/userspace-api/ioctl/cmh_mgmt.rst
@@ -0,0 +1,1295 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=============================================
+CMH Key Management ioctl Interface (cmh_mgmt)
+=============================================
+
+:Author: Cryptography Research, Inc. (CRI)
+:Maintainer: linux-crypto@vger.kernel.org
+
+Introduction
+============
+
+The ``/dev/cmh_mgmt`` character device provides user-space access to key
+management, key derivation, public-key, and post-quantum cryptographic
+operations on the CryptoManager Hub (CMH) hardware accelerator.
+
+The device is created by the ``cmh`` kernel module as a ``misc_device``.
+All operations are synchronous -- the ioctl blocks until the hardware
+completes. Opening the device requires ``CAP_SYS_ADMIN``.
+
+All ioctl argument structures are versioned: user space sets the
+``version`` field to ``CMH_MGMT_V1`` (currently 1). This allows the
+driver to extend structures in the future without breaking the ABI.
+
+Data types and ioctl numbers are defined in
+``<uapi/linux/cmh_mgmt_ioctl.h>``. The ioctl type letter is ``'J'``
+(0x4A).
+
+Error Handling
+==============
+
+Unless otherwise noted, all ioctls return 0 on success and a negative
+errno on failure. Common error codes:
+
+========== =============================================================
+``EINVAL`` Invalid ``version`` field, unsupported parameter, or
+ out-of-range length.
+``EFAULT`` Failed to copy data to/from user space.
+``ENOMEM`` Kernel memory allocation failed.
+``EIO`` Hardware returned an error (eSW command failure).
+``ENOENT`` Key not found (``KEY_FIND``, ``KEY_LIST``).
+========== =============================================================
+
+Datastore Concepts
+==================
+
+The CMH hardware maintains an embedded datastore managed by the eSW
+firmware. Objects in the datastore are identified by a 64-bit reference
+(``ref``) and optionally by a 64-bit Content ID (``cid``).
+
+Two storage classes exist:
+
+**Temporary (SYS_REF_TEMP)**
+ Lifetime is scoped to a single mailbox slot. The eSW firmware
+ reclaims the object when the slot is reused. Used for raw-key
+ provisioning via ``KEY_NEW`` + ``KEY_WRITE``.
+
+**Persistent (SYS_REF_PERSIST)**
+ Survives across mailbox slots. Requires explicit deletion via
+ ``KEY_DELETE``. Identified by CID; resolved to a per-mailbox ref
+ via ``KEY_FIND``.
+
+Mailbox Dispatch
+================
+
+All ``/dev/cmh_mgmt`` ioctls are submitted on a single management
+mailbox. This is a structural requirement of the eSW datastore model,
+not a tunable:
+
+* Datastore access control is **per-mailbox**. ``KEY_NEW`` grants the
+ creating mailbox read/write/execute access; other mailboxes have none
+ until granted. The returned 64-bit ``ref`` encodes a randomised
+ offset and does **not** carry the owning mailbox, so an operation that
+ receives only a ``ref`` (``KEY_GRANT``, ``KEY_READ``, ``KEY_DELETE``,
+ ``DS_EXPORT``) cannot itself determine which mailbox owns the object.
+ Using one fixed management mailbox guarantees that a key's create,
+ modify, grant, read and hardware-held-key compute steps all share the
+ mailbox that holds its access rights, without exposing mailbox
+ identity in the UABI. User space may still widen a key's access to
+ additional mailboxes via ``KEY_GRANT``.
+
+* The eSW ``SYS_REF_TEMP`` scratch store is per-mailbox and persists
+ across ioctl calls, so a multi-step flow that derives into
+ ``SYS_REF_TEMP`` (for example a ``KIC_*`` derivation) and later
+ consumes it (``DS_EXPORT`` with ``wrap_key = SYS_REF_TEMP``) requires
+ both calls to use the same mailbox.
+
+Per-mailbox ``rambus,cores`` device-tree affinity applies to the *stateless*
+in-kernel crypto API path, which carries no datastore state between
+requests and is balanced across mailboxes by the driver.
+
+Key Types
+=========
+
+The ``ds_type`` field in ``KEY_NEW`` and ``KEY_WRITE`` selects the
+datastore object type. Values are defined as ``CMH_DS_*`` constants:
+
+================================= ===== ==============================
+Constant Value Description
+================================= ===== ==============================
+``CMH_DS_RAW_VALUE`` 1 Raw byte array
+``CMH_DS_AES_KEY`` 2 AES key (128/192/256-bit)
+``CMH_DS_AES_XTS_KEY`` 3 AES-XTS key (256/512-bit)
+``CMH_DS_HMAC_KEY`` 4 HMAC key
+``CMH_DS_KMAC_KEY`` 5 KMAC key
+``CMH_DS_SM4_KEY`` 6 SM4 key (128-bit)
+``CMH_DS_CHACHA20_KEY`` 7 ChaCha20 key (256-bit)
+``CMH_DS_RSA_PRIV_KEY`` 10 RSA private key
+``CMH_DS_RSA_PUB_KEY`` 11 RSA public key
+``CMH_DS_RSA_CRT_KEY`` 12 RSA CRT private key
+``CMH_DS_ECDSA_PRIV_KEY`` 13 ECDSA private key
+``CMH_DS_ECDSA_PUB_KEY`` 14 ECDSA public key
+``CMH_DS_ECDH_PRIV_KEY`` 15 ECDH private key
+``CMH_DS_EDDSA_PRIV_KEY`` 16 EdDSA private key
+``CMH_DS_SHARED_SECRET`` 17 Shared secret
+``CMH_DS_SM2_PRIV_KEY`` 18 SM2 private key
+``CMH_DS_ML_KEM_DK`` 20 ML-KEM decapsulation key
+``CMH_DS_ML_DSA_SK`` 21 ML-DSA secret key
+``CMH_DS_SLHDSA_SK`` 25 SLH-DSA secret key
+================================= ===== ==============================
+
+Key Flags
+=========
+
+The ``flags`` field in ``KEY_NEW`` and ``KEY_WRITE`` is a bitmask:
+
+================== =========== ========================================
+Flag Bit Description
+================== =========== ========================================
+``CMH_FLAG_PT`` 16 Key can be read as plaintext
+``CMH_FLAG_XC`` 17 Key can be exported over XC bus
+``CMH_FLAG_SCA`` 18 SCA key stored in 2 shares
+================== =========== ========================================
+
+Elliptic Curve IDs
+==================
+
+Curve identifiers for PKE operations (``curve`` field):
+
+========================== =====
+Constant Value
+========================== =====
+``CMH_CURVE_P192`` 0x01
+``CMH_CURVE_P224`` 0x02
+``CMH_CURVE_P256`` 0x03
+``CMH_CURVE_P384`` 0x04
+``CMH_CURVE_P521`` 0x05
+``CMH_CURVE_SECP256K1`` 0x07
+``CMH_CURVE_BP192R1`` 0x11
+``CMH_CURVE_BP224R1`` 0x12
+``CMH_CURVE_BP256R1`` 0x13
+``CMH_CURVE_BP320R1`` 0x14
+``CMH_CURVE_BP384R1`` 0x15
+``CMH_CURVE_BP512R1`` 0x16
+``CMH_CURVE_SM2`` 0x18
+``CMH_CURVE_25519`` 0x21
+``CMH_CURVE_448`` 0x22
+========================== =====
+
+Key Management ioctls
+=====================
+
+CMH_IOCTL_KEY_NEW
+-----------------
+
+Create a new empty datastore object.
+
+:Direction: ``_IOWR``
+:Number: 0x01
+:Argument: ``struct cmh_ioctl_key_new``
+
+::
+
+ struct cmh_ioctl_key_new {
+ __u32 version; /* must be CMH_MGMT_V1 */
+ __u32 ds_type; /* CMH_DS_* key type */
+ __u32 len; /* key length in bytes */
+ __u32 flags; /* CMH_FLAG_* */
+ __u64 cid; /* caller ID (name) for the key */
+ __u64 ref; /* [out] key reference */
+ };
+
+The returned ``ref`` is used in subsequent ``KEY_WRITE``, ``KEY_READ``,
+and crypto operation ioctls.
+
+CMH_IOCTL_KEY_NEW_RANDOM
+------------------------
+
+Create a new datastore object filled with hardware-generated random data.
+
+:Direction: ``_IOWR``
+:Number: 0x0B
+:Argument: ``struct cmh_ioctl_key_new``
+
+Same structure as ``KEY_NEW``. The hardware DRBG fills the object with
+``len`` random bytes.
+
+CMH_IOCTL_KEY_WRITE
+-------------------
+
+Write key material into a previously created datastore object.
+
+:Direction: ``_IOW``
+:Number: 0x02
+:Argument: ``struct cmh_ioctl_key_write``
+
+::
+
+ struct cmh_ioctl_key_write {
+ __u32 version;
+ __u32 len; /* key data length */
+ __u32 ds_type; /* CMH_DS_* key type */
+ __u32 flags; /* CMH_FLAG_* */
+ __u64 ref; /* key reference from KEY_NEW */
+ __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */
+ __u64 data; /* user-space pointer to key material */
+ };
+
+If ``wrap_key`` is ``CMH_REF_NONE`` (0), key material is written in
+plaintext. Otherwise, the data is unwrapped using the specified
+wrapping key.
+
+CMH_IOCTL_KEY_READ
+------------------
+
+Read key material from a datastore object.
+
+:Direction: ``_IOWR``
+:Number: 0x03
+:Argument: ``struct cmh_ioctl_key_read``
+
+::
+
+ struct cmh_ioctl_key_read {
+ __u32 version;
+ __u32 len; /* buffer length */
+ __u64 ref; /* key reference */
+ __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */
+ __u64 data; /* user-space pointer to output buffer */
+ __u32 out_len; /* [out] actual bytes written */
+ __u32 __reserved;
+ };
+
+Plaintext reads require the ``CMH_FLAG_PT`` attribute on the key.
+The eSW prepends a 16-byte header (``CMH_SYS_WRAP_HDR_SIZE``) even
+for plaintext reads; the output buffer must accommodate this. The
+output overhead is ``CMH_DS_EXPORT_OVERHEAD_PLAIN`` (16 bytes) for
+plaintext reads and ``CMH_DS_EXPORT_OVERHEAD_WRAPPED`` (48 bytes:
+16-byte header + 16-byte nonce + 16-byte tag) for wrapped reads.
+
+CMH_IOCTL_KEY_FIND
+------------------
+
+Resolve a Content ID to a datastore reference.
+
+:Direction: ``_IOWR``
+:Number: 0x04
+:Argument: ``struct cmh_ioctl_key_find``
+
+::
+
+ struct cmh_ioctl_key_find {
+ __u32 version;
+ __u32 __reserved;
+ __u64 cid; /* caller ID to search for */
+ __u64 ref; /* [out] resolved key reference */
+ __u32 len; /* [out] key length */
+ __u32 type; /* [out] key type */
+ };
+
+Returns ``-ENOENT`` if no object with the given CID exists.
+
+CMH_IOCTL_KEY_LIST
+------------------
+
+Iterate datastore objects.
+
+:Direction: ``_IOWR``
+:Number: 0x0E
+:Argument: ``struct cmh_ioctl_key_list``
+
+::
+
+ struct cmh_ioctl_key_list {
+ __u32 version;
+ __u32 __reserved;
+ __u64 start_ref; /* starting DS reference (0 = first) */
+ __u64 ref; /* [out] object reference */
+ __u64 cid; /* [out] caller ID */
+ __u32 len; /* [out] object length */
+ __u32 type; /* [out] object type */
+ };
+
+Pass ``start_ref=0`` to begin from the first object. On return, pass
+the returned ``ref`` as ``start_ref`` in the next call. Iteration ends
+when ``ref == 0``.
+
+CMH_IOCTL_KEY_GRANT
+-------------------
+
+Set per-mailbox access permissions on a datastore object.
+
+:Direction: ``_IOW``
+:Number: 0x05
+:Argument: ``struct cmh_ioctl_key_grant``
+
+::
+
+ struct cmh_ioctl_key_grant {
+ __u32 version;
+ __u32 __reserved;
+ __u64 ref; /* key reference */
+ __u64 read; /* per-MBX read permission bitfield */
+ __u64 write; /* per-MBX write permission bitfield */
+ __u64 execute; /* per-MBX execute permission bitfield */
+ };
+
+CMH_IOCTL_KEY_DELETE
+--------------------
+
+Delete a datastore object (persistent keys only).
+
+:Direction: ``_IOW``
+:Number: 0x06
+:Argument: ``struct cmh_ioctl_key_grant``
+
+Uses the same structure as ``KEY_GRANT``; only the ``ref`` field is
+used.
+
+Datastore Export/Import ioctls
+==============================
+
+CMH_IOCTL_DS_EXPORT
+-------------------
+
+Export the entire datastore as an encrypted blob.
+
+:Direction: ``_IOWR``
+:Number: 0x07
+:Argument: ``struct cmh_ioctl_ds_export``
+
+::
+
+ struct cmh_ioctl_ds_export {
+ __u32 version;
+ __u32 len; /* buffer length */
+ __u64 cid; /* caller ID for response tagging */
+ __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */
+ __u64 data; /* user-space pointer to output buffer */
+ __u32 out_len; /* [out] actual bytes written */
+ __u32 __reserved;
+ };
+
+CMH_IOCTL_DS_IMPORT
+-------------------
+
+Import a previously exported datastore blob.
+
+:Direction: ``_IOW``
+:Number: 0x08
+:Argument: ``struct cmh_ioctl_ds_import``
+
+::
+
+ struct cmh_ioctl_ds_import {
+ __u32 version;
+ __u32 len; /* blob length */
+ __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */
+ __u64 data; /* user-space pointer to import blob */
+ };
+
+Key Derivation ioctls (KIC)
+===========================
+
+The Key Initialization Core (KIC) provides hardware key derivation from
+OTP-provisioned base keys. Up to 8 base keys are available
+(``CMH_KIC_KEY1`` through ``CMH_KIC_KEY8``).
+
+CMH_IOCTL_KIC_HKDF1
+--------------------
+
+HKDF-based key derivation (single-step, label only).
+
+:Direction: ``_IOWR``
+:Number: 0x09
+:Argument: ``struct cmh_ioctl_kic_hkdf1``
+
+::
+
+ struct cmh_ioctl_kic_hkdf1 {
+ __u32 version;
+ __u32 key_len; /* output key length */
+ __u64 base_key; /* KIC base key reference */
+ __u64 cid; /* CID for the new DS entry */
+ __u64 label; /* user-space pointer to label data */
+ __u32 label_len; /* label length in bytes */
+ __u32 flags; /* CMH_KIC_FLAG_* */
+ __u64 ref; /* [out] derived key reference */
+ };
+
+If ``CMH_KIC_FLAG_TEMP`` is set, the result is stored in the temporary
+datastore (not persistent).
+
+CMH_IOCTL_KIC_HKDF2
+--------------------
+
+HKDF-based key derivation (two-step, with salt key).
+
+:Direction: ``_IOWR``
+:Number: 0x0A
+:Argument: ``struct cmh_ioctl_kic_hkdf2``
+
+::
+
+ struct cmh_ioctl_kic_hkdf2 {
+ __u32 version;
+ __u32 key_len;
+ __u64 base_key;
+ __u64 salt_key; /* salt key reference (CMH_REF_NONE = no salt) */
+ __u64 cid;
+ __u64 label;
+ __u32 label_len;
+ __u32 flags;
+ __u64 ref; /* [out] derived key reference */
+ };
+
+CMH_IOCTL_KIC_AES_CMAC_KDF
+---------------------------
+
+AES-CMAC-based key derivation (NIST SP 800-108).
+
+:Direction: ``_IOWR``
+:Number: 0x0C
+:Argument: ``struct cmh_ioctl_kic_aes_cmac_kdf``
+
+::
+
+ struct cmh_ioctl_kic_aes_cmac_kdf {
+ __u32 version;
+ __u32 key_len; /* base & output key length (must be 32) */
+ __u64 base_key;
+ __u64 cid;
+ __u64 label;
+ __u32 label_len;
+ __u32 flags;
+ __u64 ref; /* [out] derived key reference */
+ };
+
+CMH_IOCTL_KIC_DKEK_DERIVE
+--------------------------
+
+Derive a Device Key Encryption Key (DKEK) for secure key export.
+
+:Direction: ``_IOWR``
+:Number: 0x0D
+:Argument: ``struct cmh_ioctl_kic_dkek_derive``
+
+::
+
+ struct cmh_ioctl_kic_dkek_derive {
+ __u32 version;
+ __u32 host_id; /* target host ID (0 = caller's own) */
+ __u64 base_key;
+ __u64 cid;
+ __u64 metadata; /* user-space pointer to metadata */
+ __u32 metadata_len;
+ __u32 flags;
+ __u64 ref; /* [out] derived KEK reference */
+ };
+
+PKE (Public Key Engine) ioctls
+==============================
+
+RSA Operations
+--------------
+
+CMH_IOCTL_PKE_RSA_ENC
+~~~~~~~~~~~~~~~~~~~~~~
+
+RSA public-key encryption.
+
+:Direction: ``_IOWR``
+:Number: 0x10
+:Argument: ``struct cmh_ioctl_pke_rsa_enc``
+
+::
+
+ struct cmh_ioctl_pke_rsa_enc {
+ __u32 version;
+ __u32 bits; /* RSA key size in bits (512-4096) */
+ __u64 e; /* user-space pointer to public exponent */
+ __u32 e_len; /* exponent length in bytes */
+ __u32 __reserved;
+ __u64 n; /* user-space pointer to modulus */
+ __u64 input; /* user-space pointer to input data */
+ __u64 output; /* user-space pointer to output buffer */
+ };
+
+The public key (e, n) is passed as raw user-space buffers.
+
+CMH_IOCTL_PKE_RSA_DEC
+~~~~~~~~~~~~~~~~~~~~~~
+
+RSA private-key decryption using a datastore key reference.
+
+:Direction: ``_IOWR``
+:Number: 0x11
+:Argument: ``struct cmh_ioctl_pke_rsa_dec``
+
+::
+
+ struct cmh_ioctl_pke_rsa_dec {
+ __u32 version;
+ __u32 bits;
+ __u64 e; /* public exponent */
+ __u32 e_len;
+ __u32 __reserved;
+ __u64 n; /* modulus */
+ __u64 input; /* ciphertext */
+ __u64 output; /* plaintext output */
+ __u64 key_ref; /* private key DS reference */
+ };
+
+CMH_IOCTL_PKE_RSA_CRT_DEC
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+RSA CRT private-key decryption (faster, uses CRT key format).
+
+:Direction: ``_IOWR``
+:Number: 0x12
+:Argument: ``struct cmh_ioctl_pke_rsa_crt_dec``
+
+::
+
+ struct cmh_ioctl_pke_rsa_crt_dec {
+ __u32 version;
+ __u32 bits;
+ __u64 e;
+ __u32 e_len;
+ __u32 __reserved;
+ __u64 n;
+ __u64 input;
+ __u64 output;
+ __u64 crt_ref; /* CRT key DS reference */
+ };
+
+CMH_IOCTL_PKE_RSA_KEYGEN
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Generate an RSA key pair in hardware.
+
+:Direction: ``_IOWR``
+:Number: 0x13
+:Argument: ``struct cmh_ioctl_pke_rsa_keygen``
+
+::
+
+ struct cmh_ioctl_pke_rsa_keygen {
+ __u32 version;
+ __u32 bits; /* key size in bits */
+ __u64 e; /* user-space pointer to public exponent */
+ __u32 e_len;
+ __u32 flags; /* CMH_FLAG_* */
+ __u64 n; /* [out] user-space pointer to modulus buffer */
+ __u64 d_cid; /* CID for private key DS entry */
+ __u64 d_ref; /* [out] private key reference */
+ __u64 crt_cid; /* CID for CRT key DS entry (0 = skip CRT) */
+ __u64 crt_ref; /* [out] CRT key reference */
+ };
+
+Returns private key and optional CRT key as datastore references.
+The modulus is written back to user space.
+
+ECDSA Operations
+----------------
+
+CMH_IOCTL_PKE_ECDSA_SIGN
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ECDSA signature generation using a datastore private key.
+
+:Direction: ``_IOWR``
+:Number: 0x14
+:Argument: ``struct cmh_ioctl_pke_ecdsa_sign``
+
+::
+
+ struct cmh_ioctl_pke_ecdsa_sign {
+ __u32 version;
+ __u32 curve; /* ABI curve ID (e.g. 0x03 = P-256) */
+ __u64 digest; /* user-space pointer to hash digest */
+ __u32 digest_len; /* digest length in bytes */
+ __u32 __reserved;
+ __u64 signature; /* [out] user-space pointer to (r,s) */
+ __u64 key_ref; /* private key DS reference */
+ };
+
+CMH_IOCTL_PKE_ECDH
+~~~~~~~~~~~~~~~~~~~
+
+Compute ECDH shared secret from a peer public key and a datastore
+private key.
+
+:Direction: ``_IOWR``
+:Number: 0x16
+:Argument: ``struct cmh_ioctl_pke_ecdh``
+
+::
+
+ struct cmh_ioctl_pke_ecdh {
+ __u32 version;
+ __u32 curve;
+ __u64 peer_key_x; /* user-space pointer to peer public key X */
+ __u64 key_ref; /* private key DS reference */
+ __u32 flags; /* CMH_PKE_FLAG_DS_RESULT */
+ __u32 __reserved;
+ __u64 result_cid; /* CID for DS result (if FLAG_DS_RESULT) */
+ __u64 output; /* [out] raw shared secret or DS ref */
+ };
+
+If ``CMH_PKE_FLAG_DS_RESULT`` is set, the shared secret is stored in
+the datastore and a reference is returned instead of raw bytes.
+
+CMH_IOCTL_PKE_ECDH_KEYGEN
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Derive a public key from a datastore private key.
+
+:Direction: ``_IOWR``
+:Number: 0x17
+:Argument: ``struct cmh_ioctl_pke_ecdh_keygen``
+
+::
+
+ struct cmh_ioctl_pke_ecdh_keygen {
+ __u32 version;
+ __u32 curve;
+ __u64 key_ref; /* private key DS reference */
+ __u64 public_key_x; /* [out] user-space pointer to public key X */
+ };
+
+EdDSA Operations
+----------------
+
+CMH_IOCTL_PKE_EDDSA_SIGN
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+EdDSA (Ed25519/Ed448) signature generation.
+
+:Direction: ``_IOWR``
+:Number: 0x18
+:Argument: ``struct cmh_ioctl_pke_eddsa_sign``
+
+::
+
+ struct cmh_ioctl_pke_eddsa_sign {
+ __u32 version;
+ __u32 curve; /* CURVE_25519 or CURVE_448 */
+ __u64 digest; /* user-space ptr to message (not digest) */
+ __u32 digest_len;
+ __u32 __reserved;
+ __u64 signature; /* [out] user-space pointer to signature */
+ __u64 key_ref; /* private key DS reference */
+ };
+
+Note: the ``digest`` field is the full message (pure EdDSA), not a
+pre-computed hash.
+
+CMH_IOCTL_PKE_EDDSA_VERIFY
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+EdDSA signature verification.
+
+:Direction: ``_IOW``
+:Number: 0x19
+:Argument: ``struct cmh_ioctl_pke_eddsa_verify``
+
+::
+
+ struct cmh_ioctl_pke_eddsa_verify {
+ __u32 version;
+ __u32 curve;
+ __u64 digest;
+ __u32 digest_len;
+ __u32 __reserved;
+ __u64 signature;
+ __u64 public_key_y; /* user-space pointer to public key Y */
+ };
+
+EC Key Management
+-----------------
+
+CMH_IOCTL_PKE_EC_KEYGEN
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Generate an EC private key in the hardware datastore.
+
+:Direction: ``_IOWR``
+:Number: 0x1A
+:Argument: ``struct cmh_ioctl_pke_ec_keygen``
+
+::
+
+ struct cmh_ioctl_pke_ec_keygen {
+ __u32 version;
+ __u32 curve;
+ __u32 flags; /* CMH_FLAG_* */
+ __u32 __reserved;
+ __u64 cid; /* CID for the new key DS entry */
+ __u64 ref; /* [out] private key reference */
+ };
+
+CMH_IOCTL_PKE_EC_PUBGEN
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Derive the public key from a datastore private key.
+
+:Direction: ``_IOWR``
+:Number: 0x1B
+:Argument: ``struct cmh_ioctl_pke_ec_pubgen``
+
+::
+
+ struct cmh_ioctl_pke_ec_pubgen {
+ __u32 version;
+ __u32 curve;
+ __u64 key_ref; /* private key DS reference */
+ __u64 public_key; /* [out] user-space pointer to public key */
+ };
+
+CMH_IOCTL_PKE_EDDSA_KEYGEN_SCA
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Generate a 2-share SCA-protected Ed448 private key.
+
+:Direction: ``_IOWR``
+:Number: 0x1C
+:Argument: ``struct cmh_ioctl_pke_eddsa_keygen_sca``
+
+::
+
+ struct cmh_ioctl_pke_eddsa_keygen_sca {
+ __u32 version;
+ __u32 curve; /* must be CURVE_448 */
+ __u64 key_ref; /* input: normal Ed448 private key DS ref */
+ __u64 cid; /* CID for the new SCA key DS entry */
+ __u64 sca_ref; /* [out] SCA private key reference */
+ };
+
+Post-Quantum Cryptography (PQC) ioctls
+=======================================
+
+PQC operations support the following flags in the ``flags`` field:
+
+============================ ==== ====================================
+Flag Bit Description
+============================ ==== ====================================
+``CMH_QSE_FLAG_MASKED`` 0 Use masked (SCA-resistant) HW path
+``CMH_QSE_FLAG_DS_REF`` 1 Store key output in DS, return ref
+``CMH_QSE_FLAG_HW_RNG`` 2 Use HW RNG for seed/randomness
+============================ ==== ====================================
+
+ML-KEM (FIPS 203)
+-----------------
+
+CMH_IOCTL_ML_KEM_KEYGEN
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Generate an ML-KEM key pair.
+
+:Direction: ``_IOWR``
+:Number: 0x20
+:Argument: ``struct cmh_ioctl_ml_kem_keygen``
+
+::
+
+ struct cmh_ioctl_ml_kem_keygen {
+ __u32 version;
+ __u32 k; /* security parameter: 2/3/4 */
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 __reserved;
+ __u64 seed; /* user-space pointer to seed (or 0 for HW RNG) */
+ __u64 z; /* user-space pointer to z (or 0 for HW RNG) */
+ __u64 ek; /* [out] user-space pointer to encapsulation key */
+ __u64 dk; /* [out] user-space pointer to decapsulation key,
+ * or [out] DS ref if CMH_QSE_FLAG_DS_REF */
+ __u64 dk_cid; /* CID for DS entry (if DS_REF) */
+ __u64 dk_ref; /* [out] dk DS reference (if DS_REF) */
+ };
+
+Security parameter ``k`` selects the strength: 2 (ML-KEM-512),
+3 (ML-KEM-768), or 4 (ML-KEM-1024).
+
+CMH_IOCTL_ML_KEM_ENC
+~~~~~~~~~~~~~~~~~~~~~
+
+ML-KEM encapsulation. Produces ciphertext and shared secret.
+
+:Direction: ``_IOWR``
+:Number: 0x21
+:Argument: ``struct cmh_ioctl_ml_kem_enc``
+
+::
+
+ struct cmh_ioctl_ml_kem_enc {
+ __u32 version;
+ __u32 k;
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 __reserved;
+ __u64 coin; /* user-space pointer to random coin (or 0) */
+ __u64 ek; /* user-space pointer to encapsulation key */
+ __u64 ct; /* [out] user-space pointer to ciphertext */
+ __u64 ss; /* [out] user-space pointer to shared secret */
+ __u64 __reserved2[2]; /* reserved for future use */
+ };
+
+CMH_IOCTL_ML_KEM_DEC
+~~~~~~~~~~~~~~~~~~~~~
+
+ML-KEM decapsulation. Recovers shared secret from ciphertext.
+
+:Direction: ``_IOWR``
+:Number: 0x22
+:Argument: ``struct cmh_ioctl_ml_kem_dec``
+
+::
+
+ struct cmh_ioctl_ml_kem_dec {
+ __u32 version;
+ __u32 k;
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 __reserved;
+ __u64 ct; /* user-space pointer to ciphertext */
+ __u64 dk; /* user-space pointer to dk or DS ref */
+ __u64 ss; /* [out] user-space pointer to shared secret */
+ __u64 __reserved2[2]; /* reserved for future use */
+ };
+
+ML-DSA (FIPS 204)
+-----------------
+
+CMH_IOCTL_ML_DSA_KEYGEN
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Generate an ML-DSA key pair.
+
+:Direction: ``_IOWR``
+:Number: 0x23
+:Argument: ``struct cmh_ioctl_ml_dsa_keygen``
+
+::
+
+ struct cmh_ioctl_ml_dsa_keygen {
+ __u32 version;
+ __u32 mode; /* security parameter: 2/3/5 */
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 __reserved;
+ __u64 seed; /* user-space pointer to seed (or 0 for HW RNG) */
+ __u64 pk; /* [out] user-space pointer to public key */
+ __u64 sk; /* [out] user-space pointer to secret key,
+ * or [out] DS ref if CMH_QSE_FLAG_DS_REF */
+ __u64 sk_cid; /* CID for DS entry (if DS_REF) */
+ __u64 sk_ref; /* [out] sk DS reference (if DS_REF) */
+ };
+
+Security parameter ``mode`` selects the strength: 2 (ML-DSA-44),
+3 (ML-DSA-65), or 5 (ML-DSA-87).
+
+.. note::
+
+ When ``CMH_QSE_FLAG_DS_REF`` keeps the secret key in the datastore,
+ the public key returned in ``pk`` is the only copy: there is no
+ operation to derive the public key from the secret-key reference
+ for ML-DSA. User space must persist ``pk`` at keygen time.
+
+CMH_IOCTL_ML_DSA_SIGN
+~~~~~~~~~~~~~~~~~~~~~~
+
+ML-DSA signature generation.
+
+:Direction: ``_IOWR``
+:Number: 0x24
+:Argument: ``struct cmh_ioctl_ml_dsa_sign``
+
+::
+
+ struct cmh_ioctl_ml_dsa_sign {
+ __u32 version;
+ __u32 mode;
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 mlen; /* message length in bytes */
+ __u64 m; /* user-space pointer to message */
+ __u64 sk; /* user-space pointer to sk or DS ref */
+ __u64 sig; /* [out] user-space pointer to signature */
+ __u64 rnd; /* user-space pointer to randomness (or 0) */
+ };
+
+If ``mlen`` is set to ``CMH_ML_DSA_MLEN_EXTERNAL_MU`` (0xFFFFFFFF),
+the ``m`` pointer is interpreted as a 64-byte pre-hashed mu value
+(ExternalMu mode).
+
+CMH_IOCTL_SLHDSA_KEYGEN
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Generate an SLH-DSA key pair.
+
+:Direction: ``_IOWR``
+:Number: 0x28
+:Argument: ``struct cmh_ioctl_slhdsa_keygen``
+
+::
+
+ struct cmh_ioctl_slhdsa_keygen {
+ __u32 version;
+ __u32 parameter_set; /* HCQ_SLHDSA_SHAKE_128S .. SHA2_256F */
+ __u32 flags; /* CMH_QSE_FLAG_DS_REF */
+ __u32 __reserved;
+ __u64 seed; /* user-space pointer to seed */
+ __u64 pk; /* [out] user-space pointer to public key */
+ __u64 sk; /* [out] user-space pointer to secret key,
+ * or [out] DS ref if CMH_QSE_FLAG_DS_REF */
+ __u64 sk_cid; /* CID for DS entry (if DS_REF) */
+ __u64 sk_ref; /* [out] sk DS reference (if DS_REF) */
+ };
+
+.. note::
+
+ When ``CMH_QSE_FLAG_DS_REF`` keeps the secret key in the datastore,
+ the public key returned in ``pk`` is the only copy: there is no
+ operation to derive the public key from the secret-key reference
+ for SLH-DSA. User space must persist ``pk`` at keygen time.
+
+CMH_IOCTL_SLHDSA_SIGN
+~~~~~~~~~~~~~~~~~~~~~~
+
+SLH-DSA signature generation (pure mode).
+
+:Direction: ``_IOWR``
+:Number: 0x29
+:Argument: ``struct cmh_ioctl_slhdsa_sign``
+
+::
+
+ struct cmh_ioctl_slhdsa_sign {
+ __u32 version;
+ __u32 parameter_set;
+ __u32 msg_len;
+ __u32 ctx_len;
+ __u64 msg; /* user-space pointer to message */
+ __u64 ctx; /* user-space pointer to context (or 0) */
+ __u64 sk; /* DS ref for secret key */
+ __u64 sig; /* [out] user-space pointer to signature */
+ __u64 add_random; /* user-space pointer to addl. randomness (or 0) */
+ };
+
+CMH_IOCTL_SLHDSA_SIGN_PREHASH
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+SLH-DSA pre-hash signature generation.
+
+:Direction: ``_IOWR``
+:Number: 0x2D
+:Argument: ``struct cmh_ioctl_slhdsa_sign_prehash``
+
+::
+
+ struct cmh_ioctl_slhdsa_sign_prehash {
+ __u32 version;
+ __u32 parameter_set;
+ __u32 prehash_algo; /* CMH_SLHDSA_PREHASH_* */
+ __u32 digest; /* 0 = raw msg (eSW hashes), 1 = pre-computed */
+ __u32 msg_len;
+ __u32 ctx_len;
+ __u64 msg; /* user-space pointer to message/digest */
+ __u64 ctx; /* user-space pointer to context (or 0) */
+ __u64 sk; /* DS ref for secret key */
+ __u64 sig; /* [out] user-space pointer to signature */
+ __u64 add_random; /* user-space pointer to addl. randomness (or 0) */
+ };
+
+The ``prehash_algo`` field selects the hash algorithm
+(``CMH_SLHDSA_PREHASH_SHA256``, etc.).
+
+CMH_IOCTL_SM2_ENC_POINT
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Direction: ``_IOWR``
+:Number: 0x33
+:Argument: ``struct cmh_ioctl_sm2_enc_point``
+
+::
+
+ struct cmh_ioctl_sm2_enc_point {
+ __u32 version;
+ __u32 nonce_len; /* 0 = HW generates, 32 = caller provides */
+ __u64 nonce; /* user-space pointer to nonce (or 0) */
+ __u64 public_key; /* user-space pointer to public key (64B) */
+ __u64 ciphertext; /* [out] user-space pointer to C1 (64B) */
+ __u64 enc_point; /* [out] user-space pointer to enc point (64B) */
+ };
+
+CMH_IOCTL_SM2_ENC_HASH
+~~~~~~~~~~~~~~~~~~~~~~~
+
+:Direction: ``_IOWR``
+:Number: 0x37
+:Argument: ``struct cmh_ioctl_sm2_enc_hash``
+
+::
+
+ struct cmh_ioctl_sm2_enc_hash {
+ __u32 version;
+ __u32 message_len; /* message length (1..32) */
+ __u64 message; /* user-space pointer to plaintext */
+ __u64 enc_point; /* user-space pointer to enc point (64B) */
+ __u64 ciphertext; /* [out] user-space pointer to ciphertext */
+ };
+
+CMH_IOCTL_SM2_DEC_POINT
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Direction: ``_IOWR``
+:Number: 0x32
+:Argument: ``struct cmh_ioctl_sm2_dec_point``
+
+::
+
+ struct cmh_ioctl_sm2_dec_point {
+ __u32 version;
+ __u32 ciphertext_len; /* total ciphertext length (97..128) */
+ __u64 ciphertext; /* user-space pointer to ciphertext (64B: C1) */
+ __u64 dec_point; /* [out] user-space pointer to dec point (64B) */
+ __u64 key_ref; /* private key DS reference */
+ };
+
+CMH_IOCTL_SM2_DEC_HASH
+~~~~~~~~~~~~~~~~~~~~~~~
+
+:Direction: ``_IOWR``
+:Number: 0x36
+:Argument: ``struct cmh_ioctl_sm2_dec_hash``
+
+::
+
+ struct cmh_ioctl_sm2_dec_hash {
+ __u32 version;
+ __u32 ciphertext_len; /* ciphertext length (97..128) */
+ __u64 ciphertext; /* user-space pointer to full ciphertext */
+ __u64 dec_point; /* user-space pointer to dec point (64B) */
+ __u64 plaintext; /* [out] user-space pointer to plaintext */
+ };
+
+SM2 Key Exchange (GM/T 0003.3)
+------------------------------
+
+The key exchange protocol is a multi-step flow:
+
+1. ``EC_KEYGEN(CMH_CURVE_SM2)`` -- generate a long-lived private key.
+2. ``EC_PUBGEN`` -- derive the public key.
+3. ``SM2_ID_DIGEST`` -- compute the SM3 identity digest (ZA).
+4. ``SM2_ECDH_KEYGEN`` -- generate an ephemeral session key.
+5. Exchange session keys with the peer.
+6. ``SM2_ECDH`` -- compute the shared point.
+7. ``SM2_ECDH_HASH`` -- derive the shared key from the shared point
+ and both parties' ZA digests.
+
+CMH_IOCTL_SM2_ECDH_KEYGEN
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Direction: ``_IOWR``
+:Number: 0x30
+:Argument: ``struct cmh_ioctl_sm2_ecdh_keygen``
+
+::
+
+ struct cmh_ioctl_sm2_ecdh_keygen {
+ __u32 version;
+ __u32 nonce_len; /* 0 = HW generates r (written back), 32 = caller */
+ __u64 nonce; /* [in/out] user-space pointer to nonce buffer (32B) */
+ __u64 session_key; /* [out] user-space pointer to R=r*G (64B) */
+ };
+
+``nonce_len`` must be 0 or 32. If ``nonce_len=0``, the hardware
+generates the ephemeral scalar and writes it back to the ``nonce``
+buffer.
+
+CMH_IOCTL_SM2_ECDH
+~~~~~~~~~~~~~~~~~~~
+
+:Direction: ``_IOWR``
+:Number: 0x31
+:Argument: ``struct cmh_ioctl_sm2_ecdh``
+
+::
+
+ struct cmh_ioctl_sm2_ecdh {
+ __u32 version;
+ __u32 nonce_len; /* 0 = HW generates, 32 = caller provides */
+ __u64 nonce; /* [in/out] user-space pointer to nonce r (32B) */
+ __u64 peer_public_key; /* user-space pointer to peer pub key (64B) */
+ __u64 peer_session_key; /* user-space pointer to peer session key (64B) */
+ __u64 key_ref; /* private key DS reference */
+ __u64 shared_point; /* [out] user-space pointer to shared point (64B) */
+ __u64 shared_point_ref; /* [in/out] 0 = read-back; &ref = keep DS */
+ };
+
+If ``shared_point_ref`` points to a non-zero value, the shared point
+is kept in the datastore for use by ``SM2_ECDH_HASH``.
+
+CMH_IOCTL_SM2_ID_DIGEST
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Compute the SM3 identity digest (ZA) for a public key and identity
+string.
+
+:Direction: ``_IOWR``
+:Number: 0x34
+:Argument: ``struct cmh_ioctl_sm2_id_digest``
+
+::
+
+ struct cmh_ioctl_sm2_id_digest {
+ __u32 version;
+ __u32 id_len; /* identity length in bytes (<=32) */
+ __u64 id; /* user-space pointer to identity string */
+ __u64 public_key; /* user-space pointer to public key (64B) */
+ __u64 digest; /* [out] user-space pointer to ZA digest (32B) */
+ };
+
+CMH_IOCTL_SM2_ECDH_HASH
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Derive the shared key from the shared point and ZA digests.
+
+:Direction: ``_IOWR``
+:Number: 0x35
+:Argument: ``struct cmh_ioctl_sm2_ecdh_hash``
+
+::
+
+ struct cmh_ioctl_sm2_ecdh_hash {
+ __u32 version;
+ __u32 __reserved;
+ __u64 peer_id_digest; /* ptr to Z_A -- initiator's digest (32B) */
+ __u64 id_digest; /* ptr to Z_B -- responder's digest (32B) */
+ __u64 shared_point_ref; /* DS reference from SM2_ECDH */
+ __u64 shared_key; /* [out] ptr to shared key (16B) */
+ };
+
+.. important::
+
+ The digest fields use **absolute** ordering per GM/T 0003.3, not
+ relative own/peer ordering. Both parties must pass:
+
+ - ``peer_id_digest`` = Z_A (initiator's digest) -- hashed first
+ - ``id_digest`` = Z_B (responder's digest) -- hashed second
+
+Hardware Management ioctls
+==========================
+
+CMH_IOCTL_EAC_READ
+-------------------
+
+Read and clear the hardware Error and Alarm Controller registers.
+
+:Direction: ``_IOWR``
+:Number: 0x0F
+:Argument: ``struct cmh_ioctl_eac_read``
+
+::
+
+ struct cmh_ioctl_eac_read {
+ __u32 version;
+ __u32 __reserved;
+ __u64 mailbox_notification;
+ __u32 hw_error;
+ __u32 hw_nmi;
+ __u32 hw_panic;
+ __u32 safety_fatal;
+ __u32 safety_notification;
+ __u32 sw_info0;
+ __u32 sw_info1;
+ __u32 sram_bank_errors[4];
+ __u32 __pad;
+ };
+
+The eSW atomically reads and clears the registers on each call.
+Successive reads show only new events since the last read.
+
+CMH_IOCTL_DRBG_CONFIG
+----------------------
+
+Configure the hardware DRBG before first use.
+
+:Direction: ``_IOW``
+:Number: 0x40
+:Argument: ``struct cmh_ioctl_drbg_config``
+
+::
+
+ struct cmh_ioctl_drbg_config {
+ __u32 version;
+ __u32 entropy_ratio; /* CMH_DRBG_RATIO_* */
+ __u32 security_strength; /* CMH_DRBG_STRENGTH_* */
+ __u32 __reserved;
+ };
+
+This is a management operation normally performed once at system
+startup. Must be called before any ``hwrng`` reads or DRBG generate
+operations.
+
+ioctl Number Summary
+====================
+
+====================================== ==== ==== =========================================
+ioctl Dir Seq Argument
+====================================== ==== ==== =========================================
+``CMH_IOCTL_KEY_NEW`` IOWR 0x01 ``cmh_ioctl_key_new``
+``CMH_IOCTL_KEY_WRITE`` IOW 0x02 ``cmh_ioctl_key_write``
+``CMH_IOCTL_KEY_READ`` IOWR 0x03 ``cmh_ioctl_key_read``
+``CMH_IOCTL_KEY_FIND`` IOWR 0x04 ``cmh_ioctl_key_find``
+``CMH_IOCTL_KEY_GRANT`` IOW 0x05 ``cmh_ioctl_key_grant``
+``CMH_IOCTL_KEY_DELETE`` IOW 0x06 ``cmh_ioctl_key_grant``
+``CMH_IOCTL_DS_EXPORT`` IOWR 0x07 ``cmh_ioctl_ds_export``
+``CMH_IOCTL_DS_IMPORT`` IOW 0x08 ``cmh_ioctl_ds_import``
+``CMH_IOCTL_KIC_HKDF1`` IOWR 0x09 ``cmh_ioctl_kic_hkdf1``
+``CMH_IOCTL_KIC_HKDF2`` IOWR 0x0A ``cmh_ioctl_kic_hkdf2``
+``CMH_IOCTL_KEY_NEW_RANDOM`` IOWR 0x0B ``cmh_ioctl_key_new``
+``CMH_IOCTL_KIC_AES_CMAC_KDF`` IOWR 0x0C ``cmh_ioctl_kic_aes_cmac_kdf``
+``CMH_IOCTL_KIC_DKEK_DERIVE`` IOWR 0x0D ``cmh_ioctl_kic_dkek_derive``
+``CMH_IOCTL_KEY_LIST`` IOWR 0x0E ``cmh_ioctl_key_list``
+``CMH_IOCTL_EAC_READ`` IOWR 0x0F ``cmh_ioctl_eac_read``
+``CMH_IOCTL_PKE_RSA_ENC`` IOWR 0x10 ``cmh_ioctl_pke_rsa_enc``
+``CMH_IOCTL_PKE_RSA_DEC`` IOWR 0x11 ``cmh_ioctl_pke_rsa_dec``
+``CMH_IOCTL_PKE_RSA_CRT_DEC`` IOWR 0x12 ``cmh_ioctl_pke_rsa_crt_dec``
+``CMH_IOCTL_PKE_RSA_KEYGEN`` IOWR 0x13 ``cmh_ioctl_pke_rsa_keygen``
+``CMH_IOCTL_PKE_ECDSA_SIGN`` IOWR 0x14 ``cmh_ioctl_pke_ecdsa_sign``
+``CMH_IOCTL_PKE_ECDH`` IOWR 0x16 ``cmh_ioctl_pke_ecdh``
+``CMH_IOCTL_PKE_ECDH_KEYGEN`` IOWR 0x17 ``cmh_ioctl_pke_ecdh_keygen``
+``CMH_IOCTL_PKE_EDDSA_SIGN`` IOWR 0x18 ``cmh_ioctl_pke_eddsa_sign``
+``CMH_IOCTL_PKE_EDDSA_VERIFY`` IOW 0x19 ``cmh_ioctl_pke_eddsa_verify``
+``CMH_IOCTL_PKE_EC_KEYGEN`` IOWR 0x1A ``cmh_ioctl_pke_ec_keygen``
+``CMH_IOCTL_PKE_EC_PUBGEN`` IOWR 0x1B ``cmh_ioctl_pke_ec_pubgen``
+``CMH_IOCTL_PKE_EDDSA_KEYGEN_SCA`` IOWR 0x1C ``cmh_ioctl_pke_eddsa_keygen_sca``
+``CMH_IOCTL_ML_KEM_KEYGEN`` IOWR 0x20 ``cmh_ioctl_ml_kem_keygen``
+``CMH_IOCTL_ML_KEM_ENC`` IOWR 0x21 ``cmh_ioctl_ml_kem_enc``
+``CMH_IOCTL_ML_KEM_DEC`` IOWR 0x22 ``cmh_ioctl_ml_kem_dec``
+``CMH_IOCTL_ML_DSA_KEYGEN`` IOWR 0x23 ``cmh_ioctl_ml_dsa_keygen``
+``CMH_IOCTL_ML_DSA_SIGN`` IOWR 0x24 ``cmh_ioctl_ml_dsa_sign``
+``CMH_IOCTL_SLHDSA_KEYGEN`` IOWR 0x28 ``cmh_ioctl_slhdsa_keygen``
+``CMH_IOCTL_SLHDSA_SIGN`` IOWR 0x29 ``cmh_ioctl_slhdsa_sign``
+``CMH_IOCTL_SLHDSA_SIGN_PREHASH`` IOWR 0x2D ``cmh_ioctl_slhdsa_sign_prehash``
+``CMH_IOCTL_SM2_ECDH_KEYGEN`` IOWR 0x30 ``cmh_ioctl_sm2_ecdh_keygen``
+``CMH_IOCTL_SM2_ECDH`` IOWR 0x31 ``cmh_ioctl_sm2_ecdh``
+``CMH_IOCTL_SM2_DEC_POINT`` IOWR 0x32 ``cmh_ioctl_sm2_dec_point``
+``CMH_IOCTL_SM2_ENC_POINT`` IOWR 0x33 ``cmh_ioctl_sm2_enc_point``
+``CMH_IOCTL_SM2_ID_DIGEST`` IOWR 0x34 ``cmh_ioctl_sm2_id_digest``
+``CMH_IOCTL_SM2_ECDH_HASH`` IOWR 0x35 ``cmh_ioctl_sm2_ecdh_hash``
+``CMH_IOCTL_SM2_DEC_HASH`` IOWR 0x36 ``cmh_ioctl_sm2_dec_hash``
+``CMH_IOCTL_SM2_ENC_HASH`` IOWR 0x37 ``cmh_ioctl_sm2_enc_hash``
+``CMH_IOCTL_DRBG_CONFIG`` IOW 0x40 ``cmh_ioctl_drbg_config``
+====================================== ==== ==== =========================================
+
+Relationship to the in-kernel crypto API
+=========================================
+
+The main reason these operations are exposed as ioctls, rather than
+through the standard in-kernel crypto API, is the CMH datastore key
+model: an ioctl can operate on a *datastore-referenced* (hardware-held)
+key, identified only by a ``ref`` or CID, whose raw bytes the CPU never
+sees. The standard crypto API cannot express this -- every
+``.setkey()`` takes raw key material -- so hardware key lifecycle
+(create, import, derive, grant, destroy) and compute-on-hardware-held-key
+operations have no crypto API equivalent and are only reachable here.
+
+These ioctls remain supported. Where an operation can *also* be
+expressed through a standard kernel abstraction, additional in-kernel
+crypto API bindings may be added over time; when such a binding lands the
+driver registers through it, and the ioctl continues to be maintained for
+backward compatibility:
+
+- **EdDSA** (``CMH_IOCTL_PKE_EDDSA_*``): a kernel ``sig`` binding may be
+ added once ed25519/ed448 algorithm types are accepted upstream.
+
+- **ML-KEM** (``CMH_IOCTL_ML_KEM_*``): a kernel KEM binding may be added
+ once the in-flight KEM subsystem series lands.
+
+- **Key lifecycle** (``CMH_IOCTL_KEY_*``): integration with the kernel
+ KEYS subsystem (trusted-keys / encrypted-keys) may be evaluated as a
+ follow-up series.
+
+Operations that are inherently vendor-specific (EAC Chip Authentication,
+KIC key derivation, SM2 key exchange, DRBG configuration, datastore
+export/import) have no corresponding kernel abstraction and are expected
+to remain ioctl-only.
diff --git a/Documentation/userspace-api/ioctl/index.rst b/Documentation/userspace-api/ioctl/index.rst
index 475675eae086..bf88bb6b9a6f 100644
--- a/Documentation/userspace-api/ioctl/index.rst
+++ b/Documentation/userspace-api/ioctl/index.rst
@@ -12,4 +12,5 @@ IOCTLs
ioctl-decoding
cdrom
+ cmh_mgmt
hdio
diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst
index 3f0ef1e27eb0..83fd74b6f396 100644
--- a/Documentation/userspace-api/ioctl/ioctl-number.rst
+++ b/Documentation/userspace-api/ioctl/ioctl-number.rst
@@ -170,6 +170,7 @@ Code Seq# Include File Comments
'I' all linux/isdn.h conflict!
'I' 00-0F drivers/isdn/divert/isdn_divert.h conflict!
'I' 40-4F linux/mISDNif.h conflict!
+'J' 01-40 uapi/linux/cmh_mgmt_ioctl.h Rambus CryptoManager Hub (CMH)
'K' all linux/kd.h
'L' 00-1F linux/loop.h conflict!
'L' 10-1F drivers/scsi/mpt3sas/mpt3sas_ctl.h conflict!
--
2.43.7
^ permalink raw reply related
* [PATCH v3 06/19] crypto: cmh - add CSHAKE/KMAC ahash
From: Saravanakrishnan Krishnamoorthy @ 2026-08-06 19:55 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register ahash algorithms for cSHAKE128, cSHAKE256, KMAC128, and
KMAC256 using the CMH hash core. cSHAKE supports incremental
update and export/import. KMAC has a 64KB data cap imposed by the
hardware.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 4 +-
drivers/crypto/cmh/cmh_cshake.c | 814 ++++++++++++++++++++++++
drivers/crypto/cmh/cmh_kmac.c | 630 ++++++++++++++++++
drivers/crypto/cmh/cmh_main.c | 18 +
drivers/crypto/cmh/include/cmh_cshake.h | 16 +
drivers/crypto/cmh/include/cmh_kmac.h | 16 +
6 files changed, 1497 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_cshake.c
create mode 100644 drivers/crypto/cmh/cmh_kmac.c
create mode 100644 drivers/crypto/cmh/include/cmh_cshake.h
create mode 100644 drivers/crypto/cmh/include/cmh_kmac.h
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index acd1827cc084..3ab154744658 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -16,7 +16,9 @@ cmh-y := \
cmh_key.o \
cmh_sys.o \
cmh_hash.o \
- cmh_hmac.o
+ cmh_hmac.o \
+ cmh_cshake.o \
+ cmh_kmac.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_cshake.c b/drivers/crypto/cmh/cmh_cshake.c
new file mode 100644
index 000000000000..acba3f2f6f3c
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_cshake.c
@@ -0,0 +1,814 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API CSHAKE Driver
+ *
+ * Registers cSHAKE-128 and cSHAKE-256 as ahash algorithms using the
+ * CMH Hash Core (HC) via HC_CMD_CSHAKE.
+ *
+ * CSHAKE (NIST SP 800-185) extends SHAKE with two domain separation
+ * parameters: function name N and customization string S. When both
+ * are empty, cSHAKE reduces to plain SHAKE -- the driver falls back to
+ * HC_CMD_INIT in that case (per SP 800-185 S6.2).
+ *
+ * N and S are set via .setkey() using a self-describing binary header
+ * (matching the upstream authenc precedent):
+ *
+ * struct cshake_cfg { __be32 n_len; __be32 s_len; };
+ * setkey blob: cshake_cfg || N[n_len] || S[s_len]
+ *
+ * If .setkey() is never called, the driver defaults to plain SHAKE
+ * (N="" S=""). .setkey() is per-tfm, not per-request.
+ *
+ * N is embedded inline in the HC_CMD_CSHAKE struct (max 36 bytes).
+ * S is passed as VCQ inline data following the command slot (multi-span).
+ *
+ * Uses the same self-contained transaction model as cmh_hash.c:
+ * .init() -> software-only
+ * .update() -> software-only (accumulate chunks)
+ * .final() -> CSHAKE [+ inline S] [+ RESTORE] [+ GATHER] + FINAL + FLUSH
+ * .export() -> CSHAKE [+ inline S] [+ RESTORE] [+ GATHER] + SAVE + FLUSH
+ * .import() -> restore HC context checkpoint (software-only)
+ *
+ * The HC core supports HC_CMD_SAVE / HC_CMD_RESTORE for cSHAKE mode.
+ * The cSHAKE domain-separation prefix (function name N, customization
+ * string S) is absorbed into the Keccak sponge state by HC_CMD_CSHAKE
+ * on the first submission, and preserved through save/restore.
+ * Export/import enables crypto API transform cloning.
+ *
+ * .setkey() here configures public domain-separation parameters (N, S),
+ * not a secret key.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/hash.h>
+#include <linux/scatterlist.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <asm/byteorder.h>
+
+#include "cmh_cshake.h"
+#include "cmh_vcq.h"
+#include "cmh_hc_abi.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+
+/* Algorithm Table */
+
+struct cmh_cshake_alg_info {
+ u32 hc_algo;
+ u32 digest_size;
+ const char *alg_name;
+ const char *drv_name;
+};
+
+static const struct cmh_cshake_alg_info cmh_cshake_algs_info[] = {
+ {
+ .hc_algo = HC_ALGO_SHAKE128,
+ .digest_size = CMH_SHAKE128_DIGEST_SIZE,
+ .alg_name = "cshake128",
+ .drv_name = "rambus-cmh-cshake128",
+ },
+ {
+ .hc_algo = HC_ALGO_SHAKE256,
+ .digest_size = CMH_SHAKE256_DIGEST_SIZE,
+ .alg_name = "cshake256",
+ .drv_name = "rambus-cmh-cshake256",
+ },
+};
+
+#define CMH_CSHAKE_ALG_COUNT ARRAY_SIZE(cmh_cshake_algs_info)
+
+/* Per-Request State */
+
+struct cmh_cshake_chunk {
+ struct list_head list;
+ struct list_head tfm_node; /* per-tfm orphan tracking */
+ u32 len;
+ u8 data[];
+};
+
+/*
+ * Max payload slots for CSHAKE:
+ * CSHAKE (1) + inline S (ceil(S_len/64)) + GATHER (1) + FINAL (1) + FLUSH (1)
+ * S can be up to SHAKE-128 block (168 bytes) = 3 inline slots.
+ * Conservative: 1 + 3 + 1 + 1 + 1 = 7, plus headers.
+ *
+ * Or INIT + GATHER + FINAL + FLUSH = 4 (plain SHAKE fallback).
+ */
+#define CMH_CSHAKE_MAX_PAYLOAD 8
+#define CMH_CSHAKE_MAX_PACKED (CMH_CSHAKE_MAX_PAYLOAD * 2)
+
+/*
+ * Checkpoint embedded inline: the kernel ahash API has no per-request
+ * destructor, so a heap-allocated checkpoint leaks if a request is
+ * abandoned without .final().
+ */
+struct cmh_cshake_reqctx {
+ const struct cmh_cshake_alg_info *info;
+ int error;
+ struct list_head chunks;
+ u32 num_chunks;
+ u32 total_len;
+ u32 has_checkpoint;
+ u8 checkpoint[HC_CONTEXT_SIZE];
+ /* DMA state for async final */
+ dma_addr_t digest_dma;
+ dma_addr_t ckpt_dma;
+ u8 *digest_buf;
+ struct cmh_sg_map *sgm;
+ struct vcq_cmd packed[CMH_CSHAKE_MAX_PACKED];
+};
+
+/* Per-Transform State (carries N and S across requests) */
+
+struct cmh_cshake_tfm_ctx {
+ u8 *func_name; /* N (function name), NULL if empty */
+ u32 func_name_len;
+ u8 *custom; /* S (customization string), NULL if empty */
+ u32 custom_len;
+ spinlock_t chunk_lock; /* protects all_chunks */
+ struct list_head all_chunks; /* orphan-safe chunk tracking */
+};
+
+/* VCQ Builders */
+
+/* VCQ Builders (cSHAKE-specific; shared builders in cmh_hc_abi.h / cmh_vcq.h) */
+
+static void vcq_add_hc_save(struct vcq_cmd *slot, u32 core_id,
+ u64 output_phys, u32 outlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_SAVE);
+ slot->hwc.hc.cmd_save.output = output_phys;
+ slot->hwc.hc.cmd_save.outlen = outlen;
+}
+
+static void vcq_add_hc_restore(struct vcq_cmd *slot, u32 core_id,
+ u64 input_phys, u32 inlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_RESTORE);
+ slot->hwc.hc.cmd_restore.input = input_phys;
+ slot->hwc.hc.cmd_restore.inlen = inlen;
+}
+
+static void vcq_add_hc_cshake(struct vcq_cmd *slot, u32 core_id, u32 algo,
+ const u8 *name, u32 namelen,
+ u32 customlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_CSHAKE);
+ slot->hwc.hc.cmd_cshake.custom = 0; /* inline -- CMH eSW reads from next slot(s) */
+ slot->hwc.hc.cmd_cshake.customlen = customlen;
+ slot->hwc.hc.cmd_cshake.algo = algo;
+ slot->hwc.hc.cmd_cshake.namelen = namelen;
+ if (namelen > 0 && name)
+ memcpy(slot->hwc.hc.cmd_cshake.name, name,
+ min_t(u32, namelen, HC_CSHAKE_MAX_NAMELEN));
+}
+
+/* Request Context Cleanup */
+
+static void cmh_cshake_free_chunks(struct cmh_cshake_reqctx *rctx,
+ struct cmh_cshake_tfm_ctx *tctx)
+{
+ struct cmh_cshake_chunk *chunk, *tmp;
+
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(chunk, tmp, &rctx->chunks, list) {
+ list_del(&chunk->list);
+ list_del(&chunk->tfm_node);
+ kfree(chunk);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->num_chunks = 0;
+ rctx->total_len = 0;
+}
+
+static void cmh_cshake_free_reqctx(struct cmh_cshake_reqctx *rctx,
+ struct cmh_cshake_tfm_ctx *tctx)
+{
+ cmh_cshake_free_chunks(rctx, tctx);
+ rctx->has_checkpoint = 0;
+}
+
+static struct cmh_sg_map *
+cmh_cshake_build_sg(struct cmh_cshake_reqctx *rctx, gfp_t gfp)
+{
+ struct cmh_dma_buf *bufs;
+ struct cmh_cshake_chunk *chunk;
+ struct cmh_sg_map *sgm;
+ u32 i;
+
+ bufs = kcalloc(rctx->num_chunks, sizeof(*bufs), gfp);
+ if (!bufs)
+ return NULL;
+
+ i = 0;
+ list_for_each_entry(chunk, &rctx->chunks, list) {
+ bufs[i].data = chunk->data;
+ bufs[i].len = chunk->len;
+ i++;
+ }
+
+ sgm = cmh_dma_build_sg(bufs, rctx->num_chunks, gfp);
+ kfree(bufs);
+ return sgm;
+}
+
+/* VCQ Packing + Submit */
+
+/* ahash Operations */
+
+struct cmh_cshake_alg_drv {
+ struct ahash_alg alg;
+ const struct cmh_cshake_alg_info *info;
+};
+
+static const struct cmh_cshake_alg_info *
+cmh_cshake_get_info(struct crypto_ahash *tfm)
+{
+ struct ahash_alg *alg = crypto_ahash_alg(tfm);
+
+ return container_of(alg, struct cmh_cshake_alg_drv, alg)->info;
+}
+
+/*
+ * .setkey() -- parse N and S from the self-describing cshake_cfg header.
+ *
+ * Blob format: cshake_cfg { __be32 n_len; __be32 s_len; } || N || S
+ * If never called, the driver defaults to plain SHAKE (N="" S="").
+ */
+struct cshake_cfg {
+ __be32 n_len;
+ __be32 s_len;
+};
+
+static int cmh_cshake_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cshake_cfg cfg;
+ u32 n_len, s_len;
+ const u8 *ptr;
+
+ if (keylen < sizeof(cfg))
+ return -EINVAL;
+
+ memcpy(&cfg, key, sizeof(cfg));
+ n_len = be32_to_cpu(cfg.n_len);
+ s_len = be32_to_cpu(cfg.s_len);
+
+ if (keylen != sizeof(cfg) + n_len + s_len)
+ return -EINVAL;
+
+ if (n_len > HC_CSHAKE_MAX_NAMELEN)
+ return -EINVAL;
+
+ if (s_len > HC_CSHAKE_MAX_CUSTOMLEN)
+ return -EINVAL;
+
+ /* Free previous N and S */
+ kfree(tctx->func_name);
+ kfree(tctx->custom);
+ tctx->func_name = NULL;
+ tctx->func_name_len = 0;
+ tctx->custom = NULL;
+ tctx->custom_len = 0;
+
+ ptr = key + sizeof(cfg);
+
+ if (n_len > 0) {
+ tctx->func_name = kmemdup(ptr, n_len, GFP_KERNEL);
+ if (!tctx->func_name)
+ return -ENOMEM;
+ tctx->func_name_len = n_len;
+ ptr += n_len;
+ }
+
+ if (s_len > 0) {
+ tctx->custom = kmemdup(ptr, s_len, GFP_KERNEL);
+ if (!tctx->custom) {
+ kfree(tctx->func_name);
+ tctx->func_name = NULL;
+ tctx->func_name_len = 0;
+ return -ENOMEM;
+ }
+ tctx->custom_len = s_len;
+ }
+
+ return 0;
+}
+
+static int cmh_cshake_init(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req);
+
+ rctx->info = cmh_cshake_get_info(tfm);
+ rctx->error = 0;
+ INIT_LIST_HEAD(&rctx->chunks);
+ rctx->num_chunks = 0;
+ rctx->total_len = 0;
+ rctx->has_checkpoint = 0;
+
+ return 0;
+}
+
+static int cmh_cshake_update(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_cshake_chunk *chunk;
+ int nents;
+
+ if (rctx->error)
+ return rctx->error;
+
+ if (!req->nbytes)
+ return 0;
+
+ chunk = kmalloc(sizeof(*chunk) + req->nbytes,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC);
+ if (!chunk) {
+ rctx->error = -ENOMEM;
+ goto err_free_chunks;
+ }
+
+ chunk->len = req->nbytes;
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) {
+ memcpy(chunk->data, req->svirt, req->nbytes);
+ } else {
+ nents = sg_nents_for_len(req->src, req->nbytes);
+ if (nents < 0 ||
+ sg_copy_to_buffer(req->src, nents,
+ chunk->data, req->nbytes) != req->nbytes) {
+ kfree(chunk);
+ rctx->error = -EINVAL;
+ goto err_free_chunks;
+ }
+ }
+
+ list_add_tail(&chunk->list, &rctx->chunks);
+ spin_lock_bh(&tctx->chunk_lock);
+ list_add_tail(&chunk->tfm_node, &tctx->all_chunks);
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->num_chunks++;
+ rctx->total_len += req->nbytes;
+
+ return 0;
+
+err_free_chunks:
+ /*
+ * Terminal error -- free all previously accumulated chunks.
+ * The crypto API hash path does not call .final() on error,
+ * so chunks would be orphaned otherwise.
+ */
+ cmh_cshake_free_chunks(rctx, tctx);
+ return rctx->error;
+}
+
+static void cmh_cshake_complete(void *data, int error)
+{
+ struct ahash_request *req = data;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->digest_dma, rctx->info->digest_size,
+ DMA_FROM_DEVICE);
+
+ if (!error)
+ memcpy(req->result, rctx->digest_buf,
+ rctx->info->digest_size);
+
+ kfree(rctx->digest_buf);
+ rctx->digest_buf = NULL;
+ cmh_dma_free_sg(rctx->sgm);
+ rctx->sgm = NULL;
+ cmh_cshake_free_reqctx(rctx, tctx);
+ cmh_complete(&req->base, error);
+}
+
+static int cmh_cshake_final(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_cshake_alg_info *info = rctx->info;
+ struct core_dispatch d;
+ struct vcq_cmd cmds[CMH_CSHAKE_MAX_PAYLOAD];
+ struct cmh_sg_map *sgm = NULL;
+ dma_addr_t digest_dma = DMA_MAPPING_ERROR;
+ dma_addr_t ckpt_dma = DMA_MAPPING_ERROR;
+ u8 *digest_buf;
+ u32 idx;
+ int ret;
+ gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ if (rctx->error) {
+ ret = rctx->error;
+ goto out_free;
+ }
+
+ if (rctx->num_chunks > 0) {
+ sgm = cmh_cshake_build_sg(rctx, gfp);
+ if (!sgm) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ }
+
+ digest_buf = kzalloc(info->digest_size, gfp);
+ if (!digest_buf) {
+ ret = -ENOMEM;
+ goto out_free_sg;
+ }
+ digest_dma = cmh_dma_map_single(digest_buf, info->digest_size,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(digest_dma)) {
+ ret = -ENOMEM;
+ goto out_free_digest;
+ }
+
+ /* Map checkpoint buffer if present (CMH eSW reads it) */
+ if (rctx->has_checkpoint) {
+ ckpt_dma = cmh_dma_map_single(rctx->checkpoint,
+ HC_CONTEXT_SIZE, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(ckpt_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap_digest;
+ }
+ }
+
+ d = cmh_core_select_instance(CMH_CORE_HC);
+ idx = 0;
+
+ if (rctx->has_checkpoint) {
+ /*
+ * Resuming from a saved checkpoint (after export/import):
+ * INIT + RESTORE [+ GATHER] + FINAL + FLUSH
+ * The cSHAKE prefix (N,S) is already absorbed in the
+ * saved Keccak state -- no need to replay HC_CMD_CSHAKE.
+ */
+ vcq_add_hc_init(&cmds[idx++], d.core_id, info->hc_algo);
+ vcq_add_hc_restore(&cmds[idx++], d.core_id, (u64)ckpt_dma,
+ HC_CONTEXT_SIZE);
+ } else {
+ bool use_cshake = (tctx->func_name_len > 0 ||
+ tctx->custom_len > 0);
+
+ if (use_cshake) {
+ u32 span;
+
+ vcq_add_hc_cshake(&cmds[idx], d.core_id,
+ info->hc_algo,
+ tctx->func_name,
+ tctx->func_name_len,
+ tctx->custom_len);
+ span = vcq_add_inline_data(&cmds[idx],
+ tctx->custom,
+ tctx->custom_len);
+ idx += span;
+ } else {
+ vcq_add_hc_init(&cmds[idx++], d.core_id,
+ info->hc_algo);
+ }
+ }
+
+ if (sgm)
+ vcq_add_hc_gather(&cmds[idx++], d.core_id, (u64)sgm->items_dma,
+ HC_CMD_UPDATE);
+
+ vcq_add_hc_final(&cmds[idx++], d.core_id, (u64)digest_dma, info->digest_size);
+ vcq_add_flush(&cmds[idx++], d.core_id);
+
+ rctx->digest_buf = digest_buf;
+ rctx->digest_dma = digest_dma;
+ rctx->ckpt_dma = ckpt_dma;
+ rctx->sgm = sgm;
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_CSHAKE_MAX_PACKED,
+ d.mbx_idx,
+ cmh_cshake_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_cleanup_all;
+
+ return -EINPROGRESS;
+
+out_cleanup_all:
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(ckpt_dma, HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+out_unmap_digest:
+ cmh_dma_unmap_single(digest_dma, info->digest_size,
+ DMA_FROM_DEVICE);
+out_free_digest:
+ kfree(digest_buf);
+
+out_free_sg:
+ cmh_dma_free_sg(sgm);
+
+out_free:
+ cmh_cshake_free_reqctx(rctx, tctx);
+ return ret;
+}
+
+static int cmh_cshake_finup(struct ahash_request *req)
+{
+ int ret;
+
+ ret = cmh_cshake_update(req);
+ if (ret)
+ return ret;
+
+ return cmh_cshake_final(req);
+}
+
+static int cmh_cshake_digest(struct ahash_request *req)
+{
+ int ret;
+
+ ret = cmh_cshake_init(req);
+ if (ret)
+ return ret;
+
+ return cmh_cshake_finup(req);
+}
+
+static int cmh_cshake_export(struct ahash_request *req, void *out)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_cshake_alg_info *info = rctx->info;
+ struct core_dispatch d;
+ struct vcq_cmd cmds[CMH_CSHAKE_MAX_PAYLOAD];
+ struct cmh_sg_map *sgm = NULL;
+ dma_addr_t save_dma = DMA_MAPPING_ERROR;
+ dma_addr_t ckpt_dma = DMA_MAPPING_ERROR;
+ u8 *save_buf;
+ u32 idx;
+ int ret;
+
+ if (rctx->num_chunks > 0) {
+ sgm = cmh_cshake_build_sg(rctx, GFP_KERNEL);
+ if (!sgm)
+ return -ENOMEM;
+ }
+
+ save_buf = kzalloc(HC_CONTEXT_SIZE, GFP_KERNEL);
+ if (!save_buf) {
+ cmh_dma_free_sg(sgm);
+ return -ENOMEM;
+ }
+ save_dma = cmh_dma_map_single(save_buf, HC_CONTEXT_SIZE,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(save_dma)) {
+ kfree(save_buf);
+ cmh_dma_free_sg(sgm);
+ return -ENOMEM;
+ }
+
+ /* Map checkpoint buffer if present (CMH eSW reads it) */
+ if (rctx->has_checkpoint) {
+ ckpt_dma = cmh_dma_map_single(rctx->checkpoint,
+ HC_CONTEXT_SIZE, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(ckpt_dma)) {
+ cmh_dma_unmap_single(save_dma, HC_CONTEXT_SIZE,
+ DMA_FROM_DEVICE);
+ kfree(save_buf);
+ cmh_dma_free_sg(sgm);
+ return -ENOMEM;
+ }
+ }
+
+ d = cmh_core_select_instance(CMH_CORE_HC);
+ idx = 0;
+
+ if (rctx->has_checkpoint) {
+ /*
+ * Resuming from a saved checkpoint:
+ * INIT + RESTORE [+ GATHER] + SAVE + FLUSH
+ */
+ vcq_add_hc_init(&cmds[idx++], d.core_id, info->hc_algo);
+ vcq_add_hc_restore(&cmds[idx++], d.core_id, (u64)ckpt_dma,
+ HC_CONTEXT_SIZE);
+ } else {
+ bool use_cshake = (tctx->func_name_len > 0 ||
+ tctx->custom_len > 0);
+
+ if (use_cshake) {
+ u32 span;
+
+ vcq_add_hc_cshake(&cmds[idx], d.core_id,
+ info->hc_algo,
+ tctx->func_name,
+ tctx->func_name_len,
+ tctx->custom_len);
+ span = vcq_add_inline_data(&cmds[idx],
+ tctx->custom,
+ tctx->custom_len);
+ idx += span;
+ } else {
+ vcq_add_hc_init(&cmds[idx++], d.core_id,
+ info->hc_algo);
+ }
+ }
+
+ if (sgm)
+ vcq_add_hc_gather(&cmds[idx++], d.core_id, (u64)sgm->items_dma,
+ HC_CMD_UPDATE);
+
+ vcq_add_hc_save(&cmds[idx++], d.core_id, (u64)save_dma,
+ HC_CONTEXT_SIZE);
+ vcq_add_flush(&cmds[idx++], d.core_id);
+
+ ret = cmh_vcq_pack_and_submit(cmds, idx, rctx->packed, CMH_CSHAKE_MAX_PACKED,
+ d.mbx_idx);
+
+ /* Unmap before CPU read */
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(ckpt_dma, HC_CONTEXT_SIZE, DMA_TO_DEVICE);
+ cmh_dma_unmap_single(save_dma, HC_CONTEXT_SIZE, DMA_FROM_DEVICE);
+
+ /*
+ * Free the gather map (which dma_unmaps each chunk buffer) BEFORE
+ * freeing the chunk buffers themselves -- the DMA API requires a
+ * mapping to be torn down before its memory is released.
+ */
+ cmh_dma_free_sg(sgm);
+
+ if (!ret) {
+ memcpy(out, save_buf, HC_CONTEXT_SIZE);
+ /* Checkpoint now represents all accumulated state */
+ memcpy(rctx->checkpoint, save_buf, HC_CONTEXT_SIZE);
+ rctx->has_checkpoint = 1;
+ /* Accumulated chunks are now captured in checkpoint */
+ cmh_cshake_free_chunks(rctx, tctx);
+ }
+
+ kfree(save_buf);
+ return ret;
+}
+
+static int cmh_cshake_import(struct ahash_request *req, const void *in)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req);
+
+ rctx->info = cmh_cshake_get_info(tfm);
+ rctx->error = 0;
+ INIT_LIST_HEAD(&rctx->chunks);
+ rctx->num_chunks = 0;
+ rctx->total_len = 0;
+
+ memcpy(rctx->checkpoint, in, HC_CONTEXT_SIZE);
+ rctx->has_checkpoint = 1;
+
+ return 0;
+}
+
+/* Transform init/exit */
+
+static int cmh_cshake_cra_init(struct crypto_tfm *tfm)
+{
+ struct cmh_cshake_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
+
+ tctx->func_name = NULL;
+ tctx->func_name_len = 0;
+ tctx->custom = NULL;
+ tctx->custom_len = 0;
+ spin_lock_init(&tctx->chunk_lock);
+ INIT_LIST_HEAD(&tctx->all_chunks);
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct cmh_cshake_reqctx));
+ return 0;
+}
+
+static void cmh_cshake_cra_exit(struct crypto_tfm *tfm)
+{
+ struct cmh_cshake_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
+ struct cmh_cshake_chunk *chunk, *tmp;
+
+ /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(chunk, tmp, &tctx->all_chunks, tfm_node) {
+ list_del(&chunk->tfm_node);
+ kfree(chunk);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+
+ kfree(tctx->func_name);
+ kfree(tctx->custom);
+ tctx->func_name = NULL;
+ tctx->custom = NULL;
+}
+
+/* Registration */
+
+static struct cmh_cshake_alg_drv cmh_cshake_drvs[CMH_CSHAKE_ALG_COUNT];
+
+/**
+ * cmh_cshake_register() - Register cSHAKE-128/256 hash algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_cshake_register(void)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < CMH_CSHAKE_ALG_COUNT; i++) {
+ const struct cmh_cshake_alg_info *info =
+ &cmh_cshake_algs_info[i];
+ struct cmh_cshake_alg_drv *drv = &cmh_cshake_drvs[i];
+ struct ahash_alg *alg = &drv->alg;
+
+ drv->info = info;
+
+ alg->init = cmh_cshake_init;
+ alg->update = cmh_cshake_update;
+ alg->final = cmh_cshake_final;
+ alg->finup = cmh_cshake_finup;
+ alg->digest = cmh_cshake_digest;
+ alg->export = cmh_cshake_export;
+ alg->import = cmh_cshake_import;
+ alg->setkey = cmh_cshake_setkey;
+
+ alg->halg.digestsize = info->digest_size;
+ alg->halg.statesize = HC_CONTEXT_SIZE;
+
+ strscpy(alg->halg.base.cra_name, info->alg_name,
+ CRYPTO_MAX_ALG_NAME);
+ strscpy(alg->halg.base.cra_driver_name, info->drv_name,
+ CRYPTO_MAX_ALG_NAME);
+ alg->halg.base.cra_priority = 300;
+ alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NO_FALLBACK |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_OPTIONAL_KEY |
+ CRYPTO_ALG_REQ_VIRT;
+ alg->halg.base.cra_blocksize = 1; /* XOF */
+ alg->halg.base.cra_ctxsize = sizeof(struct cmh_cshake_tfm_ctx);
+ alg->halg.base.cra_init = cmh_cshake_cra_init;
+ alg->halg.base.cra_exit = cmh_cshake_cra_exit;
+ alg->halg.base.cra_module = THIS_MODULE;
+
+ ret = crypto_register_ahash(alg);
+ if (ret) {
+ dev_err(cmh_dev(), "cshake: failed to register %s (rc=%d)\n",
+ info->drv_name, ret);
+ while (i--)
+ crypto_unregister_ahash(&cmh_cshake_drvs[i].alg);
+ return ret;
+ }
+
+ dev_dbg(cmh_dev(), "cshake: registered %s (priority 300)\n",
+ info->drv_name);
+ }
+
+ dev_info(cmh_dev(), "cshake: %zu algorithm(s) registered\n",
+ CMH_CSHAKE_ALG_COUNT);
+ return 0;
+}
+
+/**
+ * cmh_cshake_unregister() - Unregister cSHAKE hash algorithms from the crypto framework
+ */
+void cmh_cshake_unregister(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < CMH_CSHAKE_ALG_COUNT; i++) {
+ crypto_unregister_ahash(&cmh_cshake_drvs[i].alg);
+ dev_dbg(cmh_dev(), "cshake: unregistered %s\n",
+ cmh_cshake_algs_info[i].drv_name);
+ }
+
+ dev_info(cmh_dev(), "cshake: cleaned up\n");
+}
diff --git a/drivers/crypto/cmh/cmh_kmac.c b/drivers/crypto/cmh/cmh_kmac.c
new file mode 100644
index 000000000000..341f661de34e
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_kmac.c
@@ -0,0 +1,630 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API KMAC Driver
+ *
+ * Registers KMAC-128 and KMAC-256 as keyed ahash algorithms using the
+ * CMH Hash Core (HC) via HC_CMD_KMAC.
+ *
+ * KMAC (NIST SP 800-185) is a keyed variant of cSHAKE. The function
+ * name N is always "KMAC" (hardcoded by the CMH eSW). The user sets:
+ * - A key via .setkey() (raw bytes + optional S)
+ * - An optional customization string S via the setkey blob
+ *
+ * setkey blob format:
+ * struct kmac_key_param { __be32 keylen; __be32 s_len; };
+ * blob: kmac_key_param || key[keylen] || S[s_len]
+ *
+ * Uses the same self-contained transaction model as cmh_hmac.c:
+ * .setkey() -> store raw key (+ S)
+ * .init() -> software-only
+ * .update() -> software-only (accumulate chunks)
+ * .final() -> [SYS_CMD_WRITE] + HC_CMD_KMAC [+ inline S] +
+ * [GATHER] + FINAL + FLUSH
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/hash.h>
+#include <linux/scatterlist.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <asm/byteorder.h>
+
+#include "cmh_kmac.h"
+#include "cmh_vcq.h"
+#include "cmh_hc_abi.h"
+#include "cmh_sys_abi.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+/*
+ * Maximum data that can be accumulated across .update() calls.
+ * The CMH eSW rejects HC_CMD_SAVE when ctx->outlen != 0, which is
+ * always the case for KMAC (eip59_hc_kmac() sets ctx->outlen for
+ * right_encode(outlen) at finalization). All data must be buffered
+ * in kernel memory and submitted atomically in .final().
+ *
+ * The CMH eSW does not serialize outlen into the external save
+ * context, so HC_CMD_SAVE fails for KMAC mode.
+ */
+#define KMAC_MAX_DATA (64 * 1024)
+
+/* Algorithm Table */
+
+struct cmh_kmac_alg_info {
+ u32 hc_algo;
+ u32 digest_size;
+ const char *alg_name;
+ const char *drv_name;
+};
+
+static const struct cmh_kmac_alg_info cmh_kmac_algs_info[] = {
+ {
+ .hc_algo = HC_ALGO_SHAKE128,
+ .digest_size = CMH_SHAKE128_DIGEST_SIZE,
+ .alg_name = "kmac128",
+ .drv_name = "rambus-cmh-kmac128",
+ },
+ {
+ .hc_algo = HC_ALGO_SHAKE256,
+ .digest_size = CMH_SHAKE256_DIGEST_SIZE,
+ .alg_name = "kmac256",
+ .drv_name = "rambus-cmh-kmac256",
+ },
+};
+
+#define CMH_KMAC_ALG_COUNT ARRAY_SIZE(cmh_kmac_algs_info)
+
+/* Per-Request State */
+
+struct cmh_kmac_chunk {
+ struct list_head list;
+ struct list_head tfm_node; /* per-tfm orphan tracking */
+ u32 len;
+ u8 data[];
+};
+
+/*
+ * Max payload slots for KMAC:
+ * SYS_CMD_WRITE (1) + KMAC (1) + inline S (3 max) + GATHER (1) +
+ * FINAL (1) + FLUSH (1) = 8
+ */
+#define CMH_KMAC_MAX_PAYLOAD 9
+#define CMH_KMAC_MAX_PACKED (CMH_KMAC_MAX_PAYLOAD * 2)
+
+struct cmh_kmac_reqctx {
+ const struct cmh_kmac_alg_info *info;
+ int error;
+ struct list_head chunks;
+ u32 num_chunks;
+ u32 total_len;
+ /* DMA state for async final */
+ dma_addr_t digest_dma;
+ dma_addr_t key_dma;
+ u8 *digest_buf;
+ struct cmh_sg_map *sgm;
+ u32 keylen;
+ struct vcq_cmd packed[CMH_KMAC_MAX_PACKED];
+};
+
+/* Per-Transform State (carries key + S across requests) */
+
+struct cmh_kmac_tfm_ctx {
+ struct cmh_key_ctx key;
+ u8 *custom; /* S (customization string), NULL if empty */
+ u32 custom_len;
+ spinlock_t chunk_lock; /* protects all_chunks */
+ struct list_head all_chunks; /* orphan-safe chunk tracking */
+};
+
+/* VCQ Builders (KMAC-specific; shared builders in cmh_hc_abi.h / cmh_vcq.h) */
+
+static void vcq_add_hc_kmac(struct vcq_cmd *slot, u32 core_id, u64 key_ref, u32 keylen,
+ u32 customlen, u32 algo, u32 outlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_KMAC);
+ slot->hwc.hc.cmd_kmac.key = key_ref;
+ slot->hwc.hc.cmd_kmac.custom = 0; /* inline */
+ slot->hwc.hc.cmd_kmac.keylen = keylen;
+ slot->hwc.hc.cmd_kmac.customlen = customlen;
+ slot->hwc.hc.cmd_kmac.algo = algo;
+ slot->hwc.hc.cmd_kmac.outlen = outlen;
+}
+
+/* Request Context Cleanup */
+
+static void cmh_kmac_free_chunks(struct cmh_kmac_reqctx *rctx,
+ struct cmh_kmac_tfm_ctx *tctx)
+{
+ struct cmh_kmac_chunk *chunk, *tmp;
+
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(chunk, tmp, &rctx->chunks, list) {
+ list_del(&chunk->list);
+ list_del(&chunk->tfm_node);
+ kfree(chunk);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->num_chunks = 0;
+ rctx->total_len = 0;
+}
+
+static struct cmh_sg_map *
+cmh_kmac_build_sg(struct cmh_kmac_reqctx *rctx, gfp_t gfp)
+{
+ struct cmh_dma_buf *bufs;
+ struct cmh_kmac_chunk *chunk;
+ struct cmh_sg_map *sgm;
+ u32 i;
+
+ bufs = kcalloc(rctx->num_chunks, sizeof(*bufs), gfp);
+ if (!bufs)
+ return NULL;
+
+ i = 0;
+ list_for_each_entry(chunk, &rctx->chunks, list) {
+ bufs[i].data = chunk->data;
+ bufs[i].len = chunk->len;
+ i++;
+ }
+
+ sgm = cmh_dma_build_sg(bufs, rctx->num_chunks, gfp);
+ kfree(bufs);
+ return sgm;
+}
+
+/* VCQ Packing + Submit */
+
+/* ahash Operations */
+
+struct cmh_kmac_alg_drv {
+ struct ahash_alg alg;
+ const struct cmh_kmac_alg_info *info;
+};
+
+static const struct cmh_kmac_alg_info *
+cmh_kmac_get_info(struct crypto_ahash *tfm)
+{
+ struct ahash_alg *alg = crypto_ahash_alg(tfm);
+
+ return container_of(alg, struct cmh_kmac_alg_drv, alg)->info;
+}
+
+/*
+ * setkey blob for KMAC (raw key path):
+ * struct kmac_key_param { __be32 keylen; __be32 s_len; };
+ * blob: kmac_key_param || key[keylen] || S[s_len]
+ */
+struct kmac_key_param {
+ __be32 keylen;
+ __be32 s_len;
+};
+
+static int cmh_kmac_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ /* raw key bytes with optional S */
+ {
+ struct kmac_key_param hdr;
+ u32 raw_keylen, s_len;
+ const u8 *ptr;
+
+ if (keylen < sizeof(hdr))
+ return -EINVAL;
+
+ memcpy(&hdr, key, sizeof(hdr));
+ raw_keylen = be32_to_cpu(hdr.keylen);
+ s_len = be32_to_cpu(hdr.s_len);
+
+ if (keylen != sizeof(hdr) + raw_keylen + s_len)
+ return -EINVAL;
+
+ if (raw_keylen == 0)
+ return -EINVAL;
+
+ if (s_len > HC_CSHAKE_MAX_CUSTOMLEN)
+ return -EINVAL;
+
+ ptr = key + sizeof(hdr);
+
+ /* Store raw key */
+ {
+ int ret = cmh_key_setkey_raw(&tctx->key, ptr,
+ raw_keylen, CORE_ID_HC);
+ if (ret)
+ return ret;
+ }
+ ptr += raw_keylen;
+
+ /* Store S */
+ kfree(tctx->custom);
+ tctx->custom = NULL;
+ tctx->custom_len = 0;
+
+ if (s_len > 0) {
+ tctx->custom = kmemdup(ptr, s_len, GFP_KERNEL);
+ if (!tctx->custom) {
+ cmh_key_destroy(&tctx->key);
+ return -ENOMEM;
+ }
+ tctx->custom_len = s_len;
+ }
+
+ return 0;
+ }
+}
+
+static int cmh_kmac_init(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req);
+
+ rctx->info = cmh_kmac_get_info(tfm);
+ rctx->error = 0;
+ INIT_LIST_HEAD(&rctx->chunks);
+ rctx->num_chunks = 0;
+ rctx->total_len = 0;
+
+ return 0;
+}
+
+static int cmh_kmac_update(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_kmac_chunk *chunk;
+ int nents;
+
+ if (rctx->error)
+ return rctx->error;
+
+ if (!req->nbytes)
+ return 0;
+
+ if (req->nbytes > KMAC_MAX_DATA - rctx->total_len) {
+ rctx->error = -EINVAL;
+ goto err_free_chunks;
+ }
+
+ chunk = kmalloc(sizeof(*chunk) + req->nbytes,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC);
+ if (!chunk) {
+ rctx->error = -ENOMEM;
+ goto err_free_chunks;
+ }
+
+ chunk->len = req->nbytes;
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) {
+ memcpy(chunk->data, req->svirt, req->nbytes);
+ } else {
+ nents = sg_nents_for_len(req->src, req->nbytes);
+ if (nents < 0 ||
+ sg_copy_to_buffer(req->src, nents,
+ chunk->data, req->nbytes) != req->nbytes) {
+ kfree(chunk);
+ rctx->error = -EINVAL;
+ goto err_free_chunks;
+ }
+ }
+
+ list_add_tail(&chunk->list, &rctx->chunks);
+ spin_lock_bh(&tctx->chunk_lock);
+ list_add_tail(&chunk->tfm_node, &tctx->all_chunks);
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->num_chunks++;
+ rctx->total_len += req->nbytes;
+
+ return 0;
+
+err_free_chunks:
+ /*
+ * Terminal error -- free all previously accumulated chunks.
+ * The crypto API hash path does not call .final() on error,
+ * so chunks would be orphaned otherwise.
+ */
+ cmh_kmac_free_chunks(rctx, tctx);
+ return rctx->error;
+}
+
+static void cmh_kmac_complete(void *data, int error)
+{
+ struct ahash_request *req = data;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ cmh_dma_unmap_single(rctx->digest_dma, rctx->info->digest_size,
+ DMA_FROM_DEVICE);
+
+ if (!error)
+ memcpy(req->result, rctx->digest_buf,
+ rctx->info->digest_size);
+
+ kfree(rctx->digest_buf);
+ rctx->digest_buf = NULL;
+ cmh_dma_free_sg(rctx->sgm);
+ rctx->sgm = NULL;
+ cmh_kmac_free_chunks(rctx, tctx);
+ cmh_complete(&req->base, error);
+}
+
+static int cmh_kmac_final(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_kmac_alg_info *info = rctx->info;
+ struct vcq_cmd cmds[CMH_KMAC_MAX_PAYLOAD];
+ struct cmh_sg_map *sgm = NULL;
+ dma_addr_t digest_dma = DMA_MAPPING_ERROR, key_dma = DMA_MAPPING_ERROR;
+ u8 *digest_buf;
+ u64 key_ref;
+ u32 key_len;
+ struct core_dispatch d;
+ s32 target_mbx;
+ u32 core_id;
+ u32 idx;
+ int ret;
+ gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ if (rctx->error) {
+ ret = rctx->error;
+ goto out_free;
+ }
+
+ if (tctx->key.mode == CMH_KEY_NONE) {
+ ret = -ENOKEY;
+ goto out_free;
+ }
+
+ if (rctx->num_chunks > 0) {
+ sgm = cmh_kmac_build_sg(rctx, gfp);
+ if (!sgm) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ }
+
+ digest_buf = kzalloc(info->digest_size, gfp);
+ if (!digest_buf) {
+ ret = -ENOMEM;
+ goto out_free_sg;
+ }
+ digest_dma = cmh_dma_map_single(digest_buf, info->digest_size,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(digest_dma)) {
+ ret = -ENOMEM;
+ goto out_free_digest;
+ }
+
+ /* Resolve key reference */
+ idx = 0;
+
+ key_dma = tctx->key.raw.dma;
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, (u64)key_dma,
+ SYS_REF_NONE, tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ key_len = tctx->key.raw.len;
+ d = cmh_core_select_instance(CMH_CORE_HC);
+
+ target_mbx = d.mbx_idx;
+
+ core_id = d.core_id;
+
+ {
+ u32 span;
+
+ vcq_add_hc_kmac(&cmds[idx], core_id, key_ref, key_len,
+ tctx->custom_len, info->hc_algo,
+ info->digest_size);
+
+ /* Add inline S data after the KMAC slot */
+ span = vcq_add_inline_data(&cmds[idx], tctx->custom,
+ tctx->custom_len);
+ idx += span;
+ }
+
+ if (sgm)
+ vcq_add_hc_gather(&cmds[idx++], core_id, (u64)sgm->items_dma,
+ HC_CMD_UPDATE);
+
+ vcq_add_hc_final(&cmds[idx++], core_id, (u64)digest_dma, info->digest_size);
+ vcq_add_flush(&cmds[idx++], core_id);
+
+ rctx->digest_buf = digest_buf;
+ rctx->digest_dma = digest_dma;
+ rctx->sgm = sgm;
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_KMAC_MAX_PACKED,
+ target_mbx,
+ cmh_kmac_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_cleanup_all;
+
+ return -EINPROGRESS;
+
+out_cleanup_all:
+ cmh_dma_unmap_single(digest_dma, info->digest_size,
+ DMA_FROM_DEVICE);
+out_free_digest:
+ kfree(digest_buf);
+
+out_free_sg:
+ cmh_dma_free_sg(sgm);
+
+out_free:
+ cmh_kmac_free_chunks(rctx, tctx);
+ return ret;
+}
+
+static int cmh_kmac_finup(struct ahash_request *req)
+{
+ int ret;
+
+ ret = cmh_kmac_update(req);
+ if (ret)
+ return ret;
+
+ return cmh_kmac_final(req);
+}
+
+static int cmh_kmac_digest(struct ahash_request *req)
+{
+ int ret;
+
+ ret = cmh_kmac_init(req);
+ if (ret)
+ return ret;
+
+ return cmh_kmac_finup(req);
+}
+
+static int cmh_kmac_export(struct ahash_request *req, void *out)
+{
+ return -EOPNOTSUPP;
+}
+
+static int cmh_kmac_import(struct ahash_request *req, const void *in)
+{
+ return -EOPNOTSUPP;
+}
+
+/* Transform init/exit */
+
+static int cmh_kmac_cra_init(struct crypto_tfm *tfm)
+{
+ struct cmh_kmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
+
+ tctx->key.mode = CMH_KEY_NONE;
+ tctx->custom = NULL;
+ tctx->custom_len = 0;
+ spin_lock_init(&tctx->chunk_lock);
+ INIT_LIST_HEAD(&tctx->all_chunks);
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct cmh_kmac_reqctx));
+ return 0;
+}
+
+static void cmh_kmac_cra_exit(struct crypto_tfm *tfm)
+{
+ struct cmh_kmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
+ struct cmh_kmac_chunk *chunk, *tmp;
+
+ /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(chunk, tmp, &tctx->all_chunks, tfm_node) {
+ list_del(&chunk->tfm_node);
+ kfree(chunk);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+
+ cmh_key_destroy(&tctx->key);
+ kfree(tctx->custom);
+ tctx->custom = NULL;
+}
+
+/* Registration */
+
+static struct cmh_kmac_alg_drv cmh_kmac_drvs[CMH_KMAC_ALG_COUNT];
+
+/**
+ * cmh_kmac_register() - Register KMAC-128/256 hash algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_kmac_register(void)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < CMH_KMAC_ALG_COUNT; i++) {
+ const struct cmh_kmac_alg_info *info =
+ &cmh_kmac_algs_info[i];
+ struct cmh_kmac_alg_drv *drv = &cmh_kmac_drvs[i];
+ struct ahash_alg *alg = &drv->alg;
+
+ drv->info = info;
+
+ alg->init = cmh_kmac_init;
+ alg->update = cmh_kmac_update;
+ alg->final = cmh_kmac_final;
+ alg->finup = cmh_kmac_finup;
+ alg->digest = cmh_kmac_digest;
+ alg->export = cmh_kmac_export;
+ alg->import = cmh_kmac_import;
+ alg->setkey = cmh_kmac_setkey;
+
+ alg->halg.digestsize = info->digest_size;
+ alg->halg.statesize = sizeof(struct cmh_kmac_reqctx);
+
+ strscpy(alg->halg.base.cra_name, info->alg_name,
+ CRYPTO_MAX_ALG_NAME);
+ strscpy(alg->halg.base.cra_driver_name, info->drv_name,
+ CRYPTO_MAX_ALG_NAME);
+ alg->halg.base.cra_priority = 300;
+ alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NO_FALLBACK |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_REQ_VIRT;
+ alg->halg.base.cra_blocksize = 1; /* XOF/keyed XOF */
+ alg->halg.base.cra_ctxsize = sizeof(struct cmh_kmac_tfm_ctx);
+ alg->halg.base.cra_init = cmh_kmac_cra_init;
+ alg->halg.base.cra_exit = cmh_kmac_cra_exit;
+ alg->halg.base.cra_module = THIS_MODULE;
+
+ ret = crypto_register_ahash(alg);
+ if (ret) {
+ dev_err(cmh_dev(), "kmac: failed to register %s (rc=%d)\n",
+ info->drv_name, ret);
+ while (i--)
+ crypto_unregister_ahash(&cmh_kmac_drvs[i].alg);
+ return ret;
+ }
+
+ dev_dbg(cmh_dev(), "kmac: registered %s (priority 300)\n",
+ info->drv_name);
+ }
+
+ dev_info(cmh_dev(), "kmac: %zu algorithm(s) registered\n",
+ CMH_KMAC_ALG_COUNT);
+ return 0;
+}
+
+/**
+ * cmh_kmac_unregister() - Unregister KMAC hash algorithms from the crypto framework
+ */
+void cmh_kmac_unregister(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < CMH_KMAC_ALG_COUNT; i++) {
+ crypto_unregister_ahash(&cmh_kmac_drvs[i].alg);
+ dev_dbg(cmh_dev(), "kmac: unregistered %s\n",
+ cmh_kmac_algs_info[i].drv_name);
+ }
+
+ dev_info(cmh_dev(), "kmac: cleaned up\n");
+}
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index 4bd90d2d394a..bc47d5ef3f86 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -33,6 +33,8 @@
#include "cmh_rh.h"
#include "cmh_hash.h"
#include "cmh_hmac.h"
+#include "cmh_cshake.h"
+#include "cmh_kmac.h"
#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
@@ -249,6 +251,16 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_hmac_register;
+ /* Register CSHAKE hash algorithms */
+ ret = cmh_cshake_register();
+ if (ret)
+ goto err_cshake_register;
+
+ /* Register KMAC hash algorithms */
+ ret = cmh_kmac_register();
+ if (ret)
+ goto err_kmac_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -261,6 +273,10 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_kmac_unregister();
+err_kmac_register:
+ cmh_cshake_unregister();
+err_cshake_register:
cmh_hmac_unregister();
err_hmac_register:
cmh_hash_unregister();
@@ -291,6 +307,8 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_kmac_unregister();
+ cmh_cshake_unregister();
cmh_hmac_unregister();
cmh_hash_unregister();
cmh_rh_cleanup(cfg);
diff --git a/drivers/crypto/cmh/include/cmh_cshake.h b/drivers/crypto/cmh/include/cmh_cshake.h
new file mode 100644
index 000000000000..9bafe0baf52f
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_cshake.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API CSHAKE Driver
+ *
+ * Registers cSHAKE-128 and cSHAKE-256 ahash algorithms using
+ * HC_CMD_CSHAKE with inline customization string S.
+ */
+
+#ifndef CMH_CSHAKE_H
+#define CMH_CSHAKE_H
+
+int cmh_cshake_register(void);
+void cmh_cshake_unregister(void);
+
+#endif /* CMH_CSHAKE_H */
diff --git a/drivers/crypto/cmh/include/cmh_kmac.h b/drivers/crypto/cmh/include/cmh_kmac.h
new file mode 100644
index 000000000000..b3c92d71a0b6
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_kmac.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API KMAC Driver
+ *
+ * Registers KMAC-128 and KMAC-256 ahash algorithms using
+ * HC_CMD_KMAC with inline customization string S.
+ */
+
+#ifndef CMH_KMAC_H
+#define CMH_KMAC_H
+
+int cmh_kmac_register(void);
+void cmh_kmac_unregister(void);
+
+#endif /* CMH_KMAC_H */
--
2.43.7
^ permalink raw reply related
* [PATCH v3 03/19] crypto: cmh - add key provisioning and management
From: Saravanakrishnan Krishnamoorthy @ 2026-08-06 19:55 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Add the CMH key management subsystem:
- Key provisioning: create, import, derive, and destroy hardware keys
stored in the CMH datastore
- System object management: allocate and free CMH system objects
- Management ioctl interface (/dev/cmh_mgmt): ioctl commands
covering key lifecycle, KIC key derivation, PKE operations (RSA,
ECDSA, ECDH, EdDSA), PQC operations (ML-KEM, ML-DSA, SLH-DSA),
SM2, EAC, and DRBG reseeding
- SM2 ioctl handlers: SM2 encrypt, decrypt, sign, and key exchange
-- operations that require multi-step protocol flows not
expressible through the standard crypto API sig interface
- UAPI header: cmh_mgmt_ioctl.h (ioctl definitions and structures)
The misc device requires CAP_SYS_ADMIN for open().
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
Documentation/ABI/testing/cmh-mgmt | 136 ++
drivers/crypto/cmh/Kconfig | 19 +
drivers/crypto/cmh/Makefile | 11 +-
drivers/crypto/cmh/cmh_key.c | 164 +++
drivers/crypto/cmh/cmh_main.c | 9 +
drivers/crypto/cmh/cmh_mgmt.c | 1709 ++++++++++++++++++++++
drivers/crypto/cmh/cmh_mgmt_pke.c | 1134 ++++++++++++++
drivers/crypto/cmh/cmh_mgmt_pqc.c | 1325 +++++++++++++++++
drivers/crypto/cmh/cmh_pke_sm2.c | 827 +++++++++++
drivers/crypto/cmh/cmh_sys.c | 376 +++++
drivers/crypto/cmh/include/cmh_key.h | 82 ++
drivers/crypto/cmh/include/cmh_mgmt.h | 62 +
drivers/crypto/cmh/include/cmh_pke.h | 245 ++++
drivers/crypto/cmh/include/cmh_pke_sm2.h | 30 +
drivers/crypto/cmh/include/cmh_pqc.h | 25 +
drivers/crypto/cmh/include/cmh_sys.h | 111 ++
include/uapi/linux/cmh_mgmt_ioctl.h | 895 +++++++++++
17 files changed, 7159 insertions(+), 1 deletion(-)
create mode 100644 Documentation/ABI/testing/cmh-mgmt
create mode 100644 drivers/crypto/cmh/cmh_key.c
create mode 100644 drivers/crypto/cmh/cmh_mgmt.c
create mode 100644 drivers/crypto/cmh/cmh_mgmt_pke.c
create mode 100644 drivers/crypto/cmh/cmh_mgmt_pqc.c
create mode 100644 drivers/crypto/cmh/cmh_pke_sm2.c
create mode 100644 drivers/crypto/cmh/cmh_sys.c
create mode 100644 drivers/crypto/cmh/include/cmh_key.h
create mode 100644 drivers/crypto/cmh/include/cmh_mgmt.h
create mode 100644 drivers/crypto/cmh/include/cmh_pke.h
create mode 100644 drivers/crypto/cmh/include/cmh_pke_sm2.h
create mode 100644 drivers/crypto/cmh/include/cmh_pqc.h
create mode 100644 drivers/crypto/cmh/include/cmh_sys.h
create mode 100644 include/uapi/linux/cmh_mgmt_ioctl.h
diff --git a/Documentation/ABI/testing/cmh-mgmt b/Documentation/ABI/testing/cmh-mgmt
new file mode 100644
index 000000000000..7da2630edb20
--- /dev/null
+++ b/Documentation/ABI/testing/cmh-mgmt
@@ -0,0 +1,136 @@
+What: /dev/cmh_mgmt
+Date: June 2026
+KernelVersion: 7.1
+Contact: linux-crypto@vger.kernel.org
+Description:
+ Character device (misc) providing a management and
+ key-operations ioctl interface to the Rambus CryptoManager Hub
+ hardware crypto accelerator. Used for operations that
+ cannot be represented through the standard in-kernel
+ crypto API: datastore key CRUD, key derivation,
+ asymmetric crypto (EdDSA, SM2), and post-quantum crypto
+ (ML-KEM, ML-DSA, SLH-DSA).
+
+ The ioctl magic is 'J'. All struct arguments are
+ versioned via a leading __u32 version field set to
+ CMH_MGMT_V1 (1).
+
+ Ioctl commands are grouped by function:
+
+ **Key Management (0x01-0x0E):**
+
+ - CMH_IOCTL_KEY_NEW (0x01): Allocate a new datastore slot.
+ Accepts ds_type (CMH_DS_* constant), key length, flags,
+ and caller ID. Returns a 64-bit key reference.
+
+ - CMH_IOCTL_KEY_WRITE (0x02): Write key material into
+ a previously allocated datastore slot. Supports
+ plaintext or wrapped key import via a wrapping key ref.
+
+ - CMH_IOCTL_KEY_READ (0x03): Read key material from
+ a datastore slot, optionally wrapped. Returns data
+ plus a 16-byte SYS header for wrapped reads.
+
+ - CMH_IOCTL_KEY_FIND (0x04): Look up a key reference
+ by caller ID (CID).
+
+ - CMH_IOCTL_KEY_GRANT (0x05): Grant access to a key.
+
+ - CMH_IOCTL_KEY_DELETE (0x06): Delete a datastore slot.
+
+ - CMH_IOCTL_DS_EXPORT (0x07): Export the entire datastore
+ as an encrypted blob.
+
+ - CMH_IOCTL_DS_IMPORT (0x08): Import a previously exported
+ datastore blob.
+
+ - CMH_IOCTL_KEY_NEW_RANDOM (0x0B): Allocate a datastore
+ slot and fill it with hardware-generated random data.
+
+ - CMH_IOCTL_KEY_LIST (0x0E): List active datastore entries,
+ returning CIDs, types, lengths, and flags.
+
+ **Key Derivation -- KIC (0x09-0x0D):**
+
+ - CMH_IOCTL_KIC_HKDF1 (0x09): HKDF-Extract step.
+ - CMH_IOCTL_KIC_HKDF2 (0x0A): HKDF-Expand step.
+ - CMH_IOCTL_KIC_AES_CMAC_KDF (0x0C): AES-CMAC KDF.
+ - CMH_IOCTL_KIC_DKEK_DERIVE (0x0D): DKEK derivation.
+
+ **EAC -- Error and Alarm (0x0F):**
+
+ - CMH_IOCTL_EAC_READ (0x0F): Read and clear hardware
+ error, alarm, and safety notification registers.
+
+ **PKE -- Public Key Engine (0x10-0x1C):**
+
+ - CMH_IOCTL_PKE_RSA_ENC (0x10): RSA public-key encrypt.
+ - CMH_IOCTL_PKE_RSA_DEC (0x11): RSA private-key decrypt.
+ - CMH_IOCTL_PKE_RSA_CRT_DEC (0x12): RSA-CRT decrypt.
+ - CMH_IOCTL_PKE_RSA_KEYGEN (0x13): RSA key pair generation.
+ - CMH_IOCTL_PKE_ECDSA_SIGN (0x14): ECDSA sign.
+ - CMH_IOCTL_PKE_ECDH (0x16): ECDH shared secret.
+ - CMH_IOCTL_PKE_ECDH_KEYGEN (0x17): ECDH key pair generation.
+ - CMH_IOCTL_PKE_EDDSA_SIGN (0x18): EdDSA sign (Ed25519/Ed448).
+ - CMH_IOCTL_PKE_EDDSA_VERIFY (0x19): EdDSA verify.
+ - CMH_IOCTL_PKE_EC_KEYGEN (0x1A): EC key pair generation.
+ - CMH_IOCTL_PKE_EC_PUBGEN (0x1B): EC public key derivation.
+ - CMH_IOCTL_PKE_EDDSA_KEYGEN_SCA (0x1C): EdDSA SCA-protected
+ key generation.
+
+ **PQC -- Post-Quantum Crypto (0x20-0x2D):**
+
+ - CMH_IOCTL_ML_KEM_KEYGEN (0x20): ML-KEM key pair generation
+ (modes 512/768/1024).
+ - CMH_IOCTL_ML_KEM_ENC (0x21): ML-KEM encapsulation.
+ - CMH_IOCTL_ML_KEM_DEC (0x22): ML-KEM decapsulation.
+ - CMH_IOCTL_ML_DSA_KEYGEN (0x23): ML-DSA key pair generation
+ (modes 44/65/87).
+ - CMH_IOCTL_ML_DSA_SIGN (0x24): ML-DSA sign.
+ - CMH_IOCTL_SLHDSA_KEYGEN (0x28): SLH-DSA key pair generation
+ (12 parameter sets).
+ - CMH_IOCTL_SLHDSA_SIGN (0x29): SLH-DSA sign.
+ - CMH_IOCTL_SLHDSA_SIGN_PREHASH (0x2D): SLH-DSA prehash sign.
+
+ **SM2 Operations (0x30-0x37):**
+
+ - CMH_IOCTL_SM2_ECDH_KEYGEN (0x30): SM2 ephemeral key gen.
+ - CMH_IOCTL_SM2_ECDH (0x31): SM2 key exchange.
+ - CMH_IOCTL_SM2_DEC_POINT (0x32): SM2 decrypt (point step).
+ - CMH_IOCTL_SM2_ENC_POINT (0x33): SM2 encrypt (point step).
+ - CMH_IOCTL_SM2_ID_DIGEST (0x34): SM2 ID digest (ZA).
+ - CMH_IOCTL_SM2_ECDH_HASH (0x35): SM2 key exchange hash step.
+ - CMH_IOCTL_SM2_DEC_HASH (0x36): SM2 decrypt (hash step).
+ - CMH_IOCTL_SM2_ENC_HASH (0x37): SM2 encrypt (hash step).
+
+ The SM2 encrypt/decrypt hash-step ioctls accept payloads
+ of at most 32 bytes. The underlying hardware KDF emits a
+ single 32-byte SM3 block, so longer messages cannot be
+ processed in a single command and are rejected with
+ -EINVAL.
+
+ **DRBG Management (0x40):**
+
+ - CMH_IOCTL_DRBG_CONFIG (0x40): Configure the hardware
+ DRBG entropy ratio and security strength. Normally
+ called once at system start-up before hwrng reads.
+
+ All structs contain ``__reserved`` fields that must be
+ zero; the driver returns ``-EINVAL`` if any reserved field
+ is non-zero. This ensures forward compatibility when
+ reserved fields gain meaning in future versions.
+
+ All ioctls return 0 on success or a negative errno on
+ failure. Common errors:
+
+ - EINVAL: Invalid version, parameter, key type, or
+ non-zero reserved field.
+ - ENOENT: Key reference not found in datastore.
+ - ENOMEM: DMA allocation failure.
+ - EBUSY: Hardware mailbox full.
+ - ETIMEDOUT: VCQ operation timed out.
+ - EFAULT: Bad user-space pointer.
+
+ The ioctl UAPI header is <linux/cmh_mgmt_ioctl.h>.
+ All structures, constants, and type definitions are
+ documented in that header file.
diff --git a/drivers/crypto/cmh/Kconfig b/drivers/crypto/cmh/Kconfig
index fca66d5e2f89..512e7b3bdc59 100644
--- a/drivers/crypto/cmh/Kconfig
+++ b/drivers/crypto/cmh/Kconfig
@@ -63,3 +63,22 @@ config CRYPTO_DEV_CMH_DEBUG
Useful for bringup, validation, and performance analysis.
Not recommended for production.
+
+config CRYPTO_DEV_CMH_MGMT
+ bool "CMH management ioctl device (/dev/cmh_mgmt)"
+ depends on CRYPTO_DEV_CMH
+ default n
+ help
+ Expose /dev/cmh_mgmt, a misc device providing ioctl commands
+ for operations that have no kernel crypto API binding: hardware
+ key lifecycle (create, import, derive, destroy), KIC key
+ derivation, PQC keygen/encaps/decaps (ML-KEM, ML-DSA, SLH-DSA),
+ EdDSA sign/verify, SM2 key exchange, and DRBG
+ configuration.
+
+ The device requires CAP_SYS_ADMIN. Disabling this option
+ removes the ioctl interface but all kernel crypto API
+ algorithms (consumed by in-kernel users and validated by the
+ crypto test manager) remain fully functional.
+
+ If unsure, say N.
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index 742e65e3917e..bb7772555d6a 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -12,7 +12,16 @@ cmh-y := \
cmh_txn.o \
cmh_rh.o \
cmh_dma.o \
- cmh_sysfs.o
+ cmh_sysfs.o \
+ cmh_key.o \
+ cmh_sys.o
+
+# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
+cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
+ cmh_mgmt.o \
+ cmh_mgmt_pke.o \
+ cmh_mgmt_pqc.o \
+ cmh_pke_sm2.o
ccflags-y += -I$(src)/include
diff --git a/drivers/crypto/cmh/cmh_key.c b/drivers/crypto/cmh/cmh_key.c
new file mode 100644
index 000000000000..fde8be50b25c
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_key.c
@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Dual Key Path Implementation
+ *
+ * Two key provisioning paths are supported:
+ *
+ * Raw key: key bytes -> stored in tfm context ->
+ * SYS_CMD_WRITE(SYS_REF_TEMP) packed into every crypto VCQ.
+ * The raw key buffer is DMA-mapped once at setkey time and remains
+ * mapped for the lifetime of the transform (unmapped in destroy).
+ *
+ * Raw key DMA lifetime rationale
+ * ------------------------------
+ * Raw keys are DMA-mapped at setkey time and the mapping persists
+ * until the transform is destroyed (cmh_key_destroy). This is a
+ * deliberate design choice, consistent with upstream HW crypto
+ * drivers (CAAM, ccree, CCP) that also map keys at setkey for
+ * transform-lifetime reuse:
+ *
+ * - The Linux crypto framework expects setkey to prepare the
+ * transform for repeated encrypt/decrypt calls. Remapping the
+ * same key on every request would add DMA API overhead per crypto
+ * operation with no security benefit.
+ * - On destroy, kfree_sensitive() scrubs the key buffer and the
+ * DMA mapping is released. For key-by-ID (persistent), the
+ * per-MBX ref cache is zeroed with memzero_explicit().
+ * - No key material is ever logged; dev_dbg() messages only show
+ * CIDs (content identifiers), not key bytes.
+ *
+ * Hardware-required behaviors (not driver policy)
+ * ------------------------------------------------
+ * - SYS_REF_TEMP lifetime: the eSW firmware reclaims temporary
+ * datastore objects when the mailbox slot is reused. This is a
+ * hardware constraint; the driver packs SYS_CMD_WRITE into every
+ * VCQ to re-provision the raw key for each operation.
+ * - Mailbox flush (SYS_CMD_FLUSH): reclaims temp-stack space on the
+ * target MBX. Required by HW to prevent temp-stack exhaustion
+ * across multi-VCQ operations.
+ */
+
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "cmh_key.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_sys_abi.h"
+#include <uapi/linux/cmh_mgmt_ioctl.h>
+
+/**
+ * cmh_ds_type_to_core_id() - Map a datastore type to a logical core ID
+ * @ds_type: Datastore type constant (e.g. CMH_DS_AES_KEY, CMH_DS_SM4_KEY)
+ *
+ * Returns the algorithm-family identity (e.g. CORE_ID_AES = 0x03), NOT the
+ * VCQ dispatch core_id. With multi-instance, a second AES engine dispatches
+ * at CORE_ID_AES2 (0x06) but keys are still tagged with CORE_ID_AES (0x03)
+ * -- the eSW validates against the logical identity, not the dispatch ID.
+ *
+ * Return: Logical core ID on success, CORE_ID_NUM for unknown @ds_type.
+ */
+u32 cmh_ds_type_to_core_id(u32 ds_type)
+{
+ switch (ds_type) {
+ case CMH_DS_AES_KEY:
+ case CMH_DS_AES_XTS_KEY:
+ return CORE_ID_AES;
+ case CMH_DS_SM4_KEY:
+ return CORE_ID_SM4;
+ case CMH_DS_HMAC_KEY:
+ case CMH_DS_KMAC_KEY:
+ return CORE_ID_HC;
+ case CMH_DS_CHACHA20_KEY:
+ return CORE_ID_CCP;
+ case CMH_DS_RSA_PRIV_KEY:
+ case CMH_DS_RSA_PUB_KEY:
+ case CMH_DS_RSA_CRT_KEY:
+ case CMH_DS_ECDSA_PRIV_KEY:
+ case CMH_DS_ECDSA_PUB_KEY:
+ case CMH_DS_ECDH_PRIV_KEY:
+ case CMH_DS_EDDSA_PRIV_KEY:
+ case CMH_DS_SHARED_SECRET:
+ case CMH_DS_SM2_PRIV_KEY:
+ return CORE_ID_PKE;
+ case CMH_DS_ML_KEM_DK:
+ case CMH_DS_ML_DSA_SK:
+ return CORE_ID_QSE;
+ case CMH_DS_SLHDSA_SK:
+ return CORE_ID_HCQ;
+ default:
+ return CORE_ID_NUM;
+ }
+}
+
+/**
+ * cmh_key_setkey_raw() - Store a raw key in the key context
+ * @ctx: Key context to populate
+ * @key: Pointer to the raw key bytes
+ * @keylen: Length of @key in bytes
+ * @core_id: Logical core ID for SYS_TYPE tagging
+ *
+ * Duplicates the raw key, DMA-maps the copy for the lifetime of the
+ * transform, and stores the mapping in @ctx. Any previously held key
+ * is destroyed first.
+ *
+ * The DMA mapping persists until cmh_key_destroy() is called (typically
+ * from the algorithm .exit_tfm callback). This avoids per-request DMA
+ * mapping overhead and matches the setkey-to-destroy lifetime model used
+ * by other upstream HW crypto drivers (CAAM, ccree, CCP). The key
+ * buffer is freed via kfree_sensitive() on destroy.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_key_setkey_raw(struct cmh_key_ctx *ctx, const u8 *key,
+ u32 keylen, u32 core_id)
+{
+ dma_addr_t dma;
+ u8 *copy;
+
+ if (!keylen || !key)
+ return -EINVAL;
+
+ copy = kmemdup(key, keylen, GFP_KERNEL);
+ if (!copy)
+ return -ENOMEM;
+
+ /* Pre-map for the lifetime of the transform */
+ dma = cmh_dma_map_single(copy, keylen, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(dma)) {
+ kfree_sensitive(copy);
+ return -ENOMEM;
+ }
+
+ /* Clean up any previous key */
+ cmh_key_destroy(ctx);
+
+ ctx->mode = CMH_KEY_RAW;
+ ctx->raw.data = copy;
+ ctx->raw.len = keylen;
+ ctx->raw.dma = dma;
+ ctx->raw.sys_type = SYS_TYPE_SET(SYS_TYPE_FLAG_PT, core_id);
+
+ return 0;
+}
+
+/**
+ * cmh_key_destroy() - Destroy and zero-fill a key context
+ * @ctx: Key context to destroy
+ *
+ * For raw keys, unmaps the DMA buffer and securely frees the key material.
+ * Resets the key mode to CMH_KEY_NONE.
+ */
+void cmh_key_destroy(struct cmh_key_ctx *ctx)
+{
+ if (ctx->mode == CMH_KEY_RAW && ctx->raw.data) {
+ cmh_dma_unmap_single(ctx->raw.dma, ctx->raw.len,
+ DMA_TO_DEVICE);
+ kfree_sensitive(ctx->raw.data);
+ memzero_explicit(&ctx->raw, sizeof(ctx->raw));
+ }
+ ctx->mode = CMH_KEY_NONE;
+}
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index a9b0910826a3..576a0c02e6f9 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -31,6 +31,7 @@
#include "cmh_mqi.h"
#include "cmh_txn.h"
#include "cmh_rh.h"
+#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
#include "cmh_sysfs.h"
@@ -236,12 +237,19 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_rh_init;
+ /* Register key management device (/dev/cmh_mgmt) */
+ ret = cmh_mgmt_register();
+ if (ret)
+ goto err_mgmt_register;
+
g_cmh_dev = dev;
platform_set_drvdata(pdev, dev);
dev_info(cmh_dev(), "initialized successfully\n");
return 0;
+err_mgmt_register:
+ cmh_rh_cleanup(cfg);
err_rh_init:
cmh_tm_cleanup();
err_tm_init:
@@ -266,6 +274,7 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
+ cmh_mgmt_unregister();
cmh_rh_cleanup(cfg);
cmh_tm_cleanup();
cmh_mqi_cleanup(cfg);
diff --git a/drivers/crypto/cmh/cmh_mgmt.c b/drivers/crypto/cmh/cmh_mgmt.c
new file mode 100644
index 000000000000..ad1ba075e948
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_mgmt.c
@@ -0,0 +1,1709 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Key Management misc_device (/dev/cmh_mgmt)
+ *
+ * Provides ioctl interface for key provisioning (NEW, NEW_RANDOM, WRITE, READ,
+ * FIND, GRANT, DELETE) and datastore lifecycle (EXPORT, IMPORT).
+ *
+ * Each ioctl handler: copy_from_user -> validate -> DMA alloc ->
+ * build VCQ -> cmh_tm_submit_sync -> copy_to_user -> DMA free.
+ *
+ * Access requires CAP_SYS_ADMIN (checked in open()). The device node
+ * is mode 0660; DAC further limits access to owner/group.
+ * CMH eSW enforces per-MBX access control on top of this.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/miscdevice.h>
+#include <linux/uaccess.h>
+#include <linux/slab.h>
+#include <linux/fs.h>
+#include <linux/capability.h>
+#include <linux/overflow.h>
+
+#include "cmh_mgmt.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_key.h"
+#include "cmh_dma.h"
+#include "cmh_config.h"
+#include "cmh_sys_abi.h"
+#include "cmh_pke.h"
+#include "cmh_pke_sm2.h"
+#include "cmh_qse_abi.h"
+#include "cmh_hcq_abi.h"
+#include <uapi/linux/cmh_mgmt_ioctl.h>
+
+#include <crypto/utils.h>
+
+/*
+ * Pin all mgmt ioctls to a single management mailbox (MBX 0).
+ *
+ * This is a deliberate, structural choice -- not a performance default.
+ * The /dev/cmh_mgmt path is *stateful* with respect to the eSW datastore,
+ * and that state is per-mailbox, so every step of a key's lifecycle must
+ * land on the same mailbox:
+ *
+ * 1. Datastore access control is per-mailbox AND opaque to the driver.
+ * SYS_CMD_NEW grants the creating mailbox a (1 << mbx_id) access mask
+ * (read/write/execute). Crucially, the returned 64-bit ref encodes a
+ * randomised offset -- NOT the owning mailbox -- so given only a ref
+ * (as KEY_GRANT/READ/DELETE/DS_EXPORT receive), the driver cannot
+ * recover which mailbox owns the object. A fixed management mailbox
+ * is therefore the only way to guarantee that NEW, WRITE, GRANT, READ
+ * and the subsequent hardware-held-key compute ops all share the
+ * mailbox that holds the access rights, without exposing mailbox
+ * identity in the UABI. (User space may still widen access to other
+ * mailboxes explicitly via KEY_GRANT.)
+ *
+ * 2. The eSW SYS_REF_TEMP scratch store is per-mailbox and persists
+ * across ioctl calls. A derivation that writes SYS_REF_TEMP (e.g. a
+ * KIC_* derive) must be consumed by a later ioctl on the *same*
+ * mailbox (e.g. DS_EXPORT with wrap_key=SYS_REF_TEMP).
+ *
+ * Device-tree per-mailbox ``rambus,cores`` affinity applies to the *stateless*
+ * registered crypto API path (cmh_core_select_instance()), which carries
+ * no datastore state across calls and is free to balance across mailboxes.
+ *
+ * Note: MBX 0 is NOT reserved exclusively for mgmt -- registered crypto
+ * operations may also land here via TM round-robin (target_mbx = -1).
+ * This is safe because those ops do not allocate from the temp store.
+ */
+
+/* VCQ layout: header + command + flush = 3 entries */
+#define MGMT_VCQ_CMDS 3
+
+/*
+ * Tracks whether any operation has left residual state in the device's
+ * per-mailbox temporary key store since the last flush. The device
+ * reclaims temp storage only on a full mailbox flush (MBX_COMMAND_FLUSH),
+ * which also terminates any executing command queue with -EPIPE.
+ *
+ * To avoid killing concurrent in-flight operations, the flush in
+ * cmh_mgmt_ioctl() is conditional: it fires only when this flag is set.
+ * Operations that allocate temp storage (currently: KIC derivations
+ * targeting SYS_REF_TEMP) set this flag on success.
+ */
+static atomic_t mgmt_temp_dirty = ATOMIC_INIT(0);
+
+/* -- KEY_NEW -------------------------- */
+
+static int cmh_mgmt_key_new(void __user *argp)
+{
+ struct cmh_ioctl_key_new req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ u64 *ref_buf;
+ dma_addr_t ref_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.len)
+ return -EINVAL;
+
+ /* DMA buffer for CMH eSW to write back the ref */
+ ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+
+ *ref_buf = 0;
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.len);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+
+ /*
+ * Unmap before CPU read: single-phase operation (no re-use of
+ * the DMA mapping), so unmap transfers ownership back to the
+ * CPU. On SWIOTLB systems the unmap copies the bounce buffer
+ * to the original allocation. This is the correct pattern for
+ * single-shot sync submits where the buffer is not re-mapped.
+ */
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf), DMA_FROM_DEVICE);
+
+ if (ret) {
+ kfree(ref_buf);
+ return ret;
+ }
+
+ req.ref = *ref_buf;
+ kfree(ref_buf);
+
+ if (copy_to_user(argp, &req, sizeof(req))) {
+ /*
+ * The DS slot was created but its ref never reached
+ * userspace, so the caller cannot free it. Logically
+ * delete the orphaned slot before returning.
+ */
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_grant(&vcq[1], req.ref, 0, 0, 0);
+ vcq_add_sys_flush(&vcq[2]);
+ cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+ dev_warn(cmh_dev(),
+ "mgmt: KEY_NEW copy_to_user failed, DS slot cleaned up\n");
+ return -EFAULT;
+ }
+
+ dev_dbg(cmh_dev(), "mgmt: KEY_NEW cid=0x%llx len=%u -> ref=0x%llx\n",
+ req.cid, req.len, req.ref);
+ return 0;
+}
+
+/* -- KEY_WRITE ------------------------- */
+
+static int cmh_mgmt_key_write(void __user *argp)
+{
+ struct cmh_ioctl_key_write req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ void *dmabuf;
+ dma_addr_t dma_addr;
+ u32 core_id, sys_type;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.len || req.len > CMH_MGMT_MAX_DATA_LEN)
+ return -EINVAL;
+
+ core_id = cmh_ds_type_to_core_id(req.ds_type);
+ if (core_id == CORE_ID_NUM)
+ return -EINVAL;
+ sys_type = SYS_TYPE_SET(req.flags, core_id);
+
+ dmabuf = kmalloc(req.len, GFP_KERNEL);
+ if (!dmabuf)
+ return -ENOMEM;
+
+ if (copy_from_user(dmabuf, u64_to_user_ptr(req.data),
+ req.len)) {
+ kfree_sensitive(dmabuf);
+ return -EFAULT;
+ }
+
+ dma_addr = cmh_dma_map_single(dmabuf, req.len, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(dma_addr)) {
+ kfree_sensitive(dmabuf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_write(&vcq[1], req.ref, dma_addr, req.wrap_key,
+ req.len, sys_type);
+ /*
+ * PKE keys on Weierstrass curves and RSA keys must be byte-swapped
+ * when stored in the DS so they match the internal big-endian
+ * representation used by the PKE sidecar. Edwards curve keys
+ * (EdDSA) use native byte order and must NOT be swapped.
+ */
+ switch (req.ds_type) {
+ case CMH_DS_RSA_PRIV_KEY:
+ case CMH_DS_RSA_PUB_KEY:
+ case CMH_DS_RSA_CRT_KEY:
+ case CMH_DS_ECDSA_PRIV_KEY:
+ case CMH_DS_ECDSA_PUB_KEY:
+ case CMH_DS_ECDH_PRIV_KEY:
+ case CMH_DS_SHARED_SECRET:
+ case CMH_DS_SM2_PRIV_KEY:
+ vcq[1].id |= PKE_SWAP_FLAGS;
+ break;
+ default:
+ /* EdDSA, symmetric keys -- no swap */
+ break;
+ }
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(dma_addr, req.len, DMA_TO_DEVICE);
+ kfree_sensitive(dmabuf);
+
+ if (ret)
+ return ret;
+
+ dev_dbg(cmh_dev(), "mgmt: KEY_WRITE ref=0x%llx len=%u type=0x%x\n",
+ req.ref, req.len, sys_type);
+ return 0;
+}
+
+/* -- KEY_READ -------------------------- */
+
+static int cmh_mgmt_key_read(void __user *argp)
+{
+ struct cmh_ioctl_key_read req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ void *dmabuf;
+ dma_addr_t dma_addr;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (!req.len || req.len > CMH_MGMT_MAX_DATA_LEN)
+ return -EINVAL;
+
+ dmabuf = kzalloc(req.len, GFP_KERNEL);
+ if (!dmabuf)
+ return -ENOMEM;
+
+ dma_addr = cmh_dma_map_single(dmabuf, req.len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(dma_addr)) {
+ kfree(dmabuf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_read(&vcq[1], req.ref, dma_addr, req.wrap_key, req.len);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(dma_addr, req.len, DMA_FROM_DEVICE);
+
+ if (ret) {
+ kfree_sensitive(dmabuf);
+ return ret;
+ }
+
+ if (copy_to_user(u64_to_user_ptr(req.data),
+ dmabuf, req.len)) {
+ kfree_sensitive(dmabuf);
+ return -EFAULT;
+ }
+
+ req.out_len = req.len;
+ kfree_sensitive(dmabuf);
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ dev_dbg(cmh_dev(), "mgmt: KEY_READ ref=0x%llx len=%u\n",
+ req.ref, req.out_len);
+ return 0;
+}
+
+/* -- KEY_FIND -------------------------- */
+
+static int cmh_mgmt_key_find(void __user *argp)
+{
+ struct cmh_ioctl_key_find req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ struct sys_list_item *item;
+ dma_addr_t item_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+
+ item = kzalloc_obj(*item, GFP_KERNEL);
+ if (!item)
+ return -ENOMEM;
+
+ item_dma = cmh_dma_map_single(item, sizeof(*item), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(item_dma)) {
+ kfree(item);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_find(&vcq[1], req.cid, item_dma, sizeof(*item));
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(item_dma, sizeof(*item), DMA_FROM_DEVICE);
+
+ if (ret) {
+ kfree(item);
+ return ret;
+ }
+
+ req.ref = item->ref;
+ req.len = item->len;
+ req.type = item->type;
+ kfree(item);
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ dev_dbg(cmh_dev(), "mgmt: KEY_FIND cid=0x%llx -> ref=0x%llx\n",
+ req.cid, req.ref);
+ return 0;
+}
+
+/* -- KEY_LIST ------------------------- */
+
+static int cmh_mgmt_key_list(void __user *argp)
+{
+ struct cmh_ioctl_key_list req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ struct sys_list_item *item;
+ dma_addr_t item_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+
+ if (req.__reserved)
+ return -EINVAL;
+
+ item = kzalloc_obj(*item, GFP_KERNEL);
+ if (!item)
+ return -ENOMEM;
+
+ item_dma = cmh_dma_map_single(item, sizeof(*item), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(item_dma)) {
+ kfree(item);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_list(&vcq[1], req.start_ref, item_dma, sizeof(*item));
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(item_dma, sizeof(*item), DMA_FROM_DEVICE);
+
+ if (ret) {
+ kfree(item);
+ return ret;
+ }
+
+ req.ref = item->ref;
+ req.cid = item->cid;
+ req.len = item->len;
+ req.type = item->type;
+ kfree(item);
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ return 0;
+}
+
+/* -- KEY_GRANT / KEY_DELETE --------------------- */
+
+static int cmh_mgmt_key_grant(void __user *argp, bool is_delete)
+{
+ struct cmh_ioctl_key_grant req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+
+ /* DELETE = GRANT with all permissions zeroed */
+ if (is_delete) {
+ req.read = 0;
+ req.write = 0;
+ req.execute = 0;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_grant(&vcq[1], req.ref, req.read, req.write, req.execute);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+ if (ret)
+ return ret;
+
+ dev_dbg(cmh_dev(), "mgmt: KEY_%s ref=0x%llx r=0x%llx w=0x%llx x=0x%llx\n",
+ is_delete ? "DELETE" : "GRANT",
+ req.ref, req.read, req.write, req.execute);
+ return 0;
+}
+
+/* -- DS_EXPORT ------------------------- */
+
+static int cmh_mgmt_ds_export(void __user *argp)
+{
+ struct cmh_ioctl_ds_export req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ void *dmabuf;
+ dma_addr_t dma_addr;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (!req.len || req.len > CMH_MGMT_MAX_DATA_LEN)
+ return -EINVAL;
+ /*
+ * The eSW writes at least a sys_wrap_hdr into the buffer; reject a
+ * request too small to hold it up front. This also prevents the
+ * out-of-bounds read of hdr->wrap/hdr->len below on a short buffer.
+ */
+ if (req.len < sizeof(struct sys_wrap_hdr))
+ return -EINVAL;
+
+ /*
+ * req.len is the exact DMA buffer size given to the eSW.
+ * Userspace must size it to at least the export blob:
+ *
+ * wrapped: sizeof(sys_wrap_hdr) + 2*AES_BLOCK_SIZE + obj_len
+ * = 16 + 32 + obj_len = 48 + obj_len
+ * plaintext: sizeof(sys_wrap_hdr) + obj_len
+ * = 16 + obj_len
+ *
+ * obj_len is known from KEY_NEW or KEY_FIND. If req.len is
+ * too small, the eSW rejects the command and we return -EIO.
+ */
+ dmabuf = kzalloc(req.len, GFP_KERNEL);
+ if (!dmabuf)
+ return -ENOMEM;
+
+ dma_addr = cmh_dma_map_single(dmabuf, req.len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(dma_addr)) {
+ kfree(dmabuf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_export(&vcq[1], req.cid, dma_addr, req.wrap_key, req.len);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(dma_addr, req.len, DMA_FROM_DEVICE);
+
+ if (ret) {
+ kfree_sensitive(dmabuf);
+ return ret;
+ }
+
+ /* Parse actual blob size from the eSW-written header */
+ {
+ struct sys_wrap_hdr *hdr = (struct sys_wrap_hdr *)dmabuf;
+ u64 actual;
+
+ if (check_add_overflow((u64)sizeof(*hdr), (u64)hdr->wrap,
+ &actual) ||
+ check_add_overflow(actual, (u64)hdr->len, &actual) ||
+ actual > req.len) {
+ kfree_sensitive(dmabuf);
+ return -EIO;
+ }
+ req.out_len = (u32)actual;
+ }
+
+ if (copy_to_user(u64_to_user_ptr(req.data),
+ dmabuf, req.out_len)) {
+ kfree_sensitive(dmabuf);
+ return -EFAULT;
+ }
+
+ kfree_sensitive(dmabuf);
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ dev_dbg(cmh_dev(), "mgmt: DS_EXPORT wrap_key=0x%llx len=%u\n",
+ req.wrap_key, req.out_len);
+ return 0;
+}
+
+/* -- DS_IMPORT ------------------------- */
+
+static int cmh_mgmt_ds_import(void __user *argp)
+{
+ struct cmh_ioctl_ds_import req;
+ struct vcq_cmd vcq[MGMT_VCQ_CMDS];
+ void *dmabuf;
+ dma_addr_t dma_addr;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.len || req.len > CMH_MGMT_MAX_DATA_LEN)
+ return -EINVAL;
+
+ dmabuf = kmalloc(req.len, GFP_KERNEL);
+ if (!dmabuf)
+ return -ENOMEM;
+
+ if (copy_from_user(dmabuf, u64_to_user_ptr(req.data),
+ req.len)) {
+ kfree_sensitive(dmabuf);
+ return -EFAULT;
+ }
+
+ dma_addr = cmh_dma_map_single(dmabuf, req.len, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(dma_addr)) {
+ kfree_sensitive(dmabuf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], MGMT_VCQ_CMDS);
+ vcq_add_sys_import(&vcq[1], dma_addr, req.wrap_key, req.len);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(dma_addr, req.len, DMA_TO_DEVICE);
+ kfree_sensitive(dmabuf);
+
+ if (ret)
+ return ret;
+
+ dev_dbg(cmh_dev(), "mgmt: DS_IMPORT wrap_key=0x%llx len=%u\n",
+ req.wrap_key, req.len);
+ return 0;
+}
+
+/* -- KIC key derivation ioctls --------
+ *
+ * All four KIC derivation handlers (HKDF1, HKDF2, AES-CMAC-KDF,
+ * DKEK-derive) share the same two-mode structure and temp-flush pattern.
+ *
+ * Temp-storage flush rationale:
+ *
+ * The device maintains a small per-mailbox temporary key store
+ * (~960 bytes, LIFO). A derivation targeting SYS_REF_TEMP allocates
+ * from this store; the allocation persists across command-queue
+ * boundaries until either (a) a subsequent command consumes it or
+ * (b) a mailbox flush resets the store.
+ *
+ * Our single-derivation ioctls produce a temp key with no consumer
+ * in the same queue -- the key is consumed by a *later* ioctl
+ * (e.g. DS_EXPORT with wrap_key=SYS_REF_TEMP). If no consumer
+ * follows, the allocation persists. Sequential temp derivations
+ * accumulate allocations until the store is exhausted (3--8 calls
+ * depending on key size), after which the device returns ENOMEM.
+ *
+ * A mailbox flush (cmh_tm_flush_mbx / MBX_COMMAND_FLUSH) resets the
+ * temp store. It does NOT destroy persistent keys, datastore
+ * objects, or DRBG state -- only the command queue and temp store.
+ *
+ * Safe for cross-ioctl temp flows (e.g. export-to-file:
+ * HKDF1->TEMP in ioctl 1, then DS_EXPORT with wrap_key=TEMP in
+ * ioctl 2): the flush only happens in derivation handlers and in
+ * the pre-PKE dispatch path, not in DS_EXPORT/DS_IMPORT, so the
+ * temp key survives until consumed.
+ *
+ * The ioctl dispatch also flushes before PKE/SM2/PQC ioctls to
+ * protect them from temp residue left by earlier derivations on the
+ * same mailbox. The per-handler flushes here remain necessary
+ * because sequential temp derivations (without an intervening
+ * PKE/SM2/PQC ioctl) would still exhaust the store.
+ */
+
+/* -- KIC_HKDF1 ------------------------- */
+
+/*
+ * Derive a key from a KIC base key via one-step HKDF.
+ *
+ * Two modes controlled by CMH_KIC_FLAG_TEMP:
+ *
+ * TEMP (flag set) -- 3-command VCQ:
+ * [0] SYS header
+ * [1] KIC_CMD_HKDF1 (dst=SYS_REF_TEMP)
+ * [2] flush
+ * Returns SYS_REF_TEMP as ref. No DS entry created.
+ *
+ * Persistent (flag clear) -- 4-command VCQ:
+ * [0] SYS header
+ * [1] SYS_CMD_NEW (allocate DS slot, CMH eSW writes ref)
+ * [2] KIC_CMD_HKDF1 (dst=SYS_REF_LAST = just-allocated slot)
+ * [3] flush
+ * Returns the new DS reference.
+ */
+#define KDF_VCQ_MAX 4
+#define KDF_MAX_KEY_LEN 64
+#define KDF_MAX_LABEL_LEN 56
+
+static int cmh_mgmt_kic_hkdf1(void __user *argp)
+{
+ struct cmh_ioctl_kic_hkdf1 req;
+ struct vcq_cmd vcq[KDF_VCQ_MAX];
+ bool temp;
+ u64 *ref_buf = NULL;
+ void *label_buf = NULL;
+ dma_addr_t ref_dma = DMA_MAPPING_ERROR, label_dma = DMA_MAPPING_ERROR;
+ unsigned int n_cmds;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.key_len || req.key_len > KDF_MAX_KEY_LEN)
+ return -EINVAL;
+ if (req.label_len > KDF_MAX_LABEL_LEN)
+ return -EINVAL;
+
+ temp = !!(req.flags & CMH_KIC_FLAG_TEMP);
+
+ /*
+ * Persistent path: need DMA buffer for CMH eSW to write the
+ * newly-allocated DS reference.
+ */
+ if (!temp) {
+ ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+ *ref_buf = 0;
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+ }
+
+ /* DMA buffer for label data (CMH eSW DMA-reads it) */
+ if (req.label_len > 0) {
+ label_buf = kzalloc(req.label_len, GFP_KERNEL);
+ if (!label_buf) {
+ ret = -ENOMEM;
+ goto out_ref;
+ }
+ if (copy_from_user(label_buf,
+ u64_to_user_ptr(req.label),
+ req.label_len)) {
+ ret = -EFAULT;
+ goto out_label;
+ }
+ label_dma = cmh_dma_map_single(label_buf, req.label_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(label_dma)) {
+ ret = -ENOMEM;
+ goto out_label;
+ }
+ }
+
+ /* Build VCQ */
+ memset(vcq, 0, sizeof(vcq));
+
+ if (temp) {
+ /* Flush MBX to reset temp stack -- see KIC section comment */
+ ret = cmh_tm_flush_mbx(MGMT_MBX);
+ if (ret)
+ goto out_unmap_label;
+
+ n_cmds = 3;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_kic_hkdf1(&vcq[1], SYS_REF_TEMP, req.base_key,
+ label_dma, req.key_len, req.label_len,
+ SYS_TYPE_SET(0, CORE_ID_AES));
+ vcq_add_sys_flush(&vcq[2]);
+ } else {
+ n_cmds = 4;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.key_len);
+ vcq_add_kic_hkdf1(&vcq[2], SYS_REF_LAST, req.base_key,
+ label_dma, req.key_len, req.label_len,
+ SYS_TYPE_SET(0, CORE_ID_AES));
+ vcq_add_sys_flush(&vcq[3]);
+ }
+
+ ret = cmh_tm_submit_sync_mbx(vcq, n_cmds, 1, MGMT_MBX);
+
+ /* Cleanup label DMA */
+ if (label_buf) {
+ cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE);
+ kfree(label_buf);
+ label_buf = NULL;
+ }
+
+ if (ret)
+ goto out_ref;
+
+ if (temp) {
+ req.ref = SYS_REF_TEMP;
+ atomic_set(&mgmt_temp_dirty, 1);
+ } else {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ req.ref = *ref_buf;
+ kfree(ref_buf);
+ ref_buf = NULL;
+ }
+
+ if (copy_to_user(argp, &req, sizeof(req))) {
+ /*
+ * The derived-key DS slot was created but its ref never
+ * reached userspace, so the caller cannot free it. Logically
+ * delete the orphaned slot: grant-with-no-access wipes the key
+ * material and clears the CID (temp results use SYS_REF_TEMP
+ * and need no cleanup).
+ */
+ if (!temp) {
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_grant(&vcq[1], req.ref, 0, 0, 0);
+ vcq_add_sys_flush(&vcq[2]);
+ cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+ dev_warn(cmh_dev(),
+ "mgmt: KIC_HKDF1 copy_to_user failed, DS slot cleaned up\n");
+ }
+ return -EFAULT;
+ }
+
+ dev_dbg(cmh_dev(),
+ "mgmt: KIC_HKDF1 base=0x%llx len=%u flags=0x%x -> ref=0x%llx\n",
+ req.base_key, req.key_len, req.flags, req.ref);
+ return 0;
+
+out_unmap_label:
+ if (label_buf && !cmh_dma_map_error(label_dma))
+ cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE);
+out_label:
+ kfree(label_buf);
+out_ref:
+ if (ref_buf) {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ kfree(ref_buf);
+ }
+ return ret;
+}
+
+/* -- KIC_HKDF2 ------------------------- */
+
+/*
+ * Two-step HKDF key derivation. Same as HKDF1 but adds a salt key
+ * reference: Step 1: HMAC(salt, base) -> PRK; Step 2: HMAC(PRK, label) -> key.
+ */
+
+static int cmh_mgmt_kic_hkdf2(void __user *argp)
+{
+ struct cmh_ioctl_kic_hkdf2 req;
+ struct vcq_cmd vcq[KDF_VCQ_MAX];
+ bool temp;
+ u64 *ref_buf = NULL;
+ void *label_buf = NULL;
+ dma_addr_t ref_dma = DMA_MAPPING_ERROR, label_dma = DMA_MAPPING_ERROR;
+ unsigned int n_cmds;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.key_len || req.key_len > KDF_MAX_KEY_LEN)
+ return -EINVAL;
+ if (req.label_len > KDF_MAX_LABEL_LEN)
+ return -EINVAL;
+
+ temp = !!(req.flags & CMH_KIC_FLAG_TEMP);
+
+ if (!temp) {
+ ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+ *ref_buf = 0;
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+ }
+
+ if (req.label_len > 0) {
+ label_buf = kzalloc(req.label_len, GFP_KERNEL);
+ if (!label_buf) {
+ ret = -ENOMEM;
+ goto out_ref2;
+ }
+ if (copy_from_user(label_buf,
+ u64_to_user_ptr(req.label),
+ req.label_len)) {
+ ret = -EFAULT;
+ goto out_label2;
+ }
+ label_dma = cmh_dma_map_single(label_buf, req.label_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(label_dma)) {
+ ret = -ENOMEM;
+ goto out_label2;
+ }
+ }
+
+ memset(vcq, 0, sizeof(vcq));
+
+ if (temp) {
+ /* Flush MBX to reset temp stack -- see KIC section comment */
+ ret = cmh_tm_flush_mbx(MGMT_MBX);
+ if (ret)
+ goto out_unmap_label2;
+
+ n_cmds = 3;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_kic_hkdf2(&vcq[1], SYS_REF_TEMP, req.base_key,
+ req.salt_key, label_dma,
+ req.key_len, req.label_len,
+ SYS_TYPE_SET(0, CORE_ID_AES));
+ vcq_add_sys_flush(&vcq[2]);
+ } else {
+ n_cmds = 4;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.key_len);
+ vcq_add_kic_hkdf2(&vcq[2], SYS_REF_LAST, req.base_key,
+ req.salt_key, label_dma,
+ req.key_len, req.label_len,
+ SYS_TYPE_SET(0, CORE_ID_AES));
+ vcq_add_sys_flush(&vcq[3]);
+ }
+
+ ret = cmh_tm_submit_sync_mbx(vcq, n_cmds, 1, MGMT_MBX);
+
+ if (label_buf) {
+ cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE);
+ kfree(label_buf);
+ label_buf = NULL;
+ }
+
+ if (ret)
+ goto out_ref2;
+
+ if (temp) {
+ req.ref = SYS_REF_TEMP;
+ atomic_set(&mgmt_temp_dirty, 1);
+ } else {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ req.ref = *ref_buf;
+ kfree(ref_buf);
+ ref_buf = NULL;
+ }
+
+ if (copy_to_user(argp, &req, sizeof(req))) {
+ /*
+ * The derived-key DS slot was created but its ref never
+ * reached userspace, so the caller cannot free it. Logically
+ * delete the orphaned slot: grant-with-no-access wipes the key
+ * material and clears the CID (temp results use SYS_REF_TEMP
+ * and need no cleanup).
+ */
+ if (!temp) {
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_grant(&vcq[1], req.ref, 0, 0, 0);
+ vcq_add_sys_flush(&vcq[2]);
+ cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+ dev_warn(cmh_dev(),
+ "mgmt: KIC_HKDF2 copy_to_user failed, DS slot cleaned up\n");
+ }
+ return -EFAULT;
+ }
+
+ dev_dbg(cmh_dev(),
+ "mgmt: KIC_HKDF2 base=0x%llx salt=0x%llx len=%u flags=0x%x -> ref=0x%llx\n",
+ req.base_key, req.salt_key, req.key_len, req.flags, req.ref);
+ return 0;
+
+out_unmap_label2:
+ if (label_buf && !cmh_dma_map_error(label_dma))
+ cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE);
+out_label2:
+ kfree(label_buf);
+out_ref2:
+ if (ref_buf) {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ kfree(ref_buf);
+ }
+ return ret;
+}
+
+/* -- KIC_AES_CMAC_KDF ------------------ */
+
+/*
+ * Derive a key using AES-CMAC-based KDF (NIST SP800-108 style).
+ * Base key must be 32 bytes. Output is always non-PT (the hub driver
+ * rejects SYS_TYPE_FLAG_PT).
+ *
+ * VCQ layout matches HKDF: TEMP mode uses 3 commands, persistent uses 4.
+ */
+#define CMAC_KDF_KEY_LEN 32
+
+static int cmh_mgmt_kic_aes_cmac_kdf(void __user *argp)
+{
+ struct cmh_ioctl_kic_aes_cmac_kdf req;
+ struct vcq_cmd vcq[KDF_VCQ_MAX];
+ bool temp;
+ u64 *ref_buf = NULL;
+ void *label_buf = NULL;
+ dma_addr_t ref_dma = DMA_MAPPING_ERROR, label_dma = DMA_MAPPING_ERROR;
+ unsigned int n_cmds;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.key_len != CMAC_KDF_KEY_LEN)
+ return -EINVAL;
+ if (req.label_len > KDF_MAX_LABEL_LEN)
+ return -EINVAL;
+
+ temp = !!(req.flags & CMH_KIC_FLAG_TEMP);
+
+ if (!temp) {
+ ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+ *ref_buf = 0;
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+ }
+
+ if (req.label_len > 0) {
+ label_buf = kzalloc(req.label_len, GFP_KERNEL);
+ if (!label_buf) {
+ ret = -ENOMEM;
+ goto out_ref_cmac;
+ }
+ if (copy_from_user(label_buf,
+ u64_to_user_ptr(req.label),
+ req.label_len)) {
+ ret = -EFAULT;
+ goto out_label_cmac;
+ }
+ label_dma = cmh_dma_map_single(label_buf, req.label_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(label_dma)) {
+ ret = -ENOMEM;
+ goto out_label_cmac;
+ }
+ }
+
+ memset(vcq, 0, sizeof(vcq));
+
+ if (temp) {
+ /* Flush MBX to reset temp stack -- see KIC section comment */
+ ret = cmh_tm_flush_mbx(MGMT_MBX);
+ if (ret)
+ goto out_unmap_label_cmac;
+
+ n_cmds = 3;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_kic_aes_cmac_kdf(&vcq[1], SYS_REF_TEMP,
+ req.base_key, label_dma,
+ req.key_len, req.label_len,
+ SYS_TYPE_SET(0, CORE_ID_AES));
+ vcq_add_sys_flush(&vcq[2]);
+ } else {
+ n_cmds = 4;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.key_len);
+ vcq_add_kic_aes_cmac_kdf(&vcq[2], SYS_REF_LAST,
+ req.base_key, label_dma,
+ req.key_len, req.label_len,
+ SYS_TYPE_SET(0, CORE_ID_AES));
+ vcq_add_sys_flush(&vcq[3]);
+ }
+
+ ret = cmh_tm_submit_sync_mbx(vcq, n_cmds, 1, MGMT_MBX);
+
+ if (label_buf) {
+ cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE);
+ kfree(label_buf);
+ label_buf = NULL;
+ }
+
+ if (ret)
+ goto out_ref_cmac;
+
+ if (temp) {
+ req.ref = SYS_REF_TEMP;
+ atomic_set(&mgmt_temp_dirty, 1);
+ } else {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ req.ref = *ref_buf;
+ kfree(ref_buf);
+ ref_buf = NULL;
+ }
+
+ if (copy_to_user(argp, &req, sizeof(req))) {
+ /*
+ * The derived-key DS slot was created but its ref never
+ * reached userspace, so the caller cannot free it. Logically
+ * delete the orphaned slot: grant-with-no-access wipes the key
+ * material and clears the CID (temp results use SYS_REF_TEMP
+ * and need no cleanup).
+ */
+ if (!temp) {
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_grant(&vcq[1], req.ref, 0, 0, 0);
+ vcq_add_sys_flush(&vcq[2]);
+ cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+ dev_warn(cmh_dev(),
+ "mgmt: KIC_AES_CMAC_KDF copy_to_user failed, DS slot cleaned up\n");
+ }
+ return -EFAULT;
+ }
+
+ dev_dbg(cmh_dev(),
+ "mgmt: KIC_AES_CMAC_KDF base=0x%llx len=%u flags=0x%x -> ref=0x%llx\n",
+ req.base_key, req.key_len, req.flags, req.ref);
+ return 0;
+
+out_unmap_label_cmac:
+ if (label_buf && !cmh_dma_map_error(label_dma) && label_dma)
+ cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE);
+out_label_cmac:
+ kfree(label_buf);
+out_ref_cmac:
+ if (ref_buf) {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ kfree(ref_buf);
+ }
+ return ret;
+}
+
+/* -- KIC_DKEK_DERIVE ------------------- */
+
+/*
+ * Derive a Key Encryption Key (KEK) from a KIC base key.
+ * Output is tagged CORE_ID_KIC (usable for further derivation only).
+ * host_id=0 means the caller's own host; non-zero requires management
+ * host privilege (eSW enforces this).
+ */
+#define DKEK_VCQ_MAX 4
+
+static int cmh_mgmt_kic_dkek_derive(void __user *argp)
+{
+ struct cmh_ioctl_kic_dkek_derive req;
+ struct vcq_cmd vcq[DKEK_VCQ_MAX];
+ bool temp;
+ u64 *ref_buf = NULL;
+ void *meta_buf = NULL;
+ dma_addr_t ref_dma = DMA_MAPPING_ERROR, meta_dma = DMA_MAPPING_ERROR;
+ unsigned int n_cmds;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.metadata_len > KIC_DKEK_MAX_METADATA)
+ return -EINVAL;
+
+ temp = !!(req.flags & CMH_KIC_FLAG_TEMP);
+
+ if (!temp) {
+ ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+ *ref_buf = 0;
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+ }
+
+ if (req.metadata_len > 0) {
+ meta_buf = kzalloc(req.metadata_len, GFP_KERNEL);
+ if (!meta_buf) {
+ ret = -ENOMEM;
+ goto out_ref_dkek;
+ }
+ if (copy_from_user(meta_buf,
+ u64_to_user_ptr(req.metadata),
+ req.metadata_len)) {
+ ret = -EFAULT;
+ goto out_meta;
+ }
+ meta_dma = cmh_dma_map_single(meta_buf, req.metadata_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(meta_dma)) {
+ ret = -ENOMEM;
+ goto out_meta;
+ }
+ }
+
+ memset(vcq, 0, sizeof(vcq));
+
+ if (temp) {
+ /* Flush MBX to reset temp stack -- see KIC section comment */
+ ret = cmh_tm_flush_mbx(MGMT_MBX);
+ if (ret)
+ goto out_unmap_meta;
+
+ n_cmds = 3;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_kic_dkek_derive(&vcq[1], SYS_REF_TEMP,
+ req.base_key, req.host_id,
+ meta_dma, req.metadata_len);
+ vcq_add_sys_flush(&vcq[2]);
+ } else {
+ n_cmds = 4;
+ vcq_set_header(&vcq[0], n_cmds);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, KIC_KEY_SIZE);
+ vcq_add_kic_dkek_derive(&vcq[2], SYS_REF_LAST,
+ req.base_key, req.host_id,
+ meta_dma, req.metadata_len);
+ vcq_add_sys_flush(&vcq[3]);
+ }
+
+ ret = cmh_tm_submit_sync_mbx(vcq, n_cmds, 1, MGMT_MBX);
+
+ if (meta_buf) {
+ cmh_dma_unmap_single(meta_dma, req.metadata_len,
+ DMA_TO_DEVICE);
+ kfree(meta_buf);
+ meta_buf = NULL;
+ }
+
+ if (ret)
+ goto out_ref_dkek;
+
+ if (temp) {
+ req.ref = SYS_REF_TEMP;
+ atomic_set(&mgmt_temp_dirty, 1);
+ } else {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ req.ref = *ref_buf;
+ kfree(ref_buf);
+ ref_buf = NULL;
+ }
+
+ if (copy_to_user(argp, &req, sizeof(req))) {
+ /*
+ * The derived-key DS slot was created but its ref never
+ * reached userspace, so the caller cannot free it. Logically
+ * delete the orphaned slot: grant-with-no-access wipes the key
+ * material and clears the CID (temp results use SYS_REF_TEMP
+ * and need no cleanup).
+ */
+ if (!temp) {
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_grant(&vcq[1], req.ref, 0, 0, 0);
+ vcq_add_sys_flush(&vcq[2]);
+ cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+ dev_warn(cmh_dev(),
+ "mgmt: KIC_DKEK_DERIVE copy_to_user failed, DS slot cleaned up\n");
+ }
+ return -EFAULT;
+ }
+
+ dev_dbg(cmh_dev(),
+ "mgmt: KIC_DKEK_DERIVE base=0x%llx host=%u meta_len=%u flags=0x%x -> ref=0x%llx\n",
+ req.base_key, req.host_id, req.metadata_len, req.flags,
+ req.ref);
+ return 0;
+
+out_unmap_meta:
+ if (meta_buf && !cmh_dma_map_error(meta_dma) && meta_dma)
+ cmh_dma_unmap_single(meta_dma, req.metadata_len, DMA_TO_DEVICE);
+out_meta:
+ kfree(meta_buf);
+out_ref_dkek:
+ if (ref_buf) {
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ kfree(ref_buf);
+ }
+ return ret;
+}
+
+/* -- KEY_NEW_RANDOM -- DRBG-backed key generation --- */
+
+/*
+ * Allocate a new datastore slot and fill it with DRBG-generated
+ * random key material in a single atomic VCQ submission:
+ *
+ * [0] SYS header(5)
+ * [1] SYS_CMD_NEW -- allocate DS slot (CMH eSW writes ref)
+ * [2] DRBG_CMD_DATASTORE(SYS_REF_LAST) -- fill with random data
+ * [3] DRBG flush -- release DRBG core ownership
+ * [4] SYS flush
+ *
+ * The DRBG must be configured before this ioctl is used.
+ * Reuses struct cmh_ioctl_key_new (ds_type, flags, cid, len, ref).
+ */
+#define DRBG_KEYGEN_VCQ_CMDS 5
+
+static int cmh_mgmt_key_new_random(void __user *argp)
+{
+ struct cmh_ioctl_key_new req;
+ struct vcq_cmd vcq[DRBG_KEYGEN_VCQ_CMDS];
+ u64 *ref_buf;
+ dma_addr_t ref_dma;
+ u32 core_id, sys_type;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.len)
+ return -EINVAL;
+
+ core_id = cmh_ds_type_to_core_id(req.ds_type);
+ if (core_id == CORE_ID_NUM)
+ return -EINVAL;
+ sys_type = SYS_TYPE_SET(req.flags, core_id);
+
+ ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+
+ *ref_buf = 0;
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], DRBG_KEYGEN_VCQ_CMDS);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.len);
+ vcq_add_drbg_datastore(&vcq[2], SYS_REF_LAST, req.len, sys_type);
+ vcq_add_flush(&vcq[3], CORE_ID_DRBG);
+ vcq_add_sys_flush(&vcq[4]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, DRBG_KEYGEN_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf), DMA_FROM_DEVICE);
+
+ if (ret) {
+ kfree(ref_buf);
+ return ret;
+ }
+
+ req.ref = *ref_buf;
+ kfree(ref_buf);
+
+ if (copy_to_user(argp, &req, sizeof(req))) {
+ /*
+ * The DS slot was created but its ref never reached
+ * userspace, so the caller cannot free it. Logically delete
+ * the orphaned slot: grant-with-no-access wipes the key
+ * material and clears the CID (it does not reclaim the stack
+ * space -- only a datastore reset does).
+ */
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_grant(&vcq[1], req.ref, 0, 0, 0);
+ vcq_add_sys_flush(&vcq[2]);
+ cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+ dev_warn(cmh_dev(),
+ "mgmt: KEY_NEW_RANDOM copy_to_user failed, DS slot cleaned up\n");
+ return -EFAULT;
+ }
+
+ dev_dbg(cmh_dev(),
+ "mgmt: KEY_NEW_RANDOM cid=0x%llx len=%u type=0x%x -> ref=0x%llx\n",
+ req.cid, req.len, sys_type, req.ref);
+ return 0;
+}
+
+#define EAC_VCQ_CMDS 3 /* header + EAC_READ + flush */
+
+static long cmh_mgmt_eac_read(void __user *argp)
+{
+ struct cmh_ioctl_eac_read req;
+ struct eac_read_rsp *rsp;
+ struct vcq_cmd vcq[EAC_VCQ_CMDS];
+ dma_addr_t rsp_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved != 0)
+ return -EINVAL;
+ if (req.__pad != 0)
+ return -EINVAL;
+
+ /*
+ * Zero the response: the eSW writes the fields it populates, but a
+ * partial/aborted read must not copy uninitialised heap back to
+ * user space.
+ */
+ rsp = kzalloc_obj(*rsp, GFP_KERNEL);
+ if (!rsp)
+ return -ENOMEM;
+
+ rsp_dma = cmh_dma_map_single(rsp, sizeof(*rsp), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rsp_dma)) {
+ kfree(rsp);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], EAC_VCQ_CMDS);
+ vcq_add_eac_read(&vcq[1], rsp_dma, sizeof(*rsp));
+ vcq_add_flush(&vcq[2], CORE_ID_EAC);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, EAC_VCQ_CMDS, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(rsp_dma, sizeof(*rsp), DMA_FROM_DEVICE);
+
+ if (ret) {
+ kfree(rsp);
+ return ret;
+ }
+
+ /* Copy response fields into ioctl struct */
+ req.mailbox_notification = rsp->mailbox_notification;
+ req.hw_error = rsp->hw_error;
+ req.hw_nmi = rsp->hw_nmi;
+ req.hw_panic = rsp->hw_panic;
+ req.safety_fatal = rsp->safety_fatal;
+ req.safety_notification = rsp->safety_notification;
+ req.sw_info0 = rsp->sw_info0;
+ req.sw_info1 = rsp->sw_info1;
+ memcpy(req.sram_bank_errors, rsp->sram_bank_errors,
+ sizeof(req.sram_bank_errors));
+ req.__pad = 0;
+
+ kfree(rsp);
+
+ if (copy_to_user(argp, &req, sizeof(req)))
+ return -EFAULT;
+
+ return 0;
+}
+
+/* -- DRBG CONFIG (management) ------------ */
+
+#define DRBG_CONFIG_VCQ_CMDS 4 /* header + RESET + CONFIG + flush */
+
+static long cmh_mgmt_drbg_config(void __user *argp)
+{
+ struct cmh_ioctl_drbg_config req;
+ struct vcq_cmd vcq[DRBG_CONFIG_VCQ_CMDS];
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved != 0)
+ return -EINVAL;
+ if (req.entropy_ratio > 3)
+ return -EINVAL;
+ if (req.security_strength != CMH_DRBG_STRENGTH_128 &&
+ req.security_strength != CMH_DRBG_STRENGTH_256)
+ return -EINVAL;
+
+ vcq_set_header(&vcq[0], DRBG_CONFIG_VCQ_CMDS);
+ vcq_add_drbg_reset(&vcq[1]);
+ vcq_add_drbg_config(&vcq[2], req.entropy_ratio,
+ req.security_strength);
+ vcq_add_flush(&vcq[3], CORE_ID_DRBG);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, DRBG_CONFIG_VCQ_CMDS, 1, MGMT_MBX);
+ if (ret)
+ dev_warn(cmh_dev(), "mgmt: DRBG CONFIG failed (rc=%d)\n", ret);
+ else
+ dev_info(cmh_dev(), "mgmt: DRBG configured (ratio=%u strength=0x%x)\n",
+ req.entropy_ratio, req.security_strength);
+
+ return ret;
+}
+
+/* -- ioctl dispatch ------------------------ */
+
+/*
+ * PKE, SM2, and PQC ioctls use device-internal temporary storage for
+ * intermediate results. Residual allocations in the per-mailbox temp
+ * store (left by prior operations that targeted SYS_REF_TEMP) reduce
+ * the space available and can cause the device to return ENOMEM.
+ *
+ * Flush the mailbox before these operations to reset the temp store,
+ * but ONLY when the store is actually dirty (mgmt_temp_dirty flag).
+ * Unconditional flushing would kill in-flight command queues from
+ * concurrent callers on the same mailbox -- MBX_COMMAND_FLUSH
+ * terminates any executing queue with -EPIPE and discards all queued
+ * submissions.
+ *
+ * The conditional flush is safe: PKE/SM2/PQC ioctls do not consume
+ * SYS_REF_TEMP from a prior ioctl (unlike DS_EXPORT/DS_IMPORT which
+ * may reference a temp key produced by a preceding derivation), so
+ * clearing the temp store before them loses no needed state.
+ */
+static inline bool cmh_mgmt_needs_temp_flush(unsigned int cmd)
+{
+ unsigned int nr = _IOC_NR(cmd);
+
+ /*
+ * Range invariant: all PKE/SM2/PQC ioctls must have consecutive
+ * NR values between PKE_RSA_ENC (0x10) and SM2_ENC_HASH (0x37).
+ * If a new ioctl is added outside this range, update the bounds
+ * and adjust these assertions.
+ */
+ BUILD_BUG_ON(_IOC_NR(CMH_IOCTL_PKE_RSA_ENC) != 0x10);
+ BUILD_BUG_ON(_IOC_NR(CMH_IOCTL_SM2_ENC_HASH) != 0x37);
+
+ return nr >= _IOC_NR(CMH_IOCTL_PKE_RSA_ENC) &&
+ nr <= _IOC_NR(CMH_IOCTL_SM2_ENC_HASH);
+}
+
+static long cmh_mgmt_ioctl(struct file *file, unsigned int cmd,
+ unsigned long arg)
+{
+ void __user *argp = (void __user *)arg;
+ int ret;
+
+ if (cmh_mgmt_needs_temp_flush(cmd) &&
+ atomic_xchg(&mgmt_temp_dirty, 0)) {
+ ret = cmh_tm_flush_mbx(MGMT_MBX);
+ if (ret)
+ return ret;
+ }
+
+ switch (cmd) {
+ case CMH_IOCTL_KEY_NEW:
+ return cmh_mgmt_key_new(argp);
+ case CMH_IOCTL_KEY_WRITE:
+ return cmh_mgmt_key_write(argp);
+ case CMH_IOCTL_KEY_READ:
+ return cmh_mgmt_key_read(argp);
+ case CMH_IOCTL_KEY_FIND:
+ return cmh_mgmt_key_find(argp);
+ case CMH_IOCTL_KEY_GRANT:
+ return cmh_mgmt_key_grant(argp, false);
+ case CMH_IOCTL_KEY_DELETE:
+ return cmh_mgmt_key_grant(argp, true);
+ case CMH_IOCTL_DS_EXPORT:
+ return cmh_mgmt_ds_export(argp);
+ case CMH_IOCTL_DS_IMPORT:
+ return cmh_mgmt_ds_import(argp);
+ case CMH_IOCTL_KIC_HKDF1:
+ return cmh_mgmt_kic_hkdf1(argp);
+ case CMH_IOCTL_KIC_HKDF2:
+ return cmh_mgmt_kic_hkdf2(argp);
+ case CMH_IOCTL_KEY_NEW_RANDOM:
+ return cmh_mgmt_key_new_random(argp);
+ case CMH_IOCTL_KIC_AES_CMAC_KDF:
+ return cmh_mgmt_kic_aes_cmac_kdf(argp);
+ case CMH_IOCTL_KIC_DKEK_DERIVE:
+ return cmh_mgmt_kic_dkek_derive(argp);
+ case CMH_IOCTL_KEY_LIST:
+ return cmh_mgmt_key_list(argp);
+ case CMH_IOCTL_EAC_READ:
+ return cmh_mgmt_eac_read(argp);
+ /* PKE operations */
+ case CMH_IOCTL_PKE_RSA_ENC:
+ return cmh_mgmt_pke_rsa_enc(argp);
+ case CMH_IOCTL_PKE_RSA_DEC:
+ return cmh_mgmt_pke_rsa_dec(argp);
+ case CMH_IOCTL_PKE_RSA_CRT_DEC:
+ return cmh_mgmt_pke_rsa_crt_dec(argp);
+ case CMH_IOCTL_PKE_RSA_KEYGEN:
+ return cmh_mgmt_pke_rsa_keygen(argp);
+ case CMH_IOCTL_PKE_ECDSA_SIGN:
+ return cmh_mgmt_pke_ecdsa_sign(argp);
+ case CMH_IOCTL_PKE_ECDH:
+ return cmh_mgmt_pke_ecdh(argp);
+ case CMH_IOCTL_PKE_ECDH_KEYGEN:
+ return cmh_mgmt_pke_ecdh_keygen(argp);
+ case CMH_IOCTL_PKE_EDDSA_SIGN:
+ return cmh_mgmt_pke_eddsa_sign(argp);
+ case CMH_IOCTL_PKE_EDDSA_VERIFY:
+ return cmh_mgmt_pke_eddsa_verify(argp);
+ case CMH_IOCTL_PKE_EC_KEYGEN:
+ return cmh_mgmt_pke_ec_keygen(argp);
+ case CMH_IOCTL_PKE_EC_PUBGEN:
+ return cmh_mgmt_pke_ec_pubgen(argp);
+ case CMH_IOCTL_PKE_EDDSA_KEYGEN_SCA:
+ return cmh_mgmt_pke_eddsa_keygen_sca(argp);
+ /* SM2 operations */
+ case CMH_IOCTL_SM2_ECDH_KEYGEN:
+ return cmh_mgmt_sm2_ecdh_keygen(argp);
+ case CMH_IOCTL_SM2_ECDH:
+ return cmh_mgmt_sm2_ecdh(argp);
+ case CMH_IOCTL_SM2_DEC_POINT:
+ return cmh_mgmt_sm2_dec_point(argp);
+ case CMH_IOCTL_SM2_ENC_POINT:
+ return cmh_mgmt_sm2_enc_point(argp);
+ case CMH_IOCTL_SM2_ID_DIGEST:
+ return cmh_mgmt_sm2_id_digest(argp);
+ case CMH_IOCTL_SM2_ECDH_HASH:
+ return cmh_mgmt_sm2_ecdh_hash(argp);
+ case CMH_IOCTL_SM2_DEC_HASH:
+ return cmh_mgmt_sm2_dec_hash(argp);
+ case CMH_IOCTL_SM2_ENC_HASH:
+ return cmh_mgmt_sm2_enc_hash(argp);
+ /* PQC operations */
+ case CMH_IOCTL_ML_KEM_KEYGEN:
+ return cmh_mgmt_ml_kem_keygen(argp);
+ case CMH_IOCTL_ML_KEM_ENC:
+ return cmh_mgmt_ml_kem_enc(argp);
+ case CMH_IOCTL_ML_KEM_DEC:
+ return cmh_mgmt_ml_kem_dec(argp);
+ case CMH_IOCTL_ML_DSA_KEYGEN:
+ return cmh_mgmt_ml_dsa_keygen(argp);
+ case CMH_IOCTL_ML_DSA_SIGN:
+ return cmh_mgmt_ml_dsa_sign(argp);
+ case CMH_IOCTL_SLHDSA_KEYGEN:
+ return cmh_mgmt_slhdsa_keygen(argp);
+ case CMH_IOCTL_SLHDSA_SIGN:
+ return cmh_mgmt_slhdsa_sign(argp);
+ case CMH_IOCTL_SLHDSA_SIGN_PREHASH:
+ return cmh_mgmt_slhdsa_sign_prehash(argp);
+ /* DRBG management */
+ case CMH_IOCTL_DRBG_CONFIG:
+ return cmh_mgmt_drbg_config(argp);
+ default:
+ return -ENOTTY;
+ }
+}
+
+/* -- File operations ----------------------- */
+
+/*
+ * Capability is checked once at open time. A privileged process may
+ * pass the resulting fd to an unprivileged helper -- this delegation
+ * model is intentional and mirrors /dev/kvm, /dev/loop-control, etc.
+ */
+static int cmh_mgmt_open(struct inode *inode, struct file *file)
+{
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
+
+ return 0;
+}
+
+static const struct file_operations cmh_mgmt_fops = {
+ .owner = THIS_MODULE,
+ .open = cmh_mgmt_open,
+ .unlocked_ioctl = cmh_mgmt_ioctl,
+ .compat_ioctl = compat_ptr_ioctl,
+};
+
+static struct miscdevice cmh_mgmt_dev = {
+ .minor = MISC_DYNAMIC_MINOR,
+ .name = "cmh_mgmt",
+ .fops = &cmh_mgmt_fops,
+ .mode = 0660,
+};
+
+static bool cmh_mgmt_registered;
+
+/**
+ * cmh_mgmt_register() - Register the /dev/cmh_mgmt misc device
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_register(void)
+{
+ int ret;
+
+ /*
+ * ABI size guards -- catch silent layout changes at compile time.
+ * All ioctl structs use only __u32 and __u64 with explicit padding,
+ * guaranteeing identical layout on 32-bit and 64-bit (compat_ptr_ioctl).
+ */
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_new) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_write) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_read) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_find) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_list) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_grant) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_ds_export) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_ds_import) != 24);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_kic_hkdf1) != 48);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_kic_hkdf2) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_kic_aes_cmac_kdf) != 48);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_kic_dkek_derive) != 48);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_rsa_enc) != 48);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_rsa_dec) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_rsa_crt_dec) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_rsa_keygen) != 64);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ecdsa_sign) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ecdh) != 48);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ecdh_keygen) != 24);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_eddsa_sign) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_eddsa_verify) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ec_keygen) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ec_pubgen) != 24);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_eddsa_keygen_sca) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_kem_keygen) != 64);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_kem_enc) != 64);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_kem_dec) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_dsa_keygen) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_dsa_sign) != 48);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_slhdsa_keygen) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_slhdsa_sign) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_slhdsa_sign_prehash) != 64);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_ecdh_keygen) != 24);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_ecdh) != 56);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_dec_point) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_enc_point) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_id_digest) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_ecdh_hash) != 40);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_dec_hash) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_enc_hash) != 32);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_eac_read) != 64);
+ BUILD_BUG_ON(sizeof(struct cmh_ioctl_drbg_config) != 16);
+
+ ret = misc_register(&cmh_mgmt_dev);
+ if (ret) {
+ dev_err(cmh_dev(), "mgmt: misc_register failed (rc=%d)\n", ret);
+ return ret;
+ }
+
+ cmh_mgmt_registered = true;
+ dev_info(cmh_dev(), "mgmt: registered /dev/cmh_mgmt\n");
+ return 0;
+}
+
+/**
+ * cmh_mgmt_unregister() - Unregister the /dev/cmh_mgmt misc device
+ */
+void cmh_mgmt_unregister(void)
+{
+ if (!cmh_mgmt_registered)
+ return;
+
+ misc_deregister(&cmh_mgmt_dev);
+ cmh_mgmt_registered = false;
+ dev_info(cmh_dev(), "mgmt: unregistered /dev/cmh_mgmt\n");
+}
diff --git a/drivers/crypto/cmh/cmh_mgmt_pke.c b/drivers/crypto/cmh/cmh_mgmt_pke.c
new file mode 100644
index 000000000000..55e6a8ba4c75
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_mgmt_pke.c
@@ -0,0 +1,1134 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH -- PKE ioctl handlers for /dev/cmh_mgmt
+ *
+ * RSA encrypt/decrypt/CRT/keygen, ECDSA sign, ECDH/keygen,
+ * EdDSA sign/verify, EC keygen/pubgen.
+ *
+ * Split from cmh_mgmt.c for maintainability.
+ */
+
+#include <linux/kernel.h>
+#include <linux/uaccess.h>
+#include <linux/slab.h>
+#include <linux/overflow.h>
+
+#include "cmh_mgmt.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_key.h"
+#include "cmh_dma.h"
+#include "cmh_config.h"
+#include "cmh_pke.h"
+#include "cmh_pke_abi.h"
+#include "cmh_sys_abi.h"
+#include <uapi/linux/cmh_mgmt_ioctl.h>
+
+#include <crypto/utils.h>
+
+/* -- PKE ioctl helpers ------------------- */
+
+/*
+ * Maximum PKE operand size: 512 bytes (RSA 4096-bit),
+ * or 2 * 68 = 136 bytes (P-521 coordinate pair).
+ */
+#define PKE_MAX_OPERAND 512
+
+/* Validate curve ID and return coordinate length; 0 = invalid */
+static u32 cmh_pke_validate_curve(u32 curve)
+{
+ return pke_curve_clen(curve);
+}
+
+/**
+ * cmh_mgmt_pke_rsa_enc() - Handle CMH_MGMT_IOC_PKE_RSA_ENC ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_rsa_enc(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_rsa_enc req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 n_len, e_padded;
+ u8 *e_buf, *n_buf, *m_buf, *c_buf;
+ dma_addr_t e_dma, n_dma, m_dma, c_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (req.bits < PKE_RSA_MIN_BITS || req.bits > PKE_RSA_MAX_BITS)
+ return -EINVAL;
+ if (!req.e_len || req.e_len > PKE_MAX_OPERAND)
+ return -EINVAL;
+
+ n_len = req.bits / 8;
+ e_padded = ALIGN(req.e_len, 4);
+
+ e_buf = kzalloc(e_padded, GFP_KERNEL);
+ n_buf = kmalloc(n_len, GFP_KERNEL);
+ m_buf = kmalloc(n_len, GFP_KERNEL);
+ c_buf = kzalloc(n_len, GFP_KERNEL);
+ if (!e_buf || !n_buf || !m_buf || !c_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ /* Right-align exponent in zero-padded buffer for DMA alignment */
+ if (copy_from_user(e_buf + e_padded - req.e_len,
+ u64_to_user_ptr(req.e), req.e_len) ||
+ copy_from_user(n_buf, u64_to_user_ptr(req.n), n_len) ||
+ copy_from_user(m_buf, u64_to_user_ptr(req.input), n_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ e_dma = cmh_dma_map_single(e_buf, e_padded, DMA_TO_DEVICE);
+ n_dma = cmh_dma_map_single(n_buf, n_len, DMA_TO_DEVICE);
+ m_dma = cmh_dma_map_single(m_buf, n_len, DMA_TO_DEVICE);
+ c_dma = cmh_dma_map_single(c_buf, n_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(e_dma) || cmh_dma_map_error(n_dma) ||
+ cmh_dma_map_error(m_dma) || cmh_dma_map_error(c_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_rsa_enc(&vcq[1], pke_cid, req.bits, e_padded,
+ e_dma, n_dma, m_dma, c_dma, PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(c_dma))
+ cmh_dma_unmap_single(c_dma, n_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, n_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(n_dma))
+ cmh_dma_unmap_single(n_dma, n_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(e_dma))
+ cmh_dma_unmap_single(e_dma, e_padded, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.output), c_buf, n_len))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(c_buf);
+ kfree_sensitive(m_buf);
+ kfree(n_buf);
+ kfree(e_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_rsa_dec() - Handle CMH_MGMT_IOC_PKE_RSA_DEC ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_rsa_dec(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_rsa_dec req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 n_len, e_padded;
+ u8 *e_buf, *n_buf, *c_buf, *m_buf;
+ dma_addr_t e_dma, n_dma, c_dma, m_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (req.bits < PKE_RSA_MIN_BITS || req.bits > PKE_RSA_MAX_BITS)
+ return -EINVAL;
+ if (!req.e_len || req.e_len > PKE_MAX_OPERAND)
+ return -EINVAL;
+
+ n_len = req.bits / 8;
+ e_padded = ALIGN(req.e_len, 4);
+
+ e_buf = kzalloc(e_padded, GFP_KERNEL);
+ n_buf = kmalloc(n_len, GFP_KERNEL);
+ c_buf = kmalloc(n_len, GFP_KERNEL);
+ m_buf = kzalloc(n_len, GFP_KERNEL);
+ if (!e_buf || !n_buf || !c_buf || !m_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ /* Right-align exponent in zero-padded buffer for DMA alignment */
+ if (copy_from_user(e_buf + e_padded - req.e_len,
+ u64_to_user_ptr(req.e), req.e_len) ||
+ copy_from_user(n_buf, u64_to_user_ptr(req.n), n_len) ||
+ copy_from_user(c_buf, u64_to_user_ptr(req.input), n_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ e_dma = cmh_dma_map_single(e_buf, e_padded, DMA_TO_DEVICE);
+ n_dma = cmh_dma_map_single(n_buf, n_len, DMA_TO_DEVICE);
+ c_dma = cmh_dma_map_single(c_buf, n_len, DMA_TO_DEVICE);
+ m_dma = cmh_dma_map_single(m_buf, n_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(e_dma) || cmh_dma_map_error(n_dma) ||
+ cmh_dma_map_error(c_dma) || cmh_dma_map_error(m_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_rsa_dec(&vcq[1], pke_cid, req.bits, e_padded,
+ e_dma, n_dma, c_dma, m_dma, req.key_ref,
+ PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, n_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(c_dma))
+ cmh_dma_unmap_single(c_dma, n_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(n_dma))
+ cmh_dma_unmap_single(n_dma, n_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(e_dma))
+ cmh_dma_unmap_single(e_dma, e_padded, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.output), m_buf, n_len))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(m_buf);
+ kfree(c_buf);
+ kfree(n_buf);
+ kfree(e_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_rsa_crt_dec() - Handle CMH_MGMT_IOC_PKE_RSA_CRT_DEC ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_rsa_crt_dec(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_rsa_crt_dec req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 n_len, e_padded;
+ u8 *e_buf, *n_buf, *c_buf, *m_buf;
+ dma_addr_t e_dma, n_dma, c_dma, m_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (req.bits < PKE_RSA_MIN_BITS || req.bits > PKE_RSA_MAX_BITS)
+ return -EINVAL;
+ if (!req.e_len || req.e_len > PKE_MAX_OPERAND)
+ return -EINVAL;
+
+ n_len = req.bits / 8;
+ e_padded = ALIGN(req.e_len, 4);
+
+ e_buf = kzalloc(e_padded, GFP_KERNEL);
+ n_buf = kmalloc(n_len, GFP_KERNEL);
+ c_buf = kmalloc(n_len, GFP_KERNEL);
+ m_buf = kzalloc(n_len, GFP_KERNEL);
+ if (!e_buf || !n_buf || !c_buf || !m_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ /* Right-align exponent in zero-padded buffer for DMA alignment */
+ if (copy_from_user(e_buf + e_padded - req.e_len,
+ u64_to_user_ptr(req.e), req.e_len) ||
+ copy_from_user(n_buf, u64_to_user_ptr(req.n), n_len) ||
+ copy_from_user(c_buf, u64_to_user_ptr(req.input), n_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ e_dma = cmh_dma_map_single(e_buf, e_padded, DMA_TO_DEVICE);
+ n_dma = cmh_dma_map_single(n_buf, n_len, DMA_TO_DEVICE);
+ c_dma = cmh_dma_map_single(c_buf, n_len, DMA_TO_DEVICE);
+ m_dma = cmh_dma_map_single(m_buf, n_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(e_dma) || cmh_dma_map_error(n_dma) ||
+ cmh_dma_map_error(c_dma) || cmh_dma_map_error(m_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_rsa_crt_dec(&vcq[1], pke_cid, req.bits, e_padded,
+ e_dma, n_dma, c_dma, m_dma, req.crt_ref,
+ PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, n_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(c_dma))
+ cmh_dma_unmap_single(c_dma, n_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(n_dma))
+ cmh_dma_unmap_single(n_dma, n_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(e_dma))
+ cmh_dma_unmap_single(e_dma, e_padded, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.output), m_buf, n_len))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(m_buf);
+ kfree(c_buf);
+ kfree(n_buf);
+ kfree(e_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_rsa_keygen() - Handle CMH_MGMT_IOC_PKE_RSA_KEYGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_rsa_keygen(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_rsa_keygen req;
+ /*
+ * When has_crt, we use a two-VCQ approach (CRI pattern):
+ * VCQ #1: header + SYS_NEW(d) + SYS_NEW(crt) + SYS_FLUSH (4 slots)
+ * VCQ #2: header + RSA_KEYGEN + PKE_FLUSH + SYS_FLUSH (4 slots)
+ * Without CRT, single VCQ:
+ * header + SYS_NEW(d) + RSA_KEYGEN + PKE_FLUSH + SYS_FLUSH (5 slots)
+ */
+ struct vcq_cmd vcq[5];
+ u32 n_len, e_padded, key_flags, d_ds_len, crt_ds_len;
+ u8 *e_buf, *n_buf;
+ u64 *d_ref_buf, *crt_ref_buf;
+ dma_addr_t e_dma, n_dma, d_ref_dma, crt_ref_dma;
+ int idx, ret;
+ bool has_crt, is_sca;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.bits < PKE_RSA_MIN_BITS || req.bits > PKE_RSA_MAX_BITS)
+ return -EINVAL;
+ if (!req.e_len || req.e_len > PKE_MAX_OPERAND)
+ return -EINVAL;
+ if (req.flags & ~CMH_FLAG_MASK)
+ return -EINVAL;
+
+ n_len = req.bits / 8;
+ has_crt = (req.crt_cid != 0);
+ e_padded = ALIGN(req.e_len, 4);
+ key_flags = req.flags & CMH_FLAG_MASK;
+ is_sca = !!(req.flags & CMH_FLAG_SCA);
+
+ /*
+ * SCA keys are stored in 2 shares -- DS allocation must be enlarged.
+ * CRI reference formulas: cmh_pke_rsa_private_key_size().
+ */
+ if (is_sca) {
+ d_ds_len = n_len * 2;
+ crt_ds_len = (7 + n_len / 2) * 4;
+ } else {
+ d_ds_len = n_len;
+ crt_ds_len = 5 * (n_len / 2);
+ }
+
+ e_buf = kzalloc(e_padded, GFP_KERNEL);
+ n_buf = kzalloc(n_len, GFP_KERNEL);
+ d_ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ crt_ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ if (!e_buf || !n_buf || !d_ref_buf || !crt_ref_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(e_buf + e_padded - req.e_len,
+ u64_to_user_ptr(req.e), req.e_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ e_dma = cmh_dma_map_single(e_buf, e_padded, DMA_TO_DEVICE);
+ n_dma = cmh_dma_map_single(n_buf, n_len, DMA_FROM_DEVICE);
+ d_ref_dma = cmh_dma_map_single(d_ref_buf, sizeof(u64), DMA_FROM_DEVICE);
+ crt_ref_dma = cmh_dma_map_single(crt_ref_buf, sizeof(u64),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(e_dma) || cmh_dma_map_error(n_dma) ||
+ cmh_dma_map_error(d_ref_dma) || cmh_dma_map_error(crt_ref_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ if (has_crt) {
+ /*
+ * Two-VCQ approach (CRI pattern): SYS_REF_LAST can only
+ * refer to the most recently created DS object. When we
+ * need both d and crt refs, we must first allocate DS
+ * objects, read back the opaque refs, then pass them by
+ * value in the keygen VCQ.
+ *
+ * VCQ #1: allocate both DS objects.
+ */
+ idx = 0;
+ vcq_set_header(&vcq[idx++], 4);
+ vcq_add_sys_new(&vcq[idx++], req.d_cid, d_ref_dma, d_ds_len);
+ vcq_add_sys_new(&vcq[idx++], req.crt_cid, crt_ref_dma,
+ crt_ds_len);
+ vcq_add_sys_flush(&vcq[idx++]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 4, 1, MGMT_MBX);
+ if (ret)
+ goto out_unmap;
+
+ /* Sync DMA so we can read back the opaque refs */
+ cmh_dma_unmap_single(d_ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ cmh_dma_unmap_single(crt_ref_dma, sizeof(u64),
+ DMA_FROM_DEVICE);
+ d_ref_dma = 0;
+ crt_ref_dma = 0;
+
+ /*
+ * VCQ #2: keygen with resolved refs.
+ */
+ idx = 0;
+ memset(vcq, 0, sizeof(vcq));
+ vcq_set_header(&vcq[idx++], 4);
+
+ vcq[idx].magic = VCQ_CMD_MAGIC;
+ vcq[idx].id = VCQ_CMD_ID(pke_cid, PKE_SWAP_FLAGS, 1,
+ PKE_CMD_RSA_KEYGEN);
+ vcq[idx].hwc.pke.cmd_rsa_keygen.bits = req.bits;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.e = e_dma;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.n = n_dma;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.d = *d_ref_buf;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.d_type =
+ SYS_TYPE_SET(key_flags, CORE_ID_PKE);
+ vcq[idx].hwc.pke.cmd_rsa_keygen.crt = *crt_ref_buf;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.crt_type =
+ SYS_TYPE_SET(key_flags, CORE_ID_PKE);
+ idx++;
+
+ vcq_add_pke_flush(&vcq[idx++], pke_cid);
+ vcq_add_sys_flush(&vcq[idx++]);
+
+ ret = cmh_tm_submit_sync_tmo(vcq, 4, 1, MGMT_MBX,
+ cmh_tm_slow_op_timeout_jiffies());
+ } else {
+ /*
+ * Single-VCQ: only d, so SYS_REF_LAST is unambiguous.
+ */
+ idx = 0;
+ vcq_set_header(&vcq[idx++], 5);
+ vcq_add_sys_new(&vcq[idx++], req.d_cid, d_ref_dma, d_ds_len);
+
+ vcq[idx].magic = VCQ_CMD_MAGIC;
+ vcq[idx].id = VCQ_CMD_ID(pke_cid, PKE_SWAP_FLAGS, 1,
+ PKE_CMD_RSA_KEYGEN);
+ vcq[idx].hwc.pke.cmd_rsa_keygen.bits = req.bits;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.e = e_dma;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.n = n_dma;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.d = SYS_REF_LAST;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.d_type =
+ SYS_TYPE_SET(key_flags, CORE_ID_PKE);
+ vcq[idx].hwc.pke.cmd_rsa_keygen.crt = SYS_REF_NONE;
+ vcq[idx].hwc.pke.cmd_rsa_keygen.crt_type = 0;
+ idx++;
+
+ vcq_add_pke_flush(&vcq[idx++], pke_cid);
+ vcq_add_sys_flush(&vcq[idx++]);
+
+ ret = cmh_tm_submit_sync_tmo(vcq, 5, 1, MGMT_MBX,
+ cmh_tm_slow_op_timeout_jiffies());
+ }
+
+out_unmap:
+ if (crt_ref_dma && !cmh_dma_map_error(crt_ref_dma))
+ cmh_dma_unmap_single(crt_ref_dma, sizeof(u64),
+ DMA_FROM_DEVICE);
+ if (d_ref_dma && !cmh_dma_map_error(d_ref_dma))
+ cmh_dma_unmap_single(d_ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(n_dma))
+ cmh_dma_unmap_single(n_dma, n_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(e_dma))
+ cmh_dma_unmap_single(e_dma, e_padded, DMA_TO_DEVICE);
+
+ if (!ret) {
+ /* Copy generated modulus and refs back */
+ if (copy_to_user(u64_to_user_ptr(req.n), n_buf, n_len)) {
+ ret = -EFAULT;
+ } else {
+ req.d_ref = *d_ref_buf;
+ req.crt_ref = has_crt ? *crt_ref_buf : 0;
+ if (copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ }
+
+ if (ret == -EFAULT) {
+ /*
+ * The d (and CRT) DS slots were created but their
+ * refs never reached userspace, so the caller cannot
+ * free them. Logically delete the orphaned slots.
+ */
+ idx = 0;
+ vcq_set_header(&vcq[idx++], has_crt ? 4 : 3);
+ vcq_add_sys_grant(&vcq[idx++], *d_ref_buf, 0, 0, 0);
+ if (has_crt)
+ vcq_add_sys_grant(&vcq[idx++], *crt_ref_buf,
+ 0, 0, 0);
+ vcq_add_sys_flush(&vcq[idx++]);
+ cmh_tm_submit_sync_mbx(vcq, idx, 1, MGMT_MBX);
+ dev_warn(cmh_dev(),
+ "mgmt: RSA_KEYGEN copy_to_user failed, DS slots cleaned up\n");
+ }
+ }
+
+out_free:
+ kfree(crt_ref_buf);
+ kfree(d_ref_buf);
+ kfree(n_buf);
+ kfree(e_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_ecdsa_sign() - Handle CMH_MGMT_IOC_PKE_ECDSA_SIGN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_ecdsa_sign(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_ecdsa_sign req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 clen, sig_len, dig_map_len;
+ u8 *dig_buf, *sig_buf;
+ dma_addr_t dig_dma, sig_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ clen = cmh_pke_validate_curve(req.curve);
+ if (!clen || !req.digest_len ||
+ req.digest_len > CMH_MGMT_MAX_DATA_LEN)
+ return -EINVAL;
+
+ sig_len = 2 * clen;
+
+ /*
+ * eSW requires digest_len >= clen. A hash shorter than clen must
+ * be LEFT-padded with zeros (right-aligned) so it keeps its value
+ * as a big-endian integer -- trailing zeros would multiply it by
+ * 256^k. When digest_len >= clen the offset is 0 (unchanged) and
+ * the eSW applies the ECDSA bits2int leftmost-bits truncation.
+ */
+ dig_map_len = max_t(u32, req.digest_len, clen);
+
+ dig_buf = kzalloc(dig_map_len, GFP_KERNEL);
+ sig_buf = kzalloc(sig_len, GFP_KERNEL);
+ if (!dig_buf || !sig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(dig_buf + (dig_map_len - req.digest_len),
+ u64_to_user_ptr(req.digest), req.digest_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ dig_dma = cmh_dma_map_single(dig_buf, dig_map_len, DMA_TO_DEVICE);
+ sig_dma = cmh_dma_map_single(sig_buf, sig_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(dig_dma) || cmh_dma_map_error(sig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_ecdsa_sign(&vcq[1], pke_cid, req.curve, clen,
+ dig_dma, sig_dma, req.key_ref,
+ dig_map_len, pke_swap_flags(req.curve));
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(dig_dma))
+ cmh_dma_unmap_single(dig_dma, dig_map_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.signature),
+ sig_buf, sig_len))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(sig_buf);
+ kfree(dig_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_ecdh() - Handle CMH_MGMT_IOC_PKE_ECDH ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_ecdh(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_ecdh req;
+ /* Phase 1: hdr + sys_new + pke_ecdh + pke_flush; reused for Phase 2 */
+ struct vcq_cmd vcq[4];
+ u32 clen, swap, ss_type;
+ u8 *peer_buf, *ss_buf;
+ u64 *ref_buf;
+ dma_addr_t peer_dma, ss_dma, ref_dma;
+ int ret, idx;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ clen = cmh_pke_validate_curve(req.curve);
+ if (!clen)
+ return -EINVAL;
+
+ swap = PKE_SWAP_FLAGS;
+ ss_type = SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_PKE);
+
+ peer_buf = kmalloc(clen, GFP_KERNEL);
+ ss_buf = kzalloc(clen, GFP_KERNEL);
+ ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ if (!peer_buf || !ss_buf || !ref_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(peer_buf, u64_to_user_ptr(req.peer_key_x), clen)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ peer_dma = cmh_dma_map_single(peer_buf, clen, DMA_TO_DEVICE);
+ ss_dma = cmh_dma_map_single(ss_buf, clen, DMA_FROM_DEVICE);
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(peer_dma) || cmh_dma_map_error(ss_dma) ||
+ cmh_dma_map_error(ref_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ idx = 0;
+ vcq_set_header(&vcq[idx++], 4);
+ vcq_add_sys_new(&vcq[idx++], 0, ref_dma, clen);
+ vcq_add_pke_ecdh(&vcq[idx++], pke_cid, req.curve, clen, clen,
+ ss_type, peer_dma, req.key_ref,
+ SYS_REF_LAST, swap);
+ vcq_add_pke_flush(&vcq[idx++], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 4, 1, MGMT_MBX);
+ if (ret)
+ goto out_unmap;
+
+ /* Sync bounce buffer so CPU sees the DMA-written ref */
+ cmh_dma_sync_for_cpu(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+
+ /* Phase 2: extract shared secret from DS via actual ref */
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_data(&vcq[1], *ref_buf, ss_dma, clen);
+ vcq[1].id |= pke_swap_flags(req.curve);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(ref_dma))
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ss_dma))
+ cmh_dma_unmap_single(ss_dma, clen, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(peer_dma))
+ cmh_dma_unmap_single(peer_dma, clen, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.output), ss_buf, clen))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(ref_buf);
+ kfree_sensitive(ss_buf);
+ kfree(peer_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_ecdh_keygen() - Handle CMH_MGMT_IOC_PKE_ECDH_KEYGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_ecdh_keygen(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_ecdh_keygen req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 clen, out_len;
+ u8 *pkx_buf;
+ dma_addr_t pkx_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ clen = cmh_pke_validate_curve(req.curve);
+ if (!clen)
+ return -EINVAL;
+
+ /*
+ * ECDH_KEYGEN always outputs both X and Y coordinates
+ * (2 * clen bytes total) even though only X is useful for
+ * the ECDH exchange. Allocate the full output size to avoid
+ * a DMA buffer overflow, but copy only X back to userspace.
+ */
+ out_len = 2 * clen;
+
+ pkx_buf = kzalloc(out_len, GFP_KERNEL);
+ if (!pkx_buf)
+ return -ENOMEM;
+
+ pkx_dma = cmh_dma_map_single(pkx_buf, out_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(pkx_dma)) {
+ kfree(pkx_buf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_ecdh_keygen(&vcq[1], pke_cid, req.curve, clen,
+ pkx_dma, req.key_ref,
+ PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(pkx_dma, out_len, DMA_FROM_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.public_key_x),
+ pkx_buf, clen))
+ ret = -EFAULT;
+ }
+
+ kfree(pkx_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_eddsa_sign() - Handle CMH_MGMT_IOC_PKE_EDDSA_SIGN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_eddsa_sign(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_eddsa_sign req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 klen, sig_len;
+ u8 *msg_buf, *sig_buf;
+ dma_addr_t msg_dma, sig_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (!cmh_pke_validate_curve(req.curve) || !req.digest_len ||
+ req.digest_len > CMH_MGMT_MAX_DATA_LEN)
+ return -EINVAL;
+ if (!pke_curve_is_edwards(req.curve))
+ return -EINVAL;
+
+ klen = pke_eddsa_key_len(req.curve);
+ sig_len = 2 * klen;
+
+ msg_buf = kmalloc(req.digest_len, GFP_KERNEL);
+ sig_buf = kzalloc(sig_len, GFP_KERNEL);
+ if (!msg_buf || !sig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(msg_buf, u64_to_user_ptr(req.digest),
+ req.digest_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ msg_dma = cmh_dma_map_single(msg_buf, req.digest_len, DMA_TO_DEVICE);
+ sig_dma = cmh_dma_map_single(sig_buf, sig_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(msg_dma) || cmh_dma_map_error(sig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_eddsa_sign(&vcq[1], pke_cid, req.curve, klen,
+ msg_dma, sig_dma, req.key_ref,
+ req.digest_len, pke_swap_flags(req.curve));
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(msg_dma))
+ cmh_dma_unmap_single(msg_dma, req.digest_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.signature),
+ sig_buf, sig_len))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(sig_buf);
+ kfree(msg_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_eddsa_verify() - Handle CMH_MGMT_IOC_PKE_EDDSA_VERIFY ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_eddsa_verify(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_eddsa_verify req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 clen, klen, sig_len;
+ u8 *msg_buf, *sig_buf, *pky_buf, *rp_buf;
+ dma_addr_t msg_dma, sig_dma, pky_dma, rp_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ clen = cmh_pke_validate_curve(req.curve);
+ if (!clen || !req.digest_len ||
+ req.digest_len > CMH_MGMT_MAX_DATA_LEN)
+ return -EINVAL;
+ if (!pke_curve_is_edwards(req.curve))
+ return -EINVAL;
+
+ klen = pke_eddsa_key_len(req.curve);
+ sig_len = 2 * klen;
+
+ msg_buf = kmalloc(req.digest_len, GFP_KERNEL);
+ sig_buf = kmalloc(sig_len, GFP_KERNEL);
+ pky_buf = kmalloc(klen, GFP_KERNEL);
+ rp_buf = kzalloc(clen, GFP_KERNEL);
+ if (!msg_buf || !sig_buf || !pky_buf || !rp_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(msg_buf, u64_to_user_ptr(req.digest),
+ req.digest_len) ||
+ copy_from_user(sig_buf, u64_to_user_ptr(req.signature),
+ sig_len) ||
+ copy_from_user(pky_buf, u64_to_user_ptr(req.public_key_y),
+ klen)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ msg_dma = cmh_dma_map_single(msg_buf, req.digest_len, DMA_TO_DEVICE);
+ sig_dma = cmh_dma_map_single(sig_buf, sig_len, DMA_TO_DEVICE);
+ pky_dma = cmh_dma_map_single(pky_buf, klen, DMA_TO_DEVICE);
+ rp_dma = cmh_dma_map_single(rp_buf, clen, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(msg_dma) || cmh_dma_map_error(sig_dma) ||
+ cmh_dma_map_error(pky_dma) || cmh_dma_map_error(rp_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_eddsa_verify(&vcq[1], pke_cid, req.curve, req.digest_len,
+ pky_dma, msg_dma, sig_dma, rp_dma,
+ pke_swap_flags(req.curve));
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(rp_dma))
+ cmh_dma_unmap_single(rp_dma, clen, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(pky_dma))
+ cmh_dma_unmap_single(pky_dma, klen, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(msg_dma))
+ cmh_dma_unmap_single(msg_dma, req.digest_len, DMA_TO_DEVICE);
+
+out_free:
+ kfree(rp_buf);
+ kfree(pky_buf);
+ kfree(sig_buf);
+ kfree(msg_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_ec_keygen() - Handle CMH_MGMT_IOC_PKE_EC_KEYGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_ec_keygen(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_ec_keygen req;
+ /* header + SYS_NEW + ECDSA_KEYGEN + flush_pke + flush_sys */
+ struct vcq_cmd vcq[5];
+ u32 clen, key_flags, ds_len;
+ u64 *ref_buf;
+ dma_addr_t ref_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (req.flags & ~CMH_FLAG_MASK)
+ return -EINVAL;
+ clen = cmh_pke_validate_curve(req.curve);
+ if (!clen)
+ return -EINVAL;
+
+ key_flags = req.flags & CMH_FLAG_MASK;
+ /* SCA keys are stored in 2 shares -- allocate double the curve length */
+ ds_len = (req.flags & CMH_FLAG_SCA) ? clen * 2 : clen;
+
+ ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], 5);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, ds_len);
+ vcq_add_pke_ecdsa_keygen(&vcq[2], pke_cid, req.curve, clen,
+ SYS_REF_LAST,
+ SYS_TYPE_SET(key_flags, CORE_ID_PKE),
+ pke_swap_flags(req.curve));
+ vcq_add_pke_flush(&vcq[3], pke_cid);
+ vcq_add_sys_flush(&vcq[4]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 5, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+
+ if (!ret) {
+ req.ref = *ref_buf;
+ if (copy_to_user(argp, &req, sizeof(req))) {
+ /*
+ * The DS slot was created but its ref never reached
+ * userspace, so the caller cannot free it. Logically
+ * delete the orphaned slot before returning.
+ */
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_grant(&vcq[1], *ref_buf, 0, 0, 0);
+ vcq_add_sys_flush(&vcq[2]);
+ cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+ dev_warn(cmh_dev(),
+ "mgmt: PKE_EC_KEYGEN copy_to_user failed, DS slot cleaned up\n");
+ ret = -EFAULT;
+ }
+ }
+
+ kfree(ref_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_ec_pubgen() - Handle CMH_MGMT_IOC_PKE_EC_PUBGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_ec_pubgen(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_ec_pubgen req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 clen, pk_len;
+ u8 *pk_buf;
+ dma_addr_t pk_dma;
+ bool is_ed;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ clen = cmh_pke_validate_curve(req.curve);
+ if (!clen)
+ return -EINVAL;
+
+ is_ed = pke_curve_is_edwards(req.curve);
+ pk_len = is_ed ? pke_eddsa_key_len(req.curve) : 2 * clen;
+
+ pk_buf = kzalloc(pk_len, GFP_KERNEL);
+ if (!pk_buf)
+ return -ENOMEM;
+
+ pk_dma = cmh_dma_map_single(pk_buf, pk_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(pk_dma)) {
+ kfree(pk_buf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ if (is_ed)
+ vcq_add_pke_eddsa_pubgen(&vcq[1], pke_cid, req.curve,
+ pke_eddsa_key_len(req.curve),
+ pk_dma, req.key_ref,
+ pke_swap_flags(req.curve));
+ else
+ vcq_add_pke_ecdsa_pubgen(&vcq[1], pke_cid, req.curve, clen,
+ pk_dma, req.key_ref,
+ pke_swap_flags(req.curve));
+ vcq_add_pke_flush(&vcq[2], pke_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(pk_dma, pk_len, DMA_FROM_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.public_key),
+ pk_buf, pk_len))
+ ret = -EFAULT;
+ }
+
+ kfree(pk_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_pke_eddsa_keygen_sca() - Handle CMH_MGMT_IOC_PKE_EDDSA_KEYGEN_SCA ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_pke_eddsa_keygen_sca(void __user *argp)
+{
+ u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE);
+
+ struct cmh_ioctl_pke_eddsa_keygen_sca req;
+ /* header + SYS_NEW + EDDSA_KEYGEN_SCA + flush_pke + flush_sys */
+ struct vcq_cmd vcq[5];
+ u64 *ref_buf;
+ dma_addr_t ref_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ /* EdDSA SCA keygen is only supported for Ed448 */
+ if (req.curve != PKE_CURVE_448)
+ return -EINVAL;
+
+ ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ if (!ref_buf)
+ return -ENOMEM;
+
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ kfree(ref_buf);
+ return -ENOMEM;
+ }
+
+ vcq_set_header(&vcq[0], 5);
+ vcq_add_sys_new(&vcq[1], req.cid, ref_dma, PKE_ED448_SK_SCA_LEN);
+ vcq_add_pke_eddsa_keygen_sca(&vcq[2], pke_cid, req.curve, req.key_ref,
+ SYS_REF_LAST);
+ vcq_add_pke_flush(&vcq[3], pke_cid);
+ vcq_add_sys_flush(&vcq[4]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 5, 1, MGMT_MBX);
+
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+
+ if (!ret) {
+ req.sca_ref = *ref_buf;
+ if (copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ }
+
+ kfree(ref_buf);
+ return ret;
+}
diff --git a/drivers/crypto/cmh/cmh_mgmt_pqc.c b/drivers/crypto/cmh/cmh_mgmt_pqc.c
new file mode 100644
index 000000000000..df3d969980e7
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_mgmt_pqc.c
@@ -0,0 +1,1325 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH -- PQC ioctl handlers for /dev/cmh_mgmt
+ *
+ * ML-KEM keygen/encapsulate/decapsulate, ML-DSA keygen/sign,
+ * SLH-DSA keygen/sign (pure + prehash).
+ *
+ * Split from cmh_mgmt.c for maintainability.
+ */
+
+#include <linux/kernel.h>
+#include <linux/uaccess.h>
+#include <linux/slab.h>
+#include <linux/overflow.h>
+
+#include "cmh_mgmt.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_key.h"
+#include "cmh_dma.h"
+#include "cmh_config.h"
+#include "cmh_pqc.h"
+#include "cmh_qse_abi.h"
+#include "cmh_sys_abi.h"
+#include <uapi/linux/cmh_mgmt_ioctl.h>
+
+#include <crypto/utils.h>
+
+/* -- PQC -- ML-KEM -- */
+
+/**
+ * cmh_mgmt_ml_kem_keygen() - Handle CMH_MGMT_IOC_ML_KEM_KEYGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_ml_kem_keygen(void __user *argp)
+{
+ u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE);
+
+ struct cmh_ioctl_ml_kem_keygen req;
+ struct vcq_cmd vcq[QSE_VCQ_CMDS_MAX];
+ u32 ek_len, dk_len, seed_len, key_flags;
+ u32 qse_flags = 0;
+ bool masked, ds_ref, hw_rng;
+ u8 *seed_buf = NULL, *z_buf = NULL, *ek_buf, *dk_buf = NULL;
+ u64 *ref_buf = NULL;
+ dma_addr_t seed_dma = DMA_MAPPING_ERROR, z_dma = DMA_MAPPING_ERROR;
+ dma_addr_t ek_dma, dk_dma = DMA_MAPPING_ERROR, ref_dma = DMA_MAPPING_ERROR;
+ int ret, idx;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (ml_kem_k_idx(req.k) < 0)
+ return -EINVAL;
+ if (req.flags & ~(CMH_QSE_FLAG_MASK | CMH_FLAG_MASK))
+ return -EINVAL;
+
+ masked = !!(req.flags & CMH_QSE_FLAG_MASKED);
+ ds_ref = !!(req.flags & CMH_QSE_FLAG_DS_REF);
+ hw_rng = !!(req.flags & CMH_QSE_FLAG_HW_RNG);
+
+ /*
+ * QSE keys only support PT storage -- the eSW dec/sign paths
+ * hardcode SYS_TYPE_FLAG_PT when reading the key back.
+ * QSE SCA protection uses masking (CMH_QSE_FLAG_MASKED),
+ * not the 2-share mechanism (CMH_FLAG_SCA).
+ */
+ key_flags = req.flags & CMH_FLAG_MASK;
+ if (key_flags && key_flags != CMH_FLAG_PT)
+ return -EINVAL;
+ key_flags = CMH_FLAG_PT;
+
+ /* Masked keygen must store dk in DS -- polynomial unmasking not supported */
+ if (masked && !ds_ref)
+ return -EINVAL;
+
+ ek_len = ML_KEM_EK_SIZE(req.k);
+ dk_len = masked ? ML_KEM_DK_SIZE_MASKED(req.k)
+ : ML_KEM_DK_SIZE(req.k);
+ seed_len = masked ? QSE_SEED_LEN_MASKED : QSE_SEED_LEN;
+
+ if (hw_rng)
+ qse_flags |= QSE_FLAG_USE_RNG;
+ if (ds_ref)
+ qse_flags |= QSE_FLAG_USE_REF;
+
+ ek_buf = kzalloc(ek_len, GFP_KERNEL);
+ if (!ek_buf)
+ return -ENOMEM;
+
+ if (!hw_rng && req.seed && req.z) {
+ seed_buf = kmalloc(seed_len, GFP_KERNEL);
+ z_buf = kmalloc(seed_len, GFP_KERNEL);
+ if (!seed_buf || !z_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(seed_buf, u64_to_user_ptr(req.seed),
+ seed_len) ||
+ copy_from_user(z_buf, u64_to_user_ptr(req.z), seed_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ if (ds_ref) {
+ ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ if (!ref_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ } else {
+ dk_buf = kzalloc(dk_len, GFP_KERNEL);
+ if (!dk_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ }
+
+ /* DMA map */
+ ek_dma = cmh_dma_map_single(ek_buf, ek_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ek_dma)) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (seed_buf) {
+ seed_dma = cmh_dma_map_single(seed_buf, seed_len,
+ DMA_TO_DEVICE);
+ z_dma = cmh_dma_map_single(z_buf, seed_len, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(seed_dma) || cmh_dma_map_error(z_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ if (ds_ref) {
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ } else {
+ dk_dma = cmh_dma_map_single(dk_buf, dk_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(dk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ idx = 0;
+ if (ds_ref) {
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MAX);
+ idx++;
+ vcq_add_sys_new(&vcq[idx++], req.dk_cid, ref_dma, dk_len);
+ vcq_add_qse_ml_kem_keygen(&vcq[idx++], qse_cid, req.k, qse_flags,
+ seed_dma, z_dma,
+ ek_dma, SYS_REF_LAST,
+ SYS_TYPE_SET(key_flags,
+ CORE_ID_QSE),
+ masked);
+ vcq_add_qse_flush(&vcq[idx++], qse_cid);
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MAX,
+ 1, MGMT_MBX);
+ } else {
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ idx++;
+ vcq_add_qse_ml_kem_keygen(&vcq[idx++], qse_cid, req.k, qse_flags,
+ seed_dma, z_dma,
+ ek_dma, dk_dma, 0, masked);
+ vcq_add_qse_flush(&vcq[idx++], qse_cid);
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN,
+ 1, MGMT_MBX);
+ }
+
+out_unmap:
+ if (ds_ref && !cmh_dma_map_error(ref_dma))
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ if (!ds_ref && dk_buf && !cmh_dma_map_error(dk_dma))
+ cmh_dma_unmap_single(dk_dma, dk_len, DMA_FROM_DEVICE);
+ if (z_buf && !cmh_dma_map_error(z_dma))
+ cmh_dma_unmap_single(z_dma, seed_len, DMA_TO_DEVICE);
+ if (seed_buf && !cmh_dma_map_error(seed_dma))
+ cmh_dma_unmap_single(seed_dma, seed_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(ek_dma))
+ cmh_dma_unmap_single(ek_dma, ek_len, DMA_FROM_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.ek), ek_buf, ek_len)) {
+ ret = -EFAULT;
+ } else if (ds_ref) {
+ req.dk_ref = *ref_buf;
+ } else if (copy_to_user(u64_to_user_ptr(req.dk),
+ dk_buf, dk_len)) {
+ ret = -EFAULT;
+ }
+ if (!ret && copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ /*
+ * A copy_to_user faulted after the DS slot was created, so
+ * its ref never reached userspace and the caller cannot free
+ * it. Logically delete the orphaned slot: grant-with-no-access
+ * wipes the key material and clears the CID (it does not
+ * reclaim the stack space -- only a datastore reset does).
+ */
+ if (ret && ds_ref) {
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_grant(&vcq[1], *ref_buf, 0, 0, 0);
+ vcq_add_sys_flush(&vcq[2]);
+ cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+ dev_warn(cmh_dev(),
+ "mgmt: ML_KEM_KEYGEN copy_to_user failed, DS slot cleaned up\n");
+ }
+ }
+
+out_free:
+ kfree_sensitive(dk_buf);
+ kfree(ref_buf);
+ kfree_sensitive(z_buf);
+ kfree_sensitive(seed_buf);
+ kfree(ek_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_ml_kem_enc() - Handle CMH_MGMT_IOC_ML_KEM_ENC ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_ml_kem_enc(void __user *argp)
+{
+ u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE);
+
+ struct cmh_ioctl_ml_kem_enc req;
+ struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN];
+ u32 ek_len, ct_len, ss_out_len;
+ u32 qse_flags = 0;
+ bool masked, hw_rng;
+ u8 *ek_buf, *coin_buf = NULL, *ct_buf, *ss_buf;
+ dma_addr_t ek_dma, coin_dma = DMA_MAPPING_ERROR, ct_dma, ss_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved || req.__reserved2[0] || req.__reserved2[1])
+ return -EINVAL;
+ if (ml_kem_k_idx(req.k) < 0)
+ return -EINVAL;
+
+ masked = !!(req.flags & CMH_QSE_FLAG_MASKED);
+ hw_rng = !!(req.flags & CMH_QSE_FLAG_HW_RNG);
+
+ ek_len = ML_KEM_EK_SIZE(req.k);
+ ct_len = ML_KEM_CT_SIZE(req.k);
+ ss_out_len = masked ? ML_KEM_SS_LEN_MASKED : ML_KEM_SS_LEN;
+
+ if (hw_rng)
+ qse_flags |= QSE_FLAG_USE_RNG;
+
+ ek_buf = kmalloc(ek_len, GFP_KERNEL);
+ ct_buf = kzalloc(ct_len, GFP_KERNEL);
+ ss_buf = kzalloc(ss_out_len, GFP_KERNEL);
+ if (!ek_buf || !ct_buf || !ss_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(ek_buf, u64_to_user_ptr(req.ek), ek_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ if (!hw_rng && req.coin) {
+ u32 coin_len = masked ? QSE_SEED_LEN_MASKED : QSE_SEED_LEN;
+
+ coin_buf = kmalloc(coin_len, GFP_KERNEL);
+ if (!coin_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(coin_buf, u64_to_user_ptr(req.coin),
+ coin_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ coin_dma = cmh_dma_map_single(coin_buf, coin_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(coin_dma)) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ }
+
+ ek_dma = cmh_dma_map_single(ek_buf, ek_len, DMA_TO_DEVICE);
+ ct_dma = cmh_dma_map_single(ct_buf, ct_len, DMA_FROM_DEVICE);
+ ss_dma = cmh_dma_map_single(ss_buf, ss_out_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ek_dma) || cmh_dma_map_error(ct_dma) ||
+ cmh_dma_map_error(ss_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ vcq_add_qse_ml_kem_enc(&vcq[1], qse_cid, req.k, qse_flags,
+ coin_dma, ek_dma, ct_dma, ss_dma, 0, masked);
+ vcq_add_qse_flush(&vcq[2], qse_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(ss_dma))
+ cmh_dma_unmap_single(ss_dma, ss_out_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ct_dma))
+ cmh_dma_unmap_single(ct_dma, ct_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ek_dma))
+ cmh_dma_unmap_single(ek_dma, ek_len, DMA_TO_DEVICE);
+ if (coin_buf && !cmh_dma_map_error(coin_dma))
+ cmh_dma_unmap_single(coin_dma,
+ masked ? QSE_SEED_LEN_MASKED
+ : QSE_SEED_LEN,
+ DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.ct), ct_buf, ct_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ /* Unmask ss if masked: ss = share0 ^ share1 */
+ if (masked) {
+ crypto_xor(ss_buf, ss_buf + ML_KEM_SS_LEN,
+ ML_KEM_SS_LEN);
+ }
+ if (copy_to_user(u64_to_user_ptr(req.ss), ss_buf,
+ ML_KEM_SS_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ if (copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(ss_buf);
+ kfree(ct_buf);
+ kfree(coin_buf);
+ kfree(ek_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_ml_kem_dec() - Handle CMH_MGMT_IOC_ML_KEM_DEC ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_ml_kem_dec(void __user *argp)
+{
+ u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE);
+
+ struct cmh_ioctl_ml_kem_dec req;
+ struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN];
+ u32 ct_len, dk_len, ss_out_len;
+ u32 qse_flags = 0;
+ bool masked, ds_ref;
+ u8 *ct_buf, *dk_buf = NULL, *ss_buf;
+ dma_addr_t ct_dma, dk_dma = DMA_MAPPING_ERROR, ss_dma;
+ u64 dk_ref;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved || req.__reserved2[0] || req.__reserved2[1])
+ return -EINVAL;
+ if (ml_kem_k_idx(req.k) < 0)
+ return -EINVAL;
+
+ masked = !!(req.flags & CMH_QSE_FLAG_MASKED);
+ ds_ref = !!(req.flags & CMH_QSE_FLAG_DS_REF);
+
+ ct_len = ML_KEM_CT_SIZE(req.k);
+ dk_len = masked ? ML_KEM_DK_SIZE_MASKED(req.k)
+ : ML_KEM_DK_SIZE(req.k);
+ ss_out_len = masked ? ML_KEM_SS_LEN_MASKED : ML_KEM_SS_LEN;
+
+ ct_buf = kmalloc(ct_len, GFP_KERNEL);
+ ss_buf = kzalloc(ss_out_len, GFP_KERNEL);
+ if (!ct_buf || !ss_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(ct_buf, u64_to_user_ptr(req.ct), ct_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ ct_dma = cmh_dma_map_single(ct_buf, ct_len, DMA_TO_DEVICE);
+ ss_dma = cmh_dma_map_single(ss_buf, ss_out_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ct_dma) || cmh_dma_map_error(ss_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ /*
+ * dk: if DS_REF flag is set, req.dk is a DS reference.
+ * Otherwise, copy raw dk from user-space and use extmem DMA.
+ * Masked decaps requires DS ref (polynomial unmasking not supported).
+ */
+ if (ds_ref) {
+ dk_ref = req.dk;
+ qse_flags |= QSE_FLAG_USE_REF;
+ } else {
+ if (masked) {
+ ret = -EINVAL;
+ goto out_unmap;
+ }
+ dk_buf = kmalloc(dk_len, GFP_KERNEL);
+ if (!dk_buf) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ if (copy_from_user(dk_buf, u64_to_user_ptr(req.dk), dk_len)) {
+ ret = -EFAULT;
+ goto out_unmap;
+ }
+ dk_dma = cmh_dma_map_single(dk_buf, dk_len, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(dk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ dk_ref = dk_dma;
+ }
+
+ if (ds_ref) {
+ /*
+ * DS_REF decaps: CMH eSW resolves both dk and ss from DS.
+ * Phase 1: dec stores ss into SYS_REF_TEMP.
+ * Phase 2: sys_data reads ss from SYS_REF_TEMP to DMA.
+ */
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ vcq_add_qse_ml_kem_dec(&vcq[1], qse_cid, req.k, qse_flags,
+ ct_dma, dk_ref, SYS_REF_TEMP,
+ SYS_TYPE_SET(SYS_TYPE_FLAG_PT,
+ CORE_ID_QSE),
+ masked);
+ vcq_add_qse_flush(&vcq[2], qse_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN,
+ 1, MGMT_MBX);
+ if (ret)
+ goto out_unmap;
+
+ /* Phase 2: extract ss from SYS_REF_TEMP */
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ vcq_add_sys_data(&vcq[1], SYS_REF_TEMP, ss_dma,
+ ss_out_len);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN,
+ 1, MGMT_MBX);
+ } else {
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ vcq_add_qse_ml_kem_dec(&vcq[1], qse_cid, req.k, qse_flags,
+ ct_dma, dk_ref, ss_dma, 0, masked);
+ vcq_add_qse_flush(&vcq[2], qse_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN,
+ 1, MGMT_MBX);
+ }
+
+out_unmap:
+ if (dk_buf && !cmh_dma_map_error(dk_dma))
+ cmh_dma_unmap_single(dk_dma, dk_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(ss_dma))
+ cmh_dma_unmap_single(ss_dma, ss_out_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ct_dma))
+ cmh_dma_unmap_single(ct_dma, ct_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (masked) {
+ crypto_xor(ss_buf, ss_buf + ML_KEM_SS_LEN,
+ ML_KEM_SS_LEN);
+ }
+ if (copy_to_user(u64_to_user_ptr(req.ss), ss_buf,
+ ML_KEM_SS_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ if (copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(dk_buf);
+ kfree_sensitive(ss_buf);
+ kfree(ct_buf);
+ return ret;
+}
+
+/* -- PQC -- ML-DSA -- */
+
+/**
+ * cmh_mgmt_ml_dsa_keygen() - Handle CMH_MGMT_IOC_ML_DSA_KEYGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_ml_dsa_keygen(void __user *argp)
+{
+ u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE);
+
+ struct cmh_ioctl_ml_dsa_keygen req;
+ struct vcq_cmd vcq[QSE_VCQ_CMDS_MAX];
+ u32 pk_size, sk_size, seed_len, key_flags;
+ u32 qse_flags = 0;
+ bool masked, ds_ref, hw_rng;
+ u8 *seed_buf = NULL, *pk_buf, *sk_buf = NULL;
+ u64 *ref_buf = NULL;
+ dma_addr_t seed_dma = DMA_MAPPING_ERROR, pk_dma;
+ dma_addr_t sk_dma = DMA_MAPPING_ERROR, ref_dma = DMA_MAPPING_ERROR;
+ int ret, idx, mi;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ mi = ml_dsa_mode_idx(req.mode);
+ if (mi < 0)
+ return -EINVAL;
+ if (req.flags & ~(CMH_QSE_FLAG_MASK | CMH_FLAG_MASK))
+ return -EINVAL;
+
+ masked = !!(req.flags & CMH_QSE_FLAG_MASKED);
+ ds_ref = !!(req.flags & CMH_QSE_FLAG_DS_REF);
+ hw_rng = !!(req.flags & CMH_QSE_FLAG_HW_RNG);
+
+ /*
+ * QSE keys only support PT storage -- the eSW sign path
+ * hardcodes SYS_TYPE_FLAG_PT when reading the key back.
+ * QSE SCA protection uses masking (CMH_QSE_FLAG_MASKED),
+ * not the 2-share mechanism (CMH_FLAG_SCA).
+ */
+ key_flags = req.flags & CMH_FLAG_MASK;
+ if (key_flags && key_flags != CMH_FLAG_PT)
+ return -EINVAL;
+ key_flags = CMH_FLAG_PT;
+
+ if (masked && !ds_ref)
+ return -EINVAL;
+
+ pk_size = ml_dsa_pk_size[mi];
+ sk_size = masked ? ml_dsa_sk_size_masked[mi] : ml_dsa_sk_size[mi];
+ seed_len = masked ? QSE_SEED_LEN_MASKED : QSE_SEED_LEN;
+
+ if (hw_rng)
+ qse_flags |= QSE_FLAG_USE_RNG;
+ if (ds_ref)
+ qse_flags |= QSE_FLAG_USE_REF;
+
+ pk_buf = kzalloc(pk_size, GFP_KERNEL);
+ if (!pk_buf)
+ return -ENOMEM;
+
+ if (!hw_rng && req.seed) {
+ seed_buf = kmalloc(seed_len, GFP_KERNEL);
+ if (!seed_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(seed_buf, u64_to_user_ptr(req.seed),
+ seed_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ if (ds_ref) {
+ ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ if (!ref_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ } else {
+ sk_buf = kzalloc(sk_size, GFP_KERNEL);
+ if (!sk_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ }
+
+ pk_dma = cmh_dma_map_single(pk_buf, pk_size, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(pk_dma)) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (seed_buf) {
+ seed_dma = cmh_dma_map_single(seed_buf, seed_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(seed_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ if (ds_ref) {
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64),
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ref_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ } else {
+ sk_dma = cmh_dma_map_single(sk_buf, sk_size, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(sk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ idx = 0;
+ if (ds_ref) {
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MAX);
+ idx++;
+ vcq_add_sys_new(&vcq[idx++], req.sk_cid, ref_dma, sk_size);
+ vcq_add_qse_ml_dsa_keygen(&vcq[idx++], qse_cid, req.mode, qse_flags,
+ seed_dma, pk_dma,
+ SYS_REF_LAST,
+ SYS_TYPE_SET(key_flags,
+ CORE_ID_QSE),
+ masked);
+ vcq_add_qse_flush(&vcq[idx++], qse_cid);
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MAX,
+ 1, MGMT_MBX);
+ } else {
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ idx++;
+ vcq_add_qse_ml_dsa_keygen(&vcq[idx++], qse_cid, req.mode, qse_flags,
+ seed_dma, pk_dma,
+ sk_dma, 0, masked);
+ vcq_add_qse_flush(&vcq[idx++], qse_cid);
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN,
+ 1, MGMT_MBX);
+ }
+
+out_unmap:
+ if (ds_ref && !cmh_dma_map_error(ref_dma))
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ if (!ds_ref && sk_buf && !cmh_dma_map_error(sk_dma))
+ cmh_dma_unmap_single(sk_dma, sk_size, DMA_FROM_DEVICE);
+ if (seed_buf && !cmh_dma_map_error(seed_dma))
+ cmh_dma_unmap_single(seed_dma, seed_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(pk_dma))
+ cmh_dma_unmap_single(pk_dma, pk_size, DMA_FROM_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.pk), pk_buf, pk_size)) {
+ ret = -EFAULT;
+ } else if (ds_ref) {
+ req.sk_ref = *ref_buf;
+ } else if (copy_to_user(u64_to_user_ptr(req.sk),
+ sk_buf, sk_size)) {
+ ret = -EFAULT;
+ }
+ if (!ret && copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ /*
+ * A copy_to_user faulted after the DS slot was created, so
+ * its ref never reached userspace and the caller cannot free
+ * it. Logically delete the orphaned slot: grant-with-no-access
+ * wipes the key material and clears the CID (it does not
+ * reclaim the stack space -- only a datastore reset does).
+ */
+ if (ret && ds_ref) {
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_grant(&vcq[1], *ref_buf, 0, 0, 0);
+ vcq_add_sys_flush(&vcq[2]);
+ cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+ dev_warn(cmh_dev(),
+ "mgmt: ML_DSA_KEYGEN copy_to_user failed, DS slot cleaned up\n");
+ }
+ }
+
+out_free:
+ kfree_sensitive(sk_buf);
+ kfree(ref_buf);
+ kfree_sensitive(seed_buf);
+ kfree(pk_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_ml_dsa_sign() - Handle CMH_MGMT_IOC_ML_DSA_SIGN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_ml_dsa_sign(void __user *argp)
+{
+ u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE);
+
+ struct cmh_ioctl_ml_dsa_sign req;
+ struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN];
+ u32 sig_size, copy_len, rnd_len;
+ u32 qse_flags = 0;
+ bool masked;
+ u8 *m_buf, *sig_buf, *sk_buf = NULL, *rnd_buf = NULL;
+ dma_addr_t m_dma = DMA_MAPPING_ERROR, sig_dma;
+ dma_addr_t sk_dma = DMA_MAPPING_ERROR;
+ /*
+ * 0 = "no added randomness": the eSW treats a zero rnd pointer as
+ * absent. A garbage DMA_MAPPING_ERROR would be taken as a real
+ * address and DMA'd from.
+ */
+ dma_addr_t rnd_dma = 0;
+ u64 sk_ref;
+ int mi, ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ mi = ml_dsa_mode_idx(req.mode);
+ if (mi < 0)
+ return -EINVAL;
+ if (req.mlen > ML_DSA_MAX_MLEN && req.mlen != ML_DSA_MLEN_EXTERNAL_MU)
+ return -EINVAL;
+
+ masked = !!(req.flags & CMH_QSE_FLAG_MASKED);
+ rnd_len = masked ? QSE_SEED_LEN_MASKED : QSE_SEED_LEN;
+ sig_size = ml_dsa_sig_size[mi];
+ copy_len = (req.mlen == ML_DSA_MLEN_EXTERNAL_MU)
+ ? ML_DSA_EXTMU_LEN : req.mlen;
+
+ /*
+ * sk: if DS_REF, req.sk is a DS reference (masked sk lives in DS).
+ * Otherwise, copy raw sk from user-space.
+ * Masked sign requires DS ref (polynomial unmasking not supported).
+ */
+ if (req.flags & CMH_QSE_FLAG_DS_REF) {
+ sk_ref = req.sk;
+ qse_flags |= QSE_FLAG_USE_REF;
+ } else {
+ u32 sk_size;
+
+ if (masked)
+ return -EINVAL;
+ sk_size = ml_dsa_sk_size[mi];
+ sk_buf = kmalloc(sk_size, GFP_KERNEL);
+ if (!sk_buf)
+ return -ENOMEM;
+ if (copy_from_user(sk_buf, u64_to_user_ptr(req.sk), sk_size)) {
+ kfree_sensitive(sk_buf);
+ return -EFAULT;
+ }
+ }
+
+ m_buf = kmalloc(max_t(u32, copy_len, 1), GFP_KERNEL);
+ sig_buf = kzalloc(sig_size, GFP_KERNEL);
+ if (!m_buf || !sig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_len > 0 &&
+ copy_from_user(m_buf, u64_to_user_ptr(req.m), copy_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ if (req.rnd) {
+ rnd_buf = kmalloc(rnd_len, GFP_KERNEL);
+ if (!rnd_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(rnd_buf, u64_to_user_ptr(req.rnd),
+ rnd_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ if (copy_len > 0) {
+ m_dma = cmh_dma_map_single(m_buf, copy_len, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(m_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+ sig_dma = cmh_dma_map_single(sig_buf, sig_size, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(sig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ if (sk_buf) {
+ sk_dma = cmh_dma_map_single(sk_buf, ml_dsa_sk_size[mi],
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(sk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ sk_ref = sk_dma;
+ }
+
+ if (rnd_buf) {
+ rnd_dma = cmh_dma_map_single(rnd_buf, rnd_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rnd_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ vcq_add_qse_ml_dsa_sign(&vcq[1], qse_cid, req.mode, qse_flags,
+ rnd_dma, m_dma, sk_ref, sig_dma,
+ req.mlen, masked);
+ vcq_add_qse_flush(&vcq[2], qse_cid);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (rnd_buf && !cmh_dma_map_error(rnd_dma))
+ cmh_dma_unmap_single(rnd_dma, rnd_len, DMA_TO_DEVICE);
+ if (sk_buf && !cmh_dma_map_error(sk_dma))
+ cmh_dma_unmap_single(sk_dma, ml_dsa_sk_size[mi],
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_size, DMA_FROM_DEVICE);
+ if (copy_len > 0 && !cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, copy_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.sig), sig_buf, sig_size))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(rnd_buf);
+ kfree(sig_buf);
+ kfree(m_buf);
+ kfree_sensitive(sk_buf);
+ return ret;
+}
+
+/* -- PQC -- SLH-DSA -- */
+
+/**
+ * cmh_mgmt_slhdsa_keygen() - Handle CMH_MGMT_IOC_SLHDSA_KEYGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_slhdsa_keygen(void __user *argp)
+{
+ u32 hcq_cid = cmh_core_default_id(CMH_CORE_HCQ);
+
+ struct cmh_ioctl_slhdsa_keygen req;
+ struct vcq_cmd vcq[HCQ_VCQ_CMDS_MAX];
+ u32 pk_sz, sk_sz, seed_sz, sk_alloc, vcq_cnt, key_flags;
+ bool ds_ref;
+ u8 *seed_buf, *pk_buf, *sk_buf = NULL;
+ u64 *ref_buf = NULL;
+ dma_addr_t seed_dma, pk_dma, sk_dma = DMA_MAPPING_ERROR, ref_dma = DMA_MAPPING_ERROR;
+ int ret, idx;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+ if (req.parameter_set < 1 || req.parameter_set > HCQ_SLHDSA_PARAM_MAX)
+ return -EINVAL;
+ if (req.flags & ~(CMH_QSE_FLAG_DS_REF | CMH_FLAG_MASK))
+ return -EINVAL;
+
+ ds_ref = !!(req.flags & CMH_QSE_FLAG_DS_REF);
+
+ /*
+ * QSE keys only support PT storage -- the eSW sign path
+ * hardcodes SYS_TYPE_FLAG_PT when reading the key back.
+ * HCQ core sets key type internally during keygen.
+ */
+ key_flags = req.flags & CMH_FLAG_MASK;
+ if (key_flags && key_flags != CMH_FLAG_PT)
+ return -EINVAL;
+ (void)key_flags;
+
+ pk_sz = slhdsa_pk_size(req.parameter_set);
+ sk_sz = slhdsa_sk_size(req.parameter_set);
+ seed_sz = slhdsa_seed_size(req.parameter_set);
+
+ seed_buf = kmalloc(seed_sz, GFP_KERNEL);
+ pk_buf = kzalloc(pk_sz, GFP_KERNEL);
+ if (!seed_buf || !pk_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(seed_buf, u64_to_user_ptr(req.seed), seed_sz)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ /*
+ * Both paths need ref_buf for sys_new output. Non-ds_ref also
+ * needs sk_buf (+16 for SYS header) to read back via sys_read.
+ */
+ ref_buf = kzalloc(sizeof(u64), GFP_KERNEL);
+ if (!ref_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (!ds_ref) {
+ sk_alloc = sk_sz + SYS_WRAP_HDR_SIZE;
+ sk_buf = kzalloc(sk_alloc, GFP_KERNEL);
+ if (!sk_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ }
+
+ seed_dma = cmh_dma_map_single(seed_buf, seed_sz, DMA_TO_DEVICE);
+ pk_dma = cmh_dma_map_single(pk_buf, pk_sz, DMA_FROM_DEVICE);
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(seed_dma) || cmh_dma_map_error(pk_dma) ||
+ cmh_dma_map_error(ref_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ if (!ds_ref) {
+ sk_dma = cmh_dma_map_single(sk_buf, sk_alloc,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(sk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ /*
+ * SLH-DSA keygen requires seed and sk as DS references.
+ * VCQ: hdr + sys_new(sk) + sys_write(seed->TEMP) + keygen + [sys_read] + flush
+ */
+ idx = 0;
+ if (ds_ref) {
+ vcq_cnt = HCQ_VCQ_CMDS_MAX - 1; /* hdr+new+write+keygen+flush */
+ vcq_set_header(&vcq[idx++], vcq_cnt);
+ vcq_add_sys_new(&vcq[idx++], req.sk_cid, ref_dma,
+ sk_sz);
+ } else {
+ vcq_cnt = HCQ_VCQ_CMDS_MAX; /* hdr+new+write+keygen+read+flush */
+ vcq_set_header(&vcq[idx++], vcq_cnt);
+ vcq_add_sys_new(&vcq[idx++], SYS_CID_NONE, ref_dma,
+ sk_sz);
+ }
+ vcq_add_sys_write(&vcq[idx++], SYS_REF_TEMP, seed_dma, 0,
+ seed_sz,
+ SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_HCQ));
+ vcq_add_hcq_slhdsa_keygen(&vcq[idx++], hcq_cid, req.parameter_set,
+ seed_sz, pk_sz, sk_sz,
+ SYS_REF_TEMP, pk_dma, SYS_REF_LAST);
+ if (!ds_ref)
+ vcq_add_sys_read(&vcq[idx++], SYS_REF_LAST, sk_dma,
+ 0, sk_sz + SYS_WRAP_HDR_SIZE);
+ vcq_add_hcq_flush(&vcq[idx++], hcq_cid);
+
+ ret = cmh_tm_submit_sync_tmo(vcq, vcq_cnt, 1, MGMT_MBX,
+ cmh_tm_slow_op_timeout_jiffies());
+
+out_unmap:
+ if (!ds_ref && sk_buf && !cmh_dma_map_error(sk_dma))
+ cmh_dma_unmap_single(sk_dma, sk_alloc, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ref_dma))
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(pk_dma))
+ cmh_dma_unmap_single(pk_dma, pk_sz, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(seed_dma))
+ cmh_dma_unmap_single(seed_dma, seed_sz, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.pk), pk_buf, pk_sz)) {
+ ret = -EFAULT;
+ } else if (ds_ref) {
+ req.sk_ref = *ref_buf;
+ } else if (copy_to_user(u64_to_user_ptr(req.sk),
+ sk_buf + SYS_WRAP_HDR_SIZE,
+ sk_sz)) {
+ ret = -EFAULT;
+ }
+ if (!ret && copy_to_user(argp, &req, sizeof(req)))
+ ret = -EFAULT;
+ /*
+ * A copy_to_user faulted after the DS slot was created, so
+ * its ref never reached userspace and the caller cannot free
+ * it. Logically delete the orphaned slot: grant-with-no-access
+ * wipes the key material and clears the CID (it does not
+ * reclaim the stack space -- only a datastore reset does).
+ */
+ if (ret && ds_ref) {
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_grant(&vcq[1], *ref_buf, 0, 0, 0);
+ vcq_add_sys_flush(&vcq[2]);
+ cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+ dev_warn(cmh_dev(),
+ "mgmt: SLHDSA_KEYGEN copy_to_user failed, DS slot cleaned up\n");
+ }
+ }
+
+out_free:
+ kfree_sensitive(sk_buf);
+ kfree(ref_buf);
+ kfree(pk_buf);
+ kfree_sensitive(seed_buf);
+ return ret;
+}
+
+/**
+ * cmh_mgmt_slhdsa_sign() - Handle CMH_MGMT_IOC_SLHDSA_SIGN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_slhdsa_sign(void __user *argp)
+{
+ u32 hcq_cid = cmh_core_default_id(CMH_CORE_HCQ);
+
+ struct cmh_ioctl_slhdsa_sign req;
+ struct vcq_cmd vcq[HCQ_VCQ_CMDS_MIN];
+ u32 sig_sz, n_val;
+ u8 *msg_buf, *ctx_buf = NULL, *sig_buf, *rnd_buf = NULL;
+ dma_addr_t msg_dma = DMA_MAPPING_ERROR, ctx_dma = DMA_MAPPING_ERROR;
+ /*
+ * 0 = "no added randomness": the eSW treats a zero rnd pointer as
+ * absent. A garbage DMA_MAPPING_ERROR would be taken as a real
+ * address and DMA'd from.
+ */
+ dma_addr_t sig_dma, rnd_dma = 0;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.parameter_set < 1 || req.parameter_set > HCQ_SLHDSA_PARAM_MAX)
+ return -EINVAL;
+ if (req.msg_len > SLHDSA_MAX_MSG_LEN)
+ return -EINVAL;
+ if (req.ctx_len > SLHDSA_MAX_CTX_LEN)
+ return -EINVAL;
+
+ sig_sz = slhdsa_get_sig_size(req.parameter_set);
+ n_val = slhdsa_n[req.parameter_set - 1];
+
+ msg_buf = kmalloc(max_t(u32, req.msg_len, 1), GFP_KERNEL);
+ sig_buf = kzalloc(sig_sz, GFP_KERNEL);
+ if (!msg_buf || !sig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (req.msg_len > 0 &&
+ copy_from_user(msg_buf, u64_to_user_ptr(req.msg), req.msg_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ if (req.ctx_len > 0 && req.ctx) {
+ ctx_buf = kmalloc(req.ctx_len, GFP_KERNEL);
+ if (!ctx_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(ctx_buf, u64_to_user_ptr(req.ctx),
+ req.ctx_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ if (req.add_random) {
+ rnd_buf = kmalloc(n_val, GFP_KERNEL);
+ if (!rnd_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(rnd_buf, u64_to_user_ptr(req.add_random),
+ n_val)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ sig_dma = cmh_dma_map_single(sig_buf, sig_sz, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(sig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ if (req.msg_len > 0) {
+ msg_dma = cmh_dma_map_single(msg_buf, req.msg_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(msg_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ if (ctx_buf) {
+ ctx_dma = cmh_dma_map_single(ctx_buf, req.ctx_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(ctx_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ if (rnd_buf) {
+ rnd_dma = cmh_dma_map_single(rnd_buf, n_val, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rnd_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ vcq_set_header(&vcq[0], HCQ_VCQ_CMDS_MIN);
+ vcq_add_hcq_slhdsa_sign(&vcq[1], hcq_cid, req.parameter_set,
+ req.msg_len, req.ctx_len,
+ rnd_dma, msg_dma, ctx_dma,
+ req.sk, sig_dma);
+ vcq_add_hcq_flush(&vcq[2], hcq_cid);
+
+ ret = cmh_tm_submit_sync_tmo(vcq, HCQ_VCQ_CMDS_MIN, 1, MGMT_MBX,
+ cmh_tm_slow_op_timeout_jiffies());
+
+out_unmap:
+ if (rnd_buf && !cmh_dma_map_error(rnd_dma))
+ cmh_dma_unmap_single(rnd_dma, n_val, DMA_TO_DEVICE);
+ if (ctx_buf && !cmh_dma_map_error(ctx_dma))
+ cmh_dma_unmap_single(ctx_dma, req.ctx_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_sz, DMA_FROM_DEVICE);
+ if (req.msg_len > 0 && !cmh_dma_map_error(msg_dma))
+ cmh_dma_unmap_single(msg_dma, req.msg_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.sig), sig_buf, sig_sz))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(rnd_buf);
+ kfree(ctx_buf);
+ kfree(sig_buf);
+ kfree(msg_buf);
+ return ret;
+}
+
+/* -- PQC -- SLH-DSA prehash -- */
+
+/**
+ * cmh_mgmt_slhdsa_sign_prehash() - Handle CMH_MGMT_IOC_SLHDSA_SIGN_PREHASH ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_slhdsa_sign_prehash(void __user *argp)
+{
+ u32 hcq_cid = cmh_core_default_id(CMH_CORE_HCQ);
+
+ struct cmh_ioctl_slhdsa_sign_prehash req;
+ struct vcq_cmd vcq[HCQ_VCQ_CMDS_MIN];
+ u32 sig_sz, n_val, hcq_cmd;
+ u8 *msg_buf, *ctx_buf = NULL, *sig_buf, *rnd_buf = NULL;
+ dma_addr_t msg_dma = DMA_MAPPING_ERROR, ctx_dma = DMA_MAPPING_ERROR;
+ /*
+ * 0 = "no added randomness": the eSW treats a zero rnd pointer as
+ * absent. A garbage DMA_MAPPING_ERROR would be taken as a real
+ * address and DMA'd from.
+ */
+ dma_addr_t sig_dma, rnd_dma = 0;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.parameter_set < 1 || req.parameter_set > HCQ_SLHDSA_PARAM_MAX)
+ return -EINVAL;
+ if (req.prehash_algo < 1 || req.prehash_algo > HCQ_SLHDSA_PREHASH_SHAKE256)
+ return -EINVAL;
+ if (req.msg_len > SLHDSA_MAX_MSG_LEN)
+ return -EINVAL;
+ if (req.ctx_len > SLHDSA_MAX_CTX_LEN)
+ return -EINVAL;
+
+ hcq_cmd = req.digest ? HCQ_CMD_SLHDSA_SIGN_PREHASH_DIGEST
+ : HCQ_CMD_SLHDSA_SIGN_PREHASH;
+
+ sig_sz = slhdsa_get_sig_size(req.parameter_set);
+ n_val = slhdsa_n[req.parameter_set - 1];
+
+ msg_buf = kmalloc(max_t(u32, req.msg_len, 1), GFP_KERNEL);
+ sig_buf = kzalloc(sig_sz, GFP_KERNEL);
+ if (!msg_buf || !sig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (req.msg_len > 0 &&
+ copy_from_user(msg_buf, u64_to_user_ptr(req.msg), req.msg_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ if (req.ctx_len > 0 && req.ctx) {
+ ctx_buf = kmalloc(req.ctx_len, GFP_KERNEL);
+ if (!ctx_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(ctx_buf, u64_to_user_ptr(req.ctx),
+ req.ctx_len)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ if (req.add_random) {
+ rnd_buf = kmalloc(n_val, GFP_KERNEL);
+ if (!rnd_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(rnd_buf, u64_to_user_ptr(req.add_random),
+ n_val)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ sig_dma = cmh_dma_map_single(sig_buf, sig_sz, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(sig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ if (req.msg_len > 0) {
+ msg_dma = cmh_dma_map_single(msg_buf, req.msg_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(msg_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ if (ctx_buf) {
+ ctx_dma = cmh_dma_map_single(ctx_buf, req.ctx_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(ctx_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ if (rnd_buf) {
+ rnd_dma = cmh_dma_map_single(rnd_buf, n_val, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rnd_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ }
+
+ vcq_set_header(&vcq[0], HCQ_VCQ_CMDS_MIN);
+ vcq_add_hcq_slhdsa_sign_prehash(&vcq[1], hcq_cid,
+ hcq_cmd, req.parameter_set,
+ req.prehash_algo,
+ req.msg_len, req.ctx_len,
+ rnd_dma, msg_dma, ctx_dma,
+ req.sk, sig_dma);
+ vcq_add_hcq_flush(&vcq[2], hcq_cid);
+
+ ret = cmh_tm_submit_sync_tmo(vcq, HCQ_VCQ_CMDS_MIN, 1, MGMT_MBX,
+ cmh_tm_slow_op_timeout_jiffies());
+
+out_unmap:
+ if (rnd_buf && !cmh_dma_map_error(rnd_dma))
+ cmh_dma_unmap_single(rnd_dma, n_val, DMA_TO_DEVICE);
+ if (ctx_buf && !cmh_dma_map_error(ctx_dma))
+ cmh_dma_unmap_single(ctx_dma, req.ctx_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_sz, DMA_FROM_DEVICE);
+ if (req.msg_len > 0 && !cmh_dma_map_error(msg_dma))
+ cmh_dma_unmap_single(msg_dma, req.msg_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.sig), sig_buf, sig_sz))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(rnd_buf);
+ kfree(ctx_buf);
+ kfree(sig_buf);
+ kfree(msg_buf);
+ return ret;
+}
+
+/* -- EAC (Error and Alarm Controller) ---- */
+
diff --git a/drivers/crypto/cmh/cmh_pke_sm2.c b/drivers/crypto/cmh/cmh_pke_sm2.c
new file mode 100644
index 000000000000..9a6e30c7f5e5
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pke_sm2.c
@@ -0,0 +1,827 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- SM2 PKE Ioctl Handlers
+ *
+ * SM2 (GM/T 0003) is the Chinese national public-key standard over the
+ * sm2p256v1 curve (256-bit). It defines three protocols:
+ *
+ * - Signature: reuses ECDSA sign/verify with SM2_CURVE (0x18), handled
+ * by the existing cmh_mgmt_pke_ecdsa_{sign,verify}() paths.
+ * - Encryption: two-step (ENC_POINT + ENC_HASH / DEC_POINT + DEC_HASH).
+ * - Key Exchange: four-step (ECDH_KEYGEN + ID_DIGEST + ECDH + ECDH_HASH).
+ *
+ * This file implements the 8 SM2-specific ioctl handlers (0x16--0x1D).
+ * Sign/verify/keygen/pubgen use the existing ECDSA/EC paths unchanged.
+ *
+ * VCQ flag convention (from eSW API):
+ * - Most SM2 commands use flags=0 (no swap).
+ * - SM2_DEC_POINT and SM2_ECDH_HASH use PKE_SWAP_FLAGS on the
+ * PKE command itself.
+ * - SM2_ECDH and SM2_ECDH_HASH also apply PKE_SWAP_FLAGS on
+ * their sys_new/sys_data VCQ phases (Weierstrass DS format).
+ */
+
+#include <linux/uaccess.h>
+#include <linux/slab.h>
+
+#include "cmh_pke.h"
+#include "cmh_pke_sm2.h"
+#include "cmh_sys.h"
+#include "cmh_dma.h"
+#include "cmh_txn.h"
+#include "cmh_mgmt.h"
+#include "cmh_sys_abi.h"
+#include <uapi/linux/cmh_mgmt_ioctl.h>
+
+/* SM2 fixed sizes (sm2p256v1: 256-bit curve) */
+#define SM2_CLEN 32U /* coordinate length */
+#define SM2_POINT_LEN 64U /* uncompressed EC point (x||y) */
+#define SM2_SHARED_KEY_LEN 16U /* ECDH shared key output */
+#define SM2_DIGEST_LEN 32U /* SM3 ZA digest */
+#define SM2_NONCE_LEN 32U /* nonce (when caller-provided) */
+/*
+ * SM2 enc_hash/dec_hash payload limit.
+ *
+ * The eSW PKE driver expands the GM/T 0003.4 KDF by issuing a single SM3
+ * invocation per command (one 32-byte block of key stream). Messages
+ * longer than 32 bytes would require ceil(msg_len / 32) SM3 invocations
+ * with an incremented counter, which the eSW does not perform; longer
+ * inputs would silently produce incorrect ciphertext / plaintext.
+ *
+ * The eSW PKE SRAM can physically hold up to 4000 bytes of payload, but
+ * that capacity is unusable until a future eSW change implements the full
+ * KDF expansion. Until then we cap the LKM at the 32-byte limit
+ * documented in Documentation/ABI/testing/cmh-mgmt.
+ */
+#define SM2_MAX_MSG_LEN 32U /* max plaintext for encrypt/decrypt */
+#define SM2_MAX_ID_LEN 32U /* max identity string */
+#define SM2_CT_OVERHEAD 96U /* C1(64) + C3(32) */
+#define SM2_MAX_CT_LEN (SM2_CT_OVERHEAD + SM2_MAX_MSG_LEN) /* 128 */
+
+/* -- SM2_ECDH_KEYGEN ------------------- */
+
+/**
+ * cmh_mgmt_sm2_ecdh_keygen() - Handle CMH_MGMT_IOC_SM2_ECDH_KEYGEN ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_ecdh_keygen(void __user *argp)
+{
+ struct cmh_ioctl_sm2_ecdh_keygen req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 core_id = cmh_core_default_id(CMH_CORE_PKE);
+ u8 *nonce_buf, *sk_buf;
+ dma_addr_t nonce_dma, sk_dma;
+ int nonce_dir;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.nonce_len != 0 && req.nonce_len != SM2_NONCE_LEN)
+ return -EINVAL;
+
+ sk_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL);
+ nonce_buf = kzalloc(SM2_NONCE_LEN, GFP_KERNEL);
+ if (!sk_buf || !nonce_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ /*
+ * nonce_len=32: caller provides ephemeral scalar r (DMA_TO_DEVICE).
+ * nonce_len=0: HW generates r and writes it back (DMA_FROM_DEVICE).
+ * The caller MUST supply a valid nonce pointer in both cases.
+ */
+ if (req.nonce_len) {
+ if (copy_from_user(nonce_buf, u64_to_user_ptr(req.nonce),
+ SM2_NONCE_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ nonce_dir = DMA_TO_DEVICE;
+ } else {
+ nonce_dir = DMA_FROM_DEVICE;
+ }
+
+ sk_dma = cmh_dma_map_single(sk_buf, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ nonce_dma = cmh_dma_map_single(nonce_buf, SM2_NONCE_LEN, nonce_dir);
+ if (cmh_dma_map_error(sk_dma) || cmh_dma_map_error(nonce_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_sm2_ecdh_keygen(&vcq[1], core_id, nonce_dma, sk_dma,
+ req.nonce_len, 0);
+ vcq_add_pke_flush(&vcq[2], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(nonce_dma))
+ cmh_dma_unmap_single(nonce_dma, SM2_NONCE_LEN, nonce_dir);
+ if (!cmh_dma_map_error(sk_dma))
+ cmh_dma_unmap_single(sk_dma, SM2_POINT_LEN, DMA_FROM_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.session_key),
+ sk_buf, SM2_POINT_LEN))
+ ret = -EFAULT;
+ /* Write back HW-generated nonce when nonce_len=0 */
+ if (!ret && !req.nonce_len) {
+ if (copy_to_user(u64_to_user_ptr(req.nonce),
+ nonce_buf, SM2_NONCE_LEN))
+ ret = -EFAULT;
+ }
+ }
+
+out_free:
+ kfree_sensitive(nonce_buf);
+ kfree_sensitive(sk_buf);
+ return ret;
+}
+
+/* -- SM2_ECDH -------------------------- */
+
+/**
+ * cmh_mgmt_sm2_ecdh() - Handle CMH_MGMT_IOC_SM2_ECDH ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_ecdh(void __user *argp)
+{
+ struct cmh_ioctl_sm2_ecdh req;
+ /* Phase 1: hdr + sys_new + sm2_ecdh + pke_flush */
+ struct vcq_cmd vcq[4];
+ u32 sp_type, core_id;
+ u8 *nonce_buf, *peer_pk_buf, *peer_sk_buf, *sp_buf;
+ u64 *ref_buf;
+ dma_addr_t nonce_dma, peer_pk_dma, peer_sk_dma, sp_dma, ref_dma;
+ int nonce_dir, ret, idx;
+ bool keep_ds;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.nonce_len != 0 && req.nonce_len != SM2_NONCE_LEN)
+ return -EINVAL;
+
+ keep_ds = (req.shared_point_ref != 0);
+ sp_type = SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_PKE);
+ core_id = cmh_core_default_id(CMH_CORE_PKE);
+
+ peer_pk_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL);
+ peer_sk_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL);
+ sp_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL);
+ ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ nonce_buf = kzalloc(SM2_NONCE_LEN, GFP_KERNEL);
+ if (!peer_pk_buf || !peer_sk_buf || !sp_buf || !ref_buf ||
+ !nonce_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(peer_pk_buf, u64_to_user_ptr(req.peer_public_key),
+ SM2_POINT_LEN) ||
+ copy_from_user(peer_sk_buf, u64_to_user_ptr(req.peer_session_key),
+ SM2_POINT_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ if (req.nonce_len) {
+ if (copy_from_user(nonce_buf, u64_to_user_ptr(req.nonce),
+ SM2_NONCE_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ nonce_dir = DMA_TO_DEVICE;
+ } else {
+ nonce_dir = DMA_FROM_DEVICE;
+ }
+
+ peer_pk_dma = cmh_dma_map_single(peer_pk_buf, SM2_POINT_LEN,
+ DMA_TO_DEVICE);
+ peer_sk_dma = cmh_dma_map_single(peer_sk_buf, SM2_POINT_LEN,
+ DMA_TO_DEVICE);
+ sp_dma = cmh_dma_map_single(sp_buf, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE);
+ nonce_dma = cmh_dma_map_single(nonce_buf, SM2_NONCE_LEN, nonce_dir);
+
+ if (cmh_dma_map_error(peer_pk_dma) || cmh_dma_map_error(peer_sk_dma) ||
+ cmh_dma_map_error(sp_dma) || cmh_dma_map_error(ref_dma) ||
+ cmh_dma_map_error(nonce_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ /* Phase 1: sys_new(shared_point_ref) + SM2_ECDH(->SYS_REF_LAST) */
+ idx = 0;
+ vcq_set_header(&vcq[idx++], 4);
+ vcq_add_sys_new(&vcq[idx], 0, ref_dma, SM2_POINT_LEN);
+ vcq[idx++].id |= PKE_SWAP_FLAGS;
+ vcq_add_pke_sm2_ecdh(&vcq[idx++], core_id, req.nonce_len, SM2_CLEN,
+ nonce_dma, peer_pk_dma, peer_sk_dma,
+ req.key_ref, SYS_REF_LAST, sp_type, 0);
+ vcq_add_pke_flush(&vcq[idx++], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 4, 1, MGMT_MBX);
+ if (ret)
+ goto out_unmap;
+
+ if (!keep_ds) {
+ /* Sync bounce buffer so CPU sees the DMA-written ref */
+ cmh_dma_sync_for_cpu(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+
+ /* Phase 2: read shared point from DS -> DMA, consuming the slot */
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_data(&vcq[1], *ref_buf, sp_dma, SM2_POINT_LEN);
+ vcq[1].id |= PKE_SWAP_FLAGS;
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+ }
+
+out_unmap:
+ if (!cmh_dma_map_error(nonce_dma))
+ cmh_dma_unmap_single(nonce_dma, SM2_NONCE_LEN, nonce_dir);
+ if (!cmh_dma_map_error(ref_dma))
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(sp_dma))
+ cmh_dma_unmap_single(sp_dma, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(peer_sk_dma))
+ cmh_dma_unmap_single(peer_sk_dma, SM2_POINT_LEN,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(peer_pk_dma))
+ cmh_dma_unmap_single(peer_pk_dma, SM2_POINT_LEN,
+ DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (!keep_ds) {
+ if (copy_to_user(u64_to_user_ptr(req.shared_point),
+ sp_buf, SM2_POINT_LEN))
+ ret = -EFAULT;
+ } else {
+ /* Return DS ref for ECDH_HASH to consume */
+ u64 __user *sp_refp = (__u64 __user *)
+ u64_to_user_ptr(req.shared_point_ref);
+
+ if (put_user(*ref_buf, sp_refp)) {
+ /*
+ * Failed to deliver the DS ref to
+ * userspace. Logically delete the
+ * orphaned slot so it does not leak.
+ */
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_grant(&vcq[1], *ref_buf,
+ 0, 0, 0);
+ vcq_add_sys_flush(&vcq[2]);
+ cmh_tm_submit_sync_mbx(vcq, 3, 1,
+ MGMT_MBX);
+ dev_warn(cmh_dev(), "SM2 ECDH put_user failed, DS slot cleaned up\n");
+ ret = -EFAULT;
+ }
+ }
+ /* Write back HW-generated nonce when nonce_len=0 */
+ if (!ret && !req.nonce_len) {
+ if (copy_to_user(u64_to_user_ptr(req.nonce),
+ nonce_buf, SM2_NONCE_LEN))
+ ret = -EFAULT;
+ }
+ }
+
+out_free:
+ kfree_sensitive(nonce_buf);
+ kfree(ref_buf);
+ kfree_sensitive(sp_buf);
+ kfree(peer_sk_buf);
+ kfree(peer_pk_buf);
+ return ret;
+}
+
+/* -- SM2_DEC_POINT --------------------- */
+
+/**
+ * cmh_mgmt_sm2_dec_point() - Handle CMH_MGMT_IOC_SM2_DEC_POINT ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_dec_point(void __user *argp)
+{
+ struct cmh_ioctl_sm2_dec_point req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 core_id = cmh_core_default_id(CMH_CORE_PKE);
+ u8 *ct_buf, *dp_buf;
+ dma_addr_t ct_dma, dp_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.ciphertext_len <= SM2_CT_OVERHEAD ||
+ req.ciphertext_len > SM2_MAX_CT_LEN)
+ return -EINVAL;
+
+ /* Only need C1 (first 64 bytes) for the sidecar */
+ ct_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL);
+ dp_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL);
+ if (!ct_buf || !dp_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(ct_buf, u64_to_user_ptr(req.ciphertext),
+ SM2_POINT_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ ct_dma = cmh_dma_map_single(ct_buf, SM2_POINT_LEN, DMA_TO_DEVICE);
+ dp_dma = cmh_dma_map_single(dp_buf, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ct_dma) || cmh_dma_map_error(dp_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_sm2_dec_point(&vcq[1], core_id, req.ciphertext_len, SM2_CLEN,
+ ct_dma, dp_dma, req.key_ref,
+ PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[2], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(dp_dma))
+ cmh_dma_unmap_single(dp_dma, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ct_dma))
+ cmh_dma_unmap_single(ct_dma, SM2_POINT_LEN, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.dec_point),
+ dp_buf, SM2_POINT_LEN))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(dp_buf);
+ kfree(ct_buf);
+ return ret;
+}
+
+/* -- SM2_ENC_POINT --------------------- */
+
+/**
+ * cmh_mgmt_sm2_enc_point() - Handle CMH_MGMT_IOC_SM2_ENC_POINT ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_enc_point(void __user *argp)
+{
+ struct cmh_ioctl_sm2_enc_point req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 core_id = cmh_core_default_id(CMH_CORE_PKE);
+ u8 *nonce_buf = NULL, *pk_buf, *ct_buf, *ep_buf;
+ dma_addr_t nonce_dma = DMA_MAPPING_ERROR, pk_dma, ct_dma, ep_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.nonce_len != 0 && req.nonce_len != SM2_NONCE_LEN)
+ return -EINVAL;
+
+ pk_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL);
+ ct_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL);
+ ep_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL);
+ if (!pk_buf || !ct_buf || !ep_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(pk_buf, u64_to_user_ptr(req.public_key),
+ SM2_POINT_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ if (req.nonce_len) {
+ nonce_buf = kmalloc(SM2_NONCE_LEN, GFP_KERNEL);
+ if (!nonce_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ if (copy_from_user(nonce_buf, u64_to_user_ptr(req.nonce),
+ SM2_NONCE_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+ }
+
+ pk_dma = cmh_dma_map_single(pk_buf, SM2_POINT_LEN, DMA_TO_DEVICE);
+ ct_dma = cmh_dma_map_single(ct_buf, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ ep_dma = cmh_dma_map_single(ep_buf, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ if (nonce_buf)
+ nonce_dma = cmh_dma_map_single(nonce_buf, SM2_NONCE_LEN,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(pk_dma) || cmh_dma_map_error(ct_dma) ||
+ cmh_dma_map_error(ep_dma) ||
+ (nonce_buf && cmh_dma_map_error(nonce_dma))) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_sm2_enc_point(&vcq[1], core_id, nonce_dma, pk_dma, ct_dma,
+ ep_dma, req.nonce_len, 0);
+ vcq_add_pke_flush(&vcq[2], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (nonce_buf && !cmh_dma_map_error(nonce_dma))
+ cmh_dma_unmap_single(nonce_dma, SM2_NONCE_LEN, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(ep_dma))
+ cmh_dma_unmap_single(ep_dma, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ct_dma))
+ cmh_dma_unmap_single(ct_dma, SM2_POINT_LEN, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(pk_dma))
+ cmh_dma_unmap_single(pk_dma, SM2_POINT_LEN, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.ciphertext),
+ ct_buf, SM2_POINT_LEN) ||
+ copy_to_user(u64_to_user_ptr(req.enc_point),
+ ep_buf, SM2_POINT_LEN))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(nonce_buf);
+ kfree(ep_buf);
+ kfree(ct_buf);
+ kfree(pk_buf);
+ return ret;
+}
+
+/* -- SM2_ID_DIGEST --------------------- */
+
+/**
+ * cmh_mgmt_sm2_id_digest() - Handle CMH_MGMT_IOC_SM2_ID_DIGEST ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_id_digest(void __user *argp)
+{
+ struct cmh_ioctl_sm2_id_digest req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 core_id = cmh_core_default_id(CMH_CORE_PKE);
+ u8 *id_buf, *pk_buf, *dig_buf;
+ dma_addr_t id_dma, pk_dma, dig_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.id_len || req.id_len > SM2_MAX_ID_LEN)
+ return -EINVAL;
+
+ id_buf = kmalloc(req.id_len, GFP_KERNEL);
+ pk_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL);
+ dig_buf = kzalloc(SM2_DIGEST_LEN, GFP_KERNEL);
+ if (!id_buf || !pk_buf || !dig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(id_buf, u64_to_user_ptr(req.id), req.id_len) ||
+ copy_from_user(pk_buf, u64_to_user_ptr(req.public_key),
+ SM2_POINT_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ id_dma = cmh_dma_map_single(id_buf, req.id_len, DMA_TO_DEVICE);
+ pk_dma = cmh_dma_map_single(pk_buf, SM2_POINT_LEN, DMA_TO_DEVICE);
+ dig_dma = cmh_dma_map_single(dig_buf, SM2_DIGEST_LEN,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(id_dma) || cmh_dma_map_error(pk_dma) ||
+ cmh_dma_map_error(dig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_sm2_id_digest(&vcq[1], core_id, id_dma, pk_dma, dig_dma,
+ req.id_len, 0);
+ vcq_add_pke_flush(&vcq[2], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(dig_dma))
+ cmh_dma_unmap_single(dig_dma, SM2_DIGEST_LEN,
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(pk_dma))
+ cmh_dma_unmap_single(pk_dma, SM2_POINT_LEN, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(id_dma))
+ cmh_dma_unmap_single(id_dma, req.id_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.digest),
+ dig_buf, SM2_DIGEST_LEN))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(dig_buf);
+ kfree(pk_buf);
+ kfree(id_buf);
+ return ret;
+}
+
+/* -- SM2_ECDH_HASH --------------------- */
+
+/**
+ * cmh_mgmt_sm2_ecdh_hash() - Handle CMH_MGMT_IOC_SM2_ECDH_HASH ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_ecdh_hash(void __user *argp)
+{
+ struct cmh_ioctl_sm2_ecdh_hash req;
+ /* Phase 1: hdr + sys_new + sm2_ecdh_hash + pke_flush; reused for Phase 2 */
+ struct vcq_cmd vcq[4];
+ u32 sk_type, core_id;
+ u8 *peer_dig_buf, *dig_buf, *sk_buf;
+ u64 *ref_buf;
+ dma_addr_t peer_dig_dma, dig_dma, sk_dma, ref_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.__reserved)
+ return -EINVAL;
+
+ sk_type = SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_PKE);
+ core_id = cmh_core_default_id(CMH_CORE_PKE);
+
+ peer_dig_buf = kmalloc(SM2_DIGEST_LEN, GFP_KERNEL);
+ dig_buf = kmalloc(SM2_DIGEST_LEN, GFP_KERNEL);
+ sk_buf = kzalloc(SM2_SHARED_KEY_LEN, GFP_KERNEL);
+ ref_buf = kzalloc_obj(u64, GFP_KERNEL);
+ if (!peer_dig_buf || !dig_buf || !sk_buf || !ref_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(peer_dig_buf, u64_to_user_ptr(req.peer_id_digest),
+ SM2_DIGEST_LEN) ||
+ copy_from_user(dig_buf, u64_to_user_ptr(req.id_digest),
+ SM2_DIGEST_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ peer_dig_dma = cmh_dma_map_single(peer_dig_buf, SM2_DIGEST_LEN,
+ DMA_TO_DEVICE);
+ dig_dma = cmh_dma_map_single(dig_buf, SM2_DIGEST_LEN, DMA_TO_DEVICE);
+ sk_dma = cmh_dma_map_single(sk_buf, SM2_SHARED_KEY_LEN,
+ DMA_FROM_DEVICE);
+ ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(peer_dig_dma) || cmh_dma_map_error(dig_dma) ||
+ cmh_dma_map_error(sk_dma) || cmh_dma_map_error(ref_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ /*
+ * Phase 1: sys_new(shared_key_ref) + SM2_ECDH_HASH
+ * The shared_point_ref from the ECDH step is passed directly
+ * as a DS reference -- the eSW hub reads it from DS.
+ */
+ vcq_set_header(&vcq[0], 4);
+ vcq_add_sys_new(&vcq[1], 0, ref_dma, SM2_SHARED_KEY_LEN);
+ vcq[1].id |= PKE_SWAP_FLAGS;
+ vcq_add_pke_sm2_ecdh_hash(&vcq[2], core_id, peer_dig_dma, dig_dma,
+ req.shared_point_ref, SYS_REF_LAST,
+ sk_type, PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[3], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 4, 1, MGMT_MBX);
+ if (ret)
+ goto out_unmap;
+
+ /* Sync bounce buffer so CPU sees the DMA-written ref */
+ cmh_dma_sync_for_cpu(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+
+ /* Phase 2: read shared key from DS -> DMA */
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_data(&vcq[1], *ref_buf, sk_dma, SM2_SHARED_KEY_LEN);
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(ref_dma))
+ cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(sk_dma))
+ cmh_dma_unmap_single(sk_dma, SM2_SHARED_KEY_LEN,
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(dig_dma))
+ cmh_dma_unmap_single(dig_dma, SM2_DIGEST_LEN, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(peer_dig_dma))
+ cmh_dma_unmap_single(peer_dig_dma, SM2_DIGEST_LEN,
+ DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.shared_key),
+ sk_buf, SM2_SHARED_KEY_LEN))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(ref_buf);
+ kfree_sensitive(sk_buf);
+ kfree(dig_buf);
+ kfree(peer_dig_buf);
+ return ret;
+}
+
+/* -- SM2_DEC_HASH ---------------------- */
+
+/**
+ * cmh_mgmt_sm2_dec_hash() - Handle CMH_MGMT_IOC_SM2_DEC_HASH ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_dec_hash(void __user *argp)
+{
+ struct cmh_ioctl_sm2_dec_hash req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 msg_len, core_id;
+ u8 *ct_buf, *dp_buf, *pt_buf;
+ dma_addr_t ct_dma, dp_dma, pt_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (req.ciphertext_len <= SM2_CT_OVERHEAD ||
+ req.ciphertext_len > SM2_MAX_CT_LEN)
+ return -EINVAL;
+
+ msg_len = req.ciphertext_len - SM2_CT_OVERHEAD;
+ core_id = cmh_core_default_id(CMH_CORE_PKE);
+
+ ct_buf = kmalloc(req.ciphertext_len, GFP_KERNEL);
+ dp_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL);
+ pt_buf = kzalloc(msg_len, GFP_KERNEL);
+ if (!ct_buf || !dp_buf || !pt_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(ct_buf, u64_to_user_ptr(req.ciphertext),
+ req.ciphertext_len) ||
+ copy_from_user(dp_buf, u64_to_user_ptr(req.dec_point),
+ SM2_POINT_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ ct_dma = cmh_dma_map_single(ct_buf, req.ciphertext_len,
+ DMA_TO_DEVICE);
+ dp_dma = cmh_dma_map_single(dp_buf, SM2_POINT_LEN, DMA_TO_DEVICE);
+ pt_dma = cmh_dma_map_single(pt_buf, msg_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(ct_dma) || cmh_dma_map_error(dp_dma) ||
+ cmh_dma_map_error(pt_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_sm2_dec_hash(&vcq[1], core_id, ct_dma, dp_dma, pt_dma,
+ req.ciphertext_len, 0);
+ vcq_add_pke_flush(&vcq[2], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(pt_dma))
+ cmh_dma_unmap_single(pt_dma, msg_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(dp_dma))
+ cmh_dma_unmap_single(dp_dma, SM2_POINT_LEN, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(ct_dma))
+ cmh_dma_unmap_single(ct_dma, req.ciphertext_len,
+ DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.plaintext),
+ pt_buf, msg_len))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree_sensitive(pt_buf);
+ kfree_sensitive(dp_buf);
+ kfree(ct_buf);
+ return ret;
+}
+
+/* -- SM2_ENC_HASH ---------------------- */
+
+/**
+ * cmh_mgmt_sm2_enc_hash() - Handle CMH_MGMT_IOC_SM2_ENC_HASH ioctl
+ * @argp: User-space ioctl argument pointer
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_mgmt_sm2_enc_hash(void __user *argp)
+{
+ struct cmh_ioctl_sm2_enc_hash req;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ u32 ct_len, core_id;
+ u8 *msg_buf, *ep_buf, *ct_buf;
+ dma_addr_t msg_dma, ep_dma, ct_dma;
+ int ret;
+
+ if (copy_from_user(&req, argp, sizeof(req)))
+ return -EFAULT;
+ if (req.version != CMH_MGMT_V1)
+ return -EINVAL;
+ if (!req.message_len || req.message_len > SM2_MAX_MSG_LEN)
+ return -EINVAL;
+
+ ct_len = SM2_CT_OVERHEAD + req.message_len;
+ core_id = cmh_core_default_id(CMH_CORE_PKE);
+
+ msg_buf = kmalloc(req.message_len, GFP_KERNEL);
+ ep_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL);
+ ct_buf = kzalloc(ct_len, GFP_KERNEL);
+ if (!msg_buf || !ep_buf || !ct_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (copy_from_user(msg_buf, u64_to_user_ptr(req.message),
+ req.message_len) ||
+ copy_from_user(ep_buf, u64_to_user_ptr(req.enc_point),
+ SM2_POINT_LEN)) {
+ ret = -EFAULT;
+ goto out_free;
+ }
+
+ msg_dma = cmh_dma_map_single(msg_buf, req.message_len, DMA_TO_DEVICE);
+ ep_dma = cmh_dma_map_single(ep_buf, SM2_POINT_LEN, DMA_TO_DEVICE);
+ ct_dma = cmh_dma_map_single(ct_buf, ct_len, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(msg_dma) || cmh_dma_map_error(ep_dma) ||
+ cmh_dma_map_error(ct_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_sm2_enc_hash(&vcq[1], core_id, msg_dma, ep_dma, ct_dma,
+ req.message_len, 0);
+ vcq_add_pke_flush(&vcq[2], core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX);
+
+out_unmap:
+ if (!cmh_dma_map_error(ct_dma))
+ cmh_dma_unmap_single(ct_dma, ct_len, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(ep_dma))
+ cmh_dma_unmap_single(ep_dma, SM2_POINT_LEN, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(msg_dma))
+ cmh_dma_unmap_single(msg_dma, req.message_len, DMA_TO_DEVICE);
+
+ if (!ret) {
+ if (copy_to_user(u64_to_user_ptr(req.ciphertext),
+ ct_buf, ct_len))
+ ret = -EFAULT;
+ }
+
+out_free:
+ kfree(ct_buf);
+ kfree(ep_buf);
+ kfree_sensitive(msg_buf);
+ return ret;
+}
diff --git a/drivers/crypto/cmh/cmh_sys.c b/drivers/crypto/cmh/cmh_sys.c
new file mode 100644
index 000000000000..b01d058e6d89
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_sys.c
@@ -0,0 +1,376 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- SYS Core VCQ Builders
+ *
+ * VCQ builder functions for SYS core datastore commands. Each function
+ * populates a single vcq_cmd slot. Callers (cmh_mgmt.c, cmh_key.c)
+ * assemble complete VCQs by combining header + command(s) + flush,
+ * then submit via cmh_tm_submit_sync().
+ *
+ * Hardware-required datastore semantics
+ * --------------------------------------
+ * The commands below (NEW, WRITE, DATA, FIND, DELETE, FLUSH) are
+ * direct mappings of the eSW firmware SYS core command set. The
+ * eSW maintains per-mailbox datastore namespaces with two object
+ * classes:
+ *
+ * SYS_REF_TEMP -- Temporary objects. Lifetime is scoped to the
+ * current mailbox slot; reclaimed automatically
+ * when the slot is reused or on explicit FLUSH.
+ * Used for raw-key provisioning on every VCQ.
+ *
+ * SYS_REF_PERSIST -- Persistent objects. Survive across slots;
+ * require explicit DELETE to reclaim. Identified
+ * by a 64-bit Content ID (CID) and resolved to
+ * a per-MBX ref via SYS_CMD_FIND.
+ *
+ * These semantics are hardware requirements, not driver policy.
+ * The per-MBX temp-stack and per-MBX ref namespace are eSW firmware
+ * design constraints that cannot be changed by the kernel driver.
+ */
+
+#include <linux/string.h>
+
+#include "cmh_sys.h"
+
+/**
+ * vcq_add_sys_flush() - Build a SYS_FLUSH VCQ command
+ * @slot: VCQ command slot to populate
+ */
+void vcq_add_sys_flush(struct vcq_cmd *slot)
+{
+ vcq_add_flush(slot, CORE_ID_SYS);
+}
+
+/**
+ * vcq_add_sys_new() - Build a SYS_NEW VCQ command
+ * @slot: VCQ command slot to populate
+ * @cid: Content identifier for the new datastore object
+ * @ref_dma: DMA address of the object reference buffer
+ * @len: Length of the object data in bytes
+ */
+void vcq_add_sys_new(struct vcq_cmd *slot, u64 cid, u64 ref_dma, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_NEW);
+ slot->hwc.sys.cmd_new.cid = cid;
+ slot->hwc.sys.cmd_new.ref = ref_dma;
+ slot->hwc.sys.cmd_new.len = len;
+}
+
+/**
+ * vcq_add_sys_write() - Build a SYS_WRITE VCQ command
+ * @slot: VCQ command slot to populate
+ * @ref: Datastore object reference handle
+ * @src_dma: DMA address of source data buffer
+ * @wrap_key: Wrapping key reference (0 if none)
+ * @len: Length of data to write in bytes
+ * @sys_type: Datastore object type identifier
+ */
+void vcq_add_sys_write(struct vcq_cmd *slot, u64 ref, u64 src_dma,
+ u64 wrap_key, u32 len, u32 sys_type)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_WRITE);
+ slot->hwc.sys.cmd_write.ref = ref;
+ slot->hwc.sys.cmd_write.src = src_dma;
+ slot->hwc.sys.cmd_write.key = wrap_key;
+ slot->hwc.sys.cmd_write.len = len;
+ slot->hwc.sys.cmd_write.type = sys_type;
+}
+
+/**
+ * vcq_add_sys_read() - Build a SYS_READ VCQ command
+ * @slot: VCQ command slot to populate
+ * @ref: Datastore object reference handle
+ * @dst_dma: DMA address of destination buffer
+ * @wrap_key: Wrapping key reference (0 if none)
+ * @len: Length of data to read in bytes
+ */
+void vcq_add_sys_read(struct vcq_cmd *slot, u64 ref, u64 dst_dma,
+ u64 wrap_key, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_READ);
+ slot->hwc.sys.cmd_read.ref = ref;
+ slot->hwc.sys.cmd_read.dst = dst_dma;
+ slot->hwc.sys.cmd_read.key = wrap_key;
+ slot->hwc.sys.cmd_read.len = len;
+}
+
+/**
+ * vcq_add_sys_data() - Build a SYS_DATA VCQ command
+ * @slot: VCQ command slot to populate
+ * @ref: Datastore object reference handle
+ * @dst_dma: DMA address of destination buffer
+ * @len: Length of data section to read in bytes
+ */
+void vcq_add_sys_data(struct vcq_cmd *slot, u64 ref, u64 dst_dma, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_DATA);
+ slot->hwc.sys.cmd_data.ref = ref;
+ slot->hwc.sys.cmd_data.dst = dst_dma;
+ slot->hwc.sys.cmd_data.len = len;
+}
+
+/**
+ * vcq_add_sys_find() - Build a SYS_FIND VCQ command
+ * @slot: VCQ command slot to populate
+ * @cid: Content identifier to search for
+ * @dst_dma: DMA address of destination buffer for result
+ * @len: Length of destination buffer in bytes
+ */
+void vcq_add_sys_find(struct vcq_cmd *slot, u64 cid, u64 dst_dma, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_FIND);
+ slot->hwc.sys.cmd_find.cid = cid;
+ slot->hwc.sys.cmd_find.dst = dst_dma;
+ slot->hwc.sys.cmd_find.len = len;
+}
+
+/**
+ * vcq_add_sys_list() - Build a SYS_LIST VCQ command
+ * @slot: VCQ command slot to populate
+ * @ref: Datastore object reference for enumeration start
+ * @dst_dma: DMA address of destination buffer for list
+ * @len: Length of destination buffer in bytes
+ */
+void vcq_add_sys_list(struct vcq_cmd *slot, u64 ref, u64 dst_dma, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_LIST);
+ slot->hwc.sys.cmd_list.ref = ref;
+ slot->hwc.sys.cmd_list.dst = dst_dma;
+ slot->hwc.sys.cmd_list.len = len;
+}
+
+/**
+ * vcq_add_sys_grant() - Build a SYS_GRANT VCQ command
+ * @slot: VCQ command slot to populate
+ * @ref: Datastore object reference handle
+ * @read: Read permission bitmask
+ * @write: Write permission bitmask
+ * @execute: Execute permission bitmask
+ */
+void vcq_add_sys_grant(struct vcq_cmd *slot, u64 ref, u64 read,
+ u64 write, u64 execute)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_GRANT);
+ slot->hwc.sys.cmd_grant.ref = ref;
+ slot->hwc.sys.cmd_grant.read = read;
+ slot->hwc.sys.cmd_grant.write = write;
+ slot->hwc.sys.cmd_grant.execute = execute;
+}
+
+/**
+ * vcq_add_sys_export() - Build a SYS_EXPORT VCQ command
+ * @slot: VCQ command slot to populate
+ * @cid: Content identifier of object to export
+ * @dst_dma: DMA address of destination buffer for wrapped blob
+ * @wrap_key: Wrapping key reference for export
+ * @len: Length of destination buffer in bytes
+ */
+void vcq_add_sys_export(struct vcq_cmd *slot, u64 cid, u64 dst_dma,
+ u64 wrap_key, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_EXPORT);
+ slot->hwc.sys.cmd_export.cid = cid;
+ slot->hwc.sys.cmd_export.dst = dst_dma;
+ slot->hwc.sys.cmd_export.key = wrap_key;
+ slot->hwc.sys.cmd_export.len = len;
+}
+
+/**
+ * vcq_add_sys_import() - Build a SYS_IMPORT VCQ command
+ * @slot: VCQ command slot to populate
+ * @src_dma: DMA address of wrapped datastore blob to import
+ * @wrap_key: Wrapping key reference for unwrapping
+ * @len: Length of wrapped blob in bytes
+ */
+void vcq_add_sys_import(struct vcq_cmd *slot, u64 src_dma,
+ u64 wrap_key, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_IMPORT);
+ slot->hwc.sys.cmd_import.src = src_dma;
+ slot->hwc.sys.cmd_import.key = wrap_key;
+ slot->hwc.sys.cmd_import.len = len;
+}
+
+/* -- KIC Core VCQ Builders --------------------- */
+
+/**
+ * vcq_add_kic_hkdf1() - Build a KIC HKDF-Expand VCQ command
+ * @slot: VCQ command slot to populate
+ * @dst: Datastore reference for derived key output
+ * @base: Datastore reference for base key input
+ * @label_dma: DMA address of HKDF label/info buffer
+ * @key_len: Derived key length in bytes
+ * @label_len: Length of label buffer in bytes
+ * @type: Derived key datastore type
+ */
+void vcq_add_kic_hkdf1(struct vcq_cmd *slot, u64 dst, u64 base,
+ u64 label_dma, u32 key_len, u32 label_len, u32 type)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_KIC, 0, 1, KIC_CMD_HKDF1);
+ slot->hwc.kic.cmd_hkdf1.dst = dst;
+ slot->hwc.kic.cmd_hkdf1.base = base;
+ slot->hwc.kic.cmd_hkdf1.label = label_dma;
+ slot->hwc.kic.cmd_hkdf1.llen = label_len;
+ slot->hwc.kic.cmd_hkdf1.len = key_len;
+ slot->hwc.kic.cmd_hkdf1.type = type;
+}
+
+/**
+ * vcq_add_kic_hkdf2() - Build a KIC HKDF-with-salt VCQ command
+ * @slot: VCQ command slot to populate
+ * @dst: Datastore reference for derived key output
+ * @base: Datastore reference for base key input
+ * @salt: Datastore reference for HKDF salt key
+ * @label_dma: DMA address of HKDF label/info buffer
+ * @key_len: Derived key length in bytes
+ * @label_len: Length of label buffer in bytes
+ * @type: Derived key datastore type
+ */
+void vcq_add_kic_hkdf2(struct vcq_cmd *slot, u64 dst, u64 base, u64 salt,
+ u64 label_dma, u32 key_len, u32 label_len, u32 type)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_KIC, 0, 1, KIC_CMD_HKDF2);
+ slot->hwc.kic.cmd_hkdf2.dst = dst;
+ slot->hwc.kic.cmd_hkdf2.base = base;
+ slot->hwc.kic.cmd_hkdf2.salt = salt;
+ slot->hwc.kic.cmd_hkdf2.label = label_dma;
+ slot->hwc.kic.cmd_hkdf2.llen = label_len;
+ slot->hwc.kic.cmd_hkdf2.len = key_len;
+ slot->hwc.kic.cmd_hkdf2.type = type;
+}
+
+/**
+ * vcq_add_kic_aes_cmac_kdf() - Build a KIC AES-CMAC KDF VCQ command
+ * @slot: VCQ command slot to populate
+ * @out_key: Datastore reference for derived key output
+ * @base_key: Datastore reference for base key input
+ * @label_dma: DMA address of KDF label buffer
+ * @key_len: Derived key length in bytes
+ * @label_len: Length of label buffer in bytes
+ * @type: Derived key datastore type
+ */
+void vcq_add_kic_aes_cmac_kdf(struct vcq_cmd *slot, u64 out_key, u64 base_key,
+ u64 label_dma, u32 key_len, u32 label_len,
+ u32 type)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_KIC, 0, 1, KIC_CMD_AES_CMAC_KDF);
+ slot->hwc.kic.cmd_aes_cmac_kdf.base_key = base_key;
+ slot->hwc.kic.cmd_aes_cmac_kdf.out_key = out_key;
+ slot->hwc.kic.cmd_aes_cmac_kdf.label = label_dma;
+ slot->hwc.kic.cmd_aes_cmac_kdf.key_len = key_len;
+ slot->hwc.kic.cmd_aes_cmac_kdf.label_len = label_len;
+ slot->hwc.kic.cmd_aes_cmac_kdf.type = type;
+}
+
+/**
+ * vcq_add_kic_dkek_derive() - Build a KIC DKEK derivation VCQ command
+ * @slot: VCQ command slot to populate
+ * @out_key: Datastore reference for derived DKEK output
+ * @base_key: Datastore reference for base key input
+ * @host_id: Host identifier for key binding
+ * @metadata_dma: DMA address of derivation metadata buffer
+ * @metadata_len: Length of metadata buffer in bytes
+ */
+void vcq_add_kic_dkek_derive(struct vcq_cmd *slot, u64 out_key, u64 base_key,
+ u32 host_id, u64 metadata_dma, u32 metadata_len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_KIC, 0, 1, KIC_CMD_DKEK_DERIVE);
+ slot->hwc.kic.cmd_dkek_derive.base_key = base_key;
+ slot->hwc.kic.cmd_dkek_derive.out_key = out_key;
+ slot->hwc.kic.cmd_dkek_derive.host_id = host_id;
+ slot->hwc.kic.cmd_dkek_derive.metadata = metadata_dma;
+ slot->hwc.kic.cmd_dkek_derive.metadata_len = metadata_len;
+}
+
+/* -- DRBG Core VCQ Builders -------------------- */
+
+/**
+ * vcq_add_drbg_reset() - Build a DRBG reset VCQ command
+ * @slot: VCQ command slot to populate
+ *
+ * Issues DRBG_CMD_RESET which clears the instantiated state, allowing
+ * a subsequent CONFIG to proceed without a double-instantiate error.
+ */
+void vcq_add_drbg_reset(struct vcq_cmd *slot)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_DRBG, 0, 1, DRBG_CMD_RESET);
+}
+
+/**
+ * vcq_add_drbg_config() - Build a DRBG configuration VCQ command
+ * @slot: VCQ command slot to populate
+ * @ratio: Entropy-to-output ratio
+ * @strength: Security strength in bits
+ */
+void vcq_add_drbg_config(struct vcq_cmd *slot, u32 ratio, u32 strength)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_DRBG, 0, 1, DRBG_CMD_CONFIG);
+ slot->hwc.drbg.cmd_config.entropy_ratio = ratio;
+ slot->hwc.drbg.cmd_config.security_strength = strength;
+}
+
+/**
+ * vcq_add_drbg_datastore() - Build a DRBG datastore setup VCQ command
+ * @slot: VCQ command slot to populate
+ * @ref: Datastore object reference handle
+ * @len: Length of datastore allocation in bytes
+ * @type: Datastore object type
+ */
+void vcq_add_drbg_datastore(struct vcq_cmd *slot, u64 ref, u32 len, u32 type)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_DRBG, 0, 1, DRBG_CMD_DATASTORE);
+ slot->hwc.drbg.cmd_datastore.ref = ref;
+ slot->hwc.drbg.cmd_datastore.len = len;
+ slot->hwc.drbg.cmd_datastore.type = type;
+}
+
+/* -- EAC Core VCQ Builder ---------------------- */
+
+/**
+ * vcq_add_eac_read() - Build an EAC read VCQ command
+ * @slot: VCQ command slot to populate
+ * @dst_dma: DMA address of destination buffer
+ * @len: Length of data to read in bytes
+ */
+void vcq_add_eac_read(struct vcq_cmd *slot, u64 dst_dma, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(CORE_ID_EAC, 0, 1, EAC_CMD_READ);
+ slot->hwc.eac.cmd_read.dst = dst_dma;
+ slot->hwc.eac.cmd_read.len = len;
+}
diff --git a/drivers/crypto/cmh/include/cmh_key.h b/drivers/crypto/cmh/include/cmh_key.h
new file mode 100644
index 000000000000..bad69c92b892
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_key.h
@@ -0,0 +1,82 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Per-transform key context
+ *
+ * Per-transform key context used by all keyed crypto algorithms (AES,
+ * SM4, CCP, HMAC, KMAC). Stores raw key bytes supplied via the crypto
+ * API .setkey() callback: the key is DMA-mapped once at setkey time and
+ * written to SYS_REF_TEMP in every VCQ.
+ *
+ * Each keyed algorithm driver embeds a struct cmh_key_ctx in its
+ * per-transform context and calls cmh_key_setkey_raw() from its
+ * .setkey() callback.
+ *
+ * Raw-key atomicity (SYS_REF_TEMP)
+ * ---------------------------------
+ * SYS_CMD_WRITE to SYS_REF_TEMP is packed into the same VCQ as the
+ * algorithm commands (AES_CMD_INIT, HC_CMD_HMAC, etc.). SYS_REF_TEMP
+ * is per-MBX -- the CMH eSW allocates it in the tail of each mailbox's
+ * own VCQ buffer (mbx_alloc_temp), so concurrent raw-key requests on
+ * different MBXes do not interfere.
+ */
+
+#ifndef CMH_KEY_H
+#define CMH_KEY_H
+
+#include <linux/types.h>
+#include "cmh_config.h"
+#include "cmh_vcq.h"
+
+/* Key context mode */
+enum cmh_key_mode {
+ CMH_KEY_NONE = 0, /* no key set yet */
+ CMH_KEY_RAW, /* raw key bytes in memory */
+};
+
+/* Per-transform key context */
+struct cmh_key_ctx {
+ enum cmh_key_mode mode;
+ struct {
+ u8 *data; /* kmemdup'd raw key bytes */
+ u32 len; /* key length in bytes */
+ u32 sys_type; /* SYS_TYPE_SET(flags, core_id) */
+ dma_addr_t dma; /* pre-mapped DMA addr (DMA_TO_DEVICE) */
+ } raw;
+};
+
+/**
+ * cmh_key_setkey_raw() - Store raw key bytes in the transform context
+ * @ctx: Per-transform key context
+ * @key: Raw key bytes
+ * @keylen: Key length in bytes
+ * @core_id: Target algorithm core (e.g. CORE_ID_AES)
+ *
+ * SYS_TYPE_FLAG_PT is set so the written temp key
+ * can be read back as plaintext if needed. The actual SYS_CMD_WRITE
+ * to SYS_REF_TEMP is deferred to each encrypt/decrypt VCQ, where it
+ * is packed inline for atomicity.
+ *
+ * Return: 0 on success, -ENOMEM on allocation failure.
+ */
+int cmh_key_setkey_raw(struct cmh_key_ctx *ctx, const u8 *key,
+ u32 keylen, u32 core_id);
+
+/**
+ * cmh_key_destroy() - Free key resources
+ * @ctx: Per-transform key context
+ *
+ * Zeroises and frees the raw key buffer.
+ */
+void cmh_key_destroy(struct cmh_key_ctx *ctx);
+
+/**
+ * cmh_ds_type_to_core_id() - Map datastore key type to core ID
+ * @ds_type: CMH_DS_* key type constant
+ *
+ * Return: Corresponding CORE_ID_*, or CORE_ID_NUM (0x1F) on
+ * unrecognised type (caller should return -EINVAL).
+ */
+u32 cmh_ds_type_to_core_id(u32 ds_type);
+
+#endif /* CMH_KEY_H */
diff --git a/drivers/crypto/cmh/include/cmh_mgmt.h b/drivers/crypto/cmh/include/cmh_mgmt.h
new file mode 100644
index 000000000000..b211014bd71d
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_mgmt.h
@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH -- Key Management misc_device (/dev/cmh_mgmt)
+ *
+ * ioctl interface for key CRUD + datastore export/import,
+ * PKE operations (RSA, ECDSA, ECDH, EdDSA),
+ * and PQC operations (ML-KEM, ML-DSA, SLH-DSA).
+ *
+ * Registered alongside crypto algorithms in module_init,
+ * unregistered before them in module_exit.
+ */
+
+#ifndef CMH_MGMT_H
+#define CMH_MGMT_H
+
+#ifdef CONFIG_CRYPTO_DEV_CMH_MGMT
+
+/*
+ * Pin all mgmt ioctls to MBX 0 for DS ownership and SYS_REF_TEMP scope.
+ * Shared by cmh_mgmt.c, cmh_mgmt_pke.c, cmh_mgmt_pqc.c, cmh_pke_sm2.c.
+ */
+#define MGMT_MBX 0
+
+/* Maximum DMA buffer size for key data / datastore blobs */
+#define CMH_MGMT_MAX_DATA_LEN (256 * 1024) /* 256 KB */
+
+int cmh_mgmt_register(void);
+void cmh_mgmt_unregister(void);
+
+/* -- PKE ioctl handlers (cmh_mgmt_pke.c) -- */
+int cmh_mgmt_pke_rsa_enc(void __user *argp);
+int cmh_mgmt_pke_rsa_dec(void __user *argp);
+int cmh_mgmt_pke_rsa_crt_dec(void __user *argp);
+int cmh_mgmt_pke_rsa_keygen(void __user *argp);
+int cmh_mgmt_pke_ecdsa_sign(void __user *argp);
+int cmh_mgmt_pke_ecdh(void __user *argp);
+int cmh_mgmt_pke_ecdh_keygen(void __user *argp);
+int cmh_mgmt_pke_eddsa_sign(void __user *argp);
+int cmh_mgmt_pke_eddsa_verify(void __user *argp);
+int cmh_mgmt_pke_ec_keygen(void __user *argp);
+int cmh_mgmt_pke_ec_pubgen(void __user *argp);
+int cmh_mgmt_pke_eddsa_keygen_sca(void __user *argp);
+
+/* -- PQC ioctl handlers (cmh_mgmt_pqc.c) -- */
+int cmh_mgmt_ml_kem_keygen(void __user *argp);
+int cmh_mgmt_ml_kem_enc(void __user *argp);
+int cmh_mgmt_ml_kem_dec(void __user *argp);
+int cmh_mgmt_ml_dsa_keygen(void __user *argp);
+int cmh_mgmt_ml_dsa_sign(void __user *argp);
+int cmh_mgmt_slhdsa_keygen(void __user *argp);
+int cmh_mgmt_slhdsa_sign(void __user *argp);
+int cmh_mgmt_slhdsa_sign_prehash(void __user *argp);
+
+#else /* !CONFIG_CRYPTO_DEV_CMH_MGMT */
+
+static inline int cmh_mgmt_register(void) { return 0; }
+static inline void cmh_mgmt_unregister(void) { }
+
+#endif /* CONFIG_CRYPTO_DEV_CMH_MGMT */
+
+#endif /* CMH_MGMT_H */
diff --git a/drivers/crypto/cmh/include/cmh_pke.h b/drivers/crypto/cmh/include/cmh_pke.h
new file mode 100644
index 000000000000..dcfdb3fc3cd6
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_pke.h
@@ -0,0 +1,245 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- PKE Common Types and Helpers
+ *
+ * Shared definitions for RSA, ECDSA, ECDH, EdDSA, and SM2 drivers.
+ * Curve -> coordinate-length mapping, VCQ byte-swap flags, and
+ * common VCQ builder prototypes.
+ */
+
+#ifndef CMH_PKE_H
+#define CMH_PKE_H
+
+#include <linux/types.h>
+#include "cmh_vcq.h"
+#include "cmh_pke_abi.h"
+
+/* VCQ byte-swap flags for DMA transfers (per CMH VCQ ABI) */
+#define VCQ_FLAG_SWAP_BYTES 0x400000U
+#define VCQ_FLAG_SWAP_WORDS 0x200000U
+
+/* VCQ byte-swap flags for PKE -- big-endian data on LE bus */
+#define PKE_SWAP_FLAGS (VCQ_FLAG_SWAP_BYTES | VCQ_FLAG_SWAP_WORDS)
+
+/* VCQ layout: header + [SYS_WRITE] + PKE_CMD + flush */
+#define PKE_VCQ_CMDS_MIN 3 /* header + cmd + flush */
+#define PKE_VCQ_CMDS_MAX 4 /* header + SYS_WRITE + cmd + flush */
+
+/* Max RSA key size in bytes (4096 bits) */
+#define PKE_RSA_MAX_BYTES 512
+#define PKE_RSA_MIN_BITS 1024
+#define PKE_RSA_MAX_BITS 4096
+
+/* EdDSA SCA: Ed448 blinded private key length (bytes) */
+#define PKE_ED448_SK_SCA_LEN 226
+
+/**
+ * pke_curve_clen() - Get EC curve coordinate length in bytes
+ * @curve: PKE curve identifier (PKE_CURVE_*)
+ *
+ * Return: Coordinate length in bytes, or 0 for unknown curves.
+ */
+static inline u32 pke_curve_clen(u32 curve)
+{
+ switch (curve) {
+ case PKE_CURVE_P192:
+ case PKE_CURVE_BP192R1:
+ return 24;
+ case PKE_CURVE_P224:
+ case PKE_CURVE_BP224R1:
+ return 28;
+ case PKE_CURVE_P256:
+ case PKE_CURVE_SECP256K1:
+ case PKE_CURVE_BP256R1:
+ case PKE_CURVE_ANSSI_FRP256V1:
+ case PKE_CURVE_SM2:
+ case PKE_CURVE_25519:
+ return 32;
+ case PKE_CURVE_BP320R1:
+ return 40;
+ case PKE_CURVE_P384:
+ case PKE_CURVE_BP384R1:
+ return 48;
+ case PKE_CURVE_BP512R1:
+ return 64;
+ case PKE_CURVE_P521:
+ return 68; /* ceil(521/8) = 66, ABI uses ALIGN(66, 4) = 68 */
+ case PKE_CURVE_448:
+ return 56;
+ default:
+ return 0;
+ }
+}
+
+/**
+ * pke_curve_bits() - Get EC curve size in bits
+ * @curve: PKE curve identifier (PKE_CURVE_*)
+ *
+ * Return: Curve size in bits, or 0 for unknown curves.
+ */
+static inline u32 pke_curve_bits(u32 curve)
+{
+ switch (curve) {
+ case PKE_CURVE_P192:
+ case PKE_CURVE_BP192R1:
+ return 192;
+ case PKE_CURVE_P224:
+ case PKE_CURVE_BP224R1:
+ return 224;
+ case PKE_CURVE_P256:
+ case PKE_CURVE_SECP256K1:
+ case PKE_CURVE_BP256R1:
+ case PKE_CURVE_ANSSI_FRP256V1:
+ case PKE_CURVE_SM2:
+ case PKE_CURVE_25519:
+ return 256;
+ case PKE_CURVE_BP320R1:
+ return 320;
+ case PKE_CURVE_P384:
+ case PKE_CURVE_BP384R1:
+ return 384;
+ case PKE_CURVE_BP512R1:
+ return 512;
+ case PKE_CURVE_P521:
+ return 521;
+ case PKE_CURVE_448:
+ return 448;
+ default:
+ return 0;
+ }
+}
+
+/**
+ * pke_eddsa_key_len() - Get EdDSA key/pubkey length
+ * @curve: PKE curve identifier (PKE_CURVE_25519 or PKE_CURVE_448)
+ *
+ * Ed25519 uses 32 bytes (== clen), Ed448 uses 57 bytes (clen + 1
+ * flag byte per RFC 8032). Signature length is 2 * pke_eddsa_key_len().
+ *
+ * Return: Key length in bytes.
+ */
+static inline u32 pke_eddsa_key_len(u32 curve)
+{
+ u32 clen = pke_curve_clen(curve);
+
+ return (curve == PKE_CURVE_448) ? clen + 1 : clen;
+}
+
+/**
+ * pke_curve_is_edwards() - Check if curve uses Edwards form
+ * @curve: PKE curve identifier (PKE_CURVE_*)
+ *
+ * Return: true for Curve25519 and Curve448, false otherwise.
+ */
+static inline bool pke_curve_is_edwards(u32 curve)
+{
+ return curve == PKE_CURVE_25519 || curve == PKE_CURVE_448;
+}
+
+/**
+ * pke_swap_flags() - Get VCQ byte-swap flags for a given curve
+ * @curve: PKE curve identifier (PKE_CURVE_*)
+ *
+ * Weierstrass curves need byte+word swap; Edwards curves do not.
+ *
+ * Return: VCQ swap flags to OR into the command ID.
+ */
+static inline u32 pke_swap_flags(u32 curve)
+{
+ return pke_curve_is_edwards(curve) ? 0 : PKE_SWAP_FLAGS;
+}
+
+/* Common VCQ builder prototypes */
+
+void vcq_add_pke_flush(struct vcq_cmd *slot, u32 core_id);
+
+void vcq_add_pke_rsa_enc(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len,
+ u64 e_dma, u64 n_dma, u64 m_dma, u64 c_dma,
+ u32 flags);
+
+void vcq_add_pke_rsa_dec(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len,
+ u64 e_dma, u64 n_dma, u64 c_dma, u64 m_dma,
+ u64 d_ref, u32 flags);
+
+void vcq_add_pke_rsa_crt_dec(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len,
+ u64 e_dma, u64 n_dma, u64 c_dma, u64 m_dma,
+ u64 crt_ref, u32 flags);
+
+void vcq_add_pke_ecdsa_verify(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 dlen,
+ u64 pk_dma, u64 dig_dma, u64 sig_dma,
+ u64 rp_dma, u32 flags);
+
+void vcq_add_pke_ecdsa_sign(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 dig_dma, u64 sig_dma, u64 sk_ref,
+ u32 dlen, u32 flags);
+
+void vcq_add_pke_ecdsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 pk_dma, u64 sk_ref, u32 flags);
+
+void vcq_add_pke_ecdsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 sk_ref, u32 sk_type, u32 flags);
+
+void vcq_add_pke_ecdh_keygen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 pkx_dma, u64 sk_ref, u32 flags);
+
+void vcq_add_pke_ecdh(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u32 sslen, u32 ss_type, u64 peer_dma, u64 sk_ref,
+ u64 ss_ref, u32 flags);
+
+void vcq_add_pke_eddsa_verify(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 dlen,
+ u64 pky_dma, u64 dig_dma, u64 sig_dma,
+ u64 rp_dma, u32 flags);
+
+void vcq_add_pke_eddsa_sign(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 dig_dma, u64 sig_dma, u64 sk_ref,
+ u32 dlen, u32 flags);
+
+void vcq_add_pke_eddsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 pky_dma, u64 sk_ref, u32 flags);
+
+void vcq_add_pke_eddsa_keygen_sca(struct vcq_cmd *slot, u32 core_id, u32 curve,
+ u64 sk_ref, u64 sca_sk_ref);
+
+/* SM2 VCQ builders */
+
+void vcq_add_pke_sm2_ecdh_keygen(struct vcq_cmd *slot, u32 core_id, u64 nonce_dma,
+ u64 session_key_dma, u32 nonce_len, u32 flags);
+
+void vcq_add_pke_sm2_ecdh(struct vcq_cmd *slot, u32 core_id, u32 nonce_len,
+ u32 private_key_len, u64 nonce_dma,
+ u64 peer_pk_dma, u64 peer_sk_dma,
+ u64 priv_ref, u64 sp_ref, u32 sp_type, u32 flags);
+
+void vcq_add_pke_sm2_dec_point(struct vcq_cmd *slot, u32 core_id, u32 ct_len,
+ u32 pk_len, u64 ct_dma, u64 dp_dma,
+ u64 priv_ref, u32 flags);
+
+void vcq_add_pke_sm2_enc_point(struct vcq_cmd *slot, u32 core_id, u64 nonce_dma,
+ u64 pk_dma, u64 ct_dma, u64 ep_dma,
+ u32 nonce_len, u32 flags);
+
+void vcq_add_pke_sm2_id_digest(struct vcq_cmd *slot, u32 core_id, u64 id_dma,
+ u64 pk_dma, u64 dig_dma, u32 id_len,
+ u32 flags);
+
+void vcq_add_pke_sm2_ecdh_hash(struct vcq_cmd *slot, u32 core_id, u64 peer_dig_dma,
+ u64 dig_dma, u64 sp_ref, u64 sk_ref,
+ u32 sk_type, u32 flags);
+
+void vcq_add_pke_sm2_dec_hash(struct vcq_cmd *slot, u32 core_id, u64 ct_dma,
+ u64 dp_dma, u64 pt_dma, u32 ct_len, u32 flags);
+
+void vcq_add_pke_sm2_enc_hash(struct vcq_cmd *slot, u32 core_id, u64 msg_dma,
+ u64 ep_dma, u64 ct_dma, u32 msg_len, u32 flags);
+
+/* Registration */
+
+int cmh_pke_rsa_register(void);
+void cmh_pke_rsa_unregister(void);
+int cmh_pke_ecdsa_register(void);
+void cmh_pke_ecdsa_unregister(void);
+int cmh_pke_ecdh_register(void);
+void cmh_pke_ecdh_unregister(void);
+
+#endif /* CMH_PKE_H */
diff --git a/drivers/crypto/cmh/include/cmh_pke_sm2.h b/drivers/crypto/cmh/include/cmh_pke_sm2.h
new file mode 100644
index 000000000000..a2c7164b8d49
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_pke_sm2.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- SM2 PKE Ioctl Handler Declarations
+ *
+ * SM2 signature (GM/T 0003.2) requires the caller to compute
+ * ZA = SM3(ENTLA || IDA || a || b || xG || yG || xA || yA)
+ * and pass SM3(ZA || M) as the digest to the sign/verify path.
+ * The CMH eSW does NOT compute ZA internally; the full
+ * identity pre-hash is the caller's responsibility.
+ *
+ * For the in-kernel akcipher "sm2" algorithm this means the
+ * caller (e.g. asymmetric_key subsystem) must pre-hash with ZA
+ * before invoking verify. The SM2_ID_DIGEST ioctl below can
+ * compute ZA for userspace callers of the misc-device path.
+ */
+
+#ifndef CMH_PKE_SM2_H
+#define CMH_PKE_SM2_H
+
+int cmh_mgmt_sm2_ecdh_keygen(void __user *argp);
+int cmh_mgmt_sm2_ecdh(void __user *argp);
+int cmh_mgmt_sm2_dec_point(void __user *argp);
+int cmh_mgmt_sm2_enc_point(void __user *argp);
+int cmh_mgmt_sm2_id_digest(void __user *argp);
+int cmh_mgmt_sm2_ecdh_hash(void __user *argp);
+int cmh_mgmt_sm2_dec_hash(void __user *argp);
+int cmh_mgmt_sm2_enc_hash(void __user *argp);
+
+#endif /* CMH_PKE_SM2_H */
diff --git a/drivers/crypto/cmh/include/cmh_pqc.h b/drivers/crypto/cmh/include/cmh_pqc.h
new file mode 100644
index 000000000000..cd4761a0ce5c
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_pqc.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- PQC Algorithm Registration
+ *
+ * Registration/unregistration functions for PQC akcipher algorithms:
+ * ML-DSA, SLH-DSA, LMS, XMSS.
+ */
+
+#ifndef CMH_PQC_H
+#define CMH_PQC_H
+
+int cmh_pqc_mldsa_register(void);
+void cmh_pqc_mldsa_unregister(void);
+
+int cmh_pqc_slhdsa_register(void);
+void cmh_pqc_slhdsa_unregister(void);
+
+int cmh_pqc_lms_register(void);
+void cmh_pqc_lms_unregister(void);
+
+int cmh_pqc_xmss_register(void);
+void cmh_pqc_xmss_unregister(void);
+
+#endif /* CMH_PQC_H */
diff --git a/drivers/crypto/cmh/include/cmh_sys.h b/drivers/crypto/cmh/include/cmh_sys.h
new file mode 100644
index 000000000000..dd336b67bd65
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_sys.h
@@ -0,0 +1,111 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- SYS Core VCQ Builders
+ *
+ * VCQ builder functions for SYS core commands (NEW, WRITE, READ,
+ * FIND, GRANT, DATA, EXPORT, IMPORT). Each builder populates one
+ * vcq_cmd slot with the appropriate magic, command ID, and payload.
+ *
+ * Callers combine these with vcq_set_header() + vcq_add_flush()
+ * and submit via cmh_tm_submit_sync().
+ */
+
+#ifndef CMH_SYS_H
+#define CMH_SYS_H
+
+#include "cmh_vcq.h"
+
+void vcq_add_sys_new(struct vcq_cmd *slot, u64 cid, u64 ref_dma, u32 len);
+void vcq_add_sys_write(struct vcq_cmd *slot, u64 ref, u64 src_dma,
+ u64 wrap_key, u32 len, u32 sys_type);
+void vcq_add_sys_read(struct vcq_cmd *slot, u64 ref, u64 dst_dma,
+ u64 wrap_key, u32 len);
+void vcq_add_sys_data(struct vcq_cmd *slot, u64 ref, u64 dst_dma, u32 len);
+void vcq_add_sys_find(struct vcq_cmd *slot, u64 cid, u64 dst_dma, u32 len);
+void vcq_add_sys_list(struct vcq_cmd *slot, u64 ref, u64 dst_dma, u32 len);
+void vcq_add_sys_grant(struct vcq_cmd *slot, u64 ref, u64 read,
+ u64 write, u64 execute);
+void vcq_add_sys_export(struct vcq_cmd *slot, u64 cid, u64 dst_dma,
+ u64 wrap_key, u32 len);
+void vcq_add_sys_import(struct vcq_cmd *slot, u64 src_dma,
+ u64 wrap_key, u32 len);
+
+/* KIC core VCQ builders */
+void vcq_add_kic_hkdf1(struct vcq_cmd *slot, u64 dst, u64 base,
+ u64 label_dma, u32 key_len, u32 label_len, u32 type);
+void vcq_add_kic_hkdf2(struct vcq_cmd *slot, u64 dst, u64 base, u64 salt,
+ u64 label_dma, u32 key_len, u32 label_len, u32 type);
+void vcq_add_kic_aes_cmac_kdf(struct vcq_cmd *slot, u64 out_key, u64 base_key,
+ u64 label_dma, u32 key_len, u32 label_len,
+ u32 type);
+void vcq_add_kic_dkek_derive(struct vcq_cmd *slot, u64 out_key, u64 base_key,
+ u32 host_id, u64 metadata_dma, u32 metadata_len);
+
+/* DRBG core VCQ builders */
+void vcq_add_drbg_reset(struct vcq_cmd *slot);
+void vcq_add_drbg_config(struct vcq_cmd *slot, u32 ratio, u32 strength);
+void vcq_add_drbg_datastore(struct vcq_cmd *slot, u64 ref, u32 len, u32 type);
+
+/* QSE core VCQ builders */
+void vcq_add_qse_flush(struct vcq_cmd *slot, u32 core_id);
+void vcq_add_qse_ml_kem_keygen(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags,
+ u64 seed, u64 z, u64 ek, u64 dk, u32 dk_type,
+ bool masked);
+void vcq_add_qse_ml_kem_enc(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags,
+ u64 coin, u64 ek, u64 ct, u64 ss, u32 ss_type,
+ bool masked);
+void vcq_add_qse_ml_kem_dec(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags,
+ u64 ct, u64 dk, u64 ss, u32 ss_type,
+ bool masked);
+void vcq_add_qse_ml_dsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags,
+ u64 seed, u64 pk, u64 sk, u32 sk_type,
+ bool masked);
+void vcq_add_qse_ml_dsa_sign(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags,
+ u64 rnd, u64 m, u64 sk, u64 sig, u32 mlen,
+ bool masked);
+void vcq_add_qse_ml_dsa_verify(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags,
+ u64 m, u64 pk, u64 sig, u32 mlen);
+
+/* HCQ core VCQ builders */
+void vcq_add_hcq_flush(struct vcq_cmd *slot, u32 core_id);
+void vcq_add_hcq_slhdsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 seed_len, u32 pk_len, u32 sk_len,
+ u64 seed, u64 pk, u64 sk);
+void vcq_add_hcq_slhdsa_sign(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 msg_len, u32 ctx_len,
+ u64 add_random, u64 msg, u64 ctx,
+ u64 sk, u64 sig);
+void vcq_add_hcq_slhdsa_sign_internal(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 msg_len, u64 add_random,
+ u64 msg, u64 sk, u64 sig);
+void vcq_add_hcq_slhdsa_verify(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 msg_len, u32 ctx_len,
+ u64 msg, u64 ctx, u64 pk, u64 sig);
+void vcq_add_hcq_slhdsa_sign_prehash(struct vcq_cmd *slot, u32 core_id,
+ u32 cmd, u32 param_set, u32 prehash_algo,
+ u32 msg_len, u32 ctx_len,
+ u64 add_random, u64 msg, u64 ctx,
+ u64 sk, u64 sig);
+void vcq_add_hcq_slhdsa_verify_prehash(struct vcq_cmd *slot, u32 core_id,
+ u32 cmd, u32 param_set, u32 prehash_algo,
+ u32 msg_len, u32 ctx_len,
+ u64 msg, u64 ctx, u64 pk, u64 sig);
+void vcq_add_hcq_slhdsa_verify_internal(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 msg_len, u64 msg, u64 pk, u64 sig);
+void vcq_add_hcq_slhdsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 sk_len, u64 sk, u64 pk);
+void vcq_add_hcq_lms_verify(struct vcq_cmd *slot, u32 core_id, u32 lms_hss,
+ u32 pk_len, u32 sig_len, u32 dig_len,
+ u64 pk, u64 sig, u64 dig);
+void vcq_add_hcq_xmss_verify(struct vcq_cmd *slot, u32 core_id, u32 xmss_mt,
+ u32 pk_len, u32 sig_len, u32 dig_len,
+ u64 pk, u64 sig, u64 dig);
+
+/* SYS core flush */
+void vcq_add_sys_flush(struct vcq_cmd *slot);
+
+/* EAC core VCQ builder */
+void vcq_add_eac_read(struct vcq_cmd *slot, u64 dst_dma, u32 len);
+
+#endif /* CMH_SYS_H */
diff --git a/include/uapi/linux/cmh_mgmt_ioctl.h b/include/uapi/linux/cmh_mgmt_ioctl.h
new file mode 100644
index 000000000000..c3f5cb3db772
--- /dev/null
+++ b/include/uapi/linux/cmh_mgmt_ioctl.h
@@ -0,0 +1,895 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Key Management ioctl Interface (User-Space API)
+ *
+ * ioctl commands for /dev/cmh_mgmt -- key CRUD, datastore
+ * export/import, KIC key derivation, PKE, SM2, and PQC operations.
+ *
+ * Relationship to the in-kernel crypto API
+ * -----------------------------------------
+ * Most commands here have no crypto API representation (no transform
+ * type or verb exists): keystore CRUD, key generation, KIC key
+ * derivation, ML-KEM encapsulate/decapsulate, SM2 multi-step
+ * encrypt/decrypt/key-exchange, EdDSA, EAC, and DRBG configuration.
+ * For these the character device is the only available UAPI.
+ *
+ * A bounded subset names primitives the driver ALSO registers with
+ * the crypto API, and the overlap is intentional:
+ * - Hardware-held-key operations (RSA decrypt, ECDSA/ML-DSA/SLH-DSA
+ * sign, ECDH) reference a private key by datastore handle. The
+ * crypto API set_priv_key()/set_secret() take only raw key bytes
+ * and cannot name a key that never leaves the hardware; these
+ * ioctls keep the key hardware-resident. The registered
+ * transforms serve raw-key in-kernel users -- the paths are
+ * complementary.
+ *
+ * Multi-step protocol flows are documented above the PKE and SM2
+ * struct sections. Single-command ioctls are self-documenting.
+ *
+ * Versioned structs: user space sets .version = CMH_MGMT_V1 so the
+ * driver can extend structs in the future without breaking ABI.
+ */
+
+#ifndef _UAPI_CMH_MGMT_IOCTL_H
+#define _UAPI_CMH_MGMT_IOCTL_H
+
+#include <linux/types.h>
+#include <linux/ioctl.h>
+#include <linux/const.h>
+
+#define CMH_MGMT_V1 1
+
+/* Special reference values */
+#define CMH_REF_NONE 0x0000000000000000ULL /* no key (plaintext) */
+
+/* Flags for cmh_ioctl_key_new.flags / cmh_ioctl_key_write.flags */
+#define CMH_FLAG_PT _BITUL(16) /* key can be read as plaintext */
+#define CMH_FLAG_XC _BITUL(17) /* key can be exported over XC bus */
+#define CMH_FLAG_SCA _BITUL(18) /* SCA key stored in 2 shares */
+#define CMH_FLAG_MASK (CMH_FLAG_PT | CMH_FLAG_XC | CMH_FLAG_SCA)
+
+/*
+ * Datastore key types -- the LKM maps these to core IDs internally.
+ * User space passes these in cmh_ioctl_key_new.ds_type.
+ */
+#define CMH_DS_RAW_VALUE 1
+#define CMH_DS_AES_KEY 2
+#define CMH_DS_AES_XTS_KEY 3
+#define CMH_DS_HMAC_KEY 4
+#define CMH_DS_KMAC_KEY 5
+#define CMH_DS_SM4_KEY 6
+#define CMH_DS_CHACHA20_KEY 7
+
+/* PKE key types -- all map to CORE_ID_PKE (0x0A) */
+#define CMH_DS_RSA_PRIV_KEY 10
+#define CMH_DS_RSA_PUB_KEY 11
+#define CMH_DS_RSA_CRT_KEY 12
+#define CMH_DS_ECDSA_PRIV_KEY 13
+#define CMH_DS_ECDSA_PUB_KEY 14
+#define CMH_DS_ECDH_PRIV_KEY 15
+#define CMH_DS_EDDSA_PRIV_KEY 16
+#define CMH_DS_SHARED_SECRET 17
+#define CMH_DS_SM2_PRIV_KEY 18
+
+/* QSE key types -- map to CORE_ID_QSE (0x09) */
+#define CMH_DS_ML_KEM_DK 20
+#define CMH_DS_ML_DSA_SK 21
+
+/* HCQ key types -- map to CORE_ID_HCQ (0x08) */
+#define CMH_DS_SLHDSA_SK 25
+
+/* ioctl argument structures */
+
+struct cmh_ioctl_key_new {
+ __u32 version; /* must be CMH_MGMT_V1 */
+ __u32 ds_type; /* CMH_DS_* key type */
+ __u32 len; /* key length in bytes */
+ __u32 flags; /* CMH_FLAG_* (e.g. CMH_FLAG_PT) */
+ __u64 cid; /* caller ID (name) for the key */
+ __u64 ref; /* [out] CMH eSW returns key_ref here */
+};
+
+struct cmh_ioctl_key_write {
+ __u32 version;
+ __u32 len; /* key data length */
+ __u32 ds_type; /* CMH_DS_* key type */
+ __u32 flags; /* CMH_FLAG_* (e.g. CMH_FLAG_PT) */
+ __u64 ref; /* key reference from KEY_NEW */
+ __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */
+ __u64 data; /* user-space pointer to key material */
+};
+
+struct cmh_ioctl_key_read {
+ __u32 version;
+ __u32 len; /* buffer length */
+ __u64 ref; /* key reference */
+ __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */
+ __u64 data; /* user-space pointer to output buffer */
+ __u32 out_len; /* [out] actual bytes written */
+ __u32 __reserved;
+};
+
+struct cmh_ioctl_key_find {
+ __u32 version;
+ __u32 __reserved;
+ __u64 cid; /* caller ID to search for */
+ __u64 ref; /* [out] resolved key reference */
+ __u32 len; /* [out] key length */
+ __u32 type; /* [out] key type */
+};
+
+/*
+ * KEY_LIST -- iterate datastore objects.
+ *
+ * Pass start_ref=0 to begin from the first accessible object.
+ * On return, ref/cid/len/type describe that object. Pass the
+ * returned ref as start_ref in the next call to advance. Iteration
+ * ends when ref == 0 (no more objects).
+ */
+struct cmh_ioctl_key_list {
+ __u32 version;
+ __u32 __reserved;
+ __u64 start_ref; /* starting DS reference (0 = first) */
+ __u64 ref; /* [out] object reference */
+ __u64 cid; /* [out] caller ID */
+ __u32 len; /* [out] object length */
+ __u32 type; /* [out] object type */
+};
+
+struct cmh_ioctl_key_grant {
+ __u32 version;
+ __u32 __reserved;
+ __u64 ref; /* key reference */
+ __u64 read; /* per-MBX read permission bitfield */
+ __u64 write; /* per-MBX write permission bitfield */
+ __u64 execute; /* per-MBX execute permission bitfield */
+};
+
+/* Export blob overhead beyond the raw object data (bytes) */
+#define CMH_DS_EXPORT_OVERHEAD_WRAPPED 48 /* 16B hdr + 16B nonce + 16B tag */
+#define CMH_DS_EXPORT_OVERHEAD_PLAIN 16 /* 16B hdr only */
+
+/**
+ * struct cmh_ioctl_ds_export - Export a datastore object to a wrapped blob
+ * @version: protocol version (CMH_MGMT_V1)
+ * @len: DMA buffer size; must be >= export blob size:
+ * wrapped: CMH_DS_EXPORT_OVERHEAD_WRAPPED + object_len
+ * plaintext: CMH_DS_EXPORT_OVERHEAD_PLAIN + object_len
+ * object_len is known from KEY_NEW or KEY_FIND.
+ * If too small, the eSW rejects the command (-EIO).
+ * @cid: caller ID of the object to export
+ * @wrap_key: wrapping key ref (CMH_REF_NONE = plaintext export)
+ * @data: user-space pointer to output buffer (at least @len bytes)
+ * @out_len: [out] actual blob bytes written on success
+ * @__reserved: must be zero
+ */
+struct cmh_ioctl_ds_export {
+ __u32 version;
+ __u32 len; /* buffer length (see sizing rule above) */
+ __u64 cid; /* caller ID for response tagging */
+ __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */
+ __u64 data; /* user-space pointer to output buffer */
+ __u32 out_len; /* [out] actual bytes written */
+ __u32 __reserved;
+};
+
+struct cmh_ioctl_ds_import {
+ __u32 version;
+ __u32 len; /* blob length */
+ __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */
+ __u64 data; /* user-space pointer to import blob */
+};
+
+/* Flags for cmh_ioctl_kic_hkdf1.flags / cmh_ioctl_kic_hkdf2.flags */
+#define CMH_KIC_FLAG_TEMP 0x01 /* store result in TEMP (not persistent DS) */
+
+/*
+ * KIC hardware base key references.
+ *
+ * Each CMH device has up to 8 hardware base keys provisioned in OTP/fuses.
+ * These values are passed in the base_key field of KIC ioctl structs.
+ * The key valid bitmask is visible via R_KIC_KEY_VALID (MMIO 0x100).
+ */
+#define CMH_KIC_KEY1 0x0000000100000001ULL
+#define CMH_KIC_KEY2 0x0000000200000002ULL
+#define CMH_KIC_KEY3 0x0000000300000003ULL
+#define CMH_KIC_KEY4 0x0000000400000004ULL
+#define CMH_KIC_KEY5 0x0000000500000005ULL
+#define CMH_KIC_KEY6 0x0000000600000006ULL
+#define CMH_KIC_KEY7 0x0000000700000007ULL
+#define CMH_KIC_KEY8 0x0000000800000008ULL
+
+struct cmh_ioctl_kic_hkdf1 {
+ __u32 version;
+ __u32 key_len; /* output key length (e.g., 32) */
+ __u64 base_key; /* KIC base key reference */
+ __u64 cid; /* CID for the new DS entry (ignored if TEMP) */
+ __u64 label; /* user-space pointer to label data */
+ __u32 label_len; /* label length in bytes */
+ __u32 flags; /* CMH_KIC_FLAG_* */
+ __u64 ref; /* [out] derived key reference */
+};
+
+struct cmh_ioctl_kic_hkdf2 {
+ __u32 version;
+ __u32 key_len; /* output key length (e.g., 32) */
+ __u64 base_key; /* KIC base key reference */
+ __u64 salt_key; /* salt key reference (CMH_REF_NONE = no salt) */
+ __u64 cid; /* CID for the new DS entry (ignored if TEMP) */
+ __u64 label; /* user-space pointer to label data */
+ __u32 label_len; /* label length in bytes */
+ __u32 flags; /* CMH_KIC_FLAG_* */
+ __u64 ref; /* [out] derived key reference */
+};
+
+struct cmh_ioctl_kic_aes_cmac_kdf {
+ __u32 version;
+ __u32 key_len; /* base & output key length (must be 32) */
+ __u64 base_key; /* KIC base key or DS reference */
+ __u64 cid; /* CID for the new DS entry (ignored if TEMP) */
+ __u64 label; /* user-space pointer to label data */
+ __u32 label_len; /* label length in bytes */
+ __u32 flags; /* CMH_KIC_FLAG_* */
+ __u64 ref; /* [out] derived key reference */
+};
+
+#define KIC_DKEK_MAX_METADATA 64 /* max metadata length for DKEK */
+
+struct cmh_ioctl_kic_dkek_derive {
+ __u32 version;
+ __u32 host_id; /* target host ID (0 = caller's own) */
+ __u64 base_key; /* KIC base key reference */
+ __u64 cid; /* CID for the new DS entry (ignored if TEMP) */
+ __u64 metadata; /* user-space pointer to metadata */
+ __u32 metadata_len; /* metadata length in bytes */
+ __u32 flags; /* CMH_KIC_FLAG_* */
+ __u64 ref; /* [out] derived KEK reference */
+};
+
+/* -- PKE ioctl argument structures ----------- */
+
+/*
+ * PKE multi-step protocol flows
+ *
+ * RSA encrypt/decrypt:
+ * 1. KEY_NEW(CMH_DS_RSA_PRIV_KEY) + KEY_WRITE -> priv_ref (or RSA_KEYGEN -> priv_ref)
+ * 2. RSA_ENC(e, n, plaintext) -> ciphertext (public key = raw e,n)
+ * 3. RSA_DEC(e, n, ciphertext, priv_ref) -> plaintext (or RSA_CRT_DEC)
+ *
+ * ECDSA sign:
+ * 1. EC_KEYGEN(curve) -> priv_ref (or KEY_NEW + KEY_WRITE)
+ * 2. EC_PUBGEN(priv_ref) -> public_key (raw x||y returned)
+ * 3. ECDSA_SIGN(digest, priv_ref) -> signature
+ * SM2 sign uses the same path with curve=CMH_CURVE_SM2.
+ *
+ * ECDH shared secret:
+ * 1. EC_KEYGEN(curve) -> priv_ref (or KEY_NEW + KEY_WRITE)
+ * 2. ECDH_KEYGEN(priv_ref) -> public_key_x (derive pub from priv)
+ * 3. Exchange public keys with peer
+ * 4. ECDH(peer_key_x, priv_ref) -> shared_secret (raw or DS ref via FLAG_DS_RESULT)
+ *
+ * EdDSA sign/verify:
+ * 1. EC_KEYGEN(CURVE_25519 or CURVE_448) -> priv_ref
+ * 2. EC_PUBGEN(priv_ref) -> public_key
+ * 3. EDDSA_SIGN(message, priv_ref) -> signature (pure EdDSA, not prehash)
+ * 4. EDDSA_VERIFY(message, signature, public_key_y)
+ * For Ed448 SCA: EDDSA_KEYGEN_SCA(priv_ref) -> sca_ref (2-share blinded key)
+ *
+ * SM2 encryption (GM/T 0003.4):
+ * 1. EC_KEYGEN(CMH_CURVE_SM2) -> priv_ref (or KEY_NEW + KEY_WRITE)
+ * 2. EC_PUBGEN(priv_ref) -> public_key
+ * 3. SM2_ENC_POINT(public_key) -> C1, enc_point (nonce_len=0: HW ephemeral)
+ * 4. SM2_ENC_HASH(enc_point, message) -> ciphertext (C1||C3||C2)
+ * Decrypt:
+ * 5. SM2_DEC_POINT(C1, priv_ref) -> dec_point
+ * 6. SM2_DEC_HASH(ciphertext, dec_point) -> plaintext
+ * enc_point and dec_point are raw DMA buffers (64B each), not DS refs.
+ *
+ * SM2 key exchange (GM/T 0003.3):
+ * 1. EC_KEYGEN(CMH_CURVE_SM2) -> priv_ref (long-lived, persistent DS)
+ * 2. EC_PUBGEN(priv_ref) -> public_key
+ * 3. SM2_ID_DIGEST(id, public_key) -> ZA (SM3-based identity digest)
+ * 4. SM2_ECDH_KEYGEN(nonce) -> session_key, r (ephemeral scalar r)
+ * - nonce_len=32: caller supplies r (deterministic)
+ * - nonce_len=0: HW generates r, writes it back to .nonce
+ * Exchange session_key with peer.
+ * 5. SM2_ECDH(r, priv_ref, peer_pub, peer_sess) -> shared_point
+ * - Must pass the same r from step 4 (nonce_len=32)
+ * - shared_point_ref=0: reads back raw shared_point, destroys DS slot
+ * - shared_point_ref=&ref: keeps DS slot alive, writes ref for ECDH_HASH
+ * 6. SM2_ECDH_HASH(shared_point_ref, ZA_self, ZA_peer) -> shared_key (16B)
+ * - shared_point_ref is a persistent DS reference from step 5
+ * - The DS slot is consumed by the hub; caller should delete it afterward
+ * The nonce r is a raw 32-byte scalar in userspace memory between steps 4-5.
+ * The shared_point is a persistent DS ref between steps 5-6.
+ * The long-lived private key (priv_ref) persists independently.
+ */
+
+/* PKE operation flags */
+#define CMH_PKE_FLAG_DS_RESULT _BITUL(0) /* store result in DS, return ref */
+
+struct cmh_ioctl_pke_rsa_enc {
+ __u32 version;
+ __u32 bits; /* RSA key size in bits (1024-4096) */
+ __u64 e; /* user-space pointer to public exponent */
+ __u32 e_len; /* exponent length in bytes */
+ __u32 __reserved;
+ __u64 n; /* user-space pointer to modulus */
+ __u64 input; /* user-space pointer to input data */
+ __u64 output; /* user-space pointer to output buffer */
+};
+
+struct cmh_ioctl_pke_rsa_dec {
+ __u32 version;
+ __u32 bits;
+ __u64 e; /* public exponent */
+ __u32 e_len;
+ __u32 __reserved;
+ __u64 n; /* modulus */
+ __u64 input; /* ciphertext */
+ __u64 output; /* plaintext output */
+ __u64 key_ref; /* private key DS reference */
+};
+
+struct cmh_ioctl_pke_rsa_crt_dec {
+ __u32 version;
+ __u32 bits;
+ __u64 e;
+ __u32 e_len;
+ __u32 __reserved;
+ __u64 n;
+ __u64 input;
+ __u64 output;
+ __u64 crt_ref; /* CRT key DS reference */
+};
+
+struct cmh_ioctl_pke_rsa_keygen {
+ __u32 version;
+ __u32 bits; /* key size in bits */
+ __u64 e; /* user-space pointer to public exponent */
+ __u32 e_len;
+ __u32 flags; /* CMH_FLAG_* */
+ __u64 n; /* [out] user-space pointer to modulus buffer */
+ __u64 d_cid; /* CID for private key DS entry */
+ __u64 d_ref; /* [out] private key reference */
+ __u64 crt_cid; /* CID for CRT key DS entry (0 = skip CRT) */
+ __u64 crt_ref; /* [out] CRT key reference */
+};
+
+struct cmh_ioctl_pke_ecdsa_sign {
+ __u32 version;
+ __u32 curve; /* ABI curve ID (e.g. 0x03 = P-256) */
+ __u64 digest; /* user-space pointer to hash digest */
+ __u32 digest_len; /* digest length in bytes */
+ __u32 __reserved;
+ __u64 signature; /* [out] user-space pointer to (r,s) */
+ __u64 key_ref; /* private key DS reference */
+};
+
+struct cmh_ioctl_pke_ecdh {
+ __u32 version;
+ __u32 curve;
+ __u64 peer_key_x; /* user-space pointer to peer public key X */
+ __u64 key_ref; /* private key DS reference */
+ __u32 flags; /* CMH_PKE_FLAG_DS_RESULT */
+ __u32 __reserved;
+ __u64 result_cid; /* CID for DS result (if FLAG_DS_RESULT) */
+ __u64 output; /* [out] raw shared secret or DS ref */
+};
+
+struct cmh_ioctl_pke_ecdh_keygen {
+ __u32 version;
+ __u32 curve;
+ __u64 key_ref; /* private key DS reference */
+ __u64 public_key_x; /* [out] user-space pointer to public key X */
+};
+
+struct cmh_ioctl_pke_eddsa_sign {
+ __u32 version;
+ __u32 curve; /* CURVE_25519 or CURVE_448 */
+ __u64 digest; /* user-space ptr to message (not digest) */
+ __u32 digest_len;
+ __u32 __reserved;
+ __u64 signature; /* [out] user-space pointer to signature */
+ __u64 key_ref; /* private key DS reference */
+};
+
+struct cmh_ioctl_pke_eddsa_verify {
+ __u32 version;
+ __u32 curve;
+ __u64 digest;
+ __u32 digest_len;
+ __u32 __reserved;
+ __u64 signature;
+ __u64 public_key_y; /* user-space pointer to public key Y */
+};
+
+struct cmh_ioctl_pke_ec_keygen {
+ __u32 version;
+ __u32 curve;
+ __u32 flags; /* CMH_FLAG_* */
+ __u32 __reserved;
+ __u64 cid; /* CID for the new key DS entry */
+ __u64 ref; /* [out] private key reference */
+};
+
+struct cmh_ioctl_pke_ec_pubgen {
+ __u32 version;
+ __u32 curve;
+ __u64 key_ref; /* private key DS reference */
+ __u64 public_key; /* [out] user-space pointer to public key */
+};
+
+struct cmh_ioctl_pke_eddsa_keygen_sca {
+ __u32 version;
+ __u32 curve; /* must be CURVE_448 */
+ __u64 key_ref; /* input: normal Ed448 private key DS ref */
+ __u64 cid; /* CID for the new SCA key DS entry */
+ __u64 sca_ref; /* [out] SCA private key reference */
+};
+
+/*
+ * ioctl numbers -- type 'J', sequential.
+ * 'C' conflicts with OSS sound, CAPI/ISDN, and COSA WAN drivers;
+ * 'J' is unregistered in Documentation/userspace-api/ioctl/ioctl-number.rst.
+ */
+
+#define CMH_MGMT_IOC_MAGIC 'J'
+
+#define CMH_IOCTL_KEY_NEW _IOWR(CMH_MGMT_IOC_MAGIC, 0x01, struct cmh_ioctl_key_new)
+#define CMH_IOCTL_KEY_WRITE _IOW(CMH_MGMT_IOC_MAGIC, 0x02, struct cmh_ioctl_key_write)
+#define CMH_IOCTL_KEY_READ _IOWR(CMH_MGMT_IOC_MAGIC, 0x03, struct cmh_ioctl_key_read)
+#define CMH_IOCTL_KEY_FIND _IOWR(CMH_MGMT_IOC_MAGIC, 0x04, struct cmh_ioctl_key_find)
+#define CMH_IOCTL_KEY_GRANT _IOW(CMH_MGMT_IOC_MAGIC, 0x05, struct cmh_ioctl_key_grant)
+#define CMH_IOCTL_KEY_DELETE _IOW(CMH_MGMT_IOC_MAGIC, 0x06, struct cmh_ioctl_key_grant)
+#define CMH_IOCTL_DS_EXPORT _IOWR(CMH_MGMT_IOC_MAGIC, 0x07, struct cmh_ioctl_ds_export)
+#define CMH_IOCTL_DS_IMPORT _IOW(CMH_MGMT_IOC_MAGIC, 0x08, struct cmh_ioctl_ds_import)
+#define CMH_IOCTL_KIC_HKDF1 _IOWR(CMH_MGMT_IOC_MAGIC, 0x09, struct cmh_ioctl_kic_hkdf1)
+#define CMH_IOCTL_KIC_HKDF2 _IOWR(CMH_MGMT_IOC_MAGIC, 0x0A, struct cmh_ioctl_kic_hkdf2)
+#define CMH_IOCTL_KEY_NEW_RANDOM _IOWR(CMH_MGMT_IOC_MAGIC, 0x0B, struct cmh_ioctl_key_new)
+#define CMH_IOCTL_KIC_AES_CMAC_KDF _IOWR(CMH_MGMT_IOC_MAGIC, 0x0C, \
+ struct cmh_ioctl_kic_aes_cmac_kdf)
+#define CMH_IOCTL_KIC_DKEK_DERIVE _IOWR(CMH_MGMT_IOC_MAGIC, 0x0D, \
+ struct cmh_ioctl_kic_dkek_derive)
+#define CMH_IOCTL_KEY_LIST _IOWR(CMH_MGMT_IOC_MAGIC, 0x0E, struct cmh_ioctl_key_list)
+
+/* PKE operation ioctls */
+#define CMH_IOCTL_PKE_RSA_ENC _IOWR(CMH_MGMT_IOC_MAGIC, 0x10, \
+ struct cmh_ioctl_pke_rsa_enc)
+#define CMH_IOCTL_PKE_RSA_DEC _IOWR(CMH_MGMT_IOC_MAGIC, 0x11, \
+ struct cmh_ioctl_pke_rsa_dec)
+#define CMH_IOCTL_PKE_RSA_CRT_DEC _IOWR(CMH_MGMT_IOC_MAGIC, 0x12, \
+ struct cmh_ioctl_pke_rsa_crt_dec)
+#define CMH_IOCTL_PKE_RSA_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x13, \
+ struct cmh_ioctl_pke_rsa_keygen)
+#define CMH_IOCTL_PKE_ECDSA_SIGN _IOWR(CMH_MGMT_IOC_MAGIC, 0x14, \
+ struct cmh_ioctl_pke_ecdsa_sign)
+#define CMH_IOCTL_PKE_ECDH _IOWR(CMH_MGMT_IOC_MAGIC, 0x16, \
+ struct cmh_ioctl_pke_ecdh)
+#define CMH_IOCTL_PKE_ECDH_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x17, \
+ struct cmh_ioctl_pke_ecdh_keygen)
+#define CMH_IOCTL_PKE_EDDSA_SIGN _IOWR(CMH_MGMT_IOC_MAGIC, 0x18, \
+ struct cmh_ioctl_pke_eddsa_sign)
+#define CMH_IOCTL_PKE_EDDSA_VERIFY _IOW(CMH_MGMT_IOC_MAGIC, 0x19, \
+ struct cmh_ioctl_pke_eddsa_verify)
+#define CMH_IOCTL_PKE_EC_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x1A, \
+ struct cmh_ioctl_pke_ec_keygen)
+#define CMH_IOCTL_PKE_EC_PUBGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x1B, \
+ struct cmh_ioctl_pke_ec_pubgen)
+#define CMH_IOCTL_PKE_EDDSA_KEYGEN_SCA _IOWR(CMH_MGMT_IOC_MAGIC, 0x1C, \
+ struct cmh_ioctl_pke_eddsa_keygen_sca)
+
+/* -- PQC ioctl argument structures ----------- */
+
+/*
+ * PQC operation flags (bits [2:0]).
+ * PQC keygen ioctls accept CMH_FLAG_PT in bits [18:16] to explicitly
+ * set the DS key storage attribute when CMH_QSE_FLAG_DS_REF is set.
+ * CMH_FLAG_SCA and CMH_FLAG_XC are rejected -- QSE SCA protection uses
+ * polynomial masking (CMH_QSE_FLAG_MASKED), not 2-share storage,
+ * and the eSW dec/sign paths hardcode SYS_TYPE_FLAG_PT.
+ * If no CMH_FLAG_* bits are set, DS keys default to CMH_FLAG_PT.
+ */
+#define CMH_QSE_FLAG_MASKED _BITUL(0) /* use masked (SCA-resistant) HW commands */
+#define CMH_QSE_FLAG_DS_REF _BITUL(1) /* store key output in DS, return ref */
+#define CMH_QSE_FLAG_HW_RNG _BITUL(2) /* use HW RNG for seed/randomness */
+#define CMH_QSE_FLAG_MASK (_BITUL(0) | _BITUL(1) | _BITUL(2))
+
+/* -- SYS wrap header size -------------------- */
+/* sys_read prepends a 16-byte header even for plaintext reads */
+#define CMH_SYS_WRAP_HDR_SIZE 16
+
+/* -- Seed / randomness lengths --------------- */
+
+#define CMH_QSE_SEED_LEN 32 /* ML-KEM/ML-DSA seed size */
+#define CMH_QSE_SEED_LEN_MASKED 64 /* seed size for masked mode */
+
+/* -- ML-DSA ExternalMu sentinel -------------- */
+/* Pass this as mlen to use 64-byte pre-hashed mu instead of raw message */
+#define CMH_ML_DSA_MLEN_EXTERNAL_MU 0xFFFFFFFFU
+
+/* -- ML-KEM size macros ---------------------- */
+
+#define CMH_ML_KEM_EK_SIZE(k) (384U * (k) + 32U)
+#define CMH_ML_KEM_DK_SIZE(k) (768U * (k) + 96U)
+/* CT sizes: k=2 -> 768, k=3 -> 1088, k=4 -> 1568 */
+#define CMH_ML_KEM_CT_SIZE_512 768U
+#define CMH_ML_KEM_CT_SIZE_768 1088U
+#define CMH_ML_KEM_CT_SIZE_1024 1568U
+#define CMH_ML_KEM_SS_LEN 32U
+
+/* -- ML-DSA size macros ---------------------- */
+/* Indexed by mode: [0]=44 (mode=2), [1]=65 (mode=3), [2]=87 (mode=5) */
+
+#define CMH_ML_DSA_44_PK_SIZE 1312U
+#define CMH_ML_DSA_44_SK_SIZE 2560U
+#define CMH_ML_DSA_44_SIG_SIZE 2420U
+#define CMH_ML_DSA_65_PK_SIZE 1952U
+#define CMH_ML_DSA_65_SK_SIZE 4032U
+#define CMH_ML_DSA_65_SIG_SIZE 3309U
+#define CMH_ML_DSA_87_PK_SIZE 2592U
+#define CMH_ML_DSA_87_SK_SIZE 4896U
+#define CMH_ML_DSA_87_SIG_SIZE 4627U
+
+/* -- SLH-DSA parameter set IDs --------------- */
+
+#define CMH_SLHDSA_SHAKE_128S 1U
+#define CMH_SLHDSA_SHAKE_128F 2U
+#define CMH_SLHDSA_SHAKE_192S 3U
+#define CMH_SLHDSA_SHAKE_192F 4U
+#define CMH_SLHDSA_SHAKE_256S 5U
+#define CMH_SLHDSA_SHAKE_256F 6U
+#define CMH_SLHDSA_SHA2_128S 7U
+#define CMH_SLHDSA_SHA2_128F 8U
+#define CMH_SLHDSA_SHA2_192S 9U
+#define CMH_SLHDSA_SHA2_192F 10U
+#define CMH_SLHDSA_SHA2_256S 11U
+#define CMH_SLHDSA_SHA2_256F 12U
+#define CMH_SLHDSA_PARAM_MAX 12U
+
+/* SLH-DSA prehash algorithm IDs */
+#define CMH_SLHDSA_PREHASH_SHA256 1U
+#define CMH_SLHDSA_PREHASH_SHA512 2U
+#define CMH_SLHDSA_PREHASH_SHAKE128 3U
+#define CMH_SLHDSA_PREHASH_SHAKE256 4U
+#define CMH_SLHDSA_PREHASH_MAX 4U
+
+/* SLH-DSA n-value table indexed by (param_set - 1) */
+#define CMH_SLHDSA_N_128 16U
+#define CMH_SLHDSA_N_192 24U
+#define CMH_SLHDSA_N_256 32U
+
+/* SLH-DSA key sizes: pk = 2*n, sk = 4*n, seed = 3*n */
+#define CMH_SLHDSA_PK_SIZE(n) (2U * (n))
+#define CMH_SLHDSA_SK_SIZE(n) (4U * (n))
+#define CMH_SLHDSA_SEED_SIZE(n) (3U * (n))
+
+/* SLH-DSA signature sizes indexed by (param_set - 1) */
+#define CMH_SLHDSA_SIG_SIZE_SHAKE_128S 7856U
+#define CMH_SLHDSA_SIG_SIZE_SHAKE_128F 17088U
+#define CMH_SLHDSA_SIG_SIZE_SHAKE_192S 16224U
+#define CMH_SLHDSA_SIG_SIZE_SHAKE_192F 35664U
+#define CMH_SLHDSA_SIG_SIZE_SHAKE_256S 29792U
+#define CMH_SLHDSA_SIG_SIZE_SHAKE_256F 49856U
+#define CMH_SLHDSA_SIG_SIZE_SHA2_128S 7856U
+#define CMH_SLHDSA_SIG_SIZE_SHA2_128F 17088U
+#define CMH_SLHDSA_SIG_SIZE_SHA2_192S 16224U
+#define CMH_SLHDSA_SIG_SIZE_SHA2_192F 35664U
+#define CMH_SLHDSA_SIG_SIZE_SHA2_256S 29792U
+#define CMH_SLHDSA_SIG_SIZE_SHA2_256F 49856U
+
+/* -- PKE curve IDs -------------- */
+
+#define CMH_CURVE_P192 0x01U
+#define CMH_CURVE_P224 0x02U
+#define CMH_CURVE_P256 0x03U
+#define CMH_CURVE_P384 0x04U
+#define CMH_CURVE_P521 0x05U
+#define CMH_CURVE_SECP256K1 0x07U
+#define CMH_CURVE_BP192R1 0x11U
+#define CMH_CURVE_BP224R1 0x12U
+#define CMH_CURVE_BP256R1 0x13U
+#define CMH_CURVE_BP320R1 0x14U
+#define CMH_CURVE_BP384R1 0x15U
+#define CMH_CURVE_BP512R1 0x16U
+#define CMH_CURVE_SM2 0x18U
+#define CMH_CURVE_25519 0x21U
+#define CMH_CURVE_448 0x22U
+
+/* ML-KEM */
+
+struct cmh_ioctl_ml_kem_keygen {
+ __u32 version;
+ __u32 k; /* security parameter: 2/3/4 */
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 __reserved;
+ __u64 seed; /* user-space pointer to seed (or 0 for HW RNG) */
+ __u64 z; /* user-space pointer to z (or 0 for HW RNG) */
+ __u64 ek; /* [out] user-space pointer to encapsulation key */
+ __u64 dk; /* [out] user-space pointer to decapsulation key
+ * or [out] DS ref if CMH_QSE_FLAG_DS_REF
+ */
+ __u64 dk_cid; /* CID for DS entry (if DS_REF) */
+ __u64 dk_ref; /* [out] dk DS reference (if DS_REF) */
+};
+
+struct cmh_ioctl_ml_kem_enc {
+ __u32 version;
+ __u32 k;
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 __reserved;
+ __u64 coin; /* user-space pointer to random coin (or 0) */
+ __u64 ek; /* user-space pointer to encapsulation key */
+ __u64 ct; /* [out] user-space pointer to ciphertext */
+ __u64 ss; /* [out] user-space pointer to shared secret */
+ __u64 __reserved2[2]; /* reserved for future use */
+};
+
+struct cmh_ioctl_ml_kem_dec {
+ __u32 version;
+ __u32 k;
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 __reserved;
+ __u64 ct; /* user-space pointer to ciphertext */
+ __u64 dk; /* user-space pointer to dk or DS ref */
+ __u64 ss; /* [out] user-space pointer to shared secret */
+ __u64 __reserved2[2]; /* reserved for future use */
+};
+
+/* ML-DSA */
+
+struct cmh_ioctl_ml_dsa_keygen {
+ __u32 version;
+ __u32 mode; /* security parameter: 2/3/5 */
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 __reserved;
+ __u64 seed; /* user-space pointer to seed (or 0 for HW RNG) */
+ __u64 pk; /* [out] user-space pointer to public key */
+ __u64 sk; /* [out] user-space pointer to secret key
+ * or [out] DS ref if CMH_QSE_FLAG_DS_REF
+ */
+ __u64 sk_cid; /* CID for DS entry (if DS_REF) */
+ __u64 sk_ref; /* [out] sk DS reference (if DS_REF) */
+};
+
+struct cmh_ioctl_ml_dsa_sign {
+ __u32 version;
+ __u32 mode;
+ __u32 flags; /* CMH_QSE_FLAG_* */
+ __u32 mlen; /* message length in bytes */
+ __u64 m; /* user-space pointer to message */
+ __u64 sk; /* user-space pointer to sk or DS ref */
+ __u64 sig; /* [out] user-space pointer to signature */
+ __u64 rnd; /* user-space pointer to randomness (or 0) */
+};
+
+/* SLH-DSA */
+
+struct cmh_ioctl_slhdsa_keygen {
+ __u32 version;
+ __u32 parameter_set; /* HCQ_SLHDSA_SHAKE_128S .. SHA2_256F */
+ __u32 flags; /* CMH_QSE_FLAG_DS_REF */
+ __u32 __reserved;
+ __u64 seed; /* user-space pointer to seed */
+ __u64 pk; /* [out] user-space pointer to public key */
+ __u64 sk; /* [out] user-space pointer to secret key
+ * or [out] DS ref if CMH_QSE_FLAG_DS_REF
+ */
+ __u64 sk_cid; /* CID for DS entry (if DS_REF) */
+ __u64 sk_ref; /* [out] sk DS reference (if DS_REF) */
+};
+
+struct cmh_ioctl_slhdsa_sign {
+ __u32 version;
+ __u32 parameter_set;
+ __u32 msg_len;
+ __u32 ctx_len;
+ __u64 msg; /* user-space pointer to message */
+ __u64 ctx; /* user-space pointer to context (or 0) */
+ __u64 sk; /* DS ref for secret key */
+ __u64 sig; /* [out] user-space pointer to signature */
+ __u64 add_random; /* user-space pointer to addl. randomness (or 0) */
+};
+
+struct cmh_ioctl_slhdsa_sign_prehash {
+ __u32 version;
+ __u32 parameter_set;
+ __u32 prehash_algo; /* CMH_SLHDSA_PREHASH_* */
+ __u32 digest; /* 0 = raw msg (eSW hashes), 1 = pre-computed digest */
+ __u32 msg_len;
+ __u32 ctx_len;
+ __u64 msg; /* user-space pointer to message/digest */
+ __u64 ctx; /* user-space pointer to context (or 0) */
+ __u64 sk; /* DS ref for secret key */
+ __u64 sig; /* [out] user-space pointer to signature */
+ __u64 add_random; /* user-space pointer to addl. randomness (or 0) */
+};
+
+/* -- SM2 ioctl argument structures ----------- */
+
+/* SM2 fixed key sizes (sm2p256v1 curve, 256-bit) */
+#define CMH_SM2_CLEN 32U /* coordinate length */
+#define CMH_SM2_PUBKEY_LEN 64U /* uncompressed (x||y) */
+#define CMH_SM2_POINT_LEN 64U /* EC point (x||y) */
+#define CMH_SM2_SHARED_KEY_LEN 16U /* ECDH shared key */
+#define CMH_SM2_DIGEST_LEN 32U /* SM3 digest (ZA) */
+/*
+ * SM2 enc_hash/dec_hash payload limit.
+ *
+ * The eSW PKE driver implements only a single-block GM/T 0003.4 KDF
+ * (one SM3 invocation, 32 bytes of key stream). Longer messages would
+ * silently produce incorrect ciphertext / plaintext, so the driver caps
+ * the payload at 32 bytes. See Documentation/ABI/testing/cmh-mgmt.
+ */
+#define CMH_SM2_MAX_MSG_LEN 32U /* encrypt/decrypt */
+#define CMH_SM2_MAX_ID_LEN 32U /* identity string */
+#define CMH_SM2_CT_OVERHEAD 96U /* C1(64) + C3(32) */
+#define CMH_SM2_MAX_CT_LEN 128U /* 96 + max_msg = 128 */
+
+struct cmh_ioctl_sm2_ecdh_keygen {
+ __u32 version;
+ __u32 nonce_len; /* 0 = HW generates r (written back), 32 = caller provides r */
+ __u64 nonce; /* [in/out] user-space pointer to nonce buffer (32B) */
+ __u64 session_key; /* [out] user-space pointer to session key R=r*G (64B) */
+};
+
+struct cmh_ioctl_sm2_ecdh {
+ __u32 version;
+ __u32 nonce_len; /* 0 = HW generates (written back), 32 = caller provides */
+ __u64 nonce; /* [in/out] user-space pointer to nonce r (32B) */
+ __u64 peer_public_key; /* user-space pointer to peer pub key (64B) */
+ __u64 peer_session_key; /* user-space pointer to peer session key (64B) */
+ __u64 key_ref; /* private key DS reference */
+ __u64 shared_point; /* [out] user-space pointer to shared point (64B) */
+ __u64 shared_point_ref; /* [in/out] 0 = read-back mode; &ref = keep DS, write ref */
+};
+
+struct cmh_ioctl_sm2_dec_point {
+ __u32 version;
+ __u32 ciphertext_len; /* total ciphertext length (97..128) */
+ __u64 ciphertext; /* user-space pointer to ciphertext (64B: C1) */
+ __u64 dec_point; /* [out] user-space pointer to dec point (64B) */
+ __u64 key_ref; /* private key DS reference */
+};
+
+struct cmh_ioctl_sm2_enc_point {
+ __u32 version;
+ __u32 nonce_len; /* 0 = HW generates, 32 = caller provides */
+ __u64 nonce; /* user-space pointer to nonce (or 0) */
+ __u64 public_key; /* user-space pointer to public key (64B) */
+ __u64 ciphertext; /* [out] user-space pointer to C1 (64B) */
+ __u64 enc_point; /* [out] user-space pointer to enc point (64B) */
+};
+
+struct cmh_ioctl_sm2_id_digest {
+ __u32 version;
+ __u32 id_len; /* identity length in bytes (<=32) */
+ __u64 id; /* user-space pointer to identity string */
+ __u64 public_key; /* user-space pointer to public key (64B) */
+ __u64 digest; /* [out] user-space pointer to ZA digest (32B) */
+};
+
+/*
+ * SM2 ECDH_HASH -- derive shared key from shared point + ZA digests.
+ *
+ * IMPORTANT: The digest fields use ABSOLUTE ordering per GM/T 0003.3,
+ * NOT relative own/peer ordering. Both parties must pass:
+ * peer_id_digest = Z_A (initiator's digest) -- hashed FIRST
+ * id_digest = Z_B (responder's digest) -- hashed SECOND
+ * The eSW computes: KDF(shared_point || peer_id_digest || id_digest).
+ */
+struct cmh_ioctl_sm2_ecdh_hash {
+ __u32 version;
+ __u32 __reserved;
+ __u64 peer_id_digest; /* ptr to Z_A -- initiator's digest (32B) */
+ __u64 id_digest; /* ptr to Z_B -- responder's digest (32B) */
+ __u64 shared_point_ref; /* DS reference from SM2_ECDH */
+ __u64 shared_key; /* [out] ptr to shared key (16B) */
+};
+
+struct cmh_ioctl_sm2_dec_hash {
+ __u32 version;
+ __u32 ciphertext_len; /* ciphertext length (97..128) */
+ __u64 ciphertext; /* user-space pointer to full ciphertext */
+ __u64 dec_point; /* user-space pointer to dec point (64B) */
+ __u64 plaintext; /* [out] user-space pointer to plaintext */
+};
+
+struct cmh_ioctl_sm2_enc_hash {
+ __u32 version;
+ __u32 message_len; /* message length (1..32) */
+ __u64 message; /* user-space pointer to plaintext */
+ __u64 enc_point; /* user-space pointer to enc point (64B) */
+ __u64 ciphertext; /* [out] user-space pointer to ciphertext */
+};
+
+/* PQC ioctl numbers */
+#define CMH_IOCTL_ML_KEM_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x20, \
+ struct cmh_ioctl_ml_kem_keygen)
+#define CMH_IOCTL_ML_KEM_ENC _IOWR(CMH_MGMT_IOC_MAGIC, 0x21, \
+ struct cmh_ioctl_ml_kem_enc)
+#define CMH_IOCTL_ML_KEM_DEC _IOWR(CMH_MGMT_IOC_MAGIC, 0x22, \
+ struct cmh_ioctl_ml_kem_dec)
+#define CMH_IOCTL_ML_DSA_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x23, \
+ struct cmh_ioctl_ml_dsa_keygen)
+#define CMH_IOCTL_ML_DSA_SIGN _IOWR(CMH_MGMT_IOC_MAGIC, 0x24, \
+ struct cmh_ioctl_ml_dsa_sign)
+#define CMH_IOCTL_SLHDSA_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x28, \
+ struct cmh_ioctl_slhdsa_keygen)
+#define CMH_IOCTL_SLHDSA_SIGN _IOWR(CMH_MGMT_IOC_MAGIC, 0x29, \
+ struct cmh_ioctl_slhdsa_sign)
+#define CMH_IOCTL_SLHDSA_SIGN_PREHASH _IOWR(CMH_MGMT_IOC_MAGIC, 0x2D, \
+ struct cmh_ioctl_slhdsa_sign_prehash)
+
+/* SM2 operation ioctls */
+#define CMH_IOCTL_SM2_ECDH_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x30, \
+ struct cmh_ioctl_sm2_ecdh_keygen)
+#define CMH_IOCTL_SM2_ECDH _IOWR(CMH_MGMT_IOC_MAGIC, 0x31, \
+ struct cmh_ioctl_sm2_ecdh)
+#define CMH_IOCTL_SM2_DEC_POINT _IOWR(CMH_MGMT_IOC_MAGIC, 0x32, \
+ struct cmh_ioctl_sm2_dec_point)
+#define CMH_IOCTL_SM2_ENC_POINT _IOWR(CMH_MGMT_IOC_MAGIC, 0x33, \
+ struct cmh_ioctl_sm2_enc_point)
+#define CMH_IOCTL_SM2_ID_DIGEST _IOWR(CMH_MGMT_IOC_MAGIC, 0x34, \
+ struct cmh_ioctl_sm2_id_digest)
+#define CMH_IOCTL_SM2_ECDH_HASH _IOWR(CMH_MGMT_IOC_MAGIC, 0x35, \
+ struct cmh_ioctl_sm2_ecdh_hash)
+#define CMH_IOCTL_SM2_DEC_HASH _IOWR(CMH_MGMT_IOC_MAGIC, 0x36, \
+ struct cmh_ioctl_sm2_dec_hash)
+#define CMH_IOCTL_SM2_ENC_HASH _IOWR(CMH_MGMT_IOC_MAGIC, 0x37, \
+ struct cmh_ioctl_sm2_enc_hash)
+
+/*
+ * EAC (Error and Alarm Controller) -- read and clear error registers.
+ *
+ * Returns a snapshot of all hardware error/safety/notification registers.
+ * The eSW atomically reads and clears the registers on each call, so
+ * successive reads show only new events.
+ */
+struct cmh_ioctl_eac_read {
+ __u32 version; /* must be CMH_MGMT_V1 */
+ __u32 __reserved;
+ __u64 mailbox_notification; /* [out] MBX safety notification bitmask */
+ __u32 hw_error; /* [out] HWC error bitmask */
+ __u32 hw_nmi; /* [out] HWC NMI bitmask */
+ __u32 hw_panic; /* [out] HWC panic bitmask */
+ __u32 safety_fatal; /* [out] HWC fatal safety bitmask */
+ __u32 safety_notification; /* [out] HWC safety notification bitmask */
+ __u32 sw_info0; /* [out] eSW tracing info */
+ __u32 sw_info1; /* [out] eSW tracing info */
+ __u32 sram_bank_errors[4]; /* [out] correctable ECC error counts */
+ __u32 __pad; /* explicit tail padding (prevent info leak) */
+};
+
+/*
+ * DRBG CONFIG -- configure the hardware DRBG before first use.
+ *
+ * This is a management operation normally performed once at system
+ * start-up. Must be called before any hwrng reads or DRBG GENERATE
+ * operations.
+ */
+#define CMH_DRBG_RATIO_ONE 0 /* 1:1 entropy ratio */
+#define CMH_DRBG_RATIO_ONE_HALF 1 /* 1:2 */
+#define CMH_DRBG_RATIO_ONE_THIRD 2 /* 1:3 */
+#define CMH_DRBG_RATIO_ONE_FOURTH 3 /* 1:4 */
+
+#define CMH_DRBG_STRENGTH_128 0x00 /* 128-bit security */
+#define CMH_DRBG_STRENGTH_256 0x10 /* 256-bit security */
+
+struct cmh_ioctl_drbg_config {
+ __u32 version; /* must be CMH_MGMT_V1 */
+ __u32 entropy_ratio; /* CMH_DRBG_RATIO_* */
+ __u32 security_strength; /* CMH_DRBG_STRENGTH_* */
+ __u32 __reserved;
+};
+
+/* EAC ioctl number */
+#define CMH_IOCTL_EAC_READ _IOWR(CMH_MGMT_IOC_MAGIC, 0x0F, \
+ struct cmh_ioctl_eac_read)
+
+/* DRBG management ioctl number */
+#define CMH_IOCTL_DRBG_CONFIG _IOW(CMH_MGMT_IOC_MAGIC, 0x40, \
+ struct cmh_ioctl_drbg_config)
+
+#endif /* _UAPI_CMH_MGMT_IOCTL_H */
--
2.43.7
^ permalink raw reply related
* [PATCH v3 18/19] selftests: crypto: cmh - add kselftest for management ioctl
From: Saravanakrishnan Krishnamoorthy @ 2026-08-06 19:55 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Add a minimal kselftest exercising the /dev/cmh_mgmt ioctl interface:
- open/close the device node
- invalid ioctl returns -ENOTTY
- bad version field returns -EINVAL
- KEY_NEW + KEY_DELETE lifecycle
- KIC HKDF1 key derivation
- ML-KEM-768 keygen via hardware RNG
Tests use the kselftest_harness.h fixture framework and output TAP.
Tests that require hardware features not present on the device under
test are gracefully skipped (SKIP).
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
tools/testing/selftests/Makefile | 1 +
.../selftests/drivers/crypto/cmh/Makefile | 6 +
.../drivers/crypto/cmh/cmh_mgmt_test.c | 183 ++++++++++++++++++
.../selftests/drivers/crypto/cmh/config | 1 +
4 files changed, 191 insertions(+)
create mode 100644 tools/testing/selftests/drivers/crypto/cmh/Makefile
create mode 100644 tools/testing/selftests/drivers/crypto/cmh/cmh_mgmt_test.c
create mode 100644 tools/testing/selftests/drivers/crypto/cmh/config
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index 8d4db2241cc2..fe0b3a5b9769 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -17,6 +17,7 @@ TARGETS += damon
TARGETS += devices/error_logs
TARGETS += devices/probe
TARGETS += dmabuf-heaps
+TARGETS += drivers/crypto/cmh
TARGETS += drivers/dma-buf
TARGETS += drivers/ntsync
TARGETS += drivers/s390x/uvdevice
diff --git a/tools/testing/selftests/drivers/crypto/cmh/Makefile b/tools/testing/selftests/drivers/crypto/cmh/Makefile
new file mode 100644
index 000000000000..86cb63839b27
--- /dev/null
+++ b/tools/testing/selftests/drivers/crypto/cmh/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+TEST_GEN_PROGS := cmh_mgmt_test
+
+CFLAGS += -Wall -Wno-misleading-indentation -O2 $(KHDR_INCLUDES)
+
+include ../../../lib.mk
diff --git a/tools/testing/selftests/drivers/crypto/cmh/cmh_mgmt_test.c b/tools/testing/selftests/drivers/crypto/cmh/cmh_mgmt_test.c
new file mode 100644
index 000000000000..8c2269a3cd13
--- /dev/null
+++ b/tools/testing/selftests/drivers/crypto/cmh/cmh_mgmt_test.c
@@ -0,0 +1,183 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Kselftest for /dev/cmh_mgmt ioctl interface.
+ *
+ * Tests basic ioctl operations on the Rambus CryptoManager Hub management
+ * device. Requires the cmh module loaded on real or emulated hardware.
+ *
+ * Run: ./cmh_mgmt_test
+ * Output: TAP format (compatible with kselftest harness)
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+
+#include "kselftest_harness.h"
+#include <linux/cmh_mgmt_ioctl.h>
+
+#define CMH_DEV "/dev/cmh_mgmt"
+
+FIXTURE(cmh_mgmt)
+{
+ int fd;
+};
+
+FIXTURE_SETUP(cmh_mgmt)
+{
+ self->fd = open(CMH_DEV, O_RDWR);
+ if (self->fd < 0 && errno == ENOENT)
+ SKIP(return, "Device " CMH_DEV " not present (module not loaded?)");
+ if (self->fd < 0 && errno == EACCES)
+ SKIP(return, "Permission denied -- run as root or with CAP_SYS_ADMIN");
+ ASSERT_GE(self->fd, 0);
+}
+
+FIXTURE_TEARDOWN(cmh_mgmt)
+{
+ if (self->fd >= 0)
+ close(self->fd);
+}
+
+/*
+ * Test 1: open and close succeed.
+ * If we get here, FIXTURE_SETUP already validated the open.
+ */
+TEST_F(cmh_mgmt, open_close)
+{
+ ASSERT_GE(self->fd, 0);
+}
+
+/*
+ * Test 2: invalid ioctl number returns -ENOTTY.
+ */
+TEST_F(cmh_mgmt, invalid_ioctl)
+{
+ int ret;
+ unsigned long bogus_cmd = _IOC(_IOC_READ, 'J', 0xFF, 4);
+
+ ret = ioctl(self->fd, bogus_cmd, NULL);
+ ASSERT_EQ(ret, -1);
+ ASSERT_EQ(errno, ENOTTY);
+}
+
+/*
+ * Test 3: KEY_NEW with bad version field returns -EINVAL.
+ */
+TEST_F(cmh_mgmt, bad_version)
+{
+ struct cmh_ioctl_key_new req;
+ int ret;
+
+ memset(&req, 0, sizeof(req));
+ req.version = 0; /* invalid */
+ req.ds_type = CMH_DS_AES_KEY;
+ req.len = 32;
+ req.flags = CMH_FLAG_PT;
+ req.cid = 0xDEAD;
+
+ ret = ioctl(self->fd, CMH_IOCTL_KEY_NEW, &req);
+ ASSERT_EQ(ret, -1);
+ ASSERT_EQ(errno, EINVAL);
+}
+
+/*
+ * Test 4: KEY_NEW creates a key, KEY_DELETE destroys it.
+ */
+TEST_F(cmh_mgmt, key_new_delete)
+{
+ struct cmh_ioctl_key_new new_req;
+ struct cmh_ioctl_key_grant del_req;
+ int ret;
+
+ memset(&new_req, 0, sizeof(new_req));
+ new_req.version = CMH_MGMT_V1;
+ new_req.ds_type = CMH_DS_AES_KEY;
+ new_req.len = 32;
+ new_req.flags = CMH_FLAG_PT;
+ new_req.cid = 0x5E1F7E57ULL; /* "SELFTEST" */
+
+ ret = ioctl(self->fd, CMH_IOCTL_KEY_NEW, &new_req);
+ ASSERT_EQ(ret, 0);
+ ASSERT_NE(new_req.ref, (uint64_t)0);
+
+ /* Delete the key */
+ memset(&del_req, 0, sizeof(del_req));
+ del_req.version = CMH_MGMT_V1;
+ del_req.ref = new_req.ref;
+
+ ret = ioctl(self->fd, CMH_IOCTL_KEY_DELETE, &del_req);
+ ASSERT_EQ(ret, 0);
+}
+
+/*
+ * Test 5: KIC HKDF1 key derivation from hardware base key.
+ * Requires at least one KIC base key provisioned (KIC_KEY1).
+ */
+TEST_F(cmh_mgmt, kic_hkdf1)
+{
+ struct cmh_ioctl_kic_hkdf1 req;
+ static const char label[] = "kselftest-label";
+ int ret;
+
+ memset(&req, 0, sizeof(req));
+ req.version = CMH_MGMT_V1;
+ req.key_len = 32;
+ req.base_key = CMH_KIC_KEY1;
+ req.cid = 0x4B534C46ULL; /* "KSLF" */
+ req.label = (uint64_t)(uintptr_t)label;
+ req.label_len = sizeof(label) - 1;
+ req.flags = CMH_KIC_FLAG_TEMP;
+
+ ret = ioctl(self->fd, CMH_IOCTL_KIC_HKDF1, &req);
+ if (ret < 0 && errno == EIO)
+ SKIP(return, "KIC base key 1 not provisioned on this device");
+ ASSERT_EQ(ret, 0);
+ ASSERT_NE(req.ref, (uint64_t)0);
+}
+
+/*
+ * Test 6: ML-KEM-768 keygen using hardware RNG.
+ * Verifies the PQC keygen path end-to-end.
+ */
+TEST_F(cmh_mgmt, ml_kem_keygen)
+{
+ struct cmh_ioctl_ml_kem_keygen req;
+ /* ML-KEM-768: ek = 384*3+32 = 1184, dk = 768*3+96 = 2400 */
+ uint8_t ek[1184];
+ uint8_t dk[2400];
+ int ret;
+
+ memset(&req, 0, sizeof(req));
+ req.version = CMH_MGMT_V1;
+ req.k = 3; /* ML-KEM-768 */
+ req.flags = CMH_QSE_FLAG_HW_RNG;
+ req.seed = 0; /* HW RNG */
+ req.z = 0; /* HW RNG */
+ req.ek = (uint64_t)(uintptr_t)ek;
+ req.dk = (uint64_t)(uintptr_t)dk;
+ req.dk_cid = 0;
+ req.dk_ref = 0;
+
+ memset(ek, 0, sizeof(ek));
+ memset(dk, 0, sizeof(dk));
+
+ ret = ioctl(self->fd, CMH_IOCTL_ML_KEM_KEYGEN, &req);
+ if (ret < 0 && errno == ENODEV)
+ SKIP(return, "QSE core not available on this hardware");
+ ASSERT_EQ(ret, 0);
+
+ /* Verify output is non-zero (extremely unlikely for random keys) */
+ {
+ int i, nonzero = 0;
+
+ for (i = 0; i < 64; i++)
+ nonzero += (ek[i] != 0);
+ ASSERT_GT(nonzero, 0);
+ }
+}
+
+TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/drivers/crypto/cmh/config b/tools/testing/selftests/drivers/crypto/cmh/config
new file mode 100644
index 000000000000..063c1dd0e23b
--- /dev/null
+++ b/tools/testing/selftests/drivers/crypto/cmh/config
@@ -0,0 +1 @@
+CONFIG_CRYPTO_DEV_CMH=m
--
2.43.7
^ permalink raw reply related
* [PATCH v3 07/19] crypto: cmh - add SM3 ahash
From: Saravanakrishnan Krishnamoorthy @ 2026-08-06 19:55 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register the SM3 ahash algorithm using the CMH SM3 core (core ID
0x05). Supports incremental update/finup/final and export/import.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 3 +-
drivers/crypto/cmh/cmh_main.c | 9 +
drivers/crypto/cmh/cmh_sm3.c | 676 +++++++++++++++++++++++++++
drivers/crypto/cmh/include/cmh_sm3.h | 27 ++
4 files changed, 714 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_sm3.c
create mode 100644 drivers/crypto/cmh/include/cmh_sm3.h
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index 3ab154744658..664b2a20bc19 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -18,7 +18,8 @@ cmh-y := \
cmh_hash.o \
cmh_hmac.o \
cmh_cshake.o \
- cmh_kmac.o
+ cmh_kmac.o \
+ cmh_sm3.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index bc47d5ef3f86..9bb7c75d1bf3 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -35,6 +35,7 @@
#include "cmh_hmac.h"
#include "cmh_cshake.h"
#include "cmh_kmac.h"
+#include "cmh_sm3.h"
#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
@@ -261,6 +262,11 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_kmac_register;
+ /* Register SM3 hash algorithm */
+ ret = cmh_sm3_register();
+ if (ret)
+ goto err_sm3_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -273,6 +279,8 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_sm3_unregister();
+err_sm3_register:
cmh_kmac_unregister();
err_kmac_register:
cmh_cshake_unregister();
@@ -307,6 +315,7 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_sm3_unregister();
cmh_kmac_unregister();
cmh_cshake_unregister();
cmh_hmac_unregister();
diff --git a/drivers/crypto/cmh/cmh_sm3.c b/drivers/crypto/cmh/cmh_sm3.c
new file mode 100644
index 000000000000..37a66cb1508f
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_sm3.c
@@ -0,0 +1,676 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- SM3 Hash Driver (CORE_ID_SM3)
+ *
+ * Registers an asynchronous hash (ahash) algorithm for SM3
+ * (GB/T 32905-2016) using the CMH SM3 core. This is a standalone
+ * driver separate from cmh_hash.c (which handles HC-based SHA-2/3/SHAKE)
+ * because SM3 runs on a different hardware core with its own command
+ * IDs and context layout.
+ *
+ * Incremental HW update model (same pattern as cmh_hash.c):
+ *
+ * .init() -> software-only: zero per-request context
+ * .update() -> buffer data in holdback; when >= block_size bytes:
+ * SM3_CMD_INIT [+ RESTORE] + UPDATE + SAVE + FLUSH
+ * -> return -EINPROGRESS (else return 0)
+ * .final() -> SM3_CMD_INIT [+ RESTORE] [+ UPDATE] + FINAL + FLUSH
+ * .finup() -> linearise holdback + new data, then final path
+ * .digest() -> INIT + UPDATE + FINAL + FLUSH (single-shot, zero-copy)
+ * .export() -> software-only: copy checkpoint + holdback to out
+ * .import() -> software-only: restore checkpoint + holdback from in
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/hash.h>
+#include <crypto/scatterwalk.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "cmh_sm3.h"
+#include "cmh_vcq.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+
+/* Per-Request State */
+
+/*
+ * Exported SM3 state -- serialised by .export(), deserialised by
+ * .import(). This is what statesize advertises to the crypto subsystem.
+ */
+struct cmh_sm3_export_state {
+ u8 checkpoint[SM3_CONTEXT_SIZE]; /* SM3 context from last SAVE */
+ u8 buf[CMH_SM3_BLOCK_SIZE]; /* holdback buffer */
+ u32 buf_len; /* valid bytes in buf[] */
+ u32 hw_started; /* non-zero if checkpoint valid */
+};
+
+#define CMH_SM3_MAX_PAYLOAD 5 /* INIT + RESTORE + UPDATE + FINAL/SAVE + FLUSH */
+#define CMH_SM3_MAX_PACKED (CMH_SM3_MAX_PAYLOAD * 2)
+
+/*
+ * Checkpoint embedded inline: the kernel ahash API has no per-request
+ * destructor, so a heap-allocated checkpoint leaks if a request is
+ * abandoned without .final().
+ */
+struct cmh_sm3_reqctx {
+ int error;
+ u32 hw_started;
+ u32 buf_len;
+ u32 has_checkpoint;
+ u8 checkpoint[SM3_CONTEXT_SIZE]; /* SM3 context from last SAVE */
+ /* DMA state for current async operation */
+ dma_addr_t ckpt_dma;
+ dma_addr_t save_dma;
+ dma_addr_t data_dma;
+ dma_addr_t digest_dma;
+ u8 *save_buf;
+ u8 *data_buf;
+ u32 data_len;
+ u8 *digest_buf;
+ u8 buf[CMH_SM3_BLOCK_SIZE]; /* holdback for partial block */
+ struct vcq_cmd packed[CMH_SM3_MAX_PACKED];
+};
+
+/* VCQ Builders -- SM3 core (CORE_ID_SM3); generic flush from cmh_vcq.h */
+
+static void vcq_add_sm3_init(struct vcq_cmd *slot, u32 core_id)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM3_CMD_INIT);
+ /* SM3 has a single algorithm -- no algo selector field */
+}
+
+static void vcq_add_sm3_update(struct vcq_cmd *slot, u32 core_id, u64 input_phys, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM3_CMD_UPDATE);
+ slot->hwc.sm3.cmd_update.input = input_phys;
+ slot->hwc.sm3.cmd_update.inlen = len;
+}
+
+static void vcq_add_sm3_final(struct vcq_cmd *slot, u32 core_id, u64 digest_phys, u32 outlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM3_CMD_FINAL);
+ slot->hwc.sm3.cmd_final.digest = digest_phys;
+ slot->hwc.sm3.cmd_final.outlen = outlen;
+}
+
+static void vcq_add_sm3_save(struct vcq_cmd *slot, u32 core_id, u64 output_phys, u32 outlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM3_CMD_SAVE);
+ slot->hwc.sm3.cmd_save.output = output_phys;
+ slot->hwc.sm3.cmd_save.outlen = outlen;
+}
+
+static void vcq_add_sm3_restore(struct vcq_cmd *slot, u32 core_id, u64 input_phys, u32 inlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM3_CMD_RESTORE);
+ slot->hwc.sm3.cmd_restore.input = input_phys;
+ slot->hwc.sm3.cmd_restore.inlen = inlen;
+}
+
+/* Request Context Cleanup */
+
+static void cmh_sm3_free_reqctx(struct cmh_sm3_reqctx *rctx)
+{
+ rctx->has_checkpoint = 0;
+}
+
+/* VCQ Packing + Submit */
+
+/* ahash Operations */
+
+static int cmh_sm3_init(struct ahash_request *req)
+{
+ struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req);
+
+ memset(rctx, 0, sizeof(*rctx));
+ return 0;
+}
+
+/*
+ * Update completion -- takes ownership of save_buf as new checkpoint.
+ */
+static void cmh_sm3_update_complete(void *data, int error)
+{
+ struct ahash_request *req = data;
+ struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(rctx->ckpt_dma, SM3_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->save_dma, SM3_CONTEXT_SIZE,
+ DMA_FROM_DEVICE);
+ cmh_dma_unmap_single(rctx->data_dma, rctx->data_len,
+ DMA_TO_DEVICE);
+
+ if (!error) {
+ memcpy(rctx->checkpoint, rctx->save_buf, SM3_CONTEXT_SIZE);
+ rctx->has_checkpoint = 1;
+ kfree(rctx->save_buf);
+ rctx->save_buf = NULL;
+ rctx->hw_started = 1;
+ } else {
+ kfree(rctx->save_buf);
+ rctx->save_buf = NULL;
+ rctx->error = error;
+ }
+
+ kfree(rctx->data_buf);
+ rctx->data_buf = NULL;
+ rctx->data_len = 0;
+
+ cmh_complete(&req->base, error);
+}
+
+static int cmh_sm3_update(struct ahash_request *req)
+{
+ struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req);
+ struct vcq_cmd cmds[CMH_SM3_MAX_PAYLOAD];
+ struct core_dispatch d;
+ u32 total_avail, full_len, tail_len, from_src;
+ u32 idx;
+ int ret;
+ gfp_t gfp;
+
+ if (rctx->error)
+ return rctx->error;
+
+ if (!req->nbytes)
+ return 0;
+
+ /*
+ * Each update() linearises its data into a single kmalloc buffer.
+ * Guard against rctx->buf_len + req->nbytes overflowing u32, which
+ * would mis-select the "buffer only" path below and overrun the
+ * holdback. Such an update is far too large to service anyway.
+ */
+ if (req->nbytes > U32_MAX - rctx->buf_len)
+ return -EMSGSIZE;
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ total_avail = rctx->buf_len + req->nbytes;
+
+ if (total_avail < CMH_SM3_BLOCK_SIZE) {
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(rctx->buf + rctx->buf_len,
+ req->svirt, req->nbytes);
+ else
+ scatterwalk_map_and_copy(rctx->buf + rctx->buf_len,
+ req->src, 0,
+ req->nbytes, 0);
+ rctx->buf_len = total_avail;
+ return 0;
+ }
+
+ full_len = total_avail - total_avail % CMH_SM3_BLOCK_SIZE;
+ tail_len = total_avail - full_len;
+ from_src = full_len - rctx->buf_len;
+
+ /*
+ * full_len is user-controlled and can exceed KMALLOC_MAX_SIZE for a
+ * single large update; __GFP_NOWARN avoids splatting the page
+ * allocator on an oversized request.
+ */
+ rctx->data_buf = kmalloc(full_len, gfp | __GFP_NOWARN);
+ if (!rctx->data_buf)
+ return -ENOMEM;
+
+ if (rctx->buf_len > 0)
+ memcpy(rctx->data_buf, rctx->buf, rctx->buf_len);
+
+ if (from_src > 0) {
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(rctx->data_buf + rctx->buf_len,
+ req->svirt, from_src);
+ else
+ scatterwalk_map_and_copy(rctx->data_buf + rctx->buf_len,
+ req->src, 0,
+ from_src, 0);
+ }
+
+ if (tail_len > 0) {
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(rctx->buf, req->svirt + from_src,
+ tail_len);
+ else
+ scatterwalk_map_and_copy(rctx->buf, req->src,
+ from_src, tail_len,
+ 0);
+ }
+ rctx->buf_len = tail_len;
+ rctx->data_len = full_len;
+
+ rctx->save_buf = kzalloc(SM3_CONTEXT_SIZE, gfp);
+ if (!rctx->save_buf) {
+ ret = -ENOMEM;
+ goto err_free;
+ }
+
+ rctx->data_dma = cmh_dma_map_single(rctx->data_buf, full_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->data_dma)) {
+ ret = -ENOMEM;
+ goto err_free;
+ }
+
+ rctx->save_dma = cmh_dma_map_single(rctx->save_buf, SM3_CONTEXT_SIZE,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->save_dma)) {
+ ret = -ENOMEM;
+ goto err_unmap_data;
+ }
+
+ rctx->ckpt_dma = DMA_MAPPING_ERROR;
+ if (rctx->has_checkpoint) {
+ rctx->ckpt_dma = cmh_dma_map_single(rctx->checkpoint,
+ SM3_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->ckpt_dma)) {
+ ret = -ENOMEM;
+ goto err_unmap_save;
+ }
+ }
+
+ d = cmh_core_select_instance(CMH_CORE_SM3);
+ idx = 0;
+
+ vcq_add_sm3_init(&cmds[idx++], d.core_id);
+
+ if (rctx->has_checkpoint)
+ vcq_add_sm3_restore(&cmds[idx++], d.core_id,
+ (u64)rctx->ckpt_dma, SM3_CONTEXT_SIZE);
+
+ vcq_add_sm3_update(&cmds[idx++], d.core_id,
+ (u64)rctx->data_dma, full_len);
+
+ vcq_add_sm3_save(&cmds[idx++], d.core_id,
+ (u64)rctx->save_dma, SM3_CONTEXT_SIZE);
+
+ vcq_add_flush(&cmds[idx++], d.core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_SM3_MAX_PACKED,
+ d.mbx_idx,
+ cmh_sm3_update_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto err_unmap_ckpt;
+
+ return -EINPROGRESS;
+
+err_unmap_ckpt:
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(rctx->ckpt_dma, SM3_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+err_unmap_save:
+ cmh_dma_unmap_single(rctx->save_dma, SM3_CONTEXT_SIZE,
+ DMA_FROM_DEVICE);
+err_unmap_data:
+ cmh_dma_unmap_single(rctx->data_dma, full_len, DMA_TO_DEVICE);
+err_free:
+ kfree(rctx->save_buf);
+ rctx->save_buf = NULL;
+ kfree(rctx->data_buf);
+ rctx->data_buf = NULL;
+ rctx->data_len = 0;
+ return ret;
+}
+
+static void cmh_sm3_final_complete(void *data, int error)
+{
+ struct ahash_request *req = data;
+ struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(rctx->ckpt_dma, SM3_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+ if (rctx->data_buf)
+ cmh_dma_unmap_single(rctx->data_dma, rctx->data_len,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->digest_dma, CMH_SM3_DIGEST_SIZE,
+ DMA_FROM_DEVICE);
+
+ if (!error)
+ memcpy(req->result, rctx->digest_buf, CMH_SM3_DIGEST_SIZE);
+
+ kfree(rctx->digest_buf);
+ rctx->digest_buf = NULL;
+ kfree(rctx->data_buf);
+ rctx->data_buf = NULL;
+ cmh_sm3_free_reqctx(rctx);
+ cmh_complete(&req->base, error);
+}
+
+static int cmh_sm3_submit_final(struct ahash_request *req,
+ u8 *data_buf, u32 data_len)
+{
+ struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req);
+ struct vcq_cmd cmds[CMH_SM3_MAX_PAYLOAD];
+ struct core_dispatch d;
+ u32 idx;
+ int ret;
+ gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ rctx->data_buf = data_buf;
+ rctx->data_len = data_len;
+
+ rctx->digest_buf = kzalloc(CMH_SM3_DIGEST_SIZE, gfp);
+ if (!rctx->digest_buf) {
+ ret = -ENOMEM;
+ goto err_free_data;
+ }
+
+ rctx->digest_dma = cmh_dma_map_single(rctx->digest_buf,
+ CMH_SM3_DIGEST_SIZE,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->digest_dma)) {
+ ret = -ENOMEM;
+ goto err_free_digest;
+ }
+
+ rctx->data_dma = DMA_MAPPING_ERROR;
+ if (data_buf && data_len > 0) {
+ rctx->data_dma = cmh_dma_map_single(data_buf, data_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->data_dma)) {
+ ret = -ENOMEM;
+ goto err_unmap_digest;
+ }
+ }
+
+ rctx->ckpt_dma = DMA_MAPPING_ERROR;
+ if (rctx->has_checkpoint) {
+ rctx->ckpt_dma = cmh_dma_map_single(rctx->checkpoint,
+ SM3_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->ckpt_dma)) {
+ ret = -ENOMEM;
+ goto err_unmap_data;
+ }
+ }
+
+ d = cmh_core_select_instance(CMH_CORE_SM3);
+ idx = 0;
+
+ vcq_add_sm3_init(&cmds[idx++], d.core_id);
+
+ if (rctx->has_checkpoint)
+ vcq_add_sm3_restore(&cmds[idx++], d.core_id,
+ (u64)rctx->ckpt_dma, SM3_CONTEXT_SIZE);
+
+ if (data_buf && data_len > 0)
+ vcq_add_sm3_update(&cmds[idx++], d.core_id,
+ (u64)rctx->data_dma, data_len);
+
+ vcq_add_sm3_final(&cmds[idx++], d.core_id,
+ (u64)rctx->digest_dma, CMH_SM3_DIGEST_SIZE);
+
+ vcq_add_flush(&cmds[idx++], d.core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_SM3_MAX_PACKED,
+ d.mbx_idx,
+ cmh_sm3_final_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto err_unmap_ckpt;
+
+ return -EINPROGRESS;
+
+err_unmap_ckpt:
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(rctx->ckpt_dma, SM3_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+err_unmap_data:
+ if (data_buf && data_len > 0)
+ cmh_dma_unmap_single(rctx->data_dma, data_len,
+ DMA_TO_DEVICE);
+err_unmap_digest:
+ cmh_dma_unmap_single(rctx->digest_dma, CMH_SM3_DIGEST_SIZE,
+ DMA_FROM_DEVICE);
+err_free_digest:
+ kfree(rctx->digest_buf);
+ rctx->digest_buf = NULL;
+err_free_data:
+ kfree(data_buf);
+ rctx->data_buf = NULL;
+ cmh_sm3_free_reqctx(rctx);
+ return ret;
+}
+
+static int cmh_sm3_final(struct ahash_request *req)
+{
+ struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req);
+ u8 *data_buf = NULL;
+ u32 data_len = 0;
+ gfp_t gfp;
+
+ if (rctx->error)
+ return rctx->error;
+
+ if (rctx->buf_len > 0) {
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+ data_buf = kmalloc(rctx->buf_len, gfp);
+ if (!data_buf)
+ return -ENOMEM;
+ memcpy(data_buf, rctx->buf, rctx->buf_len);
+ data_len = rctx->buf_len;
+ rctx->buf_len = 0;
+ }
+
+ return cmh_sm3_submit_final(req, data_buf, data_len);
+}
+
+static int cmh_sm3_finup(struct ahash_request *req);
+
+/*
+ * One-shot digest -- delegates to init + finup so that all data is
+ * linearised and mapped through cmh_dma_map_single(), which is the
+ * only DMA mapping path aware of all supported DMA backends.
+ */
+static int cmh_sm3_digest(struct ahash_request *req)
+{
+ int ret;
+
+ ret = cmh_sm3_init(req);
+ if (ret)
+ return ret;
+ return cmh_sm3_finup(req);
+}
+
+static int cmh_sm3_finup(struct ahash_request *req)
+{
+ struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req);
+ u32 data_len;
+ u8 *data_buf;
+ gfp_t gfp;
+
+ if (rctx->error)
+ return rctx->error;
+
+ data_len = rctx->buf_len + req->nbytes;
+
+ if (data_len == 0)
+ return cmh_sm3_submit_final(req, NULL, 0);
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ data_buf = kmalloc(data_len, gfp | __GFP_NOWARN);
+ if (!data_buf)
+ return -ENOMEM;
+
+ if (rctx->buf_len > 0)
+ memcpy(data_buf, rctx->buf, rctx->buf_len);
+
+ if (req->nbytes > 0) {
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(data_buf + rctx->buf_len,
+ req->svirt, req->nbytes);
+ else
+ scatterwalk_map_and_copy(data_buf + rctx->buf_len,
+ req->src, 0,
+ req->nbytes, 0);
+ }
+
+ rctx->buf_len = 0;
+ return cmh_sm3_submit_final(req, data_buf, data_len);
+}
+
+static int cmh_sm3_export(struct ahash_request *req, void *out)
+{
+ struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_sm3_export_state *state = out;
+
+ /*
+ * Zero the whole exported state first: only rctx->buf_len bytes of
+ * buf[] are populated below and the struct may carry padding, so
+ * without this the unused tail/padding would leak kernel memory to
+ * user space through the ahash export.
+ */
+ memset(state, 0, sizeof(*state));
+
+ if (rctx->hw_started && rctx->has_checkpoint)
+ memcpy(state->checkpoint, rctx->checkpoint, SM3_CONTEXT_SIZE);
+
+ if (rctx->buf_len > 0)
+ memcpy(state->buf, rctx->buf, rctx->buf_len);
+
+ state->buf_len = rctx->buf_len;
+ state->hw_started = rctx->hw_started;
+
+ return 0;
+}
+
+static int cmh_sm3_import(struct ahash_request *req, const void *in)
+{
+ struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_sm3_export_state *state = in;
+
+ memset(rctx, 0, sizeof(*rctx));
+
+ /*
+ * Holdback invariant: buf_len < block size (a full block is always
+ * processed, never left in the holdback). Reject a corrupt state
+ * that violates it.
+ */
+ if (state->buf_len >= CMH_SM3_BLOCK_SIZE)
+ return -EINVAL;
+
+ rctx->hw_started = state->hw_started;
+ rctx->buf_len = state->buf_len;
+ memcpy(rctx->buf, state->buf, state->buf_len);
+
+ if (state->hw_started) {
+ memcpy(rctx->checkpoint, state->checkpoint, SM3_CONTEXT_SIZE);
+ rctx->has_checkpoint = 1;
+ }
+
+ return 0;
+}
+
+/* Transform init (cra_init) */
+
+static int cmh_sm3_cra_init(struct crypto_tfm *tfm)
+{
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct cmh_sm3_reqctx));
+ return 0;
+}
+
+/* Registration */
+
+static struct ahash_alg cmh_sm3_ahash_alg = {
+ .init = cmh_sm3_init,
+ .update = cmh_sm3_update,
+ .final = cmh_sm3_final,
+ .finup = cmh_sm3_finup,
+ .digest = cmh_sm3_digest,
+ .export = cmh_sm3_export,
+ .import = cmh_sm3_import,
+
+ .halg = {
+ .digestsize = CMH_SM3_DIGEST_SIZE,
+ .statesize = sizeof(struct cmh_sm3_export_state),
+ .base = {
+ .cra_name = "sm3",
+ .cra_driver_name = "rambus-cmh-sm3",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NO_FALLBACK |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_REQ_VIRT,
+ .cra_blocksize = CMH_SM3_BLOCK_SIZE,
+ .cra_ctxsize = 0,
+ .cra_init = cmh_sm3_cra_init,
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
+
+/**
+ * cmh_sm3_register() - Register SM3 hash algorithm with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_sm3_register(void)
+{
+ int ret;
+
+ ret = crypto_register_ahash(&cmh_sm3_ahash_alg);
+ if (ret) {
+ dev_err(cmh_dev(), "sm3: failed to register cmh-sm3 (rc=%d)\n",
+ ret);
+ return ret;
+ }
+
+ dev_info(cmh_dev(), "sm3: registered rambus-cmh-sm3 (priority 300)\n");
+ dev_info(cmh_dev(), "sm3: 1 algorithm(s) registered\n");
+ return 0;
+}
+
+/**
+ * cmh_sm3_unregister() - Unregister SM3 hash algorithm from the crypto framework
+ */
+void cmh_sm3_unregister(void)
+{
+ crypto_unregister_ahash(&cmh_sm3_ahash_alg);
+ dev_info(cmh_dev(), "sm3: unregistered rambus-cmh-sm3\n");
+ dev_info(cmh_dev(), "sm3: cleaned up\n");
+}
diff --git a/drivers/crypto/cmh/include/cmh_sm3.h b/drivers/crypto/cmh/include/cmh_sm3.h
new file mode 100644
index 000000000000..2f73537f9c87
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_sm3.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- SM3 Hash Driver
+ *
+ * Registers an ahash algorithm for SM3 (GB/T 32905-2016) with the
+ * Linux crypto subsystem using the CMH SM3 core (CORE_ID_SM3).
+ * Uses the same incremental HW update model as cmh_hash.c:
+ *
+ * .init() -> software-only: zero per-request context
+ * .update() -> holdback partial blocks; submit full blocks via
+ * SM3_CMD_INIT [+ RESTORE] + UPDATE + SAVE + FLUSH
+ * .final() -> SM3_CMD_INIT [+ RESTORE] [+ UPDATE] + FINAL + FLUSH
+ * .digest() -> INIT + UPDATE + FINAL + FLUSH (single-shot)
+ * .export() -> software-only: copy checkpoint + holdback
+ * .import() -> software-only: restore checkpoint + holdback
+ */
+
+#ifndef CMH_SM3_H
+#define CMH_SM3_H
+
+#include "cmh_config.h"
+
+int cmh_sm3_register(void);
+void cmh_sm3_unregister(void);
+
+#endif /* CMH_SM3_H */
--
2.43.7
^ permalink raw reply related
* [PATCH v3 10/19] crypto: cmh - add ChaCha20-Poly1305
From: Saravanakrishnan Krishnamoorthy @ 2026-08-06 19:55 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register ChaCha20-Poly1305 AEAD and ChaCha20 skcipher algorithms
using the CMH CCP core (core ID 0x18). Also registers the Poly1305
ahash for standalone use.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 5 +-
drivers/crypto/cmh/cmh_ccp.c | 356 ++++++++++++++
drivers/crypto/cmh/cmh_ccp_aead.c | 594 +++++++++++++++++++++++
drivers/crypto/cmh/cmh_ccp_poly.c | 694 +++++++++++++++++++++++++++
drivers/crypto/cmh/cmh_main.c | 25 +
drivers/crypto/cmh/include/cmh_ccp.h | 24 +
6 files changed, 1697 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_ccp.c
create mode 100644 drivers/crypto/cmh/cmh_ccp_aead.c
create mode 100644 drivers/crypto/cmh/cmh_ccp_poly.c
create mode 100644 drivers/crypto/cmh/include/cmh_ccp.h
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index 032c863d856e..ef24879ba1f1 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -25,7 +25,10 @@ cmh-y := \
cmh_aes_cmac.o \
cmh_sm4_skcipher.o \
cmh_sm4_aead.o \
- cmh_sm4_cmac.o
+ cmh_sm4_cmac.o \
+ cmh_ccp.o \
+ cmh_ccp_aead.o \
+ cmh_ccp_poly.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_ccp.c b/drivers/crypto/cmh/cmh_ccp.c
new file mode 100644
index 000000000000..babf6ffe6ff0
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_ccp.c
@@ -0,0 +1,356 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API ChaCha20 (skcipher) Driver
+ *
+ * Registers the "chacha20" skcipher algorithm with the Linux crypto
+ * subsystem, backed by the CMH CCP core.
+ *
+ * VCQ sequence:
+ * [SYS_CMD_WRITE] + CCP_CMD_CHACHA20_INIT + CCP_CMD_FINAL + CCP_CMD_FLUSH
+ *
+ * The CCP core expects a 16-byte counter+nonce (ctrnonce):
+ * bytes [0..3] = 32-bit LE counter
+ * bytes [4..15] = 12-byte nonce
+ *
+ * The Linux chacha20 skcipher interface passes a 16-byte IV in the
+ * same format, so we forward it directly.
+ *
+ * ChaCha20 is a stream cipher -- arbitrary plaintext lengths are
+ * supported (no block-alignment requirement).
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/scatterwalk.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "cmh_ccp.h"
+#include "cmh_vcq.h"
+#include "cmh_ccp_abi.h"
+#include "cmh_sys_abi.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+/* Per-transform context */
+
+struct cmh_ccp_tfm_ctx {
+ struct cmh_key_ctx key;
+};
+
+/* Per-request context (lives in skcipher_request::__ctx) */
+
+/*
+ * Maximum payload commands:
+ * [SYS_CMD_WRITE] + CCP_CMD_CHACHA20_INIT + CCP_CMD_FINAL + FLUSH = 4
+ */
+#define CMH_CCP_MAX_PAYLOAD 4
+#define CMH_CCP_MAX_PACKED (CMH_CCP_MAX_PAYLOAD * 2)
+
+struct cmh_ccp_reqctx {
+ dma_addr_t in_dma;
+ dma_addr_t out_dma;
+ dma_addr_t iv_dma;
+ dma_addr_t key_dma;
+ u8 *in_buf;
+ u8 *out_buf;
+ u8 *iv_buf;
+ u32 cryptlen;
+ u32 keylen;
+ struct vcq_cmd packed[CMH_CCP_MAX_PACKED];
+};
+
+/* VCQ Builders -- ChaCha20-specific */
+
+static void vcq_add_ccp_chacha_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref,
+ u64 ctrnonce_dma, u32 keylen, u32 op)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_CHACHA20_INIT);
+ slot->hwc.ccp.cmd_chacha.key = key_ref;
+ slot->hwc.ccp.cmd_chacha.ctrnonce = ctrnonce_dma;
+ slot->hwc.ccp.cmd_chacha.keylen = keylen;
+ slot->hwc.ccp.cmd_chacha.ctrnoncelen = CCP_CTRNONCE_SIZE;
+ slot->hwc.ccp.cmd_chacha.ctrlen = CCP_CHACHA_CTR_LEN;
+ slot->hwc.ccp.cmd_chacha.op = op;
+}
+
+static void vcq_add_ccp_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma,
+ u64 output_dma, u32 iolen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_FINAL);
+ slot->hwc.ccp.cmd_final.input = input_dma;
+ slot->hwc.ccp.cmd_final.output = output_dma;
+ slot->hwc.ccp.cmd_final.tag = 0;
+ slot->hwc.ccp.cmd_final.iolen = iolen;
+ slot->hwc.ccp.cmd_final.taglen = 0;
+}
+
+/* skcipher Operations */
+static int cmh_ccp_setkey(struct crypto_skcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_ccp_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+ /* ChaCha20 requires 32-byte key per RFC 8439 */
+ if (keylen != 32)
+ return -EINVAL;
+
+ return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_CCP);
+}
+
+static int cmh_ccp_init_tfm(struct crypto_skcipher *tfm)
+{
+ struct cmh_ccp_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+
+ memset(tctx, 0, sizeof(*tctx));
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct cmh_ccp_reqctx));
+ return 0;
+}
+
+static void cmh_ccp_exit_tfm(struct crypto_skcipher *tfm)
+{
+ struct cmh_ccp_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+
+ cmh_key_destroy(&tctx->key);
+}
+
+/* DMA unmap helper */
+static void cmh_ccp_unmap_dma(struct cmh_ccp_reqctx *rctx)
+{
+ cmh_dma_unmap_single(rctx->iv_dma, CCP_CTRNONCE_SIZE, DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen, DMA_FROM_DEVICE);
+ cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen, DMA_TO_DEVICE);
+}
+
+static void cmh_ccp_free_bufs(struct cmh_ccp_reqctx *rctx)
+{
+ kfree(rctx->iv_buf);
+ rctx->iv_buf = NULL;
+ kfree_sensitive(rctx->out_buf);
+ rctx->out_buf = NULL;
+ kfree_sensitive(rctx->in_buf);
+ rctx->in_buf = NULL;
+}
+
+static void cmh_ccp_complete(void *data, int error)
+{
+ struct skcipher_request *req = data;
+ struct cmh_ccp_reqctx *rctx = skcipher_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ cmh_ccp_unmap_dma(rctx);
+
+ /*
+ * ChaCha20 is a one-shot stream cipher. Like the generic
+ * chacha20 skcipher we leave req->iv unchanged (no counter
+ * write-back), matching testmgr's output-IV expectation.
+ */
+ if (!error)
+ scatterwalk_map_and_copy(rctx->out_buf, req->dst,
+ 0, rctx->cryptlen, 1);
+
+ cmh_ccp_free_bufs(rctx);
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * Core encrypt/decrypt -- builds a VCQ transaction and submits async.
+ *
+ * ChaCha20 is a stream cipher: encrypt and decrypt use the same
+ * underlying XOR operation.
+ */
+static int cmh_ccp_crypt(struct skcipher_request *req, u32 ccp_op)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct cmh_ccp_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+ struct cmh_ccp_reqctx *rctx = skcipher_request_ctx(req);
+ struct vcq_cmd cmds[CMH_CCP_MAX_PAYLOAD];
+ u64 key_ref;
+ u32 keylen;
+ struct core_dispatch d;
+ s32 target_mbx;
+ u32 core_id;
+ u32 idx;
+ int ret;
+ gfp_t gfp;
+
+ if (tctx->key.mode == CMH_KEY_NONE)
+ return -ENOKEY;
+
+ if (!req->cryptlen)
+ return 0;
+
+ /* Limit linearisation buffers to avoid large allocations. */
+ if (req->cryptlen > SZ_1M)
+ return -EINVAL;
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->cryptlen = req->cryptlen;
+
+ /* Linearise input from scatterlist */
+ rctx->in_buf = kmalloc(req->cryptlen, gfp);
+ if (!rctx->in_buf)
+ return -ENOMEM;
+
+ scatterwalk_map_and_copy(rctx->in_buf, req->src, 0, req->cryptlen, 0);
+
+ rctx->in_dma = cmh_dma_map_single(rctx->in_buf, req->cryptlen,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->in_dma)) {
+ ret = -ENOMEM;
+ goto out_free_in;
+ }
+
+ rctx->out_buf = kmalloc(req->cryptlen, gfp);
+ if (!rctx->out_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_in;
+ }
+
+ rctx->out_dma = cmh_dma_map_single(rctx->out_buf, req->cryptlen,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->out_dma)) {
+ ret = -ENOMEM;
+ goto out_free_out;
+ }
+
+ rctx->iv_buf = kmemdup(req->iv, CCP_CTRNONCE_SIZE, gfp);
+ if (!rctx->iv_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_out;
+ }
+
+ rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, CCP_CTRNONCE_SIZE,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->iv_dma)) {
+ ret = -ENOMEM;
+ goto out_free_iv;
+ }
+
+ /* Resolve key reference */
+ idx = 0;
+
+ rctx->key_dma = tctx->key.raw.dma;
+ rctx->keylen = tctx->key.raw.len;
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP,
+ (u64)rctx->key_dma, SYS_REF_NONE,
+ tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ keylen = tctx->key.raw.len;
+ d = cmh_core_select_instance(CMH_CORE_CCP);
+ target_mbx = d.mbx_idx;
+ core_id = d.core_id;
+
+ vcq_add_ccp_chacha_init(&cmds[idx++], core_id, key_ref,
+ (u64)rctx->iv_dma, keylen, ccp_op);
+
+ vcq_add_ccp_final(&cmds[idx++], core_id, (u64)rctx->in_dma,
+ (u64)rctx->out_dma, req->cryptlen);
+
+ vcq_add_flush(&cmds[idx++], core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_CCP_MAX_PACKED, target_mbx,
+ cmh_ccp_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ /* -EBUSY = backlogged; ownership transferred to callback. */
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_cleanup_all;
+
+ return -EINPROGRESS;
+
+out_cleanup_all:
+ cmh_dma_unmap_single(rctx->iv_dma, CCP_CTRNONCE_SIZE, DMA_TO_DEVICE);
+out_free_iv:
+ kfree(rctx->iv_buf);
+out_unmap_out:
+ cmh_dma_unmap_single(rctx->out_dma, req->cryptlen, DMA_FROM_DEVICE);
+out_free_out:
+ kfree_sensitive(rctx->out_buf);
+out_unmap_in:
+ cmh_dma_unmap_single(rctx->in_dma, req->cryptlen, DMA_TO_DEVICE);
+out_free_in:
+ kfree_sensitive(rctx->in_buf);
+ return ret;
+}
+
+static int cmh_ccp_encrypt(struct skcipher_request *req)
+{
+ return cmh_ccp_crypt(req, CCP_OP_ENCRYPT);
+}
+
+static int cmh_ccp_decrypt(struct skcipher_request *req)
+{
+ return cmh_ccp_crypt(req, CCP_OP_DECRYPT);
+}
+
+/* Registration */
+
+static struct skcipher_alg cmh_chacha20_alg = {
+ .setkey = cmh_ccp_setkey,
+ .encrypt = cmh_ccp_encrypt,
+ .decrypt = cmh_ccp_decrypt,
+ .init = cmh_ccp_init_tfm,
+ .exit = cmh_ccp_exit_tfm,
+ .min_keysize = 32,
+ .max_keysize = 32,
+ .ivsize = CCP_CTRNONCE_SIZE,
+ .base = {
+ .cra_name = "chacha20",
+ .cra_driver_name = "rambus-cmh-chacha20",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1, /* stream cipher */
+ .cra_ctxsize = sizeof(struct cmh_ccp_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ },
+};
+
+/**
+ * cmh_ccp_register() - Register ChaCha20 skcipher algorithm with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_ccp_register(void)
+{
+ int ret;
+
+ ret = crypto_register_skcipher(&cmh_chacha20_alg);
+ if (ret)
+ dev_err(cmh_dev(), "cmh_ccp: failed to register chacha20 (rc=%d)\n", ret);
+ else
+ dev_dbg(cmh_dev(), "cmh_ccp: registered chacha20\n");
+
+ return ret;
+}
+
+/**
+ * cmh_ccp_unregister() - Unregister ChaCha20 skcipher algorithm from the crypto framework
+ */
+void cmh_ccp_unregister(void)
+{
+ crypto_unregister_skcipher(&cmh_chacha20_alg);
+ dev_dbg(cmh_dev(), "cmh_ccp: unregistered chacha20\n");
+}
diff --git a/drivers/crypto/cmh/cmh_ccp_aead.c b/drivers/crypto/cmh/cmh_ccp_aead.c
new file mode 100644
index 000000000000..df5482179bdc
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_ccp_aead.c
@@ -0,0 +1,594 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API ChaCha20-Poly1305 AEAD Driver (RFC 7539)
+ *
+ * Registers "rfc7539(chacha20,poly1305)" as an AEAD algorithm with the
+ * Linux crypto subsystem, backed by the CMH CCP core.
+ *
+ * VCQ sequence:
+ * [SYS_CMD_WRITE] + CCP_CMD_AEAD_INIT + CCP_CMD_AAD_FINAL
+ * + CCP_CMD_FINAL + CCP_CMD_FLUSH
+ *
+ * CCP_CMD_AAD_FINAL is always issued -- even with no associated data --
+ * because it transitions the CCP core out of its AAD phase into the
+ * state CCP_CMD_FINAL requires.
+ *
+ * The RFC 7539 AEAD interface passes a 12-byte nonce via req->iv.
+ * The CCP core expects a 16-byte ctrnonce (4-byte LE counter + 12-byte
+ * nonce). We prepend a zero counter (per RFC 7539 S2.8: counter 0
+ * generates the Poly1305 key, counter 1 starts encryption -- the
+ * CMH eSW handles this internally from the initial counter value of 0).
+ *
+ * Tag is always 16 bytes (Poly1305 authenticator).
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/chacha.h>
+#include <crypto/internal/aead.h>
+#include <crypto/scatterwalk.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "cmh_ccp.h"
+#include "cmh_vcq.h"
+#include "cmh_ccp_abi.h"
+#include "cmh_sys_abi.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+#define CCP_AEAD_IV_SIZE 12U /* RFC 7539 nonce */
+#define CCP_ESP_IV_SIZE 8U /* RFC 7539 ESP nonce (4-byte salt at setkey) */
+#define CCP_ESP_SALT_SIZE 4U
+#define CCP_AEAD_TAG_SIZE 16U /* Poly1305 tag */
+
+struct cmh_ccp_aead_tfm_ctx {
+ struct cmh_key_ctx key;
+ u32 authsize;
+ u8 salt[CCP_ESP_SALT_SIZE]; /* ESP salt (unused for rfc7539) */
+};
+
+/* Per-request context (lives in aead_request::__ctx) */
+
+/*
+ * Maximum payload commands:
+ * [SYS_CMD_WRITE] + CCP_CMD_AEAD_INIT + CCP_CMD_AAD_FINAL
+ * + CCP_CMD_FINAL + FLUSH = 5
+ */
+#define CMH_CCP_AEAD_MAX_PAYLOAD 5
+#define CMH_CCP_AEAD_MAX_PACKED (CMH_CCP_AEAD_MAX_PAYLOAD * 2)
+
+struct cmh_ccp_aead_reqctx {
+ dma_addr_t in_dma;
+ dma_addr_t out_dma;
+ dma_addr_t iv_dma;
+ dma_addr_t key_dma;
+ dma_addr_t aad_dma;
+ dma_addr_t tag_dma;
+ u8 *in_buf;
+ u8 *out_buf;
+ u8 *iv_buf;
+ u8 *aad_buf;
+ u8 *tag_buf;
+ u32 cryptlen;
+ u32 assoclen;
+ u32 authsize;
+ u32 keylen;
+ bool encrypting;
+ struct vcq_cmd packed[CMH_CCP_AEAD_MAX_PACKED];
+};
+
+/* VCQ Builders -- CCP AEAD-specific */
+
+static void vcq_add_ccp_aead_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref,
+ u64 ctrnonce_dma, u32 keylen, u32 op)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_AEAD_INIT);
+ slot->hwc.ccp.cmd_aead.key = key_ref;
+ slot->hwc.ccp.cmd_aead.ctrnonce = ctrnonce_dma;
+ slot->hwc.ccp.cmd_aead.keylen = keylen;
+ slot->hwc.ccp.cmd_aead.ctrnoncelen = CCP_CTRNONCE_SIZE;
+ slot->hwc.ccp.cmd_aead.op = op;
+}
+
+static void vcq_add_ccp_aad_final(struct vcq_cmd *slot, u32 core_id, u64 aad_dma,
+ u32 aadlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_AAD_FINAL);
+ slot->hwc.ccp.cmd_aad_final.aad = aad_dma;
+ slot->hwc.ccp.cmd_aad_final.aadlen = aadlen;
+}
+
+static void vcq_add_ccp_aead_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma,
+ u64 output_dma, u64 tag_dma,
+ u32 iolen, u32 taglen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_FINAL);
+ slot->hwc.ccp.cmd_final.input = input_dma;
+ slot->hwc.ccp.cmd_final.output = output_dma;
+ slot->hwc.ccp.cmd_final.tag = tag_dma;
+ slot->hwc.ccp.cmd_final.iolen = iolen;
+ slot->hwc.ccp.cmd_final.taglen = taglen;
+}
+
+/* setkey */
+static int cmh_ccp_aead_setkey(struct crypto_aead *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ /* RFC 7539 AEAD requires 32-byte key */
+ if (keylen != CHACHA_KEY_SIZE)
+ return -EINVAL;
+
+ return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_CCP);
+}
+
+static int cmh_ccp_aead_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+
+ /* Poly1305 tag is always 16 bytes */
+ if (authsize != CCP_AEAD_TAG_SIZE)
+ return -EINVAL;
+
+ tctx->authsize = authsize;
+ return 0;
+}
+
+static int cmh_ccp_aead_init_tfm(struct crypto_aead *tfm)
+{
+ struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+
+ memset(tctx, 0, sizeof(*tctx));
+ tctx->authsize = CCP_AEAD_TAG_SIZE;
+ crypto_aead_set_reqsize(tfm, sizeof(struct cmh_ccp_aead_reqctx));
+ return 0;
+}
+
+static void cmh_ccp_aead_exit_tfm(struct crypto_aead *tfm)
+{
+ struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+
+ cmh_key_destroy(&tctx->key);
+}
+
+/* DMA unmap helper */
+static void cmh_ccp_aead_unmap_dma(struct cmh_ccp_aead_reqctx *rctx)
+{
+ cmh_dma_unmap_single(rctx->iv_dma, CCP_CTRNONCE_SIZE, DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->tag_dma, rctx->authsize,
+ rctx->encrypting ? DMA_FROM_DEVICE :
+ DMA_TO_DEVICE);
+ if (rctx->cryptlen > 0) {
+ cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen,
+ DMA_FROM_DEVICE);
+ cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen,
+ DMA_TO_DEVICE);
+ }
+ if (rctx->assoclen > 0)
+ cmh_dma_unmap_single(rctx->aad_dma, rctx->assoclen,
+ DMA_TO_DEVICE);
+}
+
+static void cmh_ccp_aead_free_bufs(struct cmh_ccp_aead_reqctx *rctx)
+{
+ kfree(rctx->iv_buf);
+ rctx->iv_buf = NULL;
+ kfree(rctx->tag_buf);
+ rctx->tag_buf = NULL;
+ kfree_sensitive(rctx->out_buf);
+ rctx->out_buf = NULL;
+ kfree_sensitive(rctx->in_buf);
+ rctx->in_buf = NULL;
+ kfree(rctx->aad_buf);
+ rctx->aad_buf = NULL;
+}
+
+static void cmh_ccp_aead_complete(void *data, int error)
+{
+ struct aead_request *req = data;
+ struct cmh_ccp_aead_reqctx *rctx = aead_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ cmh_ccp_aead_unmap_dma(rctx);
+
+ /*
+ * Map HW error on decrypt to -EBADMSG. The eSW CCP core uses a
+ * single error code (-EIO) for both authentication failures and
+ * other core errors (e.g. DMA timeout), so we cannot distinguish
+ * them from the MBX_STATUS alone. In practice the only error
+ * during a well-formed AEAD decrypt is auth-tag mismatch; a DMA
+ * timeout would indicate a fatal HW problem where -EBADMSG vs
+ * -EIO is moot. The kernel crypto API requires -EBADMSG for
+ * AEAD authentication failures.
+ */
+ if (error == -EIO && !rctx->encrypting)
+ error = -EBADMSG;
+
+ if (!error) {
+ if (rctx->cryptlen > 0)
+ scatterwalk_map_and_copy(rctx->out_buf, req->dst,
+ req->assoclen,
+ rctx->cryptlen, 1);
+ if (rctx->encrypting)
+ scatterwalk_map_and_copy(rctx->tag_buf, req->dst,
+ req->assoclen +
+ rctx->cryptlen,
+ rctx->authsize, 1);
+ }
+
+ cmh_ccp_aead_free_bufs(rctx);
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * Core AEAD encrypt/decrypt -- async path.
+ *
+ * Encrypt: plaintext -> ciphertext + 16-byte tag
+ * Decrypt: ciphertext + tag -> plaintext (tag verified by CMH eSW)
+ *
+ * VCQ: [SYS_CMD_WRITE] + AEAD_INIT + [AAD_FINAL] + FINAL + FLUSH
+ */
+static int cmh_ccp_aead_crypt(struct aead_request *req, u32 ccp_op)
+{
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ struct cmh_ccp_aead_reqctx *rctx = aead_request_ctx(req);
+ struct vcq_cmd cmds[CMH_CCP_AEAD_MAX_PAYLOAD];
+ u64 key_ref;
+ u32 keylen, authsize, cryptlen;
+ struct core_dispatch d;
+ s32 target_mbx;
+ u32 core_id;
+ u32 idx;
+ int ret;
+ gfp_t gfp;
+
+ if (tctx->key.mode == CMH_KEY_NONE)
+ return -ENOKEY;
+
+ authsize = tctx->authsize;
+
+ if (ccp_op == CCP_OP_ENCRYPT) {
+ cryptlen = req->cryptlen;
+ } else {
+ if (req->cryptlen < authsize)
+ return -EINVAL;
+ cryptlen = req->cryptlen - authsize;
+ }
+
+ /*
+ * HW uses a proprietary LLI scatter-gather format that is
+ * incompatible with struct scatterlist, so the payload is
+ * linearised into contiguous buffers for DMA. Cap total
+ * size to prevent excessive memory consumption.
+ */
+ if ((u64)cryptlen + req->assoclen > SZ_1M)
+ return -EINVAL;
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->cryptlen = cryptlen;
+ rctx->assoclen = req->assoclen;
+ rctx->authsize = authsize;
+ rctx->encrypting = (ccp_op == CCP_OP_ENCRYPT);
+
+ /*
+ * rfc7539esp: the last ivsize (8) bytes of the AAD region are the
+ * IV/nonce, not actual associated data. Subtract them so HW only
+ * authenticates the real AAD.
+ */
+ if (crypto_aead_ivsize(tfm) == CCP_ESP_IV_SIZE) {
+ if (rctx->assoclen < CCP_ESP_IV_SIZE)
+ return -EINVAL;
+ rctx->assoclen -= CCP_ESP_IV_SIZE;
+ }
+
+ /* Linearise AAD */
+ if (rctx->assoclen > 0) {
+ rctx->aad_buf = kmalloc(rctx->assoclen, gfp);
+ if (!rctx->aad_buf)
+ return -ENOMEM;
+ scatterwalk_map_and_copy(rctx->aad_buf, req->src,
+ 0, rctx->assoclen, 0);
+ rctx->aad_dma = cmh_dma_map_single(rctx->aad_buf,
+ rctx->assoclen,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->aad_dma)) {
+ ret = -ENOMEM;
+ goto out_free_aad;
+ }
+ }
+
+ /* Linearise input */
+ if (cryptlen > 0) {
+ rctx->in_buf = kmalloc(cryptlen, gfp);
+ if (!rctx->in_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_aad;
+ }
+ scatterwalk_map_and_copy(rctx->in_buf, req->src,
+ req->assoclen, cryptlen, 0);
+ rctx->in_dma = cmh_dma_map_single(rctx->in_buf, cryptlen,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->in_dma)) {
+ ret = -ENOMEM;
+ goto out_free_in;
+ }
+ }
+
+ /* Allocate output buffer */
+ if (cryptlen > 0) {
+ rctx->out_buf = kmalloc(cryptlen, gfp);
+ if (!rctx->out_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_in;
+ }
+ rctx->out_dma = cmh_dma_map_single(rctx->out_buf, cryptlen,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->out_dma)) {
+ ret = -ENOMEM;
+ goto out_free_out;
+ }
+ }
+
+ /* Tag buffer */
+ rctx->tag_buf = kmalloc(authsize, gfp);
+ if (!rctx->tag_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_out;
+ }
+
+ if (!rctx->encrypting) {
+ scatterwalk_map_and_copy(rctx->tag_buf, req->src,
+ req->assoclen + cryptlen,
+ authsize, 0);
+ } else {
+ memset(rctx->tag_buf, 0, authsize);
+ }
+
+ rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, authsize,
+ rctx->encrypting ?
+ DMA_FROM_DEVICE : DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->tag_dma)) {
+ ret = -ENOMEM;
+ goto out_free_tag;
+ }
+
+ /* Build 16-byte ctrnonce: 4-byte zero counter + 12-byte nonce.
+ * rfc7539: counter(4) | req->iv(12)
+ * rfc7539esp: counter(4) | salt(4) | req->iv(8)
+ */
+ rctx->iv_buf = kzalloc(CCP_CTRNONCE_SIZE, gfp);
+ if (!rctx->iv_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_tag;
+ }
+ if (crypto_aead_ivsize(tfm) == CCP_ESP_IV_SIZE) {
+ memcpy(rctx->iv_buf + CCP_CHACHA_CTR_LEN,
+ tctx->salt, CCP_ESP_SALT_SIZE);
+ memcpy(rctx->iv_buf + CCP_CHACHA_CTR_LEN + CCP_ESP_SALT_SIZE,
+ req->iv, CCP_ESP_IV_SIZE);
+ } else {
+ memcpy(rctx->iv_buf + CCP_CHACHA_CTR_LEN,
+ req->iv, CCP_AEAD_IV_SIZE);
+ }
+
+ rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, CCP_CTRNONCE_SIZE,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->iv_dma)) {
+ ret = -ENOMEM;
+ goto out_free_iv;
+ }
+
+ /* Resolve key reference */
+ idx = 0;
+
+ rctx->key_dma = tctx->key.raw.dma;
+ rctx->keylen = tctx->key.raw.len;
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP,
+ (u64)rctx->key_dma, SYS_REF_NONE,
+ tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ keylen = tctx->key.raw.len;
+ d = cmh_core_select_instance(CMH_CORE_CCP);
+ target_mbx = d.mbx_idx;
+ core_id = d.core_id;
+
+ /* AEAD_INIT */
+ vcq_add_ccp_aead_init(&cmds[idx++], core_id, key_ref,
+ (u64)rctx->iv_dma, keylen, ccp_op);
+
+ /*
+ * AAD_FINAL closes the AAD phase and moves the CCP core from
+ * STATE_AAD to STATE_UPDATE_WAIT, which CCP_CMD_FINAL requires.
+ * It must be issued even with no associated data (assoclen == 0):
+ * otherwise the core stays in STATE_AAD and FINAL fails with -EIO
+ * (e.g. testmgr rfc7539 fuzz vector "alen=0 plen=11"). With
+ * assoclen 0 the eSW skips the DMA and only records AAD_LEN = 0.
+ */
+ vcq_add_ccp_aad_final(&cmds[idx++], core_id,
+ rctx->assoclen > 0 ? (u64)rctx->aad_dma : 0,
+ rctx->assoclen);
+
+ /* FINAL with tag */
+ vcq_add_ccp_aead_final(&cmds[idx++], core_id,
+ cryptlen > 0 ? (u64)rctx->in_dma : 0,
+ cryptlen > 0 ? (u64)rctx->out_dma : 0,
+ (u64)rctx->tag_dma, cryptlen, authsize);
+
+ vcq_add_flush(&cmds[idx++], core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_CCP_AEAD_MAX_PACKED,
+ target_mbx,
+ cmh_ccp_aead_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_cleanup_all;
+
+ return -EINPROGRESS;
+
+out_cleanup_all:
+ cmh_dma_unmap_single(rctx->iv_dma, CCP_CTRNONCE_SIZE, DMA_TO_DEVICE);
+out_free_iv:
+ kfree(rctx->iv_buf);
+out_unmap_tag:
+ cmh_dma_unmap_single(rctx->tag_dma, authsize,
+ rctx->encrypting ? DMA_FROM_DEVICE :
+ DMA_TO_DEVICE);
+out_free_tag:
+ kfree(rctx->tag_buf);
+out_unmap_out:
+ if (cryptlen > 0)
+ cmh_dma_unmap_single(rctx->out_dma, cryptlen, DMA_FROM_DEVICE);
+out_free_out:
+ kfree_sensitive(rctx->out_buf);
+out_unmap_in:
+ if (cryptlen > 0)
+ cmh_dma_unmap_single(rctx->in_dma, cryptlen, DMA_TO_DEVICE);
+out_free_in:
+ kfree_sensitive(rctx->in_buf);
+out_unmap_aad:
+ if (rctx->assoclen > 0)
+ cmh_dma_unmap_single(rctx->aad_dma, rctx->assoclen,
+ DMA_TO_DEVICE);
+out_free_aad:
+ kfree(rctx->aad_buf);
+ return ret;
+}
+
+static int cmh_ccp_aead_encrypt(struct aead_request *req)
+{
+ return cmh_ccp_aead_crypt(req, CCP_OP_ENCRYPT);
+}
+
+static int cmh_ccp_aead_decrypt(struct aead_request *req)
+{
+ return cmh_ccp_aead_crypt(req, CCP_OP_DECRYPT);
+}
+
+/* -- rfc7539esp: ESP variant with 4-byte salt + 8-byte IV --------------- */
+
+/*
+ * ESP setkey: 36 bytes = 32-byte ChaCha20 key + 4-byte salt.
+ * The salt is prepended to the 8-byte per-packet IV from the ESP header
+ * to form the 12-byte RFC 7539 nonce.
+ */
+static int cmh_ccp_esp_setkey(struct crypto_aead *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+
+ if (keylen != CHACHA_KEY_SIZE + CCP_ESP_SALT_SIZE)
+ return -EINVAL;
+
+ memcpy(tctx->salt, key + CHACHA_KEY_SIZE, CCP_ESP_SALT_SIZE);
+ return cmh_key_setkey_raw(&tctx->key, key, CHACHA_KEY_SIZE, CORE_ID_CCP);
+}
+
+/* Registration */
+
+static struct aead_alg cmh_rfc7539_alg = {
+ .setkey = cmh_ccp_aead_setkey,
+ .setauthsize = cmh_ccp_aead_setauthsize,
+ .encrypt = cmh_ccp_aead_encrypt,
+ .decrypt = cmh_ccp_aead_decrypt,
+ .init = cmh_ccp_aead_init_tfm,
+ .exit = cmh_ccp_aead_exit_tfm,
+ .ivsize = CCP_AEAD_IV_SIZE,
+ .maxauthsize = CCP_AEAD_TAG_SIZE,
+ .base = {
+ .cra_name = "rfc7539(chacha20,poly1305)",
+ .cra_driver_name = "rambus-cmh-rfc7539-chacha20-poly1305",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct cmh_ccp_aead_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ },
+};
+
+static struct aead_alg cmh_rfc7539esp_alg = {
+ .setkey = cmh_ccp_esp_setkey,
+ .setauthsize = cmh_ccp_aead_setauthsize,
+ .encrypt = cmh_ccp_aead_encrypt,
+ .decrypt = cmh_ccp_aead_decrypt,
+ .init = cmh_ccp_aead_init_tfm,
+ .exit = cmh_ccp_aead_exit_tfm,
+ .ivsize = CCP_ESP_IV_SIZE,
+ .maxauthsize = CCP_AEAD_TAG_SIZE,
+ .base = {
+ .cra_name = "rfc7539esp(chacha20,poly1305)",
+ .cra_driver_name = "rambus-cmh-rfc7539esp-chacha20-poly1305",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct cmh_ccp_aead_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ },
+};
+
+/**
+ * cmh_ccp_aead_register() - Register ChaCha20-Poly1305 AEAD algorithm with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_ccp_aead_register(void)
+{
+ int ret;
+
+ ret = crypto_register_aead(&cmh_rfc7539_alg);
+ if (ret) {
+ dev_err(cmh_dev(), "cmh_ccp_aead: failed to register rfc7539 (rc=%d)\n",
+ ret);
+ return ret;
+ }
+ dev_dbg(cmh_dev(), "cmh_ccp_aead: registered rfc7539(chacha20,poly1305)\n");
+
+ ret = crypto_register_aead(&cmh_rfc7539esp_alg);
+ if (ret) {
+ dev_err(cmh_dev(), "cmh_ccp_aead: failed to register rfc7539esp (rc=%d)\n",
+ ret);
+ crypto_unregister_aead(&cmh_rfc7539_alg);
+ return ret;
+ }
+ dev_dbg(cmh_dev(), "cmh_ccp_aead: registered rfc7539esp(chacha20,poly1305)\n");
+
+ return 0;
+}
+
+/**
+ * cmh_ccp_aead_unregister() - Unregister ChaCha20-Poly1305 AEAD algorithms
+ */
+void cmh_ccp_aead_unregister(void)
+{
+ crypto_unregister_aead(&cmh_rfc7539esp_alg);
+ crypto_unregister_aead(&cmh_rfc7539_alg);
+ dev_dbg(cmh_dev(), "cmh_ccp_aead: unregistered rfc7539/rfc7539esp\n");
+}
diff --git a/drivers/crypto/cmh/cmh_ccp_poly.c b/drivers/crypto/cmh/cmh_ccp_poly.c
new file mode 100644
index 000000000000..0f783a98d54d
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_ccp_poly.c
@@ -0,0 +1,694 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API Poly1305 (ahash) Driver
+ *
+ * Registers "poly1305" as an ahash algorithm with the Linux crypto
+ * subsystem, backed by the CMH CCP core.
+ *
+ * Poly1305 is a one-time authenticator that produces a 16-byte MAC.
+ * It requires two 16-byte keys: r (clamped multiplier) and s (nonce).
+ *
+ * Key format: 32 bytes = r_key[0..15] || s_key[16..31]
+ * This matches the Poly1305 key layout in RFC 7539 S2.5.
+ *
+ * VCQ sequence:
+ * SYS_CMD_WRITE(s_key) + SYS_CMD_WRITE(r_key)
+ * + CCP_CMD_POLY1305_INIT + CCP_CMD_FINAL + CCP_CMD_FLUSH
+ *
+ * Both keys are written to SYS_REF_TEMP; the CMH eSW stacks them
+ * so that POLY1305_INIT finds r_key (most recent) as rkey and
+ * s_key (previous) as skey.
+ *
+ * The ahash interface accumulates data via .update() and submits the
+ * full VCQ asynchronously in .final(). Because the CCP core exposes no
+ * external save/restore, input is buffered in kernel memory and capped
+ * at POLY_MAX_DATA (64 KB). Past that cap -- or when the flat export
+ * window is exceeded -- the request transparently switches to the
+ * Poly1305 library (<crypto/poly1305.h>), which keeps arbitrary-length
+ * MACs and transform clone (export/import) conformant with O(1) memory.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/hash.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/poly1305.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "cmh_ccp.h"
+#include "cmh_vcq.h"
+#include "cmh_ccp_abi.h"
+#include "cmh_sys_abi.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+/*
+ * Maximum accumulated data for Poly1305 -- driver-imposed, not HW.
+ *
+ * The CCP core does not expose external save/restore VCQ commands,
+ * so the driver must accumulate all data in kernel memory via
+ * .update() and submit it atomically in .final(). This cap limits
+ * the per-request kernel allocation.
+ */
+#define POLY_MAX_DATA (64 * 1024)
+
+/*
+ * Per-transform context -- stores the raw 32-byte key (r || s).
+ *
+ * Only the raw-key path is supported for standalone Poly1305.
+ */
+struct cmh_poly_tfm_ctx {
+ u8 key[POLY1305_KEY_SIZE];
+ dma_addr_t rkey_dma;
+ dma_addr_t skey_dma;
+ u32 keylen;
+ bool has_key;
+ spinlock_t chunk_lock; /* protects all_chunks */
+ struct list_head all_chunks; /* orphan-safe chunk tracking */
+};
+
+/* Chunk node for O(1) update() appends */
+struct cmh_poly_chunk {
+ struct list_head list;
+ struct list_head tfm_node; /* per-tfm orphan tracking */
+ u32 len;
+ u8 data[];
+};
+
+/* Per-request context (lives in ahash_request::__ctx) */
+
+/*
+ * Maximum payload commands:
+ * SYS_CMD_WRITE(s) + SYS_CMD_WRITE(r) + POLY1305_INIT
+ * + CCP_CMD_FINAL + FLUSH = 5
+ */
+#define CMH_POLY_MAX_PAYLOAD 5
+#define CMH_POLY_MAX_PACKED (CMH_POLY_MAX_PAYLOAD * 2)
+
+struct cmh_poly_reqctx {
+ struct list_head chunks;
+ u32 total_len;
+ bool switched; /* handed off to the Poly1305 library */
+ struct poly1305_desc_ctx fb_state; /* SW fallback running state */
+ u8 *buf; /* linearised in final() */
+ /* DMA state for async final */
+ dma_addr_t in_dma;
+ dma_addr_t tag_dma;
+ u8 *tag_buf;
+ struct vcq_cmd packed[CMH_POLY_MAX_PACKED];
+};
+
+/*
+ * Export/import (transform clone): the CCP core lacks external
+ * save/restore, so the driver serialises its accumulated input for the
+ * common (bounded) case (CMH_POLY_FMT_RAW). When the input exceeds the
+ * HW cap (POLY_MAX_DATA, 64 KB) or the flat export window, the request
+ * switches to the Poly1305 library and serialises the library's
+ * fixed-size running state (CMH_POLY_FMT_FB), so arbitrary-length MACs
+ * and transform clone both stay conformant with O(1) driver memory.
+ */
+#define CMH_POLY_FMT_RAW 0
+#define CMH_POLY_FMT_FB 1
+
+struct cmh_poly_export_state {
+ u8 format;
+ u8 __pad[3];
+ u32 total_len;
+ u8 data[];
+};
+
+#define CMH_POLY_STATE_SIZE 4096
+#define CMH_POLY_EXPORT_MAX \
+ (CMH_POLY_STATE_SIZE - sizeof(struct cmh_poly_export_state))
+
+static void vcq_add_ccp_poly_init(struct vcq_cmd *slot, u32 core_id,
+ u64 rkey_ref, u32 rkeylen,
+ u64 skey_ref, u32 skeylen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_POLY1305_INIT);
+ slot->hwc.ccp.cmd_poly.rkey = rkey_ref;
+ slot->hwc.ccp.cmd_poly.rkeylen = rkeylen;
+ slot->hwc.ccp.cmd_poly.skey = skey_ref;
+ slot->hwc.ccp.cmd_poly.skeylen = skeylen;
+}
+
+static void vcq_add_ccp_poly_final(struct vcq_cmd *slot, u32 core_id,
+ u64 input_dma, u64 tag_dma,
+ u32 iolen, u32 taglen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_FINAL);
+ slot->hwc.ccp.cmd_final.input = input_dma;
+ slot->hwc.ccp.cmd_final.output = 0;
+ slot->hwc.ccp.cmd_final.tag = tag_dma;
+ slot->hwc.ccp.cmd_final.iolen = iolen;
+ slot->hwc.ccp.cmd_final.taglen = taglen;
+}
+
+static int cmh_poly_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+
+ /* Poly1305: exactly 32 bytes (r[16] + s[16]) */
+ if (keylen != POLY1305_KEY_SIZE)
+ return -EINVAL;
+
+ /* Unmap old key DMA if re-keying */
+ if (tctx->has_key) {
+ cmh_dma_unmap_single(tctx->rkey_dma, CCP_POLY_KEY_SIZE,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(tctx->skey_dma, CCP_POLY_KEY_SIZE,
+ DMA_TO_DEVICE);
+ }
+
+ memcpy(tctx->key, key, POLY1305_KEY_SIZE);
+ tctx->keylen = POLY1305_KEY_SIZE;
+
+ /*
+ * Pre-map both key halves for DMA. The key buffer lives in
+ * the tfm context and is stable until exit_tfm() or re-setkey.
+ */
+ tctx->skey_dma = cmh_dma_map_single(tctx->key + CCP_POLY_KEY_SIZE,
+ CCP_POLY_KEY_SIZE,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(tctx->skey_dma)) {
+ tctx->has_key = false;
+ return -ENOMEM;
+ }
+
+ tctx->rkey_dma = cmh_dma_map_single(tctx->key, CCP_POLY_KEY_SIZE,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(tctx->rkey_dma)) {
+ cmh_dma_unmap_single(tctx->skey_dma, CCP_POLY_KEY_SIZE,
+ DMA_TO_DEVICE);
+ tctx->has_key = false;
+ return -ENOMEM;
+ }
+
+ tctx->has_key = true;
+ return 0;
+}
+
+static void cmh_poly_free_chunks(struct cmh_poly_reqctx *rctx,
+ struct cmh_poly_tfm_ctx *tctx)
+{
+ struct cmh_poly_chunk *c, *tmp;
+
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(c, tmp, &rctx->chunks, list) {
+ list_del(&c->list);
+ list_del(&c->tfm_node);
+ kfree_sensitive(c);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->total_len = 0;
+}
+
+/* Software-fallback helpers (arbitrary-length + transform-clone support) */
+
+/* Feed a scatterlist to the Poly1305 library, segment by segment. */
+static void cmh_poly_fb_feed_sg(struct poly1305_desc_ctx *desc,
+ struct scatterlist *sg, u32 len)
+{
+ struct sg_mapping_iter miter;
+ u32 remaining = len;
+
+ sg_miter_start(&miter, sg, sg_nents(sg),
+ SG_MITER_FROM_SG | SG_MITER_ATOMIC);
+ while (remaining && sg_miter_next(&miter)) {
+ u32 n = min_t(u32, miter.length, remaining);
+
+ poly1305_update(desc, miter.addr, n);
+ remaining -= n;
+ }
+ sg_miter_stop(&miter);
+}
+
+/*
+ * Switch a request from the HW-buffered path to the Poly1305 library:
+ * initialise the library state with the transform key, replay every
+ * accumulated chunk through it, then drop the chunks. Afterwards the
+ * request is O(1) in memory and no longer input-capped.
+ */
+static int cmh_poly_switch_to_fb(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_poly_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_poly_chunk *c;
+
+ if (!tctx->has_key)
+ return -ENOKEY;
+
+ poly1305_init(&rctx->fb_state, tctx->key);
+ list_for_each_entry(c, &rctx->chunks, list)
+ poly1305_update(&rctx->fb_state, c->data, c->len);
+ cmh_poly_free_chunks(rctx, tctx);
+ rctx->switched = true;
+ return 0;
+}
+
+/* Forward the current update() payload to the library. */
+static void cmh_poly_fb_forward(struct ahash_request *req,
+ struct cmh_poly_reqctx *rctx)
+{
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ poly1305_update(&rctx->fb_state, req->svirt, req->nbytes);
+ else
+ cmh_poly_fb_feed_sg(&rctx->fb_state, req->src, req->nbytes);
+}
+
+static int cmh_poly_init(struct ahash_request *req)
+{
+ struct cmh_poly_reqctx *rctx = ahash_request_ctx(req);
+
+ memset(rctx, 0, sizeof(*rctx));
+ INIT_LIST_HEAD(&rctx->chunks);
+ return 0;
+}
+
+static int cmh_poly_update(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_poly_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_poly_chunk *chunk;
+ gfp_t gfp;
+ int ret;
+
+ if (!req->nbytes)
+ return 0;
+
+ /* Already handed off to the library: forward directly (O(1) mem). */
+ if (rctx->switched) {
+ cmh_poly_fb_forward(req, rctx);
+ return 0;
+ }
+
+ /*
+ * Exceeding the HW input cap: switch to the Poly1305 library
+ * (replaying the buffered chunks) rather than failing, then
+ * forward this update.
+ */
+ if (req->nbytes > POLY_MAX_DATA - rctx->total_len) {
+ ret = cmh_poly_switch_to_fb(req);
+ if (ret)
+ goto err_free_chunks;
+ cmh_poly_fb_forward(req, rctx);
+ return 0;
+ }
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+ chunk = kmalloc(sizeof(*chunk) + req->nbytes, gfp);
+ if (!chunk) {
+ ret = -ENOMEM;
+ goto err_free_chunks;
+ }
+
+ chunk->len = req->nbytes;
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(chunk->data, req->svirt, req->nbytes);
+ else
+ scatterwalk_map_and_copy(chunk->data, req->src,
+ 0, req->nbytes, 0);
+ list_add_tail(&chunk->list, &rctx->chunks);
+ spin_lock_bh(&tctx->chunk_lock);
+ list_add_tail(&chunk->tfm_node, &tctx->all_chunks);
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->total_len += req->nbytes;
+ return 0;
+
+err_free_chunks:
+ /*
+ * Terminal error -- free all previously accumulated chunks.
+ * Callers may not call .final() on error, so they would leak.
+ */
+ cmh_poly_free_chunks(rctx, tctx);
+ return ret;
+}
+
+static void cmh_poly_complete(void *data, int error)
+{
+ struct ahash_request *req = data;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_poly_reqctx *rctx = ahash_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ if (rctx->total_len > 0)
+ cmh_dma_unmap_single(rctx->in_dma, rctx->total_len,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->tag_dma, POLY1305_DIGEST_SIZE,
+ DMA_FROM_DEVICE);
+
+ if (!error)
+ memcpy(req->result, rctx->tag_buf, POLY1305_DIGEST_SIZE);
+
+ kfree(rctx->tag_buf);
+ rctx->tag_buf = NULL;
+ cmh_poly_free_chunks(rctx, tctx);
+ kfree_sensitive(rctx->buf);
+ rctx->buf = NULL;
+ rctx->total_len = 0;
+ cmh_complete(&req->base, error);
+}
+
+static int cmh_poly_final(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_poly_reqctx *rctx = ahash_request_ctx(req);
+ struct vcq_cmd cmds[CMH_POLY_MAX_PAYLOAD];
+ struct core_dispatch d;
+ s32 target_mbx;
+ u32 core_id;
+ u32 idx;
+ int ret;
+ gfp_t gfp;
+
+ /* Switched to the Poly1305 library: complete there (synchronous). */
+ if (rctx->switched) {
+ poly1305_final(&rctx->fb_state, req->result);
+ return 0;
+ }
+
+ if (!tctx->has_key) {
+ ret = -ENOKEY;
+ goto out_free_chunks;
+ }
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ /* Linearise chunks into a single contiguous buffer for DMA */
+ if (rctx->total_len > 0) {
+ struct cmh_poly_chunk *c;
+ u32 off = 0;
+
+ rctx->buf = kmalloc(rctx->total_len, gfp);
+ if (!rctx->buf) {
+ ret = -ENOMEM;
+ goto out_free_chunks;
+ }
+ list_for_each_entry(c, &rctx->chunks, list) {
+ memcpy(rctx->buf + off, c->data, c->len);
+ off += c->len;
+ }
+ }
+
+ /* Tag output buffer */
+ rctx->tag_buf = kzalloc(POLY1305_DIGEST_SIZE, gfp);
+ if (!rctx->tag_buf) {
+ ret = -ENOMEM;
+ goto out_free_buf;
+ }
+
+ rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf,
+ POLY1305_DIGEST_SIZE,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->tag_dma)) {
+ ret = -ENOMEM;
+ goto out_free_tag;
+ }
+
+ /* Map input data */
+ if (rctx->total_len > 0) {
+ rctx->in_dma = cmh_dma_map_single(rctx->buf, rctx->total_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->in_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap_tag;
+ }
+ }
+
+ /*
+ * Key DMA handles are pre-mapped in setkey() and live in
+ * the tfm context. Use them directly for the VCQ writes.
+ */
+
+ d = cmh_core_select_instance(CMH_CORE_CCP);
+ target_mbx = d.mbx_idx;
+ core_id = d.core_id;
+ idx = 0;
+
+ /* Write s_key to SYS_REF_TEMP first (bottom of stack) */
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP,
+ (u64)tctx->skey_dma, SYS_REF_NONE,
+ CCP_POLY_KEY_SIZE,
+ SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_CCP));
+
+ /* Write r_key to SYS_REF_TEMP second (top of stack) */
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP,
+ (u64)tctx->rkey_dma, SYS_REF_NONE,
+ CCP_POLY_KEY_SIZE,
+ SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_CCP));
+
+ /* POLY1305_INIT: rkey=TEMP (top), skey=TEMP (next) */
+ vcq_add_ccp_poly_init(&cmds[idx++], core_id, SYS_REF_TEMP,
+ CCP_POLY_KEY_SIZE, SYS_REF_TEMP,
+ CCP_POLY_KEY_SIZE);
+
+ /* FINAL: data -> tag */
+ vcq_add_ccp_poly_final(&cmds[idx++], core_id,
+ rctx->total_len > 0 ? (u64)rctx->in_dma : 0,
+ (u64)rctx->tag_dma, rctx->total_len,
+ POLY1305_DIGEST_SIZE);
+
+ vcq_add_flush(&cmds[idx++], core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_POLY_MAX_PACKED, target_mbx,
+ cmh_poly_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_unmap_in;
+
+ return -EINPROGRESS;
+
+out_unmap_in:
+ if (rctx->total_len > 0)
+ cmh_dma_unmap_single(rctx->in_dma, rctx->total_len,
+ DMA_TO_DEVICE);
+out_unmap_tag:
+ cmh_dma_unmap_single(rctx->tag_dma, POLY1305_DIGEST_SIZE,
+ DMA_FROM_DEVICE);
+out_free_tag:
+ kfree(rctx->tag_buf);
+out_free_buf:
+ kfree_sensitive(rctx->buf);
+ rctx->buf = NULL;
+out_free_chunks:
+ cmh_poly_free_chunks(rctx, tctx);
+ rctx->total_len = 0;
+ return ret;
+}
+
+static int cmh_poly_export(struct ahash_request *req, void *out)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_poly_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_poly_export_state *state = out;
+ struct cmh_poly_chunk *chunk;
+ u32 offset = 0;
+ int ret;
+
+ BUILD_BUG_ON(sizeof(struct poly1305_desc_ctx) > CMH_POLY_EXPORT_MAX);
+
+ /*
+ * If more data is buffered than the flat window holds, switch to
+ * the Poly1305 library so a bounded, fixed-size state can be
+ * exported -- making export/import (clone) work at any length.
+ */
+ if (!rctx->switched && rctx->total_len > CMH_POLY_EXPORT_MAX) {
+ ret = cmh_poly_switch_to_fb(req);
+ if (ret)
+ return ret;
+ }
+
+ /* Zero the whole state buffer so no kernel memory leaks out. */
+ memset(state, 0, crypto_ahash_statesize(tfm));
+
+ if (rctx->switched) {
+ state->format = CMH_POLY_FMT_FB;
+ memcpy(state->data, &rctx->fb_state, sizeof(rctx->fb_state));
+ return 0;
+ }
+
+ state->format = CMH_POLY_FMT_RAW;
+ state->total_len = rctx->total_len;
+ list_for_each_entry(chunk, &rctx->chunks, list) {
+ memcpy(state->data + offset, chunk->data, chunk->len);
+ offset += chunk->len;
+ }
+ return 0;
+}
+
+static int cmh_poly_import(struct ahash_request *req, const void *in)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_poly_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_poly_export_state *state = in;
+ struct cmh_poly_chunk *chunk;
+
+ memset(rctx, 0, sizeof(*rctx));
+ INIT_LIST_HEAD(&rctx->chunks);
+
+ /* Fallback-format state: restore the Poly1305 library state. */
+ if (state->format == CMH_POLY_FMT_FB) {
+ memcpy(&rctx->fb_state, state->data, sizeof(rctx->fb_state));
+ rctx->switched = true;
+ return 0;
+ }
+
+ if (state->format != CMH_POLY_FMT_RAW)
+ return -EINVAL;
+
+ if (state->total_len > CMH_POLY_EXPORT_MAX)
+ return -EINVAL;
+
+ if (state->total_len) {
+ chunk = kmalloc(sizeof(*chunk) + state->total_len,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC);
+ if (!chunk)
+ return -ENOMEM;
+ chunk->len = state->total_len;
+ memcpy(chunk->data, state->data, state->total_len);
+ list_add_tail(&chunk->list, &rctx->chunks);
+ spin_lock_bh(&tctx->chunk_lock);
+ list_add_tail(&chunk->tfm_node, &tctx->all_chunks);
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->total_len = state->total_len;
+ }
+ return 0;
+}
+
+static int cmh_poly_finup(struct ahash_request *req)
+{
+ int err;
+
+ err = cmh_poly_update(req);
+ if (err)
+ return err;
+ return cmh_poly_final(req);
+}
+
+static int cmh_poly_digest(struct ahash_request *req)
+{
+ int err;
+
+ err = cmh_poly_init(req);
+ if (err)
+ return err;
+ return cmh_poly_finup(req);
+}
+
+static int cmh_poly_init_tfm(struct crypto_ahash *tfm)
+{
+ struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+
+ memset(tctx, 0, sizeof(*tctx));
+ spin_lock_init(&tctx->chunk_lock);
+ INIT_LIST_HEAD(&tctx->all_chunks);
+ crypto_ahash_set_reqsize(tfm, sizeof(struct cmh_poly_reqctx));
+ return 0;
+}
+
+static void cmh_poly_exit_tfm(struct crypto_ahash *tfm)
+{
+ struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_poly_chunk *c, *tmp;
+
+ /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(c, tmp, &tctx->all_chunks, tfm_node) {
+ list_del(&c->tfm_node);
+ kfree_sensitive(c);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+
+ if (tctx->has_key) {
+ cmh_dma_unmap_single(tctx->rkey_dma, CCP_POLY_KEY_SIZE,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(tctx->skey_dma, CCP_POLY_KEY_SIZE,
+ DMA_TO_DEVICE);
+ }
+ memzero_explicit(tctx->key, POLY1305_KEY_SIZE);
+}
+
+static struct ahash_alg cmh_poly1305_alg = {
+ .init = cmh_poly_init,
+ .update = cmh_poly_update,
+ .final = cmh_poly_final,
+ .finup = cmh_poly_finup,
+ .digest = cmh_poly_digest,
+ .export = cmh_poly_export,
+ .import = cmh_poly_import,
+ .setkey = cmh_poly_setkey,
+ .init_tfm = cmh_poly_init_tfm,
+ .exit_tfm = cmh_poly_exit_tfm,
+ .halg = {
+ .digestsize = POLY1305_DIGEST_SIZE,
+ .statesize = CMH_POLY_STATE_SIZE,
+ .base = {
+ .cra_name = "poly1305",
+ .cra_driver_name = "rambus-cmh-poly1305",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NO_FALLBACK |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_REQ_VIRT,
+ .cra_blocksize = POLY1305_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct cmh_poly_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
+
+/**
+ * cmh_ccp_poly_register() - Register Poly1305 hash algorithm with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_ccp_poly_register(void)
+{
+ int ret;
+
+ ret = crypto_register_ahash(&cmh_poly1305_alg);
+ if (ret)
+ dev_err(cmh_dev(), "cmh_ccp_poly: failed to register poly1305 (rc=%d)\n",
+ ret);
+ else
+ dev_dbg(cmh_dev(), "cmh_ccp_poly: registered poly1305\n");
+
+ return ret;
+}
+
+/**
+ * cmh_ccp_poly_unregister() - Unregister Poly1305 hash algorithm from the crypto framework
+ */
+void cmh_ccp_poly_unregister(void)
+{
+ crypto_unregister_ahash(&cmh_poly1305_alg);
+ dev_dbg(cmh_dev(), "cmh_ccp_poly: unregistered poly1305\n");
+}
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index fec7992bf58d..73b25ed7edcb 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -38,6 +38,7 @@
#include "cmh_sm3.h"
#include "cmh_aes.h"
#include "cmh_sm4.h"
+#include "cmh_ccp.h"
#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
@@ -299,6 +300,21 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_sm4_cmac_register;
+ /* Register CCP ChaCha20 skcipher algorithm */
+ ret = cmh_ccp_register();
+ if (ret)
+ goto err_ccp_register;
+
+ /* Register CCP ChaCha20-Poly1305 AEAD (RFC 7539) */
+ ret = cmh_ccp_aead_register();
+ if (ret)
+ goto err_ccp_aead_register;
+
+ /* Register CCP Poly1305 shash algorithm */
+ ret = cmh_ccp_poly_register();
+ if (ret)
+ goto err_ccp_poly_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -311,6 +327,12 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_ccp_poly_unregister();
+err_ccp_poly_register:
+ cmh_ccp_aead_unregister();
+err_ccp_aead_register:
+ cmh_ccp_unregister();
+err_ccp_register:
cmh_sm4_cmac_unregister();
err_sm4_cmac_register:
cmh_sm4_aead_unregister();
@@ -359,6 +381,9 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_ccp_poly_unregister();
+ cmh_ccp_aead_unregister();
+ cmh_ccp_unregister();
cmh_sm4_cmac_unregister();
cmh_sm4_aead_unregister();
cmh_sm4_unregister();
diff --git a/drivers/crypto/cmh/include/cmh_ccp.h b/drivers/crypto/cmh/include/cmh_ccp.h
new file mode 100644
index 000000000000..363d208cbceb
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_ccp.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- CCP Crypto API Drivers
+ *
+ * Registers CCP algorithms with the Linux crypto subsystem:
+ * skcipher: chacha20
+ * shash: poly1305
+ * aead: rfc7539(chacha20poly1305)
+ */
+
+#ifndef CMH_CCP_H
+#define CMH_CCP_H
+
+int cmh_ccp_register(void);
+void cmh_ccp_unregister(void);
+
+int cmh_ccp_aead_register(void);
+void cmh_ccp_aead_unregister(void);
+
+int cmh_ccp_poly_register(void);
+void cmh_ccp_poly_unregister(void);
+
+#endif /* CMH_CCP_H */
--
2.43.7
^ permalink raw reply related
* [PATCH v3 12/19] crypto: cmh - add RSA akcipher
From: Saravanakrishnan Krishnamoorthy @ 2026-08-06 19:55 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register the RSA akcipher algorithm using the CMH PKE core (core ID
0x0a). Supports encrypt, decrypt, sign, and verify operations with
2048, 3072, and 4096-bit keys. 512- and 1024-bit keys are also
accepted for legacy/test interoperability. Includes common PKE
helpers shared by subsequent ECDSA and ECDH patches.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 4 +-
drivers/crypto/cmh/cmh_main.c | 9 +
drivers/crypto/cmh/cmh_pke_common.c | 578 +++++++++++++++++++++++++
drivers/crypto/cmh/cmh_pke_rsa.c | 638 ++++++++++++++++++++++++++++
4 files changed, 1228 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_pke_common.c
create mode 100644 drivers/crypto/cmh/cmh_pke_rsa.c
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index 407fd2870f6d..cdbcc8cdac5f 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -29,7 +29,9 @@ cmh-y := \
cmh_ccp.o \
cmh_ccp_aead.o \
cmh_ccp_poly.o \
- cmh_rng.o
+ cmh_rng.o \
+ cmh_pke_common.o \
+ cmh_pke_rsa.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index 1e0f5dad8575..46738193d354 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -40,6 +40,7 @@
#include "cmh_aes.h"
#include "cmh_sm4.h"
#include "cmh_ccp.h"
+#include "cmh_pke.h"
#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
@@ -321,6 +322,11 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_ccp_poly_register;
+ /* Register PKE RSA akcipher */
+ ret = cmh_pke_rsa_register();
+ if (ret)
+ goto err_pke_rsa_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -333,6 +339,8 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_pke_rsa_unregister();
+err_pke_rsa_register:
cmh_ccp_poly_unregister();
err_ccp_poly_register:
cmh_ccp_aead_unregister();
@@ -389,6 +397,7 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_pke_rsa_unregister();
cmh_ccp_poly_unregister();
cmh_ccp_aead_unregister();
cmh_ccp_unregister();
diff --git a/drivers/crypto/cmh/cmh_pke_common.c b/drivers/crypto/cmh/cmh_pke_common.c
new file mode 100644
index 000000000000..ab3e2eb7d3f8
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pke_common.c
@@ -0,0 +1,578 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- PKE Common VCQ Builders
+ *
+ * VCQ builder functions for all PKE core commands. Each builder
+ * populates a single vcq_cmd slot with the appropriate magic,
+ * command ID, byte-swap flags, and command-specific payload.
+ *
+ * RSA commands always use PKE_SWAP_FLAGS (VCQ_FLAG_SWAP_BYTES |
+ * VCQ_FLAG_SWAP_WORDS). EC Weierstrass curves (NIST P-*, Brainpool,
+ * secp256k1, SM2) use PKE_SWAP_FLAGS; Edwards curves (25519, 448)
+ * use no swap flags. SM2 commands use per-command flags documented
+ * in the eSW ABI.
+ *
+ * Callers combine these with vcq_set_header() + vcq_add_flush()
+ * and submit via cmh_tm_submit_sync().
+ */
+
+#include <linux/string.h>
+
+#include "cmh_pke.h"
+
+/**
+ * vcq_add_pke_flush() - Add a PKE flush command to a VCQ slot
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ *
+ * Populates @slot with a flush command for the specified PKE core.
+ */
+void vcq_add_pke_flush(struct vcq_cmd *slot, u32 core_id)
+{
+ vcq_add_flush(slot, core_id);
+}
+
+/* RSA */
+
+/**
+ * vcq_add_pke_rsa_enc() - Build a VCQ command for RSA public-key encryption
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @bits: RSA key size in bits
+ * @e_len: Length of the public exponent in bytes
+ * @e_dma: DMA address of public exponent buffer
+ * @n_dma: DMA address of modulus buffer
+ * @m_dma: DMA address of plaintext message buffer
+ * @c_dma: DMA address of ciphertext output buffer
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_rsa_enc(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len,
+ u64 e_dma, u64 n_dma, u64 m_dma, u64 c_dma,
+ u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_RSA_ENC);
+ slot->hwc.pke.cmd_rsa_enc.bits = bits;
+ slot->hwc.pke.cmd_rsa_enc.e_len = e_len;
+ slot->hwc.pke.cmd_rsa_enc.e = e_dma;
+ slot->hwc.pke.cmd_rsa_enc.n = n_dma;
+ slot->hwc.pke.cmd_rsa_enc.m = m_dma;
+ slot->hwc.pke.cmd_rsa_enc.c = c_dma;
+}
+
+/**
+ * vcq_add_pke_rsa_dec() - Build a VCQ command for RSA private-key decryption
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @bits: RSA key size in bits
+ * @e_len: Length of the public exponent in bytes
+ * @e_dma: DMA address of public exponent buffer
+ * @n_dma: DMA address of modulus buffer
+ * @c_dma: DMA address of ciphertext input buffer
+ * @m_dma: DMA address of plaintext output buffer
+ * @d_ref: Datastore reference for the private exponent
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_rsa_dec(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len,
+ u64 e_dma, u64 n_dma, u64 c_dma, u64 m_dma,
+ u64 d_ref, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_RSA_DEC);
+ slot->hwc.pke.cmd_rsa_dec.bits = bits;
+ slot->hwc.pke.cmd_rsa_dec.e_len = e_len;
+ slot->hwc.pke.cmd_rsa_dec.e = e_dma;
+ slot->hwc.pke.cmd_rsa_dec.n = n_dma;
+ slot->hwc.pke.cmd_rsa_dec.c = c_dma;
+ slot->hwc.pke.cmd_rsa_dec.m = m_dma;
+ slot->hwc.pke.cmd_rsa_dec.d = d_ref;
+}
+
+/**
+ * vcq_add_pke_rsa_crt_dec() - Build a VCQ command for RSA-CRT decryption
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @bits: RSA key size in bits
+ * @e_len: Length of the public exponent in bytes
+ * @e_dma: DMA address of public exponent buffer
+ * @n_dma: DMA address of modulus buffer
+ * @c_dma: DMA address of ciphertext input buffer
+ * @m_dma: DMA address of plaintext output buffer
+ * @crt_ref: Datastore reference for CRT private key components
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_rsa_crt_dec(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len,
+ u64 e_dma, u64 n_dma, u64 c_dma, u64 m_dma,
+ u64 crt_ref, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_RSA_CRT_DEC);
+ slot->hwc.pke.cmd_rsa_crt_dec.bits = bits;
+ slot->hwc.pke.cmd_rsa_crt_dec.e_len = e_len;
+ slot->hwc.pke.cmd_rsa_crt_dec.e = e_dma;
+ slot->hwc.pke.cmd_rsa_crt_dec.n = n_dma;
+ slot->hwc.pke.cmd_rsa_crt_dec.c = c_dma;
+ slot->hwc.pke.cmd_rsa_crt_dec.m = m_dma;
+ slot->hwc.pke.cmd_rsa_crt_dec.crt = crt_ref;
+}
+
+/* ECDSA */
+
+/**
+ * vcq_add_pke_ecdsa_verify() - Build a VCQ command for ECDSA signature verification
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521)
+ * @dlen: Digest length in bytes
+ * @pk_dma: DMA address of public key buffer
+ * @dig_dma: DMA address of digest buffer
+ * @sig_dma: DMA address of signature buffer
+ * @rp_dma: DMA address of r-prime verification output buffer
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_ecdsa_verify(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 dlen,
+ u64 pk_dma, u64 dig_dma, u64 sig_dma,
+ u64 rp_dma, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDSA_VERIFY);
+ slot->hwc.pke.cmd_ecdsa_verify.curve = curve;
+ slot->hwc.pke.cmd_ecdsa_verify.digest_len = dlen;
+ slot->hwc.pke.cmd_ecdsa_verify.public_key = pk_dma;
+ slot->hwc.pke.cmd_ecdsa_verify.digest = dig_dma;
+ slot->hwc.pke.cmd_ecdsa_verify.signature = sig_dma;
+ slot->hwc.pke.cmd_ecdsa_verify.rprime = rp_dma;
+}
+
+/**
+ * vcq_add_pke_ecdsa_sign() - Build a VCQ command for ECDSA signing
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521)
+ * @sklen: Secret key length in bytes
+ * @dig_dma: DMA address of digest buffer
+ * @sig_dma: DMA address of signature output buffer
+ * @sk_ref: Datastore reference for the secret key
+ * @dlen: Digest length in bytes
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_ecdsa_sign(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 dig_dma, u64 sig_dma, u64 sk_ref,
+ u32 dlen, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDSA_SIGN);
+ slot->hwc.pke.cmd_ecdsa_sign.curve = curve;
+ slot->hwc.pke.cmd_ecdsa_sign.secret_key_len = sklen;
+ slot->hwc.pke.cmd_ecdsa_sign.digest = dig_dma;
+ slot->hwc.pke.cmd_ecdsa_sign.signature = sig_dma;
+ slot->hwc.pke.cmd_ecdsa_sign.secret_key = sk_ref;
+ slot->hwc.pke.cmd_ecdsa_sign.digest_len = dlen;
+}
+
+/**
+ * vcq_add_pke_ecdsa_pubgen() - Build a VCQ command for ECDSA public key generation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521)
+ * @sklen: Secret key length in bytes
+ * @pk_dma: DMA address of public key output buffer
+ * @sk_ref: Datastore reference for the secret key
+ * @flags: VCQ command flags
+ *
+ * Generates the public key from an existing private key stored in the
+ * datastore.
+ */
+void vcq_add_pke_ecdsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 pk_dma, u64 sk_ref, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDSA_PUBGEN);
+ slot->hwc.pke.cmd_ecdsa_pubgen.curve = curve;
+ slot->hwc.pke.cmd_ecdsa_pubgen.secret_key_len = sklen;
+ slot->hwc.pke.cmd_ecdsa_pubgen.public_key = pk_dma;
+ slot->hwc.pke.cmd_ecdsa_pubgen.secret_key = sk_ref;
+}
+
+/**
+ * vcq_add_pke_ecdsa_keygen() - Build a VCQ command for ECDSA key pair generation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521)
+ * @sklen: Secret key length in bytes
+ * @sk_ref: Datastore reference for the generated secret key
+ * @sk_type: Datastore type for the secret key object
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_ecdsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 sk_ref, u32 sk_type, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDSA_KEYGEN);
+ slot->hwc.pke.cmd_ecdsa_keygen.curve = curve;
+ slot->hwc.pke.cmd_ecdsa_keygen.secret_key_len = sklen;
+ slot->hwc.pke.cmd_ecdsa_keygen.secret_key = sk_ref;
+ slot->hwc.pke.cmd_ecdsa_keygen.secret_key_type = sk_type;
+}
+
+/* ECDH */
+
+/**
+ * vcq_add_pke_ecdh_keygen() - Build a VCQ command for ECDH key pair generation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521, X25519, X448)
+ * @sklen: Secret key length in bytes
+ * @pkx_dma: DMA address of public key X-coordinate output buffer
+ * @sk_ref: Datastore reference for the generated secret key
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_ecdh_keygen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 pkx_dma, u64 sk_ref, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDH_KEYGEN);
+ slot->hwc.pke.cmd_ecdh_keygen.curve = curve;
+ slot->hwc.pke.cmd_ecdh_keygen.secret_key_len = sklen;
+ slot->hwc.pke.cmd_ecdh_keygen.public_key_x = pkx_dma;
+ slot->hwc.pke.cmd_ecdh_keygen.secret_key = sk_ref;
+}
+
+/**
+ * vcq_add_pke_ecdh() - Build a VCQ command for ECDH shared secret computation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521, X25519, X448)
+ * @sklen: Secret key length in bytes
+ * @sslen: Shared secret length in bytes
+ * @ss_type: Datastore type for the shared secret object
+ * @peer_dma: DMA address of peer public key buffer
+ * @sk_ref: Datastore reference for the local secret key
+ * @ss_ref: Datastore reference for the computed shared secret
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_ecdh(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u32 sslen, u32 ss_type, u64 peer_dma, u64 sk_ref,
+ u64 ss_ref, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDH);
+ slot->hwc.pke.cmd_ecdh.curve = curve;
+ slot->hwc.pke.cmd_ecdh.secret_key_len = sklen;
+ slot->hwc.pke.cmd_ecdh.shared_secret_len = sslen;
+ slot->hwc.pke.cmd_ecdh.shared_secret_type = ss_type;
+ slot->hwc.pke.cmd_ecdh.peer_key_x = peer_dma;
+ slot->hwc.pke.cmd_ecdh.secret_key = sk_ref;
+ slot->hwc.pke.cmd_ecdh.shared_secret = ss_ref;
+}
+
+/* EdDSA */
+
+/**
+ * vcq_add_pke_eddsa_verify() - Build a VCQ command for EdDSA signature verification
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (Ed25519 or Ed448)
+ * @dlen: Digest (message) length in bytes
+ * @pky_dma: DMA address of public key Y-coordinate buffer
+ * @dig_dma: DMA address of digest buffer
+ * @sig_dma: DMA address of signature buffer
+ * @rp_dma: DMA address of r-prime verification output buffer
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_eddsa_verify(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 dlen,
+ u64 pky_dma, u64 dig_dma, u64 sig_dma,
+ u64 rp_dma, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_EDDSA_VERIFY);
+ slot->hwc.pke.cmd_eddsa_verify.curve = curve;
+ slot->hwc.pke.cmd_eddsa_verify.digest_len = dlen;
+ slot->hwc.pke.cmd_eddsa_verify.public_key_y = pky_dma;
+ slot->hwc.pke.cmd_eddsa_verify.digest = dig_dma;
+ slot->hwc.pke.cmd_eddsa_verify.signature = sig_dma;
+ slot->hwc.pke.cmd_eddsa_verify.rprime = rp_dma;
+}
+
+/**
+ * vcq_add_pke_eddsa_sign() - Build a VCQ command for EdDSA signing
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (Ed25519 or Ed448)
+ * @sklen: Secret key length in bytes
+ * @dig_dma: DMA address of digest (message) buffer
+ * @sig_dma: DMA address of signature output buffer
+ * @sk_ref: Datastore reference for the secret key
+ * @dlen: Digest (message) length in bytes
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_eddsa_sign(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 dig_dma, u64 sig_dma, u64 sk_ref,
+ u32 dlen, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_EDDSA_SIGN);
+ slot->hwc.pke.cmd_eddsa_sign.curve = curve;
+ slot->hwc.pke.cmd_eddsa_sign.secret_key_len = sklen;
+ slot->hwc.pke.cmd_eddsa_sign.digest = dig_dma;
+ slot->hwc.pke.cmd_eddsa_sign.signature = sig_dma;
+ slot->hwc.pke.cmd_eddsa_sign.secret_key = sk_ref;
+ slot->hwc.pke.cmd_eddsa_sign.digest_len = dlen;
+}
+
+/**
+ * vcq_add_pke_eddsa_pubgen() - Build a VCQ command for EdDSA public key generation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (Ed25519 or Ed448)
+ * @sklen: Secret key length in bytes
+ * @pky_dma: DMA address of public key Y-coordinate output buffer
+ * @sk_ref: Datastore reference for the secret key
+ * @flags: VCQ command flags
+ *
+ * Generates the public key from an existing private key stored in the
+ * datastore.
+ */
+void vcq_add_pke_eddsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen,
+ u64 pky_dma, u64 sk_ref, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_EDDSA_PUBGEN);
+ slot->hwc.pke.cmd_eddsa_pubgen.curve = curve;
+ slot->hwc.pke.cmd_eddsa_pubgen.secret_key_len = sklen;
+ slot->hwc.pke.cmd_eddsa_pubgen.public_key_y = pky_dma;
+ slot->hwc.pke.cmd_eddsa_pubgen.secret_key = sk_ref;
+}
+
+/**
+ * vcq_add_pke_eddsa_keygen_sca() - Build a VCQ command for EdDSA SCA key generation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @curve: Curve identifier (Ed448)
+ * @sk_ref: Datastore reference for the input secret key
+ * @sca_sk_ref: Datastore reference for the SCA-masked output key
+ *
+ * Blinds an Ed448 private key into a side-channel-protected masked
+ * form. No byte-swap flags are used (CRI reference uses flags=0).
+ */
+void vcq_add_pke_eddsa_keygen_sca(struct vcq_cmd *slot, u32 core_id, u32 curve,
+ u64 sk_ref, u64 sca_sk_ref)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1,
+ PKE_CMD_EDDSA_PRIV_KEYGEN_SCA);
+ slot->hwc.pke.cmd_eddsa_keygen_sca.curve = curve;
+ slot->hwc.pke.cmd_eddsa_keygen_sca.secret_key = sk_ref;
+ slot->hwc.pke.cmd_eddsa_keygen_sca.sca_secret_key = sca_sk_ref;
+}
+
+/* SM2 */
+
+/**
+ * vcq_add_pke_sm2_ecdh_keygen() - Build a VCQ command for SM2 ECDH ephemeral key generation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @nonce_dma: DMA address of nonce input buffer
+ * @session_key_dma: DMA address of session key output buffer
+ * @nonce_len: Nonce length in bytes
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_ecdh_keygen(struct vcq_cmd *slot, u32 core_id, u64 nonce_dma,
+ u64 session_key_dma, u32 nonce_len, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1,
+ PKE_CMD_SM2_ECDH_KEYGEN);
+ slot->hwc.pke.cmd_sm2_ecdh_keygen.nonce = nonce_dma;
+ slot->hwc.pke.cmd_sm2_ecdh_keygen.session_key = session_key_dma;
+ slot->hwc.pke.cmd_sm2_ecdh_keygen.nonce_len = nonce_len;
+}
+
+/**
+ * vcq_add_pke_sm2_ecdh() - Build a VCQ command for SM2 ECDH shared secret computation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @nonce_len: Nonce length in bytes
+ * @private_key_len: Private key length in bytes
+ * @nonce_dma: DMA address of nonce buffer
+ * @peer_pk_dma: DMA address of peer public key buffer
+ * @peer_sk_dma: DMA address of peer session key buffer
+ * @priv_ref: Datastore reference for the local private key
+ * @sp_ref: Datastore reference for the shared point output
+ * @sp_type: Datastore type for the shared point object
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_ecdh(struct vcq_cmd *slot, u32 core_id, u32 nonce_len,
+ u32 private_key_len, u64 nonce_dma,
+ u64 peer_pk_dma, u64 peer_sk_dma,
+ u64 priv_ref, u64 sp_ref, u32 sp_type, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ECDH);
+ slot->hwc.pke.cmd_sm2_ecdh.nonce_len = nonce_len;
+ slot->hwc.pke.cmd_sm2_ecdh.private_key_len = private_key_len;
+ slot->hwc.pke.cmd_sm2_ecdh.nonce = nonce_dma;
+ slot->hwc.pke.cmd_sm2_ecdh.peer_public_key = peer_pk_dma;
+ slot->hwc.pke.cmd_sm2_ecdh.peer_session_key = peer_sk_dma;
+ slot->hwc.pke.cmd_sm2_ecdh.private_key = priv_ref;
+ slot->hwc.pke.cmd_sm2_ecdh.shared_point = sp_ref;
+ slot->hwc.pke.cmd_sm2_ecdh.shared_point_type = sp_type;
+}
+
+/**
+ * vcq_add_pke_sm2_dec_point() - Build a VCQ command for SM2 decryption point multiplication
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @ct_len: Ciphertext length in bytes
+ * @pk_len: Private key length in bytes
+ * @ct_dma: DMA address of ciphertext input buffer
+ * @dp_dma: DMA address of decryption point output buffer
+ * @priv_ref: Datastore reference for the private key
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_dec_point(struct vcq_cmd *slot, u32 core_id, u32 ct_len,
+ u32 pk_len, u64 ct_dma, u64 dp_dma,
+ u64 priv_ref, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_DEC_POINT);
+ slot->hwc.pke.cmd_sm2_dec_point.ciphertext_len = ct_len;
+ slot->hwc.pke.cmd_sm2_dec_point.private_key_len = pk_len;
+ slot->hwc.pke.cmd_sm2_dec_point.ciphertext = ct_dma;
+ slot->hwc.pke.cmd_sm2_dec_point.dec_point = dp_dma;
+ slot->hwc.pke.cmd_sm2_dec_point.private_key = priv_ref;
+}
+
+/**
+ * vcq_add_pke_sm2_enc_point() - Build a VCQ command for SM2 encryption point multiplication
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @nonce_dma: DMA address of nonce buffer
+ * @pk_dma: DMA address of public key buffer
+ * @ct_dma: DMA address of ciphertext header output buffer
+ * @ep_dma: DMA address of encryption point output buffer
+ * @nonce_len: Nonce length in bytes
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_enc_point(struct vcq_cmd *slot, u32 core_id, u64 nonce_dma,
+ u64 pk_dma, u64 ct_dma, u64 ep_dma,
+ u32 nonce_len, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ENC_POINT);
+ slot->hwc.pke.cmd_sm2_enc_point.nonce = nonce_dma;
+ slot->hwc.pke.cmd_sm2_enc_point.public_key = pk_dma;
+ slot->hwc.pke.cmd_sm2_enc_point.ciphertext = ct_dma;
+ slot->hwc.pke.cmd_sm2_enc_point.enc_point = ep_dma;
+ slot->hwc.pke.cmd_sm2_enc_point.nonce_len = nonce_len;
+}
+
+/**
+ * vcq_add_pke_sm2_id_digest() - Build a VCQ command for SM2 identity digest computation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @id_dma: DMA address of identity string buffer
+ * @pk_dma: DMA address of public key buffer
+ * @dig_dma: DMA address of digest output buffer
+ * @id_len: Identity string length in bytes
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_id_digest(struct vcq_cmd *slot, u32 core_id, u64 id_dma,
+ u64 pk_dma, u64 dig_dma, u32 id_len,
+ u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ID_DIGEST);
+ slot->hwc.pke.cmd_sm2_id_digest.id = id_dma;
+ slot->hwc.pke.cmd_sm2_id_digest.public_key = pk_dma;
+ slot->hwc.pke.cmd_sm2_id_digest.digest = dig_dma;
+ slot->hwc.pke.cmd_sm2_id_digest.id_len = id_len;
+}
+
+/**
+ * vcq_add_pke_sm2_ecdh_hash() - Build a VCQ command for SM2 ECDH key derivation hash
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @peer_dig_dma: DMA address of peer identity digest buffer
+ * @dig_dma: DMA address of local identity digest buffer
+ * @sp_ref: Datastore reference for the shared point
+ * @sk_ref: Datastore reference for the derived shared key output
+ * @sk_type: Datastore type for the shared key object
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_ecdh_hash(struct vcq_cmd *slot, u32 core_id, u64 peer_dig_dma,
+ u64 dig_dma, u64 sp_ref, u64 sk_ref,
+ u32 sk_type, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ECDH_HASH);
+ slot->hwc.pke.cmd_sm2_ecdh_hash.peer_id_digest = peer_dig_dma;
+ slot->hwc.pke.cmd_sm2_ecdh_hash.id_digest = dig_dma;
+ slot->hwc.pke.cmd_sm2_ecdh_hash.shared_point = sp_ref;
+ slot->hwc.pke.cmd_sm2_ecdh_hash.shared_key = sk_ref;
+ slot->hwc.pke.cmd_sm2_ecdh_hash.shared_key_type = sk_type;
+}
+
+/**
+ * vcq_add_pke_sm2_dec_hash() - Build a VCQ command for SM2 decryption hash verification
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @ct_dma: DMA address of ciphertext input buffer
+ * @dp_dma: DMA address of decryption point buffer
+ * @pt_dma: DMA address of plaintext output buffer
+ * @ct_len: Ciphertext length in bytes
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_dec_hash(struct vcq_cmd *slot, u32 core_id, u64 ct_dma,
+ u64 dp_dma, u64 pt_dma, u32 ct_len, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_DEC_HASH);
+ slot->hwc.pke.cmd_sm2_dec_hash.ciphertext = ct_dma;
+ slot->hwc.pke.cmd_sm2_dec_hash.dec_point = dp_dma;
+ slot->hwc.pke.cmd_sm2_dec_hash.plaintext = pt_dma;
+ slot->hwc.pke.cmd_sm2_dec_hash.ciphertext_len = ct_len;
+}
+
+/**
+ * vcq_add_pke_sm2_enc_hash() - Build a VCQ command for SM2 encryption hash computation
+ * @slot: VCQ command slot to populate
+ * @core_id: PKE hardware core ID
+ * @msg_dma: DMA address of plaintext message buffer
+ * @ep_dma: DMA address of encryption point buffer
+ * @ct_dma: DMA address of ciphertext output buffer
+ * @msg_len: Message length in bytes
+ * @flags: VCQ command flags
+ */
+void vcq_add_pke_sm2_enc_hash(struct vcq_cmd *slot, u32 core_id, u64 msg_dma,
+ u64 ep_dma, u64 ct_dma, u32 msg_len, u32 flags)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ENC_HASH);
+ slot->hwc.pke.cmd_sm2_enc_hash.message = msg_dma;
+ slot->hwc.pke.cmd_sm2_enc_hash.enc_point = ep_dma;
+ slot->hwc.pke.cmd_sm2_enc_hash.ciphertext = ct_dma;
+ slot->hwc.pke.cmd_sm2_enc_hash.message_len = msg_len;
+}
diff --git a/drivers/crypto/cmh/cmh_pke_rsa.c b/drivers/crypto/cmh/cmh_pke_rsa.c
new file mode 100644
index 000000000000..a848b7eddc8e
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pke_rsa.c
@@ -0,0 +1,638 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- RSA akcipher Driver
+ *
+ * Registers "rsa" akcipher algorithm with the Linux crypto subsystem
+ * (priority 300, overrides software rsa-generic at 100).
+ *
+ * Raw RSA operations only (m^e mod n / c^d mod n). The kernel's
+ * pkcs1pad() template wraps this for PKCS#1 v1.5 / PSS / OAEP.
+ *
+ * Key format: DER-encoded ASN.1, parsed by kernel rsa_parse_pub_key()
+ * / rsa_parse_priv_key() helpers.
+ *
+ * Private key via cmh_key_ctx: raw keys written via SYS_REF_TEMP.
+ * Datastore-referenced keys are only reachable through the ioctl
+ * path (cmh_mgmt.c).
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/scatterlist.h>
+#include <crypto/akcipher.h>
+#include <crypto/internal/akcipher.h>
+#include <crypto/internal/rsa.h>
+
+#include "cmh_pke.h"
+#include "cmh_sys.h"
+#include "cmh_sys_abi.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+struct cmh_rsa_tfm_ctx {
+ struct cmh_key_ctx key; /* private key (raw d only) */
+ u8 *n; /* modulus (big-endian) */
+ u8 *e; /* public exponent (big-endian) */
+ size_t n_sz;
+ size_t e_sz;
+ u32 bits; /* key size in bits */
+};
+
+static inline struct cmh_rsa_tfm_ctx *cmh_rsa_ctx(struct crypto_akcipher *tfm)
+{
+ return akcipher_tfm_ctx(tfm);
+}
+
+struct cmh_rsa_reqctx {
+ u8 *e_buf;
+ u8 *n_buf;
+ u8 *m_buf;
+ u8 *c_buf;
+ dma_addr_t e_dma;
+ dma_addr_t n_dma;
+ dma_addr_t m_dma;
+ dma_addr_t c_dma;
+ u32 key_bytes;
+ u32 e_padded;
+ u32 n_sz;
+};
+
+static u32 cmh_rsa_key_bits(size_t n_sz)
+{
+ /*
+ * Only accept exact modulus sizes supported by the hardware.
+ * The programmed RSA width must match the actual modulus buffer
+ * length; rounding a shorter modulus up to the next size would
+ * let the device read past the end of the DMA buffer.
+ */
+ switch (n_sz) {
+ case 64:
+ return 512;
+ case 128:
+ return 1024;
+ case 256:
+ return 2048;
+ case 384:
+ return 3072;
+ case 512:
+ return 4096;
+ default:
+ return 0;
+ }
+}
+
+static void cmh_rsa_enc_complete(void *data, int error)
+{
+ struct akcipher_request *req = data;
+ struct cmh_rsa_reqctx *rctx = akcipher_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ if (!cmh_dma_map_error(rctx->c_dma))
+ cmh_dma_unmap_single(rctx->c_dma, rctx->key_bytes,
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(rctx->m_dma))
+ cmh_dma_unmap_single(rctx->m_dma, rctx->key_bytes,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->n_dma))
+ cmh_dma_unmap_single(rctx->n_dma, rctx->n_sz,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->e_dma))
+ cmh_dma_unmap_single(rctx->e_dma, rctx->e_padded,
+ DMA_TO_DEVICE);
+
+ if (!error) {
+ int nents;
+
+ nents = sg_nents_for_len(req->dst, rctx->key_bytes);
+ if (nents < 0 ||
+ sg_copy_from_buffer(req->dst, nents,
+ rctx->c_buf,
+ rctx->key_bytes) != rctx->key_bytes)
+ error = -EINVAL;
+ else
+ req->dst_len = rctx->key_bytes;
+ }
+
+ kfree(rctx->c_buf);
+ rctx->c_buf = NULL;
+ kfree_sensitive(rctx->m_buf);
+ rctx->m_buf = NULL;
+ kfree(rctx->n_buf);
+ rctx->n_buf = NULL;
+ kfree(rctx->e_buf);
+ rctx->e_buf = NULL;
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * RSA encrypt: c = m^e mod n (public key operation)
+ * Also used for signature verification (verify = encrypt for raw RSA).
+ */
+static int cmh_rsa_enc(struct akcipher_request *req)
+{
+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+ struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm);
+ struct cmh_rsa_reqctx *rctx = akcipher_request_ctx(req);
+ u32 key_bytes = ctx->bits / 8;
+ u32 e_padded = ALIGN(ctx->e_sz, 4);
+ struct core_dispatch d = cmh_core_select_instance(CMH_CORE_PKE);
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN];
+ int ret, nents;
+ gfp_t gfp;
+
+ if (!ctx->n || !ctx->e)
+ return -EINVAL;
+ if (req->src_len > key_bytes)
+ return -EINVAL;
+ if (req->dst_len < key_bytes) {
+ req->dst_len = key_bytes;
+ return -EOVERFLOW;
+ }
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->key_bytes = key_bytes;
+ rctx->e_padded = e_padded;
+ rctx->n_sz = ctx->n_sz;
+ rctx->e_dma = DMA_MAPPING_ERROR;
+ rctx->n_dma = DMA_MAPPING_ERROR;
+ rctx->m_dma = DMA_MAPPING_ERROR;
+ rctx->c_dma = DMA_MAPPING_ERROR;
+
+ rctx->e_buf = kzalloc(e_padded, gfp);
+ rctx->n_buf = kmemdup(ctx->n, ctx->n_sz, gfp);
+ rctx->m_buf = kzalloc(key_bytes, gfp);
+ rctx->c_buf = kzalloc(key_bytes, gfp);
+ if (!rctx->e_buf || !rctx->n_buf || !rctx->m_buf || !rctx->c_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ memcpy(rctx->e_buf + e_padded - ctx->e_sz, ctx->e, ctx->e_sz);
+
+ nents = sg_nents_for_len(req->src, req->src_len);
+ if (nents < 0 ||
+ sg_pcopy_to_buffer(req->src, nents,
+ rctx->m_buf + key_bytes - req->src_len,
+ req->src_len, 0) != req->src_len) {
+ ret = -EINVAL;
+ goto out_free;
+ }
+
+ rctx->e_dma = cmh_dma_map_single(rctx->e_buf, e_padded,
+ DMA_TO_DEVICE);
+ rctx->n_dma = cmh_dma_map_single(rctx->n_buf, ctx->n_sz,
+ DMA_TO_DEVICE);
+ rctx->m_dma = cmh_dma_map_single(rctx->m_buf, key_bytes,
+ DMA_TO_DEVICE);
+ rctx->c_dma = cmh_dma_map_single(rctx->c_buf, key_bytes,
+ DMA_FROM_DEVICE);
+
+ if (cmh_dma_map_error(rctx->e_dma) ||
+ cmh_dma_map_error(rctx->n_dma) ||
+ cmh_dma_map_error(rctx->m_dma) ||
+ cmh_dma_map_error(rctx->c_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN);
+ vcq_add_pke_rsa_enc(&vcq[1], d.core_id, ctx->bits, e_padded,
+ rctx->e_dma, rctx->n_dma, rctx->m_dma,
+ rctx->c_dma, PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[2], d.core_id);
+
+ ret = cmh_tm_submit_async(vcq, PKE_VCQ_CMDS_MIN, 1, d.mbx_idx,
+ cmh_rsa_enc_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG), 0);
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (!ret)
+ return -EINPROGRESS;
+
+out_unmap:
+ if (!cmh_dma_map_error(rctx->c_dma))
+ cmh_dma_unmap_single(rctx->c_dma, key_bytes,
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(rctx->m_dma))
+ cmh_dma_unmap_single(rctx->m_dma, key_bytes,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->n_dma))
+ cmh_dma_unmap_single(rctx->n_dma, ctx->n_sz,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->e_dma))
+ cmh_dma_unmap_single(rctx->e_dma, e_padded,
+ DMA_TO_DEVICE);
+
+out_free:
+ kfree(rctx->c_buf);
+ kfree_sensitive(rctx->m_buf);
+ kfree(rctx->n_buf);
+ kfree(rctx->e_buf);
+ return ret;
+}
+
+static void cmh_rsa_dec_complete(void *data, int error)
+{
+ struct akcipher_request *req = data;
+ struct cmh_rsa_reqctx *rctx = akcipher_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ if (!cmh_dma_map_error(rctx->m_dma))
+ cmh_dma_unmap_single(rctx->m_dma, rctx->key_bytes,
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(rctx->c_dma))
+ cmh_dma_unmap_single(rctx->c_dma, rctx->key_bytes,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->n_dma))
+ cmh_dma_unmap_single(rctx->n_dma, rctx->n_sz,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->e_dma))
+ cmh_dma_unmap_single(rctx->e_dma, rctx->e_padded,
+ DMA_TO_DEVICE);
+
+ if (!error) {
+ int nents;
+
+ nents = sg_nents_for_len(req->dst, rctx->key_bytes);
+ if (nents < 0 ||
+ sg_copy_from_buffer(req->dst, nents,
+ rctx->m_buf,
+ rctx->key_bytes) != rctx->key_bytes)
+ error = -EINVAL;
+ else
+ req->dst_len = rctx->key_bytes;
+ }
+
+ kfree_sensitive(rctx->m_buf);
+ rctx->m_buf = NULL;
+ kfree(rctx->c_buf);
+ rctx->c_buf = NULL;
+ kfree(rctx->n_buf);
+ rctx->n_buf = NULL;
+ kfree(rctx->e_buf);
+ rctx->e_buf = NULL;
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * RSA decrypt: m = c^d mod n (private key operation)
+ * Also used for signing (sign = decrypt for raw RSA).
+ *
+ * Private key 'd' is written via SYS_REF_TEMP inline.
+ */
+static int cmh_rsa_dec(struct akcipher_request *req)
+{
+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+ struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm);
+ struct cmh_rsa_reqctx *rctx = akcipher_request_ctx(req);
+ u32 key_bytes = ctx->bits / 8;
+ u32 e_padded = ALIGN(ctx->e_sz, 4);
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MAX];
+ struct core_dispatch dd;
+ int ret, idx, nents;
+ gfp_t gfp;
+
+ if (ctx->key.mode != CMH_KEY_RAW)
+ return -EINVAL;
+ if (!ctx->n || !ctx->e)
+ return -EINVAL;
+ if (req->src_len > key_bytes)
+ return -EINVAL;
+ if (req->dst_len < key_bytes) {
+ req->dst_len = key_bytes;
+ return -EOVERFLOW;
+ }
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->key_bytes = key_bytes;
+ rctx->e_padded = e_padded;
+ rctx->n_sz = ctx->n_sz;
+ rctx->e_dma = DMA_MAPPING_ERROR;
+ rctx->n_dma = DMA_MAPPING_ERROR;
+ rctx->m_dma = DMA_MAPPING_ERROR;
+ rctx->c_dma = DMA_MAPPING_ERROR;
+
+ rctx->e_buf = kzalloc(e_padded, gfp);
+ rctx->n_buf = kmemdup(ctx->n, ctx->n_sz, gfp);
+ rctx->c_buf = kzalloc(key_bytes, gfp);
+ rctx->m_buf = kzalloc(key_bytes, gfp);
+ if (!rctx->e_buf || !rctx->n_buf || !rctx->c_buf || !rctx->m_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ memcpy(rctx->e_buf + e_padded - ctx->e_sz, ctx->e, ctx->e_sz);
+
+ nents = sg_nents_for_len(req->src, req->src_len);
+ if (nents < 0 ||
+ sg_pcopy_to_buffer(req->src, nents,
+ rctx->c_buf + key_bytes - req->src_len,
+ req->src_len, 0) != req->src_len) {
+ ret = -EINVAL;
+ goto out_free;
+ }
+
+ rctx->e_dma = cmh_dma_map_single(rctx->e_buf, e_padded,
+ DMA_TO_DEVICE);
+ rctx->n_dma = cmh_dma_map_single(rctx->n_buf, ctx->n_sz,
+ DMA_TO_DEVICE);
+ rctx->c_dma = cmh_dma_map_single(rctx->c_buf, key_bytes,
+ DMA_TO_DEVICE);
+ rctx->m_dma = cmh_dma_map_single(rctx->m_buf, key_bytes,
+ DMA_FROM_DEVICE);
+
+ if (cmh_dma_map_error(rctx->e_dma) ||
+ cmh_dma_map_error(rctx->n_dma) ||
+ cmh_dma_map_error(rctx->c_dma) ||
+ cmh_dma_map_error(rctx->m_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ dd = cmh_core_select_instance(CMH_CORE_PKE);
+
+ idx = 1;
+ vcq_add_sys_write(&vcq[idx], SYS_REF_TEMP, ctx->key.raw.dma,
+ SYS_REF_NONE, ctx->key.raw.len,
+ ctx->key.raw.sys_type);
+ vcq[idx].id |= PKE_SWAP_FLAGS;
+ idx++;
+ vcq_add_pke_rsa_dec(&vcq[idx++], dd.core_id, ctx->bits, e_padded,
+ rctx->e_dma, rctx->n_dma, rctx->c_dma,
+ rctx->m_dma, SYS_REF_TEMP, PKE_SWAP_FLAGS);
+ vcq_add_pke_flush(&vcq[idx++], dd.core_id);
+ vcq_set_header(&vcq[0], idx);
+
+ ret = cmh_tm_submit_async(vcq, idx, 1, dd.mbx_idx,
+ cmh_rsa_dec_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG), 0);
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (!ret)
+ return -EINPROGRESS;
+
+out_unmap:
+ if (!cmh_dma_map_error(rctx->m_dma))
+ cmh_dma_unmap_single(rctx->m_dma, key_bytes,
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(rctx->c_dma))
+ cmh_dma_unmap_single(rctx->c_dma, key_bytes,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->n_dma))
+ cmh_dma_unmap_single(rctx->n_dma, ctx->n_sz,
+ DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(rctx->e_dma))
+ cmh_dma_unmap_single(rctx->e_dma, e_padded,
+ DMA_TO_DEVICE);
+
+out_free:
+ kfree_sensitive(rctx->m_buf);
+ kfree(rctx->c_buf);
+ kfree(rctx->n_buf);
+ kfree(rctx->e_buf);
+ return ret;
+}
+
+static int cmh_rsa_set_pub_key(struct crypto_akcipher *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm);
+ struct rsa_key rsa = {};
+ int ret;
+
+ /*
+ * Re-keying: release any private-key state (and its persistent DMA
+ * mapping) left by a prior set_priv_key so it is not leaked until
+ * exit_tfm and cannot be mistaken for the new public key.
+ */
+ cmh_key_destroy(&ctx->key);
+
+ ret = rsa_parse_pub_key(&rsa, key, keylen);
+ if (ret)
+ return ret;
+
+ /* Strip ASN.1 leading zero padding from modulus */
+ while (rsa.n_sz > 0 && rsa.n[0] == 0) {
+ rsa.n++;
+ rsa.n_sz--;
+ }
+
+ ctx->bits = cmh_rsa_key_bits(rsa.n_sz);
+ if (!ctx->bits)
+ return -EINVAL;
+
+ /* Reject an exponent wider than the modulus (HW buffer bound). */
+ if (!rsa.e_sz || rsa.e_sz > ctx->bits / 8)
+ return -EINVAL;
+
+ kfree(ctx->n);
+ kfree(ctx->e);
+ ctx->n = NULL;
+ ctx->e = NULL;
+ ctx->n_sz = 0;
+ ctx->e_sz = 0;
+
+ ctx->n = kmemdup(rsa.n, rsa.n_sz, GFP_KERNEL);
+ ctx->e = kmemdup(rsa.e, rsa.e_sz, GFP_KERNEL);
+ if (!ctx->n || !ctx->e) {
+ kfree(ctx->n);
+ kfree(ctx->e);
+ ctx->n = NULL;
+ ctx->e = NULL;
+ return -ENOMEM;
+ }
+
+ ctx->n_sz = rsa.n_sz;
+ ctx->e_sz = rsa.e_sz;
+
+ return 0;
+}
+
+static int cmh_rsa_set_priv_key(struct crypto_akcipher *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm);
+ struct rsa_key rsa = {};
+ u32 key_bytes;
+ u8 *d_padded;
+ int ret;
+
+ ret = rsa_parse_priv_key(&rsa, key, keylen);
+ if (ret)
+ return ret;
+
+ /* Strip ASN.1 leading zero padding from modulus */
+ while (rsa.n_sz > 0 && rsa.n[0] == 0) {
+ rsa.n++;
+ rsa.n_sz--;
+ }
+
+ ctx->bits = cmh_rsa_key_bits(rsa.n_sz);
+ if (!ctx->bits || !rsa.d_sz)
+ return -EINVAL;
+
+ key_bytes = ctx->bits / 8;
+
+ /* Strip ASN.1 leading zero padding from private exponent */
+ while (rsa.d_sz > 0 && rsa.d[0] == 0) {
+ rsa.d++;
+ rsa.d_sz--;
+ }
+
+ if (!rsa.d_sz || rsa.d_sz > key_bytes)
+ return -EINVAL;
+
+ /* Reject an exponent wider than the modulus (HW buffer bound). */
+ if (!rsa.e_sz || rsa.e_sz > key_bytes)
+ return -EINVAL;
+
+ kfree(ctx->n);
+ kfree(ctx->e);
+ ctx->n = NULL;
+ ctx->e = NULL;
+ ctx->n_sz = 0;
+ ctx->e_sz = 0;
+
+ ctx->n = kmemdup(rsa.n, rsa.n_sz, GFP_KERNEL);
+ ctx->e = kmemdup(rsa.e, rsa.e_sz, GFP_KERNEL);
+ if (!ctx->n || !ctx->e) {
+ ret = -ENOMEM;
+ goto err;
+ }
+
+ ctx->n_sz = rsa.n_sz;
+ ctx->e_sz = rsa.e_sz;
+
+ /*
+ * Left-pad d to key_bytes (big-endian alignment).
+ * The CMH eSW resolves SYS_REF_TEMP by checking
+ * hdr->len >= key_bytes, so the written buffer must
+ * be at least key_bytes wide.
+ */
+ d_padded = kzalloc(key_bytes, GFP_KERNEL);
+ if (!d_padded) {
+ ret = -ENOMEM;
+ goto err;
+ }
+ memcpy(d_padded + key_bytes - rsa.d_sz, rsa.d, rsa.d_sz);
+
+ ret = cmh_key_setkey_raw(&ctx->key, d_padded, key_bytes,
+ CORE_ID_PKE);
+ kfree_sensitive(d_padded);
+ if (ret)
+ goto err;
+
+ return 0;
+err:
+ kfree(ctx->n);
+ kfree(ctx->e);
+ ctx->n = NULL;
+ ctx->e = NULL;
+ ctx->n_sz = 0;
+ ctx->e_sz = 0;
+ ctx->bits = 0;
+ return ret;
+}
+
+static unsigned int cmh_rsa_max_size(struct crypto_akcipher *tfm)
+{
+ struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm);
+
+ return ctx->n_sz;
+}
+
+static int cmh_rsa_init_tfm(struct crypto_akcipher *tfm)
+{
+ struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ tfm->reqsize = sizeof(struct cmh_rsa_reqctx);
+ return 0;
+}
+
+static void cmh_rsa_exit_tfm(struct crypto_akcipher *tfm)
+{
+ struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm);
+
+ cmh_key_destroy(&ctx->key);
+ kfree(ctx->n);
+ kfree(ctx->e);
+ ctx->n = NULL;
+ ctx->e = NULL;
+}
+
+/*
+ * Raw RSA stays as akcipher (encrypt/decrypt only). The kernel's
+ * rsassa-pkcs1 sig template wraps our akcipher for sign/verify,
+ * matching the upstream split (rsa.c = akcipher,
+ * rsassa-pkcs1.c = sig template).
+ */
+static struct akcipher_alg cmh_rsa_alg = {
+ .encrypt = cmh_rsa_enc,
+ .decrypt = cmh_rsa_dec,
+ .set_pub_key = cmh_rsa_set_pub_key,
+ .set_priv_key = cmh_rsa_set_priv_key,
+ .max_size = cmh_rsa_max_size,
+ .init = cmh_rsa_init_tfm,
+ .exit = cmh_rsa_exit_tfm,
+ .base = {
+ .cra_name = "rsa",
+ .cra_driver_name = "rambus-cmh-rsa",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_rsa_tfm_ctx),
+ },
+};
+
+static bool cmh_rsa_registered;
+
+/**
+ * cmh_pke_rsa_register() - Register RSA akcipher algorithm with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_pke_rsa_register(void)
+{
+ int ret;
+
+ ret = crypto_register_akcipher(&cmh_rsa_alg);
+ if (ret) {
+ dev_err(cmh_dev(),
+ "cmh: failed to register rsa akcipher (%d)\n",
+ ret);
+ return ret;
+ }
+
+ cmh_rsa_registered = true;
+ return 0;
+}
+
+/**
+ * cmh_pke_rsa_unregister() - Unregister RSA akcipher algorithm from the crypto framework
+ */
+void cmh_pke_rsa_unregister(void)
+{
+ if (cmh_rsa_registered)
+ crypto_unregister_akcipher(&cmh_rsa_alg);
+ cmh_rsa_registered = false;
+}
--
2.43.7
^ permalink raw reply related
* [PATCH v3 09/19] crypto: cmh - add SM4 skcipher/aead/cmac/xcbc
From: Saravanakrishnan Krishnamoorthy @ 2026-08-06 19:55 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register SM4 algorithms using the CMH SM4 core (core ID 0x04):
- skcipher: SM4-ECB, SM4-CBC, SM4-CTR, SM4-XTS, SM4-CFB
- aead: SM4-GCM, SM4-CCM
- ahash: SM4-CMAC, SM4-XCBC
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 5 +-
drivers/crypto/cmh/cmh_main.c | 25 +
drivers/crypto/cmh/cmh_sm4_aead.c | 870 ++++++++++++++++++++++++
drivers/crypto/cmh/cmh_sm4_cmac.c | 929 ++++++++++++++++++++++++++
drivers/crypto/cmh/cmh_sm4_skcipher.c | 695 +++++++++++++++++++
drivers/crypto/cmh/include/cmh_sm4.h | 24 +
6 files changed, 2547 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_sm4_aead.c
create mode 100644 drivers/crypto/cmh/cmh_sm4_cmac.c
create mode 100644 drivers/crypto/cmh/cmh_sm4_skcipher.c
create mode 100644 drivers/crypto/cmh/include/cmh_sm4.h
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index c5dcfa4f7794..032c863d856e 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -22,7 +22,10 @@ cmh-y := \
cmh_sm3.o \
cmh_aes.o \
cmh_aes_aead.o \
- cmh_aes_cmac.o
+ cmh_aes_cmac.o \
+ cmh_sm4_skcipher.o \
+ cmh_sm4_aead.o \
+ cmh_sm4_cmac.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index 5feab88aa0bc..fec7992bf58d 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -37,6 +37,7 @@
#include "cmh_kmac.h"
#include "cmh_sm3.h"
#include "cmh_aes.h"
+#include "cmh_sm4.h"
#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
@@ -283,6 +284,21 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_aes_cmac_register;
+ /* Register SM4 skcipher algorithms */
+ ret = cmh_sm4_register();
+ if (ret)
+ goto err_sm4_register;
+
+ /* Register SM4 AEAD algorithms (GCM, CCM) */
+ ret = cmh_sm4_aead_register();
+ if (ret)
+ goto err_sm4_aead_register;
+
+ /* Register SM4 CMAC/XCBC algorithms */
+ ret = cmh_sm4_cmac_register();
+ if (ret)
+ goto err_sm4_cmac_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -295,6 +311,12 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_sm4_cmac_unregister();
+err_sm4_cmac_register:
+ cmh_sm4_aead_unregister();
+err_sm4_aead_register:
+ cmh_sm4_unregister();
+err_sm4_register:
cmh_aes_cmac_unregister();
err_aes_cmac_register:
cmh_aes_aead_unregister();
@@ -337,6 +359,9 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_sm4_cmac_unregister();
+ cmh_sm4_aead_unregister();
+ cmh_sm4_unregister();
cmh_aes_cmac_unregister();
cmh_aes_aead_unregister();
cmh_aes_unregister();
diff --git a/drivers/crypto/cmh/cmh_sm4_aead.c b/drivers/crypto/cmh/cmh_sm4_aead.c
new file mode 100644
index 000000000000..90d46c9f2fa7
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_sm4_aead.c
@@ -0,0 +1,870 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API SM4 AEAD Driver (GCM/CCM)
+ *
+ * Registers AEAD algorithms with the Linux crypto subsystem:
+ * gcm(sm4), ccm(sm4)
+ *
+ * GCM: SM4_CMD_INIT(mode=GCM) + [AAD_FINAL] + SM4_CMD_FINAL + FLUSH
+ * CCM: SM4_CMD_CCM_INIT + [AAD_FINAL] + SM4_CMD_FINAL + FLUSH
+ * - SM4 CCM uses a distinct sm4_cmd_ccm_init struct
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/cipher.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/utils.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "cmh_sm4.h"
+#include "cmh_vcq.h"
+#include "cmh_sm4_abi.h"
+#include "cmh_sys_abi.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+/*
+ * GCM IV contract:
+ *
+ * The SM4 core requires exactly 16 bytes loaded into its IV register.
+ * For standard 96-bit nonce GCM, the driver passes:
+ *
+ * IV[0..11] = user-supplied 12-byte nonce
+ * IV[12..15] = 0x00000000
+ *
+ * The hardware internally sets the last 32 bits to the big-endian
+ * counter value 1 (forming J0 = nonce || 0x00000001) before
+ * processing AAD. The driver must NOT pre-set the counter.
+ *
+ * If the IV format is incorrect, GCM authentication will fail
+ * (encrypt produces wrong ciphertext/tag, decrypt rejects).
+ */
+#define SM4_GCM_IV_SIZE 12U /* GCM nonce size (standard) */
+#define SM4_GCM_HW_IV_SIZE 16U /* HW requires 16-byte IV buffer */
+#define SM4_GCM_TAG_SIZE 16U
+
+/* CCM: callers pass a 16-byte IV in RFC 3610 format:
+ * iv[0] = L-1, iv[1..14-iv[0]] = nonce, rest = counter (zeroed).
+ * Nonce length = 14 - iv[0], range 7..13.
+ */
+#define SM4_CCM_IV_SIZE 16U
+
+enum cmh_sm4_aead_type {
+ CMH_SM4_AEAD_GCM,
+ CMH_SM4_AEAD_CCM,
+};
+
+struct cmh_sm4_aead_info {
+ enum cmh_sm4_aead_type type;
+ u32 sm4_mode;
+ u32 ivsize;
+ u32 maxauthsize;
+ const char *alg_name;
+ const char *drv_name;
+};
+
+static const struct cmh_sm4_aead_info sm4_aead_algs[] = {
+ { CMH_SM4_AEAD_GCM, SM4_MODE_GCM, SM4_GCM_IV_SIZE,
+ SM4_GCM_TAG_SIZE, "gcm(sm4)", "rambus-cmh-gcm-sm4" },
+ { CMH_SM4_AEAD_CCM, SM4_MODE_CCM, SM4_CCM_IV_SIZE,
+ SM4_GCM_TAG_SIZE, "ccm(sm4)", "rambus-cmh-ccm-sm4" },
+};
+
+struct cmh_sm4_aead_tfm_ctx {
+ struct cmh_key_ctx key;
+ u32 authsize;
+ struct crypto_cipher *sw_cipher; /* CCM empty-input fallback */
+};
+
+/* Per-request context (lives in aead_request::__ctx) */
+
+#define CMH_SM4_AEAD_MAX_PAYLOAD 5
+#define CMH_SM4_AEAD_MAX_PACKED (CMH_SM4_AEAD_MAX_PAYLOAD * 2)
+
+struct cmh_sm4_aead_reqctx {
+ dma_addr_t in_dma;
+ dma_addr_t out_dma;
+ dma_addr_t iv_dma;
+ dma_addr_t key_dma;
+ dma_addr_t aad_dma;
+ dma_addr_t tag_dma;
+ u8 *in_buf;
+ u8 *out_buf;
+ u8 *iv_buf;
+ u8 *aad_buf;
+ u8 *tag_buf;
+ u32 cryptlen;
+ u32 assoclen;
+ u32 authsize;
+ u32 iv_map_len;
+ u32 keylen;
+ bool encrypting;
+ bool empty_gcm_fallback;
+ struct vcq_cmd packed[CMH_SM4_AEAD_MAX_PACKED];
+};
+
+struct cmh_sm4_aead_drv {
+ struct aead_alg alg;
+ const struct cmh_sm4_aead_info *info;
+};
+
+static const struct cmh_sm4_aead_info *
+cmh_sm4_aead_get_info(struct crypto_aead *tfm)
+{
+ struct aead_alg *alg = crypto_aead_alg(tfm);
+
+ return container_of(alg, struct cmh_sm4_aead_drv, alg)->info;
+}
+
+/* VCQ Builders -- SM4 AEAD-specific */
+
+static void vcq_add_sm4_aead_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref,
+ u64 iv_dma, u32 keylen, u32 ivlen,
+ u32 mode, u32 op, u32 aadlen, u32 iolen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_INIT);
+ slot->hwc.sm4.cmd_init.key = key_ref;
+ slot->hwc.sm4.cmd_init.iv = iv_dma;
+ slot->hwc.sm4.cmd_init.keylen = keylen;
+ slot->hwc.sm4.cmd_init.ivlen = ivlen;
+ slot->hwc.sm4.cmd_init.mode = mode;
+ slot->hwc.sm4.cmd_init.op = op;
+ slot->hwc.sm4.cmd_init.aadlen = aadlen;
+ slot->hwc.sm4.cmd_init.iolen = iolen;
+}
+
+static void vcq_add_sm4_ccm_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref,
+ u64 nonce_dma, u32 keylen, u32 noncelen,
+ u32 op, u32 aadlen, u32 iolen, u32 taglen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_CCM_INIT);
+ slot->hwc.sm4.cmd_ccm_init.key = key_ref;
+ slot->hwc.sm4.cmd_ccm_init.nonce = nonce_dma;
+ slot->hwc.sm4.cmd_ccm_init.keylen = keylen;
+ slot->hwc.sm4.cmd_ccm_init.noncelen = noncelen;
+ slot->hwc.sm4.cmd_ccm_init.op = op;
+ slot->hwc.sm4.cmd_ccm_init.aadlen = aadlen;
+ slot->hwc.sm4.cmd_ccm_init.iolen = iolen;
+ slot->hwc.sm4.cmd_ccm_init.taglen = taglen;
+}
+
+static void vcq_add_sm4_aad_final(struct vcq_cmd *slot, u32 core_id, u64 aad_dma,
+ u32 aadlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_AAD_FINAL);
+ slot->hwc.sm4.cmd_aad_final.data = aad_dma;
+ slot->hwc.sm4.cmd_aad_final.datalen = aadlen;
+}
+
+static void vcq_add_sm4_aead_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma,
+ u64 output_dma, u64 tag_dma,
+ u32 iolen, u32 taglen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_FINAL);
+ slot->hwc.sm4.cmd_final.input = input_dma;
+ slot->hwc.sm4.cmd_final.output = output_dma;
+ slot->hwc.sm4.cmd_final.tag = tag_dma;
+ slot->hwc.sm4.cmd_final.iolen = iolen;
+ slot->hwc.sm4.cmd_final.taglen = taglen;
+}
+
+/* setkey */
+static int cmh_sm4_aead_setkey(struct crypto_aead *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_sm4_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ /* SM4 always uses 128-bit keys */
+ if (keylen != CMH_SM4_KEY_SIZE)
+ return -EINVAL;
+
+ if (tctx->sw_cipher) {
+ int ret;
+
+ ret = crypto_cipher_setkey(tctx->sw_cipher, key, keylen);
+ if (ret)
+ return ret;
+ }
+
+ return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_SM4);
+}
+
+static int cmh_sm4_aead_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ struct cmh_sm4_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ const struct cmh_sm4_aead_info *info = cmh_sm4_aead_get_info(tfm);
+
+ if (info->type == CMH_SM4_AEAD_GCM) {
+ /* eSW enforces taglen == 16 for SM4 GCM (EIP40_SM4_TAG_SIZE) */
+ if (authsize != 16)
+ return -EINVAL;
+ } else {
+ /* CCM: accept 4, 6, 8, 10, 12, 14, 16 per RFC 3610 */
+ if (authsize < 4 || authsize > 16 || (authsize & 1))
+ return -EINVAL;
+ }
+
+ tctx->authsize = authsize;
+ return 0;
+}
+
+static int cmh_sm4_aead_init_tfm(struct crypto_aead *tfm)
+{
+ struct cmh_sm4_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ const struct cmh_sm4_aead_info *info = cmh_sm4_aead_get_info(tfm);
+
+ memset(tctx, 0, sizeof(*tctx));
+ tctx->authsize = info->maxauthsize;
+
+ if (info->type == CMH_SM4_AEAD_CCM) {
+ struct crypto_cipher *ci;
+
+ ci = crypto_alloc_cipher("sm4", 0, 0);
+ if (IS_ERR(ci))
+ return PTR_ERR(ci);
+ tctx->sw_cipher = ci;
+ }
+
+ crypto_aead_set_reqsize(tfm, sizeof(struct cmh_sm4_aead_reqctx));
+ return 0;
+}
+
+static void cmh_sm4_aead_exit_tfm(struct crypto_aead *tfm)
+{
+ struct cmh_sm4_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+
+ if (tctx->sw_cipher)
+ crypto_free_cipher(tctx->sw_cipher);
+ cmh_key_destroy(&tctx->key);
+}
+
+/* DMA unmap helper */
+static void cmh_sm4_aead_unmap_dma(struct cmh_sm4_aead_reqctx *rctx)
+{
+ u32 tag_map_len;
+
+ cmh_dma_unmap_single(rctx->iv_dma, rctx->iv_map_len, DMA_TO_DEVICE);
+ tag_map_len = rctx->empty_gcm_fallback ?
+ SM4_GCM_HW_IV_SIZE : rctx->authsize;
+ cmh_dma_unmap_single(rctx->tag_dma, tag_map_len,
+ (rctx->encrypting || rctx->empty_gcm_fallback) ?
+ DMA_FROM_DEVICE : DMA_TO_DEVICE);
+ if (rctx->cryptlen > 0) {
+ cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen,
+ DMA_FROM_DEVICE);
+ cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen,
+ DMA_TO_DEVICE);
+ }
+ if (rctx->assoclen > 0)
+ cmh_dma_unmap_single(rctx->aad_dma, rctx->assoclen,
+ DMA_TO_DEVICE);
+}
+
+static void cmh_sm4_aead_free_bufs(struct cmh_sm4_aead_reqctx *rctx)
+{
+ kfree(rctx->iv_buf);
+ rctx->iv_buf = NULL;
+ kfree(rctx->tag_buf);
+ rctx->tag_buf = NULL;
+ kfree_sensitive(rctx->out_buf);
+ rctx->out_buf = NULL;
+ kfree_sensitive(rctx->in_buf);
+ rctx->in_buf = NULL;
+ kfree(rctx->aad_buf);
+ rctx->aad_buf = NULL;
+}
+
+static void cmh_sm4_aead_complete(void *data, int error)
+{
+ struct aead_request *req = data;
+ struct cmh_sm4_aead_reqctx *rctx = aead_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ cmh_sm4_aead_unmap_dma(rctx);
+
+ /*
+ * Map HW error on decrypt to -EBADMSG. The eSW SM4 core uses a
+ * single error code (-EIO) for both authentication failures and
+ * other core errors (e.g. DMA timeout), so we cannot distinguish
+ * them from the MBX_STATUS alone. In practice the only error
+ * during a well-formed AEAD decrypt is auth-tag mismatch; a DMA
+ * timeout would indicate a fatal HW problem where -EBADMSG vs
+ * -EIO is moot. The kernel crypto API requires -EBADMSG for
+ * AEAD authentication failures.
+ */
+ if (error == -EIO && !rctx->encrypting)
+ error = -EBADMSG;
+
+ if (!error) {
+ if (rctx->empty_gcm_fallback && !rctx->encrypting) {
+ if (crypto_memneq(rctx->tag_buf, rctx->in_buf,
+ rctx->authsize))
+ error = -EBADMSG;
+ }
+ if (!error && rctx->cryptlen > 0)
+ scatterwalk_map_and_copy(rctx->out_buf, req->dst,
+ req->assoclen,
+ rctx->cryptlen, 1);
+ if (!error && rctx->encrypting)
+ scatterwalk_map_and_copy(rctx->tag_buf, req->dst,
+ req->assoclen +
+ rctx->cryptlen,
+ rctx->authsize, 1);
+ }
+
+ cmh_sm4_aead_free_bufs(rctx);
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * GCM empty-input fallback (SM4).
+ *
+ * When both AAD and plaintext are empty, GCM reduces to:
+ * tag = E(K, J0) where J0 = nonce || 0x00000001
+ *
+ * The eSW GCM engine rejects this degenerate case, so we compute it
+ * via a single ECB block encryption of J0.
+ *
+ * VCQ: [SYS_CMD_WRITE] + SM4_CMD_INIT(ECB) + SM4_CMD_FINAL + FLUSH
+ */
+static int cmh_sm4_gcm_empty(struct aead_request *req, u32 sm4_op)
+{
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct cmh_sm4_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ struct cmh_sm4_aead_reqctx *rctx = aead_request_ctx(req);
+ struct vcq_cmd cmds[CMH_SM4_AEAD_MAX_PAYLOAD];
+ u64 key_ref;
+ u32 keylen, authsize;
+ struct core_dispatch d;
+ s32 target_mbx;
+ u32 core_id;
+ u32 idx;
+ int ret;
+ gfp_t gfp;
+
+ authsize = tctx->authsize;
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->cryptlen = 0;
+ rctx->assoclen = 0;
+ rctx->authsize = authsize;
+ rctx->encrypting = (sm4_op == SM4_OP_ENCRYPT);
+ rctx->empty_gcm_fallback = true;
+
+ /* Build J0 = nonce || 0x00000001 in iv_buf */
+ rctx->iv_buf = kzalloc(SM4_GCM_HW_IV_SIZE, gfp);
+ if (!rctx->iv_buf)
+ return -ENOMEM;
+ memcpy(rctx->iv_buf, req->iv, SM4_GCM_IV_SIZE);
+ rctx->iv_buf[15] = 0x01;
+ rctx->iv_map_len = SM4_GCM_HW_IV_SIZE;
+
+ rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, SM4_GCM_HW_IV_SIZE,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->iv_dma)) {
+ ret = -ENOMEM;
+ goto out_free_iv;
+ }
+
+ /* Tag buffer -- receives E(K, J0) output */
+ rctx->tag_buf = kzalloc(SM4_GCM_HW_IV_SIZE, gfp);
+ if (!rctx->tag_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_iv;
+ }
+ rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, SM4_GCM_HW_IV_SIZE,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->tag_dma)) {
+ ret = -ENOMEM;
+ goto out_free_tag;
+ }
+
+ /* For decrypt: read expected tag from request */
+ if (!rctx->encrypting) {
+ rctx->in_buf = kmalloc(authsize, gfp);
+ if (!rctx->in_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_tag;
+ }
+ scatterwalk_map_and_copy(rctx->in_buf, req->src, 0,
+ authsize, 0);
+ }
+
+ /* Resolve key */
+ idx = 0;
+ rctx->key_dma = tctx->key.raw.dma;
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP,
+ (u64)rctx->key_dma, SYS_REF_NONE,
+ tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ keylen = tctx->key.raw.len;
+ d = cmh_core_select_instance(CMH_CORE_SM4);
+ target_mbx = d.mbx_idx;
+ core_id = d.core_id;
+
+ /* ECB INIT: single block encryption of J0 */
+ vcq_add_sm4_aead_init(&cmds[idx++], core_id, key_ref,
+ 0, keylen, 0, SM4_MODE_ECB, SM4_OP_ENCRYPT,
+ 0, SM4_GCM_HW_IV_SIZE);
+
+ /* FINAL: J0 in, E(K,J0) out */
+ vcq_add_sm4_aead_final(&cmds[idx++], core_id,
+ (u64)rctx->iv_dma, (u64)rctx->tag_dma,
+ 0, SM4_GCM_HW_IV_SIZE, 0);
+
+ vcq_add_flush(&cmds[idx++], core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_SM4_AEAD_MAX_PACKED,
+ target_mbx,
+ cmh_sm4_aead_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_free_in;
+
+ return -EINPROGRESS;
+
+out_free_in:
+ kfree_sensitive(rctx->in_buf);
+out_unmap_tag:
+ cmh_dma_unmap_single(rctx->tag_dma, SM4_GCM_HW_IV_SIZE,
+ DMA_FROM_DEVICE);
+out_free_tag:
+ kfree(rctx->tag_buf);
+out_unmap_iv:
+ cmh_dma_unmap_single(rctx->iv_dma, SM4_GCM_HW_IV_SIZE, DMA_TO_DEVICE);
+out_free_iv:
+ kfree(rctx->iv_buf);
+ return ret;
+}
+
+/*
+ * CCM empty-input fallback (SM4).
+ *
+ * When both AAD and plaintext are empty, CCM reduces to:
+ * T = E(K, B0) -- CBC-MAC of the single formatting block
+ * S0 = E(K, A0) -- CTR block zero
+ * tag = (T XOR S0)[0..authsize-1]
+ *
+ * The eSW rejects this degenerate case, so the driver computes it
+ * synchronously via two crypto_cipher single-block encryptions.
+ */
+static int cmh_sm4_ccm_empty(struct aead_request *req, u32 sm4_op)
+{
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct cmh_sm4_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ u32 authsize = tctx->authsize;
+ u8 b0[CMH_SM4_BLOCK_SIZE], a0[CMH_SM4_BLOCK_SIZE];
+ u8 t[CMH_SM4_BLOCK_SIZE], s0[CMH_SM4_BLOCK_SIZE];
+ u8 tag[CMH_SM4_BLOCK_SIZE];
+ u8 L;
+ u32 i;
+
+ /* Defense-in-depth: iv[0] = L-1, valid L is 2..8 per RFC 3610 S2.1 */
+ if (WARN_ON_ONCE(req->iv[0] < 1 || req->iv[0] > 7))
+ return -EINVAL;
+
+ L = req->iv[0] + 1;
+
+ if (tctx->key.mode != CMH_KEY_RAW)
+ return -EOPNOTSUPP;
+
+ /* B0: flags || nonce || Q(=0). Adata=0, t=authsize, q=L. */
+ memset(b0, 0, CMH_SM4_BLOCK_SIZE);
+ b0[0] = (u8)(8 * ((authsize - 2) / 2) + (L - 1));
+ memcpy(&b0[1], &req->iv[1], 15 - L);
+
+ /* A0: (L-1) || nonce || counter(=0) */
+ memset(a0, 0, CMH_SM4_BLOCK_SIZE);
+ a0[0] = (u8)(L - 1);
+ memcpy(&a0[1], &req->iv[1], 15 - L);
+
+ crypto_cipher_encrypt_one(tctx->sw_cipher, t, b0);
+ crypto_cipher_encrypt_one(tctx->sw_cipher, s0, a0);
+
+ for (i = 0; i < authsize; i++)
+ tag[i] = t[i] ^ s0[i];
+
+ if (sm4_op == SM4_OP_ENCRYPT) {
+ scatterwalk_map_and_copy(tag, req->dst,
+ req->assoclen, authsize, 1);
+ } else {
+ u8 expected[CMH_SM4_BLOCK_SIZE];
+
+ scatterwalk_map_and_copy(expected, req->src,
+ req->assoclen, authsize, 0);
+ if (crypto_memneq(tag, expected, authsize))
+ return -EBADMSG;
+ }
+
+ return 0;
+}
+
+static int cmh_sm4_aead_crypt(struct aead_request *req, u32 sm4_op)
+{
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct cmh_sm4_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm);
+ const struct cmh_sm4_aead_info *info = cmh_sm4_aead_get_info(tfm);
+ struct cmh_sm4_aead_reqctx *rctx = aead_request_ctx(req);
+ struct vcq_cmd cmds[CMH_SM4_AEAD_MAX_PAYLOAD];
+ u64 key_ref;
+ u32 keylen, authsize, cryptlen;
+ struct core_dispatch d;
+ s32 target_mbx;
+ u32 core_id;
+ u32 idx;
+ int ret;
+ gfp_t gfp;
+
+ if (tctx->key.mode == CMH_KEY_NONE)
+ return -ENOKEY;
+
+ authsize = tctx->authsize;
+
+ if (sm4_op == SM4_OP_ENCRYPT) {
+ cryptlen = req->cryptlen;
+ } else {
+ if (req->cryptlen < authsize)
+ return -EINVAL;
+ cryptlen = req->cryptlen - authsize;
+ }
+
+ /*
+ * Validate CCM IV format early -- the empty-input fallback and
+ * nonce extraction both depend on iv[0] being in range [1,7].
+ */
+ if (info->type == CMH_SM4_AEAD_CCM) {
+ if (req->iv[0] < 1 || req->iv[0] > 7)
+ return -EINVAL;
+ }
+
+ /*
+ * The CMH eSW rejects SM4 GCM/CCM when both aadlen and iolen
+ * are zero. For GCM, the tag is simply E(K, J0) -- use ECB
+ * fallback. For CCM, compute tag = E(K,B0) XOR E(K,A0) in SW.
+ */
+ if (cryptlen == 0 && req->assoclen == 0) {
+ if (info->type == CMH_SM4_AEAD_GCM)
+ return cmh_sm4_gcm_empty(req, sm4_op);
+ return cmh_sm4_ccm_empty(req, sm4_op);
+ }
+
+ /*
+ * HW uses a proprietary LLI scatter-gather format that is
+ * incompatible with struct scatterlist, so the payload is
+ * linearised into contiguous buffers for DMA. Cap total
+ * size to prevent excessive memory consumption.
+ */
+ if ((u64)cryptlen + req->assoclen > SZ_1M)
+ return -EINVAL;
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->cryptlen = cryptlen;
+ rctx->assoclen = req->assoclen;
+ rctx->authsize = authsize;
+ rctx->encrypting = (sm4_op == SM4_OP_ENCRYPT);
+
+ /* Linearise AAD */
+ if (req->assoclen > 0) {
+ rctx->aad_buf = kmalloc(req->assoclen, gfp);
+ if (!rctx->aad_buf)
+ return -ENOMEM;
+ scatterwalk_map_and_copy(rctx->aad_buf, req->src,
+ 0, req->assoclen, 0);
+ rctx->aad_dma = cmh_dma_map_single(rctx->aad_buf,
+ req->assoclen,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->aad_dma)) {
+ ret = -ENOMEM;
+ goto out_free_aad;
+ }
+ }
+
+ /* Linearise input */
+ if (cryptlen > 0) {
+ rctx->in_buf = kmalloc(cryptlen, gfp);
+ if (!rctx->in_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_aad;
+ }
+ scatterwalk_map_and_copy(rctx->in_buf, req->src,
+ req->assoclen, cryptlen, 0);
+ rctx->in_dma = cmh_dma_map_single(rctx->in_buf, cryptlen,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->in_dma)) {
+ ret = -ENOMEM;
+ goto out_free_in;
+ }
+ }
+
+ /* Allocate output buffer */
+ if (cryptlen > 0) {
+ rctx->out_buf = kmalloc(cryptlen, gfp);
+ if (!rctx->out_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_in;
+ }
+ rctx->out_dma = cmh_dma_map_single(rctx->out_buf, cryptlen,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->out_dma)) {
+ ret = -ENOMEM;
+ goto out_free_out;
+ }
+ }
+
+ /* Tag buffer */
+ rctx->tag_buf = kmalloc(authsize, gfp);
+ if (!rctx->tag_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_out;
+ }
+
+ if (!rctx->encrypting) {
+ scatterwalk_map_and_copy(rctx->tag_buf, req->src,
+ req->assoclen + cryptlen,
+ authsize, 0);
+ } else {
+ memset(rctx->tag_buf, 0, authsize);
+ }
+
+ rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, authsize,
+ rctx->encrypting ?
+ DMA_FROM_DEVICE : DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->tag_dma)) {
+ ret = -ENOMEM;
+ goto out_free_tag;
+ }
+
+ /* Map IV/nonce */
+ if (info->type == CMH_SM4_AEAD_GCM) {
+ rctx->iv_buf = kzalloc(SM4_GCM_HW_IV_SIZE, gfp);
+ if (!rctx->iv_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_tag;
+ }
+ memcpy(rctx->iv_buf, req->iv, SM4_GCM_IV_SIZE);
+ rctx->iv_map_len = SM4_GCM_HW_IV_SIZE;
+ rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf,
+ rctx->iv_map_len,
+ DMA_TO_DEVICE);
+ } else {
+ u32 noncelen;
+
+ if (req->iv[0] < 1 || req->iv[0] > 7) {
+ ret = -EINVAL;
+ goto out_unmap_tag;
+ }
+ noncelen = 14 - req->iv[0];
+
+ rctx->iv_buf = kmemdup(req->iv + 1, noncelen, gfp);
+ if (!rctx->iv_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_tag;
+ }
+ rctx->iv_map_len = noncelen;
+ rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf,
+ rctx->iv_map_len,
+ DMA_TO_DEVICE);
+ }
+ if (cmh_dma_map_error(rctx->iv_dma)) {
+ ret = -ENOMEM;
+ goto out_free_iv;
+ }
+
+ /* Resolve key reference */
+ idx = 0;
+
+ rctx->key_dma = tctx->key.raw.dma;
+ rctx->keylen = tctx->key.raw.len;
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP,
+ (u64)rctx->key_dma, SYS_REF_NONE,
+ tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ keylen = tctx->key.raw.len;
+ d = cmh_core_select_instance(CMH_CORE_SM4);
+ target_mbx = d.mbx_idx;
+ core_id = d.core_id;
+
+ /* Build INIT command */
+ if (info->type == CMH_SM4_AEAD_CCM) {
+ vcq_add_sm4_ccm_init(&cmds[idx++], core_id, key_ref,
+ (u64)rctx->iv_dma, keylen,
+ rctx->iv_map_len, sm4_op,
+ req->assoclen, cryptlen, authsize);
+ } else {
+ vcq_add_sm4_aead_init(&cmds[idx++], core_id, key_ref,
+ (u64)rctx->iv_dma, keylen,
+ SM4_GCM_HW_IV_SIZE, info->sm4_mode,
+ sm4_op, req->assoclen, cryptlen);
+ }
+
+ if (req->assoclen > 0)
+ vcq_add_sm4_aad_final(&cmds[idx++], core_id,
+ (u64)rctx->aad_dma, req->assoclen);
+
+ vcq_add_sm4_aead_final(&cmds[idx++], core_id,
+ cryptlen > 0 ? (u64)rctx->in_dma : 0,
+ cryptlen > 0 ? (u64)rctx->out_dma : 0,
+ (u64)rctx->tag_dma, cryptlen, authsize);
+
+ vcq_add_flush(&cmds[idx++], core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_SM4_AEAD_MAX_PACKED,
+ target_mbx,
+ cmh_sm4_aead_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_cleanup_all;
+
+ return -EINPROGRESS;
+
+out_cleanup_all:
+ cmh_dma_unmap_single(rctx->iv_dma, rctx->iv_map_len, DMA_TO_DEVICE);
+out_free_iv:
+ kfree(rctx->iv_buf);
+out_unmap_tag:
+ cmh_dma_unmap_single(rctx->tag_dma, authsize,
+ rctx->encrypting ? DMA_FROM_DEVICE :
+ DMA_TO_DEVICE);
+out_free_tag:
+ kfree(rctx->tag_buf);
+out_unmap_out:
+ if (cryptlen > 0)
+ cmh_dma_unmap_single(rctx->out_dma, cryptlen, DMA_FROM_DEVICE);
+out_free_out:
+ kfree_sensitive(rctx->out_buf);
+out_unmap_in:
+ if (cryptlen > 0)
+ cmh_dma_unmap_single(rctx->in_dma, cryptlen, DMA_TO_DEVICE);
+out_free_in:
+ kfree_sensitive(rctx->in_buf);
+out_unmap_aad:
+ if (req->assoclen > 0)
+ cmh_dma_unmap_single(rctx->aad_dma, req->assoclen,
+ DMA_TO_DEVICE);
+out_free_aad:
+ kfree(rctx->aad_buf);
+ return ret;
+}
+
+static int cmh_sm4_aead_encrypt(struct aead_request *req)
+{
+ return cmh_sm4_aead_crypt(req, SM4_OP_ENCRYPT);
+}
+
+static int cmh_sm4_aead_decrypt(struct aead_request *req)
+{
+ return cmh_sm4_aead_crypt(req, SM4_OP_DECRYPT);
+}
+
+/* Registration */
+
+static struct cmh_sm4_aead_drv sm4_aead_drv_algs[ARRAY_SIZE(sm4_aead_algs)];
+
+/**
+ * cmh_sm4_aead_register() - Register SM4-GCM/CCM AEAD algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_sm4_aead_register(void)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < ARRAY_SIZE(sm4_aead_algs); i++) {
+ const struct cmh_sm4_aead_info *info = &sm4_aead_algs[i];
+ struct cmh_sm4_aead_drv *drv = &sm4_aead_drv_algs[i];
+ struct aead_alg *alg = &drv->alg;
+
+ drv->info = info;
+
+ memset(alg, 0, sizeof(*alg));
+
+ alg->setkey = cmh_sm4_aead_setkey;
+ alg->setauthsize = cmh_sm4_aead_setauthsize;
+ alg->encrypt = cmh_sm4_aead_encrypt;
+ alg->decrypt = cmh_sm4_aead_decrypt;
+ alg->init = cmh_sm4_aead_init_tfm;
+ alg->exit = cmh_sm4_aead_exit_tfm;
+ alg->ivsize = info->ivsize;
+ alg->maxauthsize = info->maxauthsize;
+
+ strscpy(alg->base.cra_name, info->alg_name,
+ CRYPTO_MAX_ALG_NAME);
+ strscpy(alg->base.cra_driver_name, info->drv_name,
+ CRYPTO_MAX_ALG_NAME);
+ alg->base.cra_priority = 300;
+ alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC;
+ alg->base.cra_blocksize = 1;
+ alg->base.cra_ctxsize = sizeof(struct cmh_sm4_aead_tfm_ctx);
+ alg->base.cra_module = THIS_MODULE;
+
+ ret = crypto_register_aead(alg);
+ if (ret) {
+ dev_err(cmh_dev(), "cmh_sm4_aead: failed to register %s (rc=%d)\n",
+ info->alg_name, ret);
+ goto err_unregister;
+ }
+
+ dev_dbg(cmh_dev(), "cmh_sm4_aead: registered %s\n", info->alg_name);
+ }
+
+ return 0;
+
+err_unregister:
+ while (i--)
+ crypto_unregister_aead(&sm4_aead_drv_algs[i].alg);
+ return ret;
+}
+
+/**
+ * cmh_sm4_aead_unregister() - Unregister SM4 AEAD algorithms from the crypto framework
+ */
+void cmh_sm4_aead_unregister(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(sm4_aead_algs); i++) {
+ crypto_unregister_aead(&sm4_aead_drv_algs[i].alg);
+ dev_dbg(cmh_dev(), "cmh_sm4_aead: unregistered %s\n",
+ sm4_aead_algs[i].alg_name);
+ }
+}
diff --git a/drivers/crypto/cmh/cmh_sm4_cmac.c b/drivers/crypto/cmh/cmh_sm4_cmac.c
new file mode 100644
index 000000000000..e6c8ec6b70f7
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_sm4_cmac.c
@@ -0,0 +1,929 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API SM4-CMAC / SM4-XCBC (ahash) Driver
+ *
+ * Registers cmac(sm4) and xcbc(sm4) as ahash algorithms.
+ *
+ * Both produce a 16-byte tag (MAC) from a key and message.
+ * VCQ sequence: [SYS_CMD_WRITE] + SM4_CMD_INIT(CMAC/XCBC) +
+ * SM4_CMD_AAD_FINAL + SM4_CMD_FINAL + FLUSH
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/cipher.h>
+#include <crypto/internal/hash.h>
+#include <crypto/hash.h>
+#include <crypto/scatterwalk.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "cmh_sm4.h"
+#include "cmh_vcq.h"
+#include "cmh_sm4_abi.h"
+#include "cmh_sys_abi.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+#define SM4_MAC_DIGEST_SIZE 16U
+#define SM4_MAC_BLOCK_SIZE 16U
+/*
+ * Maximum accumulated data for SM4 MAC -- driver-imposed, not HW.
+ *
+ * The SM4 core does not expose external save/restore VCQ commands,
+ * so the driver must accumulate all data in kernel memory via
+ * .update() and submit it atomically in .final(). This cap limits
+ * the per-request kernel allocation.
+ */
+#define SM4_MAC_MAX_DATA (64 * 1024)
+
+struct cmh_sm4_mac_alg_info {
+ u32 sm4_mode; /* SM4_MODE_CMAC or SM4_MODE_XCBC */
+ const char *alg_name;
+ const char *drv_name;
+};
+
+static const struct cmh_sm4_mac_alg_info sm4_mac_algs[] = {
+ { SM4_MODE_CMAC, "cmac(sm4)", "rambus-cmh-cmac-sm4" },
+ { SM4_MODE_XCBC, "xcbc(sm4)", "rambus-cmh-xcbc-sm4" },
+};
+
+struct cmh_sm4_mac_tfm_ctx {
+ struct cmh_key_ctx key;
+ u32 sm4_mode;
+ struct crypto_cipher *sw_cipher; /* empty-input fallback (CMAC/XCBC) */
+ struct crypto_ahash *fb; /* generic SW fallback (oversized ops) */
+ /* Cached subkeys (derived at setkey time for concurrency safety) */
+ u8 xcbc_k1[CMH_SM4_BLOCK_SIZE]; /* K1 = E(K, 0x01..01) */
+ u8 xcbc_k3[CMH_SM4_BLOCK_SIZE]; /* K3 = E(K, 0x03..03) */
+ u8 cmac_k2[CMH_SM4_BLOCK_SIZE]; /* K2 = dbl(dbl(E(K, 0))) */
+ bool subkeys_valid;
+ spinlock_t chunk_lock; /* protects all_chunks */
+ struct list_head all_chunks; /* orphan-safe chunk tracking */
+};
+
+/* Per-request context (lives in ahash_request::__ctx) */
+/* Chunk node for O(1) update() appends */
+struct cmh_sm4_mac_chunk {
+ struct list_head list;
+ struct list_head tfm_node; /* per-tfm orphan tracking */
+ u32 len;
+ u8 data[];
+};
+
+/* Per-request context (lives in ahash_request::__ctx) */
+
+#define CMH_SM4_MAC_MAX_PAYLOAD 5
+#define CMH_SM4_MAC_MAX_PACKED (CMH_SM4_MAC_MAX_PAYLOAD * 2)
+
+struct cmh_sm4_mac_reqctx {
+ struct list_head chunks;
+ u32 total_len;
+ bool switched; /* handed off to SW fallback */
+ u8 *buf; /* linearised in final() */
+ /* DMA state for async final */
+ dma_addr_t key_dma;
+ dma_addr_t in_dma;
+ dma_addr_t tag_dma;
+ u8 *tag_buf;
+ u32 keylen;
+ struct vcq_cmd packed[CMH_SM4_MAC_MAX_PACKED];
+};
+
+/*
+ * Flat state for export/import, tagged by the leading @format byte:
+ * CMH_SM4_MAC_FMT_RAW is the accumulated input bytes (flat window);
+ * CMH_SM4_MAC_FMT_FB is the software fallback's own exported state,
+ * used once the request has switched to the fallback so export/import
+ * (transform clone) works at any input length.
+ */
+#define CMH_SM4_MAC_FMT_RAW 0
+#define CMH_SM4_MAC_FMT_FB 1
+
+struct cmh_sm4_mac_export_state {
+ u8 format;
+ u8 __pad[3];
+ u32 total_len;
+ u8 data[];
+};
+
+/*
+ * The crypto subsystem pre-allocates statesize bytes per request.
+ * CMH_SM4_MAC_STATE_SIZE (4096) sizes both the CMH_SM4_MAC_FMT_RAW
+ * window (CMH_SM4_MAC_EXPORT_MAX accumulated bytes) and the
+ * CMH_SM4_MAC_FMT_FB software state. A RAW export past
+ * CMH_SM4_MAC_EXPORT_MAX transparently switches to the fallback and
+ * emits CMH_SM4_MAC_FMT_FB instead, so export/import is not capped.
+ */
+#define CMH_SM4_MAC_STATE_SIZE 4096
+#define CMH_SM4_MAC_EXPORT_MAX \
+ (CMH_SM4_MAC_STATE_SIZE - sizeof(struct cmh_sm4_mac_export_state))
+
+struct cmh_sm4_mac_drv {
+ struct ahash_alg alg;
+ const struct cmh_sm4_mac_alg_info *info;
+};
+
+/*
+ * GF(2^128) doubling used to derive the CMAC subkeys (NIST SP 800-38B).
+ * Shift the 128-bit big-endian value left by one bit and, if the top bit
+ * was set, reduce with Rb = 0x87.
+ */
+static void cmh_sm4_cmac_dbl(u8 out[CMH_SM4_BLOCK_SIZE],
+ const u8 in[CMH_SM4_BLOCK_SIZE])
+{
+ u8 carry = in[0] >> 7;
+ unsigned int i;
+
+ for (i = 0; i < CMH_SM4_BLOCK_SIZE - 1; i++)
+ out[i] = (in[i] << 1) | (in[i + 1] >> 7);
+ out[CMH_SM4_BLOCK_SIZE - 1] = (in[CMH_SM4_BLOCK_SIZE - 1] << 1) ^
+ (carry ? 0x87 : 0x00);
+}
+
+static int cmh_sm4_mac_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ int ret;
+
+ if (keylen != CMH_SM4_KEY_SIZE)
+ return -EINVAL;
+
+ /*
+ * Program the HW key first; only (re)derive the software subkeys on
+ * success so a failed HW step cannot leave the SW subkeys (new)
+ * inconsistent with the HW key (old).
+ */
+ if (tctx->sw_cipher)
+ tctx->subkeys_valid = false;
+
+ ret = cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_SM4);
+ if (ret)
+ return ret;
+
+ if (tctx->sw_cipher && tctx->sm4_mode == SM4_MODE_XCBC) {
+ u8 const1[CMH_SM4_BLOCK_SIZE], const3[CMH_SM4_BLOCK_SIZE];
+
+ ret = crypto_cipher_setkey(tctx->sw_cipher, key, keylen);
+ if (ret)
+ return ret;
+
+ /* Pre-derive XCBC subkeys for concurrent-safe final() */
+ memset(const1, 0x01, CMH_SM4_BLOCK_SIZE);
+ memset(const3, 0x03, CMH_SM4_BLOCK_SIZE);
+ crypto_cipher_encrypt_one(tctx->sw_cipher, tctx->xcbc_k1,
+ const1);
+ crypto_cipher_encrypt_one(tctx->sw_cipher, tctx->xcbc_k3,
+ const3);
+
+ /*
+ * Leave sw_cipher keyed with K1 permanently.
+ * final() only needs E(K1, block) and never touches the
+ * original key again, so no re-keying in the hot path
+ * eliminates the per-tfm concurrency race entirely.
+ */
+ ret = crypto_cipher_setkey(tctx->sw_cipher, tctx->xcbc_k1,
+ CMH_SM4_BLOCK_SIZE);
+ if (ret)
+ return ret;
+ } else if (tctx->sw_cipher && tctx->sm4_mode == SM4_MODE_CMAC) {
+ u8 zero[CMH_SM4_BLOCK_SIZE] = { 0 };
+ u8 l[CMH_SM4_BLOCK_SIZE], k1[CMH_SM4_BLOCK_SIZE];
+
+ ret = crypto_cipher_setkey(tctx->sw_cipher, key, keylen);
+ if (ret)
+ return ret;
+
+ /*
+ * Pre-derive the CMAC subkey K2 for the empty-message
+ * fallback (NIST SP 800-38B):
+ * L = E(K, 0^128); K1 = dbl(L); K2 = dbl(K1)
+ * sw_cipher is left keyed with the original K, so final()
+ * computes E(K, K2 ^ pad) with no hot-path re-keying.
+ */
+ crypto_cipher_encrypt_one(tctx->sw_cipher, l, zero);
+ cmh_sm4_cmac_dbl(k1, l);
+ cmh_sm4_cmac_dbl(tctx->cmac_k2, k1);
+ memzero_explicit(l, sizeof(l));
+ memzero_explicit(k1, sizeof(k1));
+ }
+
+ if (tctx->sw_cipher)
+ tctx->subkeys_valid = true;
+
+ /* Keep the software fallback keyed in lock-step for oversized ops. */
+ return crypto_ahash_setkey(tctx->fb, key, keylen);
+}
+
+static void cmh_sm4_mac_free_chunks(struct cmh_sm4_mac_reqctx *rctx,
+ struct cmh_sm4_mac_tfm_ctx *tctx)
+{
+ struct cmh_sm4_mac_chunk *c, *tmp;
+
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(c, tmp, &rctx->chunks, list) {
+ list_del(&c->list);
+ list_del(&c->tfm_node);
+ kfree_sensitive(c);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->total_len = 0;
+}
+
+/* Software-fallback helpers (arbitrary-length + transform-clone support) */
+
+/*
+ * The fallback ahash_request lives immediately after the reqctx;
+ * cmh_sm4_mac_init_tfm() reserves crypto_ahash_reqsize(fb) bytes for it.
+ */
+static struct ahash_request *
+cmh_sm4_mac_fb_req(struct cmh_sm4_mac_reqctx *rctx)
+{
+ return PTR_ALIGN((void *)(rctx + 1), crypto_tfm_ctx_alignment());
+}
+
+static int cmh_sm4_mac_fb_update_virt(struct ahash_request *fb_req,
+ const u8 *data, u32 len)
+{
+ ahash_request_set_virt(fb_req, data, NULL, len);
+ return crypto_ahash_update(fb_req);
+}
+
+/*
+ * Switch a request from the HW-buffered path to the software fallback
+ * (the generic cmac(sm4)/xcbc(sm4) matching this transform): initialise
+ * the fallback request, replay every accumulated chunk through it, then
+ * drop the chunks. The fallback transform was keyed by
+ * cmh_sm4_mac_setkey() when the caller installed the MAC key.
+ */
+static int cmh_sm4_mac_switch_to_fb(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_sm4_mac_reqctx *rctx = ahash_request_ctx(req);
+ struct ahash_request *fb_req = cmh_sm4_mac_fb_req(rctx);
+ struct cmh_sm4_mac_chunk *c;
+ int ret;
+
+ ahash_request_set_tfm(fb_req, tctx->fb);
+ ahash_request_set_callback(fb_req, 0, NULL, NULL);
+
+ ret = crypto_ahash_init(fb_req);
+ if (ret)
+ return ret;
+
+ list_for_each_entry(c, &rctx->chunks, list) {
+ ret = cmh_sm4_mac_fb_update_virt(fb_req, c->data, c->len);
+ if (ret)
+ return ret;
+ }
+
+ cmh_sm4_mac_free_chunks(rctx, tctx);
+ rctx->switched = true;
+ return 0;
+}
+
+/* Forward the current update() payload to the fallback. */
+static int cmh_sm4_mac_fb_forward(struct ahash_request *req,
+ struct cmh_sm4_mac_reqctx *rctx)
+{
+ struct ahash_request *fb_req = cmh_sm4_mac_fb_req(rctx);
+
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ return cmh_sm4_mac_fb_update_virt(fb_req, req->svirt,
+ req->nbytes);
+ ahash_request_set_crypt(fb_req, req->src, NULL, req->nbytes);
+ return crypto_ahash_update(fb_req);
+}
+
+static int cmh_sm4_mac_init(struct ahash_request *req)
+{
+ struct cmh_sm4_mac_reqctx *rctx = ahash_request_ctx(req);
+
+ memset(rctx, 0, sizeof(*rctx));
+ INIT_LIST_HEAD(&rctx->chunks);
+ return 0;
+}
+
+static int cmh_sm4_mac_update(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_sm4_mac_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_sm4_mac_chunk *chunk;
+ gfp_t gfp;
+ int ret;
+
+ if (!req->nbytes)
+ return 0;
+
+ /* Already handed off to the fallback: forward directly (O(1) mem). */
+ if (rctx->switched)
+ return cmh_sm4_mac_fb_forward(req, rctx);
+
+ /*
+ * Exceeding the HW input cap: switch to the software fallback
+ * (replaying the buffered chunks) rather than failing, then
+ * forward this update.
+ */
+ if (req->nbytes > SM4_MAC_MAX_DATA - rctx->total_len) {
+ ret = cmh_sm4_mac_switch_to_fb(req);
+ if (ret)
+ goto err_free_chunks;
+ return cmh_sm4_mac_fb_forward(req, rctx);
+ }
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+ chunk = kmalloc(sizeof(*chunk) + req->nbytes, gfp);
+ if (!chunk) {
+ ret = -ENOMEM;
+ goto err_free_chunks;
+ }
+
+ chunk->len = req->nbytes;
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(chunk->data, req->svirt, req->nbytes);
+ else
+ scatterwalk_map_and_copy(chunk->data, req->src,
+ 0, req->nbytes, 0);
+ list_add_tail(&chunk->list, &rctx->chunks);
+ spin_lock_bh(&tctx->chunk_lock);
+ list_add_tail(&chunk->tfm_node, &tctx->all_chunks);
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->total_len += req->nbytes;
+ return 0;
+
+err_free_chunks:
+ /*
+ * Terminal error -- free all previously accumulated chunks.
+ * callers may not call .final() on error, so they would leak.
+ */
+ cmh_sm4_mac_free_chunks(rctx, tctx);
+ return ret;
+}
+
+static void cmh_sm4_mac_complete(void *data, int error)
+{
+ struct ahash_request *req = data;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_sm4_mac_reqctx *rctx = ahash_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ if (rctx->total_len > 0)
+ cmh_dma_unmap_single(rctx->in_dma, rctx->total_len,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->tag_dma, SM4_MAC_DIGEST_SIZE,
+ DMA_FROM_DEVICE);
+
+ if (!error)
+ memcpy(req->result, rctx->tag_buf, SM4_MAC_DIGEST_SIZE);
+
+ kfree(rctx->tag_buf);
+ rctx->tag_buf = NULL;
+ cmh_sm4_mac_free_chunks(rctx, tctx);
+ kfree_sensitive(rctx->buf);
+ rctx->buf = NULL;
+ rctx->total_len = 0;
+ cmh_complete(&req->base, error);
+}
+
+static int cmh_sm4_mac_final(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_sm4_mac_reqctx *rctx = ahash_request_ctx(req);
+ struct vcq_cmd cmds[CMH_SM4_MAC_MAX_PAYLOAD];
+ u64 key_ref;
+ u32 keylen;
+ struct core_dispatch d;
+ s32 target_mbx;
+ u32 core_id;
+ u32 idx;
+ int ret;
+ gfp_t gfp;
+
+ if (tctx->key.mode == CMH_KEY_NONE) {
+ ret = -ENOKEY;
+ goto out_free_chunks;
+ }
+
+ /*
+ * Switched to the generic software fallback (input over the HW cap,
+ * or an oversized clone was imported): complete there. Must precede
+ * the empty-input paths below -- a switch frees the chunks, so
+ * total_len is 0 even though data was streamed to the fallback.
+ */
+ if (rctx->switched) {
+ struct ahash_request *fb_req = cmh_sm4_mac_fb_req(rctx);
+
+ ahash_request_set_crypt(fb_req, NULL, req->result, 0);
+ return crypto_ahash_final(fb_req);
+ }
+
+ /*
+ * XCBC empty-input SW fallback (RFC 3566).
+ *
+ * For a zero-length message:
+ * K1 = E(K, 0x01010101...) -- encryption subkey
+ * K3 = E(K, 0x03030303...) -- incomplete-block subkey
+ * pad = 0x80 00...00 -- single 1 bit + 127 zero bits
+ * tag = E(K1, pad XOR K3)
+ *
+ * The eSW produces incorrect output for this case, so the driver
+ * computes it synchronously using crypto_cipher.
+ *
+ * For DS keys we cannot derive subkeys (no raw key material),
+ * and the HW also cannot handle empty XCBC correctly, so
+ * return -EOPNOTSUPP.
+ */
+ if (rctx->total_len == 0 && tctx->sm4_mode == SM4_MODE_XCBC) {
+ u8 block[CMH_SM4_BLOCK_SIZE];
+ u32 i;
+
+ if (tctx->key.mode != CMH_KEY_RAW ||
+ !tctx->subkeys_valid) {
+ cmh_sm4_mac_free_chunks(rctx, tctx);
+ return -EOPNOTSUPP;
+ }
+
+ /* block = pad XOR K3 */
+ memset(block, 0, CMH_SM4_BLOCK_SIZE);
+ block[0] = 0x80;
+ for (i = 0; i < CMH_SM4_BLOCK_SIZE; i++)
+ block[i] ^= tctx->xcbc_k3[i];
+
+ /*
+ * tag = E(K1, block)
+ *
+ * sw_cipher is permanently keyed with K1 (set at setkey
+ * time), so this is safe for concurrent requests sharing
+ * the same tfm -- no re-keying, no race.
+ */
+ crypto_cipher_encrypt_one(tctx->sw_cipher, req->result,
+ block);
+
+ cmh_sm4_mac_free_chunks(rctx, tctx);
+ return 0;
+ }
+
+ /*
+ * CMAC empty-input SW fallback (NIST SP 800-38B).
+ *
+ * For a zero-length message the sole block is incomplete, so the
+ * K2 subkey is used:
+ * pad = 0x80 00...00 -- single 1 bit + 127 zero bits
+ * tag = E(K, pad XOR K2)
+ *
+ * The eSW produces incorrect output for this case, so the driver
+ * computes it synchronously using crypto_cipher.
+ *
+ * For DS keys we cannot derive subkeys (no raw key material),
+ * and the HW also cannot handle empty CMAC correctly, so
+ * return -EOPNOTSUPP.
+ */
+ if (rctx->total_len == 0 && tctx->sm4_mode == SM4_MODE_CMAC) {
+ u8 block[CMH_SM4_BLOCK_SIZE];
+ u32 i;
+
+ if (tctx->key.mode != CMH_KEY_RAW || !tctx->subkeys_valid) {
+ cmh_sm4_mac_free_chunks(rctx, tctx);
+ return -EOPNOTSUPP;
+ }
+
+ /* block = pad XOR K2 */
+ memset(block, 0, CMH_SM4_BLOCK_SIZE);
+ block[0] = 0x80;
+ for (i = 0; i < CMH_SM4_BLOCK_SIZE; i++)
+ block[i] ^= tctx->cmac_k2[i];
+
+ /*
+ * tag = E(K, block). sw_cipher is keyed with the original
+ * key K (set at setkey time, never re-keyed), so this is
+ * safe for concurrent requests sharing the same tfm.
+ */
+ crypto_cipher_encrypt_one(tctx->sw_cipher, req->result,
+ block);
+
+ cmh_sm4_mac_free_chunks(rctx, tctx);
+ return 0;
+ }
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ /* Linearise chunks into a single contiguous buffer for DMA */
+ if (rctx->total_len > 0) {
+ struct cmh_sm4_mac_chunk *c;
+ u32 off = 0;
+
+ rctx->buf = kmalloc(rctx->total_len, gfp);
+ if (!rctx->buf) {
+ ret = -ENOMEM;
+ goto out_free_chunks;
+ }
+ list_for_each_entry(c, &rctx->chunks, list) {
+ memcpy(rctx->buf + off, c->data, c->len);
+ off += c->len;
+ }
+ }
+
+ rctx->tag_buf = kzalloc(SM4_MAC_DIGEST_SIZE, gfp);
+ if (!rctx->tag_buf) {
+ ret = -ENOMEM;
+ goto out_free_buf;
+ }
+
+ rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf,
+ SM4_MAC_DIGEST_SIZE,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->tag_dma)) {
+ ret = -ENOMEM;
+ goto out_free_tag;
+ }
+
+ if (rctx->total_len > 0) {
+ rctx->in_dma = cmh_dma_map_single(rctx->buf, rctx->total_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->in_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap_tag;
+ }
+ }
+
+ idx = 0;
+
+ rctx->key_dma = tctx->key.raw.dma;
+ rctx->keylen = tctx->key.raw.len;
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP,
+ (u64)rctx->key_dma, SYS_REF_NONE,
+ tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ keylen = tctx->key.raw.len;
+ d = cmh_core_select_instance(CMH_CORE_SM4);
+ target_mbx = d.mbx_idx;
+ core_id = d.core_id;
+
+ /*
+ * INIT: mode=CMAC or XCBC
+ * CMAC/XCBC data goes through the AAD path:
+ * aadlen = total data length, iolen = 0
+ */
+ {
+ struct vcq_cmd *slot = &cmds[idx++];
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_INIT);
+ slot->hwc.sm4.cmd_init.key = key_ref;
+ slot->hwc.sm4.cmd_init.iv = 0;
+ slot->hwc.sm4.cmd_init.keylen = keylen;
+ slot->hwc.sm4.cmd_init.ivlen = 0;
+ slot->hwc.sm4.cmd_init.mode = tctx->sm4_mode;
+ slot->hwc.sm4.cmd_init.op = SM4_OP_ENCRYPT;
+ slot->hwc.sm4.cmd_init.aadlen = rctx->total_len;
+ slot->hwc.sm4.cmd_init.iolen = 0;
+ }
+
+ /* AAD_FINAL: send data through the AAD path */
+ if (rctx->total_len > 0) {
+ struct vcq_cmd *slot = &cmds[idx++];
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_AAD_FINAL);
+ slot->hwc.sm4.cmd_aad_final.data = (u64)rctx->in_dma;
+ slot->hwc.sm4.cmd_aad_final.datalen = rctx->total_len;
+ }
+
+ /* FINAL: tag extraction only (no data) */
+ {
+ struct vcq_cmd *slot = &cmds[idx++];
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_FINAL);
+ slot->hwc.sm4.cmd_final.input = 0;
+ slot->hwc.sm4.cmd_final.output = 0;
+ slot->hwc.sm4.cmd_final.tag = (u64)rctx->tag_dma;
+ slot->hwc.sm4.cmd_final.iolen = 0;
+ slot->hwc.sm4.cmd_final.taglen = SM4_MAC_DIGEST_SIZE;
+ }
+
+ vcq_add_flush(&cmds[idx++], core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_SM4_MAC_MAX_PACKED,
+ target_mbx,
+ cmh_sm4_mac_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_cleanup_all;
+
+ return -EINPROGRESS;
+
+out_cleanup_all:
+ if (rctx->total_len > 0 && !cmh_dma_map_error(rctx->in_dma))
+ cmh_dma_unmap_single(rctx->in_dma, rctx->total_len,
+ DMA_TO_DEVICE);
+out_unmap_tag:
+ cmh_dma_unmap_single(rctx->tag_dma, SM4_MAC_DIGEST_SIZE,
+ DMA_FROM_DEVICE);
+out_free_tag:
+ kfree(rctx->tag_buf);
+out_free_buf:
+ kfree_sensitive(rctx->buf);
+ rctx->buf = NULL;
+out_free_chunks:
+ cmh_sm4_mac_free_chunks(rctx, tctx);
+ rctx->total_len = 0;
+ return ret;
+}
+
+/*
+ * ahash .export()/.import(): serialise/deserialise the software
+ * accumulation buffer, or the generic fallback's own state once the
+ * request has switched to it (oversized input or clone).
+ */
+
+static int cmh_sm4_mac_export(struct ahash_request *req, void *out)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_sm4_mac_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_sm4_mac_export_state *state = out;
+ struct cmh_sm4_mac_chunk *chunk;
+ u32 offset = 0;
+ int ret;
+
+ /*
+ * If more data is buffered than the flat window holds, switch to
+ * the software fallback so a bounded, fixed-size state can be
+ * exported -- making export/import (clone) work at any length.
+ */
+ if (!rctx->switched && rctx->total_len > CMH_SM4_MAC_EXPORT_MAX) {
+ ret = cmh_sm4_mac_switch_to_fb(req);
+ if (ret)
+ return ret;
+ }
+
+ /* Zero the whole state buffer so no kernel memory leaks out. */
+ memset(state, 0, crypto_ahash_statesize(tfm));
+
+ if (rctx->switched) {
+ state->format = CMH_SM4_MAC_FMT_FB;
+ return crypto_ahash_export(cmh_sm4_mac_fb_req(rctx),
+ state->data);
+ }
+
+ state->format = CMH_SM4_MAC_FMT_RAW;
+ state->total_len = rctx->total_len;
+ list_for_each_entry(chunk, &rctx->chunks, list) {
+ memcpy(state->data + offset, chunk->data, chunk->len);
+ offset += chunk->len;
+ }
+ return 0;
+}
+
+static int cmh_sm4_mac_import(struct ahash_request *req, const void *in)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_sm4_mac_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_sm4_mac_export_state *state = in;
+ struct cmh_sm4_mac_chunk *chunk;
+
+ /*
+ * Do NOT call free_chunks() here: the crypto API does not
+ * guarantee the request context is in a valid state before
+ * import(), so the list pointers may be stale or invalid.
+ * Re-initialize from scratch instead. Any pre-existing chunks
+ * are tracked on tctx->all_chunks and freed in exit_tfm.
+ */
+ memset(rctx, 0, sizeof(*rctx));
+ INIT_LIST_HEAD(&rctx->chunks);
+
+ /* Fallback-format state: replay it into a fallback request. */
+ if (state->format == CMH_SM4_MAC_FMT_FB) {
+ struct ahash_request *fb_req = cmh_sm4_mac_fb_req(rctx);
+ int ret;
+
+ ahash_request_set_tfm(fb_req, tctx->fb);
+ ahash_request_set_callback(fb_req, 0, NULL, NULL);
+ ret = crypto_ahash_import(fb_req, state->data);
+ if (ret)
+ return ret;
+ rctx->switched = true;
+ return 0;
+ }
+
+ if (state->format != CMH_SM4_MAC_FMT_RAW)
+ return -EINVAL;
+
+ if (state->total_len > CMH_SM4_MAC_EXPORT_MAX)
+ return -EINVAL;
+
+ if (state->total_len) {
+ chunk = kmalloc(sizeof(*chunk) + state->total_len,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC);
+ if (!chunk)
+ return -ENOMEM;
+ chunk->len = state->total_len;
+ memcpy(chunk->data, state->data, state->total_len);
+ list_add_tail(&chunk->list, &rctx->chunks);
+ spin_lock_bh(&tctx->chunk_lock);
+ list_add_tail(&chunk->tfm_node, &tctx->all_chunks);
+ spin_unlock_bh(&tctx->chunk_lock);
+ rctx->total_len = state->total_len;
+ }
+ return 0;
+}
+
+static int cmh_sm4_mac_finup(struct ahash_request *req)
+{
+ int err;
+
+ err = cmh_sm4_mac_update(req);
+ if (err)
+ return err;
+ return cmh_sm4_mac_final(req);
+}
+
+static int cmh_sm4_mac_digest(struct ahash_request *req)
+{
+ int err;
+
+ err = cmh_sm4_mac_init(req);
+ if (err)
+ return err;
+ return cmh_sm4_mac_finup(req);
+}
+
+/* Registration */
+
+static struct cmh_sm4_mac_drv sm4_mac_drv_algs[ARRAY_SIZE(sm4_mac_algs)];
+
+static int cmh_sm4_mac_init_tfm(struct crypto_ahash *tfm)
+{
+ struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct ahash_alg *alg = crypto_ahash_alg(tfm);
+ struct cmh_sm4_mac_drv *drv =
+ container_of(alg, struct cmh_sm4_mac_drv, alg);
+ struct crypto_ahash *fb;
+
+ memset(tctx, 0, sizeof(*tctx));
+ tctx->sm4_mode = drv->info->sm4_mode;
+ spin_lock_init(&tctx->chunk_lock);
+ INIT_LIST_HEAD(&tctx->all_chunks);
+
+ /* Allocate SW cipher for the CMAC/XCBC empty-input fallback */
+ if (tctx->sm4_mode == SM4_MODE_XCBC ||
+ tctx->sm4_mode == SM4_MODE_CMAC) {
+ struct crypto_cipher *ci;
+
+ ci = crypto_alloc_cipher("sm4", 0, 0);
+ if (IS_ERR(ci))
+ return PTR_ERR(ci);
+ tctx->sw_cipher = ci;
+ }
+
+ /*
+ * Generic software fallback for oversized input / clone. The
+ * generic cmac(sm4)/xcbc(sm4) has no core export/import state, so it
+ * cannot serve as a CRYPTO_ALG_NEED_FALLBACK fallback -- allocate it
+ * explicitly (masking out CRYPTO_ALG_ASYNC to exclude this driver).
+ */
+ fb = crypto_alloc_ahash(crypto_ahash_alg_name(tfm), 0,
+ CRYPTO_ALG_ASYNC);
+ if (IS_ERR(fb)) {
+ if (tctx->sw_cipher)
+ crypto_free_cipher(tctx->sw_cipher);
+ tctx->sw_cipher = NULL;
+ return PTR_ERR(fb);
+ }
+ tctx->fb = fb;
+
+ crypto_ahash_set_reqsize(tfm,
+ sizeof(struct cmh_sm4_mac_reqctx) +
+ crypto_tfm_ctx_alignment() +
+ sizeof(struct ahash_request) +
+ crypto_ahash_reqsize(fb));
+ return 0;
+}
+
+static void cmh_sm4_mac_exit_tfm(struct crypto_ahash *tfm)
+{
+ struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct cmh_sm4_mac_chunk *c, *tmp;
+
+ /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */
+ spin_lock_bh(&tctx->chunk_lock);
+ list_for_each_entry_safe(c, tmp, &tctx->all_chunks, tfm_node) {
+ list_del(&c->tfm_node);
+ kfree_sensitive(c);
+ }
+ spin_unlock_bh(&tctx->chunk_lock);
+
+ if (tctx->sw_cipher)
+ crypto_free_cipher(tctx->sw_cipher);
+ if (tctx->fb)
+ crypto_free_ahash(tctx->fb);
+ memzero_explicit(tctx->xcbc_k1, sizeof(tctx->xcbc_k1));
+ memzero_explicit(tctx->xcbc_k3, sizeof(tctx->xcbc_k3));
+ memzero_explicit(tctx->cmac_k2, sizeof(tctx->cmac_k2));
+ cmh_key_destroy(&tctx->key);
+}
+
+/**
+ * cmh_sm4_cmac_register() - Register SM4-CMAC/XCBC hash algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_sm4_cmac_register(void)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < ARRAY_SIZE(sm4_mac_algs); i++) {
+ const struct cmh_sm4_mac_alg_info *info = &sm4_mac_algs[i];
+ struct cmh_sm4_mac_drv *drv = &sm4_mac_drv_algs[i];
+ struct ahash_alg *alg = &drv->alg;
+
+ drv->info = info;
+
+ memset(alg, 0, sizeof(*alg));
+
+ alg->init = cmh_sm4_mac_init;
+ alg->update = cmh_sm4_mac_update;
+ alg->final = cmh_sm4_mac_final;
+ alg->finup = cmh_sm4_mac_finup;
+ alg->digest = cmh_sm4_mac_digest;
+ alg->export = cmh_sm4_mac_export;
+ alg->import = cmh_sm4_mac_import;
+ alg->setkey = cmh_sm4_mac_setkey;
+ alg->init_tfm = cmh_sm4_mac_init_tfm;
+ alg->exit_tfm = cmh_sm4_mac_exit_tfm;
+
+ alg->halg.digestsize = SM4_MAC_DIGEST_SIZE;
+ alg->halg.statesize = CMH_SM4_MAC_STATE_SIZE;
+
+ strscpy(alg->halg.base.cra_name, info->alg_name,
+ CRYPTO_MAX_ALG_NAME);
+ strscpy(alg->halg.base.cra_driver_name, info->drv_name,
+ CRYPTO_MAX_ALG_NAME);
+ alg->halg.base.cra_priority = 300;
+ alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NO_FALLBACK |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_REQ_VIRT;
+ alg->halg.base.cra_blocksize = SM4_MAC_BLOCK_SIZE;
+ alg->halg.base.cra_ctxsize = sizeof(struct cmh_sm4_mac_tfm_ctx);
+ alg->halg.base.cra_module = THIS_MODULE;
+
+ ret = crypto_register_ahash(alg);
+ if (ret) {
+ dev_err(cmh_dev(), "cmh_sm4_mac: failed to register %s (rc=%d)\n",
+ info->alg_name, ret);
+ goto err_unregister;
+ }
+
+ dev_dbg(cmh_dev(), "cmh_sm4_mac: registered %s\n",
+ info->alg_name);
+ }
+
+ return 0;
+
+err_unregister:
+ while (i--)
+ crypto_unregister_ahash(&sm4_mac_drv_algs[i].alg);
+ return ret;
+}
+
+/**
+ * cmh_sm4_cmac_unregister() - Unregister SM4 MAC hash algorithms from the crypto framework
+ */
+void cmh_sm4_cmac_unregister(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(sm4_mac_algs); i++) {
+ crypto_unregister_ahash(&sm4_mac_drv_algs[i].alg);
+ dev_dbg(cmh_dev(), "cmh_sm4_mac: unregistered %s\n",
+ sm4_mac_algs[i].alg_name);
+ }
+}
diff --git a/drivers/crypto/cmh/cmh_sm4_skcipher.c b/drivers/crypto/cmh/cmh_sm4_skcipher.c
new file mode 100644
index 000000000000..5168ad12716e
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_sm4_skcipher.c
@@ -0,0 +1,695 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API SM4 (skcipher) Driver
+ *
+ * Registers skcipher algorithms with the Linux crypto subsystem:
+ * ecb(sm4), cbc(sm4), ctr(sm4), cfb(sm4), xts(sm4)
+ *
+ * Uses the CMH SM4 Core via VCQ commands:
+ * [SYS_CMD_WRITE] + SM4_CMD_INIT + SM4_CMD_FINAL + VCQ_CMD_FLUSH
+ *
+ * The SM4 core requires bidirectional DMA -- both input and output
+ * buffers are mapped and passed in a single SM4_CMD_FINAL command.
+ *
+ * Raw-key atomicity: SYS_CMD_WRITE to SYS_REF_TEMP is packed into
+ * the same VCQ as SM4 commands (see cmh_key.h for details).
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/algapi.h>
+#include <crypto/xts.h>
+#include <crypto/scatterwalk.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/unaligned.h>
+
+#include "cmh_sm4.h"
+#include "cmh_vcq.h"
+#include "cmh_sm4_abi.h"
+#include "cmh_sys_abi.h"
+#include "cmh_sys.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+/* Algorithm Table */
+
+struct cmh_sm4_alg_info {
+ u32 sm4_mode; /* SM4_MODE_* */
+ u32 ivsize; /* bytes (0 for ECB) */
+ u32 min_keysize;
+ u32 max_keysize;
+ const char *alg_name; /* Linux crypto name: "ecb(sm4)" */
+ const char *drv_name; /* driver name: "rambus-cmh-ecb-sm4" */
+};
+
+static const struct cmh_sm4_alg_info sm4_algs[] = {
+ { SM4_MODE_ECB, 0, CMH_SM4_KEY_SIZE, CMH_SM4_KEY_SIZE,
+ "ecb(sm4)", "rambus-cmh-ecb-sm4" },
+ { SM4_MODE_CBC, CMH_SM4_IV_SIZE, CMH_SM4_KEY_SIZE, CMH_SM4_KEY_SIZE,
+ "cbc(sm4)", "rambus-cmh-cbc-sm4" },
+ { SM4_MODE_CTR, CMH_SM4_IV_SIZE, CMH_SM4_KEY_SIZE, CMH_SM4_KEY_SIZE,
+ "ctr(sm4)", "rambus-cmh-ctr-sm4" },
+ { SM4_MODE_CFB, CMH_SM4_IV_SIZE, CMH_SM4_KEY_SIZE, CMH_SM4_KEY_SIZE,
+ "cfb(sm4)", "rambus-cmh-cfb-sm4" },
+ { SM4_MODE_XTS, CMH_SM4_IV_SIZE, CMH_SM4_KEY_SIZE * 2,
+ CMH_SM4_KEY_SIZE * 2,
+ "xts(sm4)", "rambus-cmh-xts-sm4" },
+};
+
+/* Per-transform context (allocated by crypto framework) */
+
+struct cmh_sm4_tfm_ctx {
+ struct cmh_key_ctx key;
+};
+
+/* Per-request context (lives in skcipher_request::__ctx) */
+
+/*
+ * Maximum payload commands:
+ * [SYS_CMD_WRITE] + SM4_CMD_INIT + [SM4_CMD_UPDATE] + SM4_CMD_FINAL
+ * + VCQ_CMD_FLUSH = 5
+ * UPDATE is used for XTS data > 2 blocks (see cmh_sm4_crypt).
+ */
+#define CMH_SM4_MAX_PAYLOAD 5
+#define CMH_SM4_MAX_PACKED (CMH_SM4_MAX_PAYLOAD * 2)
+
+struct cmh_sm4_reqctx {
+ dma_addr_t in_dma;
+ dma_addr_t out_dma;
+ dma_addr_t iv_dma;
+ dma_addr_t iv2_dma;
+ dma_addr_t key_dma;
+ u8 *in_buf;
+ u8 *out_buf;
+ u8 *iv_buf;
+ u8 *iv2_buf;
+ u32 cryptlen;
+ u32 ivsize;
+ u32 keylen;
+ u32 sm4_mode;
+ u32 sm4_op;
+ /* CTR counter-wrap split state */
+ u32 ctr_chunk1_len;
+ u32 core_id;
+ s32 target_mbx;
+ u64 key_ref;
+ struct vcq_cmd packed[CMH_SM4_MAX_PACKED];
+};
+
+/* VCQ Builders -- SM4-specific */
+
+static void vcq_add_sm4_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, u64 iv_dma,
+ u32 keylen, u32 ivlen, u32 mode, u32 op,
+ u32 iolen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_INIT);
+ slot->hwc.sm4.cmd_init.key = key_ref;
+ slot->hwc.sm4.cmd_init.iv = iv_dma;
+ slot->hwc.sm4.cmd_init.keylen = keylen;
+ slot->hwc.sm4.cmd_init.ivlen = ivlen;
+ slot->hwc.sm4.cmd_init.mode = mode;
+ slot->hwc.sm4.cmd_init.op = op;
+ slot->hwc.sm4.cmd_init.aadlen = 0;
+ slot->hwc.sm4.cmd_init.iolen = iolen;
+}
+
+static void vcq_add_sm4_update(struct vcq_cmd *slot, u32 core_id, u64 input_dma,
+ u64 output_dma, u32 iolen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_UPDATE);
+ slot->hwc.sm4.cmd_update.input = input_dma;
+ slot->hwc.sm4.cmd_update.output = output_dma;
+ slot->hwc.sm4.cmd_update.iolen = iolen;
+}
+
+static void vcq_add_sm4_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma,
+ u64 output_dma, u32 iolen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_FINAL);
+ slot->hwc.sm4.cmd_final.input = input_dma;
+ slot->hwc.sm4.cmd_final.output = output_dma;
+ slot->hwc.sm4.cmd_final.iolen = iolen;
+ slot->hwc.sm4.cmd_final.tag = 0;
+ slot->hwc.sm4.cmd_final.taglen = 0;
+}
+
+/*
+ * We wrap each skcipher_alg with its info pointer in a compound struct,
+ * then use container_of() in cmh_sm4_get_info() to recover it.
+ */
+struct cmh_sm4_alg_drv {
+ struct skcipher_alg alg;
+ const struct cmh_sm4_alg_info *info;
+};
+
+static bool sm4_is_stream_mode(u32 mode)
+{
+ return mode == SM4_MODE_CTR || mode == SM4_MODE_CFB;
+}
+
+/*
+ * Update req->iv after a successful encrypt/decrypt.
+ * Same semantics as cmh_aes_update_iv -- see cmh_aes.c.
+ */
+static void cmh_sm4_update_iv(struct skcipher_request *req, u32 mode,
+ u32 op, const u8 *in_buf, const u8 *out_buf)
+{
+ u32 bs = CMH_SM4_BLOCK_SIZE;
+ u32 nblocks;
+
+ switch (mode) {
+ case SM4_MODE_CBC:
+ if (op == SM4_OP_ENCRYPT)
+ memcpy(req->iv, out_buf + req->cryptlen - bs, bs);
+ else
+ memcpy(req->iv, in_buf + req->cryptlen - bs, bs);
+ break;
+ case SM4_MODE_CTR:
+ /* Arithmetic big-endian 128-bit counter increment */
+ nblocks = DIV_ROUND_UP(req->cryptlen, bs);
+ {
+ u8 *iv = req->iv;
+ int i;
+
+ for (i = bs - 1; i >= 0 && nblocks; i--) {
+ u32 sum = (u32)iv[i] + (nblocks & 0xff);
+
+ iv[i] = (u8)sum;
+ nblocks = (nblocks >> 8) + (sum >> 8);
+ }
+ }
+ break;
+ case SM4_MODE_CFB:
+ /*
+ * For sub-block requests (cryptlen < 16), there is no
+ * complete ciphertext block to chain, so the IV is left
+ * unchanged -- CFB-128 has no defined chaining semantic
+ * for partial blocks (shift-register CFB-n is a different
+ * mode). Without this guard the pointer arithmetic
+ * underflows and reads before the buffer.
+ */
+ if (req->cryptlen >= bs) {
+ if (op == SM4_OP_ENCRYPT)
+ memcpy(req->iv, out_buf + req->cryptlen - bs,
+ bs);
+ else
+ memcpy(req->iv, in_buf + req->cryptlen - bs,
+ bs);
+ }
+ break;
+ default:
+ break;
+ }
+}
+
+/* skcipher Operations */
+
+static const struct cmh_sm4_alg_info *
+cmh_sm4_get_info(struct crypto_skcipher *tfm)
+{
+ struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
+
+ return container_of(alg, struct cmh_sm4_alg_drv, alg)->info;
+}
+
+static int cmh_sm4_setkey(struct crypto_skcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct cmh_sm4_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+ const struct cmh_sm4_alg_info *info = cmh_sm4_get_info(tfm);
+
+ if (info->sm4_mode == SM4_MODE_XTS) {
+ int err;
+
+ /* XTS: double key (32 bytes) */
+ if (keylen != CMH_SM4_KEY_SIZE * 2)
+ return -EINVAL;
+ err = xts_verify_key(tfm, key, keylen);
+ if (err)
+ return err;
+ } else {
+ /* SM4 always uses 128-bit (16-byte) keys */
+ if (keylen != CMH_SM4_KEY_SIZE)
+ return -EINVAL;
+ }
+
+ return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_SM4);
+}
+
+static int cmh_sm4_init_tfm(struct crypto_skcipher *tfm)
+{
+ struct cmh_sm4_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+
+ memset(tctx, 0, sizeof(*tctx));
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct cmh_sm4_reqctx));
+ return 0;
+}
+
+static void cmh_sm4_exit_tfm(struct crypto_skcipher *tfm)
+{
+ struct cmh_sm4_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+
+ cmh_key_destroy(&tctx->key);
+}
+
+#define CMH_SM4_MAX_CRYPTLEN SZ_32M
+
+/* DMA unmap helper */
+static void cmh_sm4_unmap_dma(struct cmh_sm4_reqctx *rctx)
+{
+ if (rctx->iv2_buf)
+ cmh_dma_unmap_single(rctx->iv2_dma, rctx->ivsize,
+ DMA_TO_DEVICE);
+ if (rctx->ivsize > 0)
+ cmh_dma_unmap_single(rctx->iv_dma, rctx->ivsize,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen, DMA_FROM_DEVICE);
+ cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen, DMA_TO_DEVICE);
+}
+
+static void cmh_sm4_free_bufs(struct cmh_sm4_reqctx *rctx)
+{
+ kfree(rctx->iv2_buf);
+ rctx->iv2_buf = NULL;
+ kfree(rctx->iv_buf);
+ rctx->iv_buf = NULL;
+ kfree_sensitive(rctx->out_buf);
+ rctx->out_buf = NULL;
+ kfree_sensitive(rctx->in_buf);
+ rctx->in_buf = NULL;
+}
+
+/*
+ * Submit the second CTR chunk after the first completes.
+ * Called from cmh_sm4_complete when ctr_chunk1_len > 0.
+ */
+static int cmh_sm4_ctr_submit_chunk2(struct skcipher_request *req);
+
+static void cmh_sm4_complete(void *data, int error)
+{
+ struct skcipher_request *req = data;
+ struct cmh_sm4_reqctx *rctx = skcipher_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ /*
+ * CTR counter-wrap: first chunk completed, submit second.
+ * DMA mappings remain valid (they cover the full buffer).
+ *
+ * Recursion depth bounded: chunk2 clears ctr_chunk1_len before
+ * submission, so the second cmh_sm4_complete invocation sees 0
+ * and finalizes (max depth = 2).
+ */
+ if (rctx->ctr_chunk1_len && !error) {
+ int ret = cmh_sm4_ctr_submit_chunk2(req);
+
+ if (!ret || ret == -EBUSY)
+ return;
+ /* Submission failed; clean up below */
+ error = ret;
+ }
+
+ cmh_sm4_unmap_dma(rctx);
+
+ if (!error) {
+ scatterwalk_map_and_copy(rctx->out_buf, req->dst,
+ 0, rctx->cryptlen, 1);
+ cmh_sm4_update_iv(req, rctx->sm4_mode, rctx->sm4_op,
+ rctx->in_buf, rctx->out_buf);
+ }
+
+ cmh_sm4_free_bufs(rctx);
+ cmh_complete(&req->base, error);
+}
+
+static int cmh_sm4_ctr_submit_chunk2(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct cmh_sm4_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+ struct cmh_sm4_reqctx *rctx = skcipher_request_ctx(req);
+ struct vcq_cmd cmds[CMH_SM4_MAX_PAYLOAD];
+ u32 chunk1 = rctx->ctr_chunk1_len;
+ u32 chunk2 = rctx->cryptlen - chunk1;
+ u64 key_ref;
+ u32 keylen;
+ u32 idx = 0;
+
+ /* Clear split flag so next completion is final */
+ rctx->ctr_chunk1_len = 0;
+
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP,
+ (u64)rctx->key_dma, SYS_REF_NONE,
+ tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ keylen = tctx->key.raw.len;
+
+ vcq_add_sm4_init(&cmds[idx++], rctx->core_id, key_ref,
+ (u64)rctx->iv2_dma, keylen, rctx->ivsize,
+ rctx->sm4_mode, rctx->sm4_op, chunk2);
+ vcq_add_sm4_final(&cmds[idx++], rctx->core_id,
+ (u64)(rctx->in_dma + chunk1),
+ (u64)(rctx->out_dma + chunk1), chunk2);
+ vcq_add_flush(&cmds[idx++], rctx->core_id);
+
+ return cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_SM4_MAX_PACKED,
+ rctx->target_mbx,
+ cmh_sm4_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+}
+
+static int cmh_sm4_crypt(struct skcipher_request *req, u32 sm4_op)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct cmh_sm4_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+ const struct cmh_sm4_alg_info *info = cmh_sm4_get_info(tfm);
+ struct cmh_sm4_reqctx *rctx = skcipher_request_ctx(req);
+ struct vcq_cmd cmds[CMH_SM4_MAX_PAYLOAD];
+ u64 key_ref;
+ u32 keylen;
+ struct core_dispatch d;
+ s32 target_mbx;
+ u32 core_id;
+ u32 idx;
+ int ret;
+ gfp_t gfp;
+
+ if (tctx->key.mode == CMH_KEY_NONE)
+ return -ENOKEY;
+
+ if (!req->cryptlen)
+ return 0;
+
+ if (req->cryptlen > CMH_SM4_MAX_CRYPTLEN)
+ return -EINVAL;
+
+ switch (info->sm4_mode) {
+ case SM4_MODE_CTR:
+ case SM4_MODE_CFB:
+ break;
+ case SM4_MODE_XTS:
+ if (req->cryptlen < CMH_SM4_BLOCK_SIZE)
+ return -EINVAL;
+ break;
+ default:
+ if (req->cryptlen & (CMH_SM4_BLOCK_SIZE - 1))
+ return -EINVAL;
+ break;
+ }
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->cryptlen = req->cryptlen;
+ rctx->ivsize = info->ivsize;
+ rctx->sm4_mode = info->sm4_mode;
+ rctx->sm4_op = sm4_op;
+ rctx->iv2_buf = NULL;
+
+ /*
+ * cryptlen is user-controlled up to CMH_SM4_MAX_CRYPTLEN (well above
+ * KMALLOC_MAX_SIZE), so use __GFP_NOWARN: an oversized request fails
+ * cleanly with -ENOMEM instead of splatting the page allocator.
+ */
+ rctx->in_buf = kmalloc(req->cryptlen, gfp | __GFP_NOWARN);
+ if (!rctx->in_buf)
+ return -ENOMEM;
+
+ scatterwalk_map_and_copy(rctx->in_buf, req->src, 0, req->cryptlen, 0);
+
+ rctx->in_dma = cmh_dma_map_single(rctx->in_buf, req->cryptlen,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->in_dma)) {
+ ret = -ENOMEM;
+ goto out_free_in;
+ }
+
+ rctx->out_buf = kmalloc(req->cryptlen, gfp | __GFP_NOWARN);
+ if (!rctx->out_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_in;
+ }
+
+ rctx->out_dma = cmh_dma_map_single(rctx->out_buf, req->cryptlen,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->out_dma)) {
+ ret = -ENOMEM;
+ goto out_free_out;
+ }
+
+ if (info->ivsize > 0) {
+ rctx->iv_buf = kmemdup(req->iv, info->ivsize, gfp);
+ if (!rctx->iv_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_out;
+ }
+ rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, info->ivsize,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->iv_dma)) {
+ ret = -ENOMEM;
+ goto out_free_iv;
+ }
+ }
+
+ idx = 0;
+
+ rctx->key_dma = tctx->key.raw.dma;
+ rctx->keylen = tctx->key.raw.len;
+ vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP,
+ (u64)rctx->key_dma, SYS_REF_NONE,
+ tctx->key.raw.len,
+ tctx->key.raw.sys_type);
+ key_ref = SYS_REF_TEMP;
+ keylen = tctx->key.raw.len;
+ d = cmh_core_select_instance(CMH_CORE_SM4);
+ target_mbx = d.mbx_idx;
+ core_id = d.core_id;
+
+ /*
+ * iolen in INIT: passed for all modes. The EIP-40 eSW ignores
+ * it for CTR (stream cipher), but uses it for XTS/CBC/ECB to
+ * know the total data length. Pass cryptlen unconditionally.
+ */
+ vcq_add_sm4_init(&cmds[idx++], core_id, key_ref, (u64)rctx->iv_dma,
+ keylen, info->ivsize, info->sm4_mode, sm4_op,
+ req->cryptlen);
+
+ if (info->sm4_mode == SM4_MODE_XTS &&
+ req->cryptlen > 2 * CMH_SM4_BLOCK_SIZE) {
+ u32 final_len, update_len;
+
+ if (req->cryptlen & (CMH_SM4_BLOCK_SIZE - 1))
+ final_len = CMH_SM4_BLOCK_SIZE +
+ (req->cryptlen & (CMH_SM4_BLOCK_SIZE - 1));
+ else
+ final_len = 2 * CMH_SM4_BLOCK_SIZE;
+
+ update_len = req->cryptlen - final_len;
+
+ vcq_add_sm4_update(&cmds[idx++], core_id,
+ (u64)rctx->in_dma,
+ (u64)rctx->out_dma, update_len);
+ vcq_add_sm4_final(&cmds[idx++], core_id,
+ (u64)(rctx->in_dma + update_len),
+ (u64)(rctx->out_dma + update_len),
+ final_len);
+ } else if (info->sm4_mode == SM4_MODE_CTR) {
+ /*
+ * CTR counter-wrap: split at the 64-bit boundary,
+ * consistent with the AES-SCA driver. The completion
+ * callback submits chunk2 with IV = {upper64+1, 0}.
+ */
+ u64 lower64 = get_unaligned_be64(rctx->iv_buf + 8);
+ u32 nblocks = DIV_ROUND_UP(req->cryptlen,
+ CMH_SM4_BLOCK_SIZE);
+ u64 bwrap = lower64 ? (~lower64 + 1ULL) : U64_MAX;
+
+ if (nblocks > bwrap) {
+ u32 chunk1 = (u32)bwrap * CMH_SM4_BLOCK_SIZE;
+ u64 upper64;
+
+ /* Prepare second IV for chained submission */
+ rctx->iv2_buf = kmalloc(info->ivsize, gfp);
+ if (!rctx->iv2_buf) {
+ ret = -ENOMEM;
+ goto out_unmap_iv;
+ }
+ upper64 = get_unaligned_be64(rctx->iv_buf);
+ put_unaligned_be64(upper64 + 1, rctx->iv2_buf);
+ put_unaligned_be64(0, rctx->iv2_buf + 8);
+
+ rctx->iv2_dma =
+ cmh_dma_map_single(rctx->iv2_buf,
+ info->ivsize,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->iv2_dma)) {
+ ret = -ENOMEM;
+ goto out_free_iv2;
+ }
+
+ /* Store state for the chained second submission */
+ rctx->ctr_chunk1_len = chunk1;
+ rctx->core_id = core_id;
+ rctx->target_mbx = target_mbx;
+ rctx->key_ref = key_ref;
+
+ /* First transaction: only chunk1 */
+ vcq_add_sm4_final(&cmds[idx++], core_id,
+ (u64)rctx->in_dma,
+ (u64)rctx->out_dma, chunk1);
+ } else {
+ /* No wrap: single FINAL with all data */
+ vcq_add_sm4_final(&cmds[idx++], core_id,
+ (u64)rctx->in_dma,
+ (u64)rctx->out_dma,
+ req->cryptlen);
+ }
+ } else {
+ vcq_add_sm4_final(&cmds[idx++], core_id,
+ (u64)rctx->in_dma,
+ (u64)rctx->out_dma, req->cryptlen);
+ }
+
+ vcq_add_flush(&cmds[idx++], core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_SM4_MAX_PACKED, target_mbx,
+ cmh_sm4_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto out_cleanup_all;
+
+ return -EINPROGRESS;
+
+out_cleanup_all:
+ if (rctx->iv2_buf) {
+ cmh_dma_unmap_single(rctx->iv2_dma, info->ivsize,
+ DMA_TO_DEVICE);
+ }
+out_free_iv2:
+ kfree(rctx->iv2_buf);
+out_unmap_iv:
+ if (info->ivsize > 0)
+ cmh_dma_unmap_single(rctx->iv_dma, info->ivsize,
+ DMA_TO_DEVICE);
+out_free_iv:
+ kfree(rctx->iv_buf);
+out_unmap_out:
+ cmh_dma_unmap_single(rctx->out_dma, req->cryptlen, DMA_FROM_DEVICE);
+out_free_out:
+ kfree_sensitive(rctx->out_buf);
+out_unmap_in:
+ cmh_dma_unmap_single(rctx->in_dma, req->cryptlen, DMA_TO_DEVICE);
+out_free_in:
+ kfree_sensitive(rctx->in_buf);
+ return ret;
+}
+
+static int cmh_sm4_encrypt(struct skcipher_request *req)
+{
+ return cmh_sm4_crypt(req, SM4_OP_ENCRYPT);
+}
+
+static int cmh_sm4_decrypt(struct skcipher_request *req)
+{
+ return cmh_sm4_crypt(req, SM4_OP_DECRYPT);
+}
+
+/* Registration */
+
+static struct cmh_sm4_alg_drv sm4_drv_algs[ARRAY_SIZE(sm4_algs)];
+
+/**
+ * cmh_sm4_register() - Register SM4-CBC/CTR/ECB/XTS skcipher algorithms
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_sm4_register(void)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < ARRAY_SIZE(sm4_algs); i++) {
+ const struct cmh_sm4_alg_info *info = &sm4_algs[i];
+ struct cmh_sm4_alg_drv *drv = &sm4_drv_algs[i];
+ struct skcipher_alg *alg = &drv->alg;
+
+ drv->info = info;
+
+ memset(alg, 0, sizeof(*alg));
+
+ alg->setkey = cmh_sm4_setkey;
+ alg->encrypt = cmh_sm4_encrypt;
+ alg->decrypt = cmh_sm4_decrypt;
+ alg->init = cmh_sm4_init_tfm;
+ alg->exit = cmh_sm4_exit_tfm;
+ alg->min_keysize = info->min_keysize;
+ alg->max_keysize = info->max_keysize;
+ alg->ivsize = info->ivsize;
+
+ strscpy(alg->base.cra_name, info->alg_name,
+ CRYPTO_MAX_ALG_NAME);
+ strscpy(alg->base.cra_driver_name, info->drv_name,
+ CRYPTO_MAX_ALG_NAME);
+ alg->base.cra_priority = 300;
+ alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC;
+ alg->base.cra_blocksize = sm4_is_stream_mode(info->sm4_mode)
+ ? 1 : CMH_SM4_BLOCK_SIZE;
+ alg->base.cra_ctxsize = sizeof(struct cmh_sm4_tfm_ctx);
+ alg->base.cra_module = THIS_MODULE;
+
+ ret = crypto_register_skcipher(alg);
+ if (ret) {
+ dev_err(cmh_dev(), "cmh_sm4: failed to register %s (rc=%d)\n",
+ info->alg_name, ret);
+ goto err_unregister;
+ }
+
+ dev_dbg(cmh_dev(), "cmh_sm4: registered %s\n", info->alg_name);
+ }
+
+ return 0;
+
+err_unregister:
+ while (i--)
+ crypto_unregister_skcipher(&sm4_drv_algs[i].alg);
+ return ret;
+}
+
+/**
+ * cmh_sm4_unregister() - Unregister SM4 skcipher algorithms from the crypto framework
+ */
+void cmh_sm4_unregister(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(sm4_algs); i++) {
+ crypto_unregister_skcipher(&sm4_drv_algs[i].alg);
+ dev_dbg(cmh_dev(), "cmh_sm4: unregistered %s\n", sm4_algs[i].alg_name);
+ }
+}
diff --git a/drivers/crypto/cmh/include/cmh_sm4.h b/drivers/crypto/cmh/include/cmh_sm4.h
new file mode 100644
index 000000000000..9f4b0fb918db
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_sm4.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- SM4 Crypto API Drivers
+ *
+ * Registers SM4 algorithms with the Linux crypto subsystem:
+ * skcipher: ecb/cbc/ctr/cfb/xts(sm4)
+ * aead: gcm/ccm(sm4)
+ * shash: cmac/xcbc(sm4)
+ */
+
+#ifndef CMH_SM4_H
+#define CMH_SM4_H
+
+int cmh_sm4_register(void);
+void cmh_sm4_unregister(void);
+
+int cmh_sm4_aead_register(void);
+void cmh_sm4_aead_unregister(void);
+
+int cmh_sm4_cmac_register(void);
+void cmh_sm4_cmac_unregister(void);
+
+#endif /* CMH_SM4_H */
--
2.43.7
^ permalink raw reply related
* [PATCH v3 14/19] crypto: cmh - add ECDH/X25519 kpp
From: Saravanakrishnan Krishnamoorthy @ 2026-08-06 19:55 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register ECDH and X25519 kpp algorithms using the CMH PKE core.
Supports P-256, P-384, and Curve25519 for key agreement.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 3 +-
drivers/crypto/cmh/cmh_main.c | 8 +
drivers/crypto/cmh/cmh_pke_ecdh.c | 663 ++++++++++++++++++++++++++++++
3 files changed, 673 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_pke_ecdh.c
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index ae1f74a93e99..da420d0ec1fe 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -32,7 +32,8 @@ cmh-y := \
cmh_rng.o \
cmh_pke_common.o \
cmh_pke_rsa.o \
- cmh_pke_ecdsa.o
+ cmh_pke_ecdsa.o \
+ cmh_pke_ecdh.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index 82b734b42805..302d975256e8 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -332,6 +332,11 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_pke_ecdsa_register;
+ /* Register PKE ECDH/X25519 kpp */
+ ret = cmh_pke_ecdh_register();
+ if (ret)
+ goto err_pke_ecdh_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -344,6 +349,8 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_pke_ecdh_unregister();
+err_pke_ecdh_register:
cmh_pke_ecdsa_unregister();
err_pke_ecdsa_register:
cmh_pke_rsa_unregister();
@@ -404,6 +411,7 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_pke_ecdh_unregister();
cmh_pke_ecdsa_unregister();
cmh_pke_rsa_unregister();
cmh_ccp_poly_unregister();
diff --git a/drivers/crypto/cmh/cmh_pke_ecdh.c b/drivers/crypto/cmh/cmh_pke_ecdh.c
new file mode 100644
index 000000000000..1b49dd060e52
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pke_ecdh.c
@@ -0,0 +1,663 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- ECDH / X25519 kpp Driver
+ *
+ * Registers "ecdh-nist-p256", "ecdh-nist-p384", and "curve25519"
+ * kpp algorithms with priority 300.
+ *
+ * - set_secret: decodes private key from kpp_secret + ecdh struct
+ * (NIST curves) or raw 32-byte scalar (Curve25519).
+ * Stores in cmh_key_ctx: raw keys written via SYS_REF_TEMP.
+ * Datastore-referenced keys are only reachable through the ioctl
+ * path (cmh_mgmt.c).
+ *
+ * - generate_public_key: PKE_CMD_ECDH_KEYGEN -> outputs X coordinate
+ * (NIST Weierstrass) or full public key (Edwards/Montgomery).
+ * For NIST curves, we generate X||Y by calling ECDSA_PUBGEN instead,
+ * matching the kernel ecdh.c pattern that outputs uncompressed X||Y.
+ *
+ * - compute_shared_secret: PKE_CMD_ECDH -> shared secret X coordinate.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/scatterlist.h>
+#include <crypto/kpp.h>
+#include <crypto/ecdh.h>
+#include <crypto/internal/kpp.h>
+#include <crypto/internal/ecc.h>
+
+#include "cmh_pke.h"
+#include "cmh_sys.h"
+#include "cmh_sys_abi.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+
+/*
+ * ECDH key format: kpp_secret header + key_size(u16) + key data.
+ * We decode this inline to avoid depending on CONFIG_CRYPTO_ECDH.
+ */
+#define ECDH_KPP_SECRET_MIN_SIZE (sizeof(struct kpp_secret) + sizeof(unsigned short))
+
+struct cmh_ecdh_tfm_ctx {
+ struct cmh_key_ctx key;
+ u32 curve; /* PKE_CURVE_* */
+ u32 clen; /* coordinate length in bytes */
+};
+
+static inline struct cmh_ecdh_tfm_ctx *cmh_ecdh_ctx(struct crypto_kpp *tfm)
+{
+ return kpp_tfm_ctx(tfm);
+}
+
+/*
+ * Per-request context for ECDH/X25519 operations.
+ *
+ * generate_public_key: single-phase async VCQ.
+ * compute_shared_secret: 2-phase async VCQ with callback chaining.
+ * Phase 1: sys_write(sk) + sys_new(ref) + ecdh(peer) + pflush
+ * -> phase1 callback reads ref, submits Phase 2.
+ * Phase 2: sys_data(ref, ss_dma) + sys_flush
+ * -> phase2 callback extracts shared secret, completes req.
+ *
+ * Both phases target the same mbx_idx so the DS reference remains
+ * valid, since DS objects are MBX-scoped.
+ */
+struct cmh_ecdh_reqctx {
+ /* Buffers */
+ u8 *pk_buf; /* keygen: output public key */
+ u8 *peer_buf; /* compute: peer public key */
+ u8 *ss_buf; /* compute: shared secret output */
+ u64 *ref_buf; /* compute: DS ref from Phase 1 */
+ /* DMA handles */
+ dma_addr_t pk_dma;
+ dma_addr_t peer_dma;
+ dma_addr_t ss_dma;
+ dma_addr_t ref_dma;
+ /* Sizes and params for Phase 2 re-submit */
+ u32 out_len; /* keygen: public key size */
+ u32 clen;
+ u32 peer_len;
+ u32 dma_swap;
+ int mbx_idx; /* pinned MBX for Phase 2 */
+};
+
+/*
+ * set_secret: NIST curves decode kpp_secret + u16 key_size + raw scalar.
+ * Curve25519 uses raw 32-byte scalar directly.
+ */
+static int cmh_ecdh_set_secret_nist(struct crypto_kpp *tfm,
+ const void *buf, unsigned int len)
+{
+ struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm);
+ const u8 *ptr = buf;
+ struct kpp_secret secret;
+ unsigned short key_size;
+ int ret;
+
+ if (!buf || len < ECDH_KPP_SECRET_MIN_SIZE)
+ return -EINVAL;
+
+ memcpy(&secret, ptr, sizeof(secret));
+ ptr += sizeof(secret);
+
+ if (secret.type != CRYPTO_KPP_SECRET_TYPE_ECDH)
+ return -EINVAL;
+ if (len < secret.len)
+ return -EINVAL;
+
+ memcpy(&key_size, ptr, sizeof(key_size));
+ ptr += sizeof(key_size);
+
+ if (key_size == 0) {
+ /*
+ * key_size == 0: generate a validated random private key.
+ * Uses the kernel ECC library (FIPS 186-5 A.2.2) to ensure
+ * the scalar is in the valid range [2, n-3] for the curve.
+ */
+ u64 priv[ECC_MAX_DIGITS];
+ unsigned int ndigits = ctx->clen / sizeof(u64);
+ unsigned int curve_id;
+ u8 *rnd;
+
+ if (secret.len != ECDH_KPP_SECRET_MIN_SIZE)
+ return -EINVAL;
+ if (ndigits > ECC_MAX_DIGITS)
+ return -EINVAL;
+ /* Reject non-limb-aligned clen to prevent ndigits truncation */
+ if (ctx->clen % sizeof(u64))
+ return -EINVAL;
+
+ if (ctx->curve == PKE_CURVE_P256)
+ curve_id = ECC_CURVE_NIST_P256;
+ else if (ctx->curve == PKE_CURVE_P384)
+ curve_id = ECC_CURVE_NIST_P384;
+ else
+ return -EINVAL;
+
+ ret = ecc_gen_privkey(curve_id, ndigits, priv);
+ if (ret) {
+ memzero_explicit(priv, sizeof(priv));
+ return ret;
+ }
+
+ rnd = kmalloc(ctx->clen, GFP_KERNEL);
+ if (!rnd) {
+ memzero_explicit(priv, sizeof(priv));
+ return -ENOMEM;
+ }
+
+ /* Convert VLI (native LE-digit-order) to big-endian bytes */
+ ecc_swap_digits(priv, (u64 *)rnd, ndigits);
+ memzero_explicit(priv, sizeof(priv));
+
+ ret = cmh_key_setkey_raw(&ctx->key, rnd, ctx->clen,
+ CORE_ID_PKE);
+ kfree_sensitive(rnd);
+ return ret;
+ }
+
+ if (key_size != ctx->clen)
+ return -EINVAL;
+
+ if (secret.len != ECDH_KPP_SECRET_MIN_SIZE + key_size)
+ return -EINVAL;
+
+ return cmh_key_setkey_raw(&ctx->key, ptr, key_size, CORE_ID_PKE);
+}
+
+static int cmh_ecdh_set_secret_x25519(struct crypto_kpp *tfm,
+ const void *buf, unsigned int len)
+{
+ struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm);
+
+ if (len != pke_curve_clen(PKE_CURVE_25519))
+ return -EINVAL;
+
+ return cmh_key_setkey_raw(&ctx->key, buf, len, CORE_ID_PKE);
+}
+
+static void cmh_ecdh_keygen_complete(void *data, int error)
+{
+ struct kpp_request *req = data;
+ struct cmh_ecdh_reqctx *rctx = kpp_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ if (!cmh_dma_map_error(rctx->pk_dma))
+ cmh_dma_unmap_single(rctx->pk_dma, rctx->out_len,
+ DMA_FROM_DEVICE);
+
+ if (!error) {
+ int nents;
+
+ nents = sg_nents_for_len(req->dst, rctx->out_len);
+ if (nents < 0 ||
+ sg_copy_from_buffer(req->dst, nents,
+ rctx->pk_buf,
+ rctx->out_len) != rctx->out_len)
+ error = -EINVAL;
+ else
+ req->dst_len = rctx->out_len;
+ }
+
+ kfree(rctx->pk_buf);
+ rctx->pk_buf = NULL;
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * generate_public_key: For NIST ECDH, use ECDH_KEYGEN which outputs
+ * the public key X-coordinate. But the kernel kpp interface expects
+ * uncompressed X||Y, so we use ECDSA_PUBGEN which gives us (X,Y).
+ * For Curve25519, ECDH_KEYGEN gives us the Montgomery u-coordinate
+ * which is the full public key.
+ */
+static int cmh_ecdh_generate_public_key(struct kpp_request *req)
+{
+ struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
+ struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm);
+ struct cmh_ecdh_reqctx *rctx = kpp_request_ctx(req);
+ u32 clen = ctx->clen;
+ bool is_25519 = (ctx->curve == PKE_CURVE_25519);
+ u32 out_len = is_25519 ? clen : 2 * clen;
+ struct vcq_cmd vcq[PKE_VCQ_CMDS_MAX];
+ struct core_dispatch dd;
+ u32 swap, dma_swap;
+ int ret, idx;
+ gfp_t gfp;
+
+ if (ctx->key.mode != CMH_KEY_RAW)
+ return -EINVAL;
+ if (req->dst_len < out_len)
+ return -EINVAL;
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->out_len = out_len;
+ rctx->pk_dma = DMA_MAPPING_ERROR;
+
+ rctx->pk_buf = kzalloc(out_len, gfp);
+ if (!rctx->pk_buf)
+ return -ENOMEM;
+
+ rctx->pk_dma = cmh_dma_map_single(rctx->pk_buf, out_len,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->pk_dma)) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ /*
+ * The SYS_REF_TEMP key write uses per-curve dma_swap (0 for the
+ * little-endian Curve25519 secret), but the PKE compute command
+ * itself takes PKE_SWAP_FLAGS for every curve: the HW emits the
+ * public-key / shared-secret coordinate in the order PKE_SWAP_FLAGS
+ * maps to the expected encoding (the mgmt ECDH keygen does the same
+ * and matches RFC 7748). Do NOT switch the command to dma_swap.
+ */
+ swap = PKE_SWAP_FLAGS;
+ dma_swap = pke_swap_flags(ctx->curve);
+
+ dd = cmh_core_select_instance(CMH_CORE_PKE);
+
+ vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MAX);
+ idx = 1;
+ vcq_add_sys_write(&vcq[idx], SYS_REF_TEMP, ctx->key.raw.dma,
+ SYS_REF_NONE, ctx->key.raw.len,
+ ctx->key.raw.sys_type);
+ vcq[idx].id |= dma_swap;
+ idx++;
+ if (is_25519)
+ vcq_add_pke_ecdh_keygen(&vcq[idx++], dd.core_id, ctx->curve,
+ clen, rctx->pk_dma, SYS_REF_TEMP,
+ swap);
+ else
+ vcq_add_pke_ecdsa_pubgen(&vcq[idx++], dd.core_id,
+ ctx->curve, clen, rctx->pk_dma,
+ SYS_REF_TEMP, swap);
+ vcq_add_pke_flush(&vcq[idx++], dd.core_id);
+
+ ret = cmh_tm_submit_async(vcq, PKE_VCQ_CMDS_MAX, 1, dd.mbx_idx,
+ cmh_ecdh_keygen_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG), 0);
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (!ret)
+ return -EINPROGRESS;
+
+ if (!cmh_dma_map_error(rctx->pk_dma))
+ cmh_dma_unmap_single(rctx->pk_dma, out_len,
+ DMA_FROM_DEVICE);
+
+out_free:
+ kfree(rctx->pk_buf);
+ return ret;
+}
+
+static void cmh_ecdh_ss_phase2_complete(void *data, int error)
+{
+ struct kpp_request *req = data;
+ struct cmh_ecdh_reqctx *rctx = kpp_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ if (!cmh_dma_map_error(rctx->ss_dma))
+ cmh_dma_unmap_single(rctx->ss_dma, rctx->clen,
+ DMA_FROM_DEVICE);
+
+ if (!error) {
+ int nents;
+
+ nents = sg_nents_for_len(req->dst, rctx->clen);
+ if (nents < 0 ||
+ sg_copy_from_buffer(req->dst, nents,
+ rctx->ss_buf,
+ rctx->clen) != rctx->clen)
+ error = -EINVAL;
+ else
+ req->dst_len = rctx->clen;
+ }
+
+ kfree(rctx->ref_buf);
+ rctx->ref_buf = NULL;
+ kfree_sensitive(rctx->ss_buf);
+ rctx->ss_buf = NULL;
+ cmh_complete(&req->base, error);
+}
+
+static void cmh_ecdh_ss_phase1_complete(void *data, int error)
+{
+ struct kpp_request *req = data;
+ struct cmh_ecdh_reqctx *rctx = kpp_request_ctx(req);
+ struct vcq_cmd vcq[3];
+ int ret;
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ /* Phase 1-only resource: peer -- always clean up */
+ if (!cmh_dma_map_error(rctx->peer_dma))
+ cmh_dma_unmap_single(rctx->peer_dma, rctx->peer_len,
+ DMA_TO_DEVICE);
+ kfree(rctx->peer_buf);
+ rctx->peer_buf = NULL;
+
+ if (error)
+ goto out_cleanup;
+
+ /* Read the DS reference written by Phase 1 */
+ cmh_dma_sync_for_cpu(rctx->ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ cmh_dma_unmap_single(rctx->ref_dma, sizeof(u64), DMA_FROM_DEVICE);
+ rctx->ref_dma = DMA_MAPPING_ERROR;
+
+ /* Phase 2: extract shared secret from DS */
+ vcq_set_header(&vcq[0], 3);
+ vcq_add_sys_data(&vcq[1], *rctx->ref_buf, rctx->ss_dma,
+ rctx->clen);
+ vcq[1].id |= rctx->dma_swap;
+ vcq_add_sys_flush(&vcq[2]);
+
+ ret = cmh_tm_submit_async(vcq, 3, 1, rctx->mbx_idx,
+ cmh_ecdh_ss_phase2_complete, req,
+ true, 0);
+ if (ret == -EBUSY || !ret)
+ return;
+
+ error = ret;
+
+out_cleanup:
+ if (!cmh_dma_map_error(rctx->ref_dma))
+ cmh_dma_unmap_single(rctx->ref_dma, sizeof(u64),
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(rctx->ss_dma))
+ cmh_dma_unmap_single(rctx->ss_dma, rctx->clen,
+ DMA_FROM_DEVICE);
+ kfree(rctx->ref_buf);
+ rctx->ref_buf = NULL;
+ kfree_sensitive(rctx->ss_buf);
+ rctx->ss_buf = NULL;
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * compute_shared_secret: PKE_CMD_ECDH.
+ *
+ * req->src = peer public key (X||Y for NIST, raw 32B for Curve25519).
+ * Output = shared secret X coordinate (clen bytes).
+ *
+ * The CMH ECDH command stores the shared secret in a DS object,
+ * not directly to DMA. We create a DS slot with SYS_CMD_NEW,
+ * reference it via SYS_REF_LAST, then extract the result with a
+ * second VCQ submission using SYS_CMD_DATA with the actual ref.
+ */
+static int cmh_ecdh_compute_shared_secret(struct kpp_request *req)
+{
+ struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
+ struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm);
+ struct cmh_ecdh_reqctx *rctx = kpp_request_ctx(req);
+ u32 clen = ctx->clen;
+ bool is_25519 = (ctx->curve == PKE_CURVE_25519);
+ u32 peer_len = is_25519 ? clen : 2 * clen;
+ u32 ss_type = SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_PKE);
+ struct vcq_cmd vcq[5];
+ struct core_dispatch dd;
+ u32 swap, dma_swap;
+ int ret, idx, nents;
+ gfp_t gfp;
+
+ if (ctx->key.mode != CMH_KEY_RAW)
+ return -EINVAL;
+ if (req->src_len < peer_len || req->dst_len < clen)
+ return -EINVAL;
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->clen = clen;
+ rctx->peer_len = peer_len;
+ rctx->pk_dma = DMA_MAPPING_ERROR;
+ rctx->peer_dma = DMA_MAPPING_ERROR;
+ rctx->ss_dma = DMA_MAPPING_ERROR;
+ rctx->ref_dma = DMA_MAPPING_ERROR;
+
+ rctx->peer_buf = kmalloc(peer_len, gfp);
+ rctx->ss_buf = kzalloc(clen, gfp);
+ rctx->ref_buf = kzalloc_obj(u64, gfp);
+ if (!rctx->peer_buf || !rctx->ss_buf || !rctx->ref_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ nents = sg_nents_for_len(req->src, peer_len);
+ if (nents < 0 ||
+ sg_pcopy_to_buffer(req->src, nents, rctx->peer_buf,
+ peer_len, 0) != peer_len) {
+ ret = -EINVAL;
+ goto out_free;
+ }
+
+ rctx->peer_dma = cmh_dma_map_single(rctx->peer_buf, peer_len,
+ DMA_TO_DEVICE);
+ rctx->ss_dma = cmh_dma_map_single(rctx->ss_buf, clen,
+ DMA_FROM_DEVICE);
+ rctx->ref_dma = cmh_dma_map_single(rctx->ref_buf, sizeof(u64),
+ DMA_FROM_DEVICE);
+
+ if (cmh_dma_map_error(rctx->peer_dma) ||
+ cmh_dma_map_error(rctx->ss_dma) ||
+ cmh_dma_map_error(rctx->ref_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ /*
+ * The SYS_REF_TEMP key write uses per-curve dma_swap (0 for the
+ * little-endian Curve25519 secret), but the PKE compute command
+ * itself takes PKE_SWAP_FLAGS for every curve: the HW emits the
+ * public-key / shared-secret coordinate in the order PKE_SWAP_FLAGS
+ * maps to the expected encoding (the mgmt ECDH keygen does the same
+ * and matches RFC 7748). Do NOT switch the command to dma_swap.
+ */
+ swap = PKE_SWAP_FLAGS;
+ dma_swap = pke_swap_flags(ctx->curve);
+ rctx->dma_swap = dma_swap;
+
+ dd = cmh_core_select_instance(CMH_CORE_PKE);
+ rctx->mbx_idx = dd.mbx_idx;
+
+ vcq_set_header(&vcq[0], 5);
+ idx = 1;
+ vcq_add_sys_write(&vcq[idx], SYS_REF_TEMP, ctx->key.raw.dma,
+ SYS_REF_NONE, ctx->key.raw.len,
+ ctx->key.raw.sys_type);
+ vcq[idx].id |= dma_swap;
+ idx++;
+ vcq_add_sys_new(&vcq[idx++], 0, rctx->ref_dma, clen);
+ vcq_add_pke_ecdh(&vcq[idx++], dd.core_id, ctx->curve, clen,
+ clen, ss_type, rctx->peer_dma,
+ SYS_REF_TEMP, SYS_REF_LAST, swap);
+ vcq_add_pke_flush(&vcq[idx++], dd.core_id);
+
+ ret = cmh_tm_submit_async(vcq, 5, 1, dd.mbx_idx,
+ cmh_ecdh_ss_phase1_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG), 0);
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (!ret)
+ return -EINPROGRESS;
+
+out_unmap:
+ if (!cmh_dma_map_error(rctx->ss_dma))
+ cmh_dma_unmap_single(rctx->ss_dma, clen,
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(rctx->ref_dma))
+ cmh_dma_unmap_single(rctx->ref_dma, sizeof(u64),
+ DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(rctx->peer_dma))
+ cmh_dma_unmap_single(rctx->peer_dma, peer_len,
+ DMA_TO_DEVICE);
+
+out_free:
+ kfree(rctx->ref_buf);
+ kfree_sensitive(rctx->ss_buf);
+ kfree(rctx->peer_buf);
+ return ret;
+}
+
+static unsigned int cmh_ecdh_max_size(struct crypto_kpp *tfm)
+{
+ struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm);
+
+ /* Max output = X||Y for generate_public_key (NIST) */
+ return 2 * ctx->clen;
+}
+
+static unsigned int cmh_x25519_max_size(struct crypto_kpp *tfm)
+{
+ return pke_curve_clen(PKE_CURVE_25519); /* single coordinate */
+}
+
+static int cmh_ecdh_p256_init(struct crypto_kpp *tfm)
+{
+ struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->curve = PKE_CURVE_P256;
+ ctx->clen = pke_curve_clen(PKE_CURVE_P256);
+ tfm->reqsize = sizeof(struct cmh_ecdh_reqctx);
+ return 0;
+}
+
+static int cmh_ecdh_p384_init(struct crypto_kpp *tfm)
+{
+ struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->curve = PKE_CURVE_P384;
+ ctx->clen = pke_curve_clen(PKE_CURVE_P384);
+ tfm->reqsize = sizeof(struct cmh_ecdh_reqctx);
+ return 0;
+}
+
+static int cmh_x25519_init(struct crypto_kpp *tfm)
+{
+ struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->curve = PKE_CURVE_25519;
+ ctx->clen = pke_curve_clen(PKE_CURVE_25519);
+ tfm->reqsize = sizeof(struct cmh_ecdh_reqctx);
+ return 0;
+}
+
+static void cmh_ecdh_exit(struct crypto_kpp *tfm)
+{
+ struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm);
+
+ cmh_key_destroy(&ctx->key);
+}
+
+static struct kpp_alg cmh_ecdh_algs[] = {
+ {
+ .set_secret = cmh_ecdh_set_secret_nist,
+ .generate_public_key = cmh_ecdh_generate_public_key,
+ .compute_shared_secret = cmh_ecdh_compute_shared_secret,
+ .max_size = cmh_ecdh_max_size,
+ .init = cmh_ecdh_p256_init,
+ .exit = cmh_ecdh_exit,
+ .base = {
+ .cra_name = "ecdh-nist-p256",
+ .cra_driver_name = "rambus-cmh-ecdh-nist-p256",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_ecdh_tfm_ctx),
+ },
+ },
+ {
+ .set_secret = cmh_ecdh_set_secret_nist,
+ .generate_public_key = cmh_ecdh_generate_public_key,
+ .compute_shared_secret = cmh_ecdh_compute_shared_secret,
+ .max_size = cmh_ecdh_max_size,
+ .init = cmh_ecdh_p384_init,
+ .exit = cmh_ecdh_exit,
+ .base = {
+ .cra_name = "ecdh-nist-p384",
+ .cra_driver_name = "rambus-cmh-ecdh-nist-p384",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_ecdh_tfm_ctx),
+ },
+ },
+ {
+ .set_secret = cmh_ecdh_set_secret_x25519,
+ .generate_public_key = cmh_ecdh_generate_public_key,
+ .compute_shared_secret = cmh_ecdh_compute_shared_secret,
+ .max_size = cmh_x25519_max_size,
+ .init = cmh_x25519_init,
+ .exit = cmh_ecdh_exit,
+ .base = {
+ .cra_name = "curve25519",
+ .cra_driver_name = "rambus-cmh-curve25519",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_ecdh_tfm_ctx),
+ },
+ },
+};
+
+/**
+ * cmh_pke_ecdh_register() - Register ECDH kpp algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_pke_ecdh_register(void)
+{
+ int ret, i;
+
+ for (i = 0; i < ARRAY_SIZE(cmh_ecdh_algs); i++) {
+ ret = crypto_register_kpp(&cmh_ecdh_algs[i]);
+ if (ret) {
+ dev_err(cmh_dev(), "cmh: failed to register %s (%d)\n",
+ cmh_ecdh_algs[i].base.cra_name, ret);
+ goto err_unregister;
+ }
+ }
+
+ return 0;
+
+err_unregister:
+ while (i--)
+ crypto_unregister_kpp(&cmh_ecdh_algs[i]);
+ return ret;
+}
+
+/**
+ * cmh_pke_ecdh_unregister() - Unregister ECDH kpp algorithms from the crypto framework
+ */
+void cmh_pke_ecdh_unregister(void)
+{
+ int i = ARRAY_SIZE(cmh_ecdh_algs);
+
+ while (i--)
+ crypto_unregister_kpp(&cmh_ecdh_algs[i]);
+}
--
2.43.7
^ permalink raw reply related
* [PATCH v3 16/19] crypto: cmh - add SLH-DSA/LMS/XMSS (HCQ)
From: Saravanakrishnan Krishnamoorthy @ 2026-08-06 19:55 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register SLH-DSA, LMS, LMS-HSS, XMSS, and XMSS-MT algorithms
using the CMH HCQ core (core ID 0x08). SLH-DSA is registered as a
sig algorithm with sign and verify support. LMS, LMS-HSS, XMSS,
and XMSS-MT are registered as verify-only sig algorithms: their
stateful signing semantics (one-time-key private state the signer
must track) are not modeled by the kernel crypto API, so only
verification is exposed.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 6 +-
drivers/crypto/cmh/cmh_hcq.c | 313 +++++++++++++++++++++++
drivers/crypto/cmh/cmh_main.c | 24 ++
drivers/crypto/cmh/cmh_pqc_lms.c | 238 ++++++++++++++++++
drivers/crypto/cmh/cmh_pqc_slhdsa.c | 377 ++++++++++++++++++++++++++++
drivers/crypto/cmh/cmh_pqc_xmss.c | 238 ++++++++++++++++++
6 files changed, 1195 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_hcq.c
create mode 100644 drivers/crypto/cmh/cmh_pqc_lms.c
create mode 100644 drivers/crypto/cmh/cmh_pqc_slhdsa.c
create mode 100644 drivers/crypto/cmh/cmh_pqc_xmss.c
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index 7196d992487e..a78311bf3195 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -36,7 +36,11 @@ cmh-y := \
cmh_pke_ecdh.o \
cmh_qse.o \
cmh_pqc_mldsa.o \
- cmh_pqc_sizes.o
+ cmh_pqc_sizes.o \
+ cmh_hcq.o \
+ cmh_pqc_slhdsa.o \
+ cmh_pqc_lms.o \
+ cmh_pqc_xmss.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_hcq.c b/drivers/crypto/cmh/cmh_hcq.c
new file mode 100644
index 000000000000..8fc3a5cb0f9f
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_hcq.c
@@ -0,0 +1,313 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- HCQ Core VCQ Builders
+ *
+ * VCQ builder functions for SLH-DSA, LMS, and XMSS commands.
+ * Each function populates a single vcq_cmd slot. Callers assemble
+ * complete VCQs with header + command(s) + flush, then submit via
+ * cmh_tm_submit_sync().
+ */
+
+#include <linux/string.h>
+
+#include "cmh_sys.h"
+
+/* -- HCQ flush -- */
+
+/**
+ * vcq_add_hcq_flush() - Build an HCQ flush VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ */
+void vcq_add_hcq_flush(struct vcq_cmd *slot, u32 core_id)
+{
+ vcq_add_flush(slot, core_id);
+}
+
+/* -- SLH-DSA -- */
+
+/**
+ * vcq_add_hcq_slhdsa_keygen() - Build an SLH-DSA key generation VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @param_set: SLH-DSA parameter set identifier
+ * @seed_len: Length of seed buffer in bytes
+ * @pk_len: Length of public key buffer in bytes
+ * @sk_len: Length of secret key buffer in bytes
+ * @seed: DMA address of seed input buffer
+ * @pk: DMA address of public key output buffer
+ * @sk: DMA address of secret key output buffer
+ */
+void vcq_add_hcq_slhdsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 seed_len, u32 pk_len, u32 sk_len,
+ u64 seed, u64 pk, u64 sk)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HCQ_CMD_SLHDSA_KEYGEN);
+ slot->hwc.hcq.cmd_slhdsa_keygen.parameter_set = param_set;
+ slot->hwc.hcq.cmd_slhdsa_keygen.seed_len = seed_len;
+ slot->hwc.hcq.cmd_slhdsa_keygen.pk_len = pk_len;
+ slot->hwc.hcq.cmd_slhdsa_keygen.sk_len = sk_len;
+ slot->hwc.hcq.cmd_slhdsa_keygen.seed = seed;
+ slot->hwc.hcq.cmd_slhdsa_keygen.pk = pk;
+ slot->hwc.hcq.cmd_slhdsa_keygen.sk = sk;
+}
+
+/**
+ * vcq_add_hcq_slhdsa_sign() - Build an SLH-DSA signing VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @param_set: SLH-DSA parameter set identifier
+ * @msg_len: Length of message buffer in bytes
+ * @ctx_len: Length of context string in bytes
+ * @add_random: DMA address of additional randomness buffer
+ * @msg: DMA address of message buffer
+ * @ctx: DMA address of context string buffer
+ * @sk: DMA address of secret key buffer
+ * @sig: DMA address of signature output buffer
+ */
+void vcq_add_hcq_slhdsa_sign(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 msg_len, u32 ctx_len,
+ u64 add_random, u64 msg, u64 ctx,
+ u64 sk, u64 sig)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HCQ_CMD_SLHDSA_SIGN);
+ slot->hwc.hcq.cmd_slhdsa_sign.parameter_set = param_set;
+ slot->hwc.hcq.cmd_slhdsa_sign.message_len = msg_len;
+ slot->hwc.hcq.cmd_slhdsa_sign.add_random = add_random;
+ slot->hwc.hcq.cmd_slhdsa_sign.message = msg;
+ slot->hwc.hcq.cmd_slhdsa_sign.context = ctx;
+ slot->hwc.hcq.cmd_slhdsa_sign.sk = sk;
+ slot->hwc.hcq.cmd_slhdsa_sign.sig = sig;
+ slot->hwc.hcq.cmd_slhdsa_sign.context_len = ctx_len;
+}
+
+/**
+ * vcq_add_hcq_slhdsa_sign_internal() - Build an SLH-DSA internal signing VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @param_set: SLH-DSA parameter set identifier
+ * @msg_len: Length of message buffer in bytes
+ * @add_random: DMA address of additional randomness buffer
+ * @msg: DMA address of message buffer
+ * @sk: DMA address of secret key buffer
+ * @sig: DMA address of signature output buffer
+ */
+void vcq_add_hcq_slhdsa_sign_internal(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 msg_len, u64 add_random,
+ u64 msg, u64 sk, u64 sig)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HCQ_CMD_SLHDSA_SIGN_INTERNAL);
+ slot->hwc.hcq.cmd_slhdsa_sign_internal.parameter_set = param_set;
+ slot->hwc.hcq.cmd_slhdsa_sign_internal.message_len = msg_len;
+ slot->hwc.hcq.cmd_slhdsa_sign_internal.add_random = add_random;
+ slot->hwc.hcq.cmd_slhdsa_sign_internal.message = msg;
+ slot->hwc.hcq.cmd_slhdsa_sign_internal.sk = sk;
+ slot->hwc.hcq.cmd_slhdsa_sign_internal.sig = sig;
+}
+
+/**
+ * vcq_add_hcq_slhdsa_verify() - Build an SLH-DSA verification VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @param_set: SLH-DSA parameter set identifier
+ * @msg_len: Length of message buffer in bytes
+ * @ctx_len: Length of context string in bytes
+ * @msg: DMA address of message buffer
+ * @ctx: DMA address of context string buffer
+ * @pk: DMA address of public key buffer
+ * @sig: DMA address of signature buffer to verify
+ */
+void vcq_add_hcq_slhdsa_verify(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 msg_len, u32 ctx_len,
+ u64 msg, u64 ctx, u64 pk, u64 sig)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HCQ_CMD_SLHDSA_VERIFY);
+ slot->hwc.hcq.cmd_slhdsa_verify.parameter_set = param_set;
+ slot->hwc.hcq.cmd_slhdsa_verify.message_len = msg_len;
+ slot->hwc.hcq.cmd_slhdsa_verify.message = msg;
+ slot->hwc.hcq.cmd_slhdsa_verify.context = ctx;
+ slot->hwc.hcq.cmd_slhdsa_verify.pk = pk;
+ slot->hwc.hcq.cmd_slhdsa_verify.sig = sig;
+ slot->hwc.hcq.cmd_slhdsa_verify.context_len = ctx_len;
+}
+
+/**
+ * vcq_add_hcq_slhdsa_sign_prehash() - Build an SLH-DSA prehash signing VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @cmd: VCQ command ID (sign-prehash variant)
+ * @param_set: SLH-DSA parameter set identifier
+ * @prehash_algo: Prehash algorithm identifier
+ * @msg_len: Length of message buffer in bytes
+ * @ctx_len: Length of context string in bytes
+ * @add_random: DMA address of additional randomness buffer
+ * @msg: DMA address of message buffer
+ * @ctx: DMA address of context string buffer
+ * @sk: DMA address of secret key buffer
+ * @sig: DMA address of signature output buffer
+ */
+void vcq_add_hcq_slhdsa_sign_prehash(struct vcq_cmd *slot, u32 core_id,
+ u32 cmd, u32 param_set, u32 prehash_algo,
+ u32 msg_len, u32 ctx_len,
+ u64 add_random, u64 msg, u64 ctx,
+ u64 sk, u64 sig)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd);
+ slot->hwc.hcq.cmd_slhdsa_sign_prehash.parameter_set = param_set;
+ slot->hwc.hcq.cmd_slhdsa_sign_prehash.prehash_algo = prehash_algo;
+ slot->hwc.hcq.cmd_slhdsa_sign_prehash.message_len = msg_len;
+ slot->hwc.hcq.cmd_slhdsa_sign_prehash.context_len = ctx_len;
+ slot->hwc.hcq.cmd_slhdsa_sign_prehash.add_random = add_random;
+ slot->hwc.hcq.cmd_slhdsa_sign_prehash.message = msg;
+ slot->hwc.hcq.cmd_slhdsa_sign_prehash.context = ctx;
+ slot->hwc.hcq.cmd_slhdsa_sign_prehash.sk = sk;
+ slot->hwc.hcq.cmd_slhdsa_sign_prehash.sig = sig;
+}
+
+/**
+ * vcq_add_hcq_slhdsa_verify_prehash() - Build an SLH-DSA prehash verify VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @cmd: VCQ command ID (verify-prehash variant)
+ * @param_set: SLH-DSA parameter set identifier
+ * @prehash_algo: Prehash algorithm identifier
+ * @msg_len: Length of message buffer in bytes
+ * @ctx_len: Length of context string in bytes
+ * @msg: DMA address of message buffer
+ * @ctx: DMA address of context string buffer
+ * @pk: DMA address of public key buffer
+ * @sig: DMA address of signature buffer to verify
+ */
+void vcq_add_hcq_slhdsa_verify_prehash(struct vcq_cmd *slot, u32 core_id,
+ u32 cmd, u32 param_set, u32 prehash_algo,
+ u32 msg_len, u32 ctx_len,
+ u64 msg, u64 ctx, u64 pk, u64 sig)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd);
+ slot->hwc.hcq.cmd_slhdsa_verify_prehash.parameter_set = param_set;
+ slot->hwc.hcq.cmd_slhdsa_verify_prehash.prehash_algo = prehash_algo;
+ slot->hwc.hcq.cmd_slhdsa_verify_prehash.message_len = msg_len;
+ slot->hwc.hcq.cmd_slhdsa_verify_prehash.context_len = ctx_len;
+ slot->hwc.hcq.cmd_slhdsa_verify_prehash.message = msg;
+ slot->hwc.hcq.cmd_slhdsa_verify_prehash.context = ctx;
+ slot->hwc.hcq.cmd_slhdsa_verify_prehash.pk = pk;
+ slot->hwc.hcq.cmd_slhdsa_verify_prehash.sig = sig;
+}
+
+/**
+ * vcq_add_hcq_slhdsa_verify_internal() - Build an SLH-DSA internal verify VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @param_set: SLH-DSA parameter set identifier
+ * @msg_len: Length of message buffer in bytes
+ * @msg: DMA address of message buffer
+ * @pk: DMA address of public key buffer
+ * @sig: DMA address of signature buffer to verify
+ */
+void vcq_add_hcq_slhdsa_verify_internal(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 msg_len, u64 msg, u64 pk, u64 sig)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1,
+ HCQ_CMD_SLHDSA_VERIFY_INTERNAL);
+ slot->hwc.hcq.cmd_slhdsa_verify_internal.parameter_set = param_set;
+ slot->hwc.hcq.cmd_slhdsa_verify_internal.message_len = msg_len;
+ slot->hwc.hcq.cmd_slhdsa_verify_internal.message = msg;
+ slot->hwc.hcq.cmd_slhdsa_verify_internal.pk = pk;
+ slot->hwc.hcq.cmd_slhdsa_verify_internal.sig = sig;
+}
+
+/**
+ * vcq_add_hcq_slhdsa_pubgen() - Build an SLH-DSA public key generation VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @param_set: SLH-DSA parameter set identifier
+ * @sk_len: Length of secret key buffer in bytes
+ * @sk: DMA address of secret key input buffer
+ * @pk: DMA address of public key output buffer
+ */
+void vcq_add_hcq_slhdsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 param_set,
+ u32 sk_len, u64 sk, u64 pk)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HCQ_CMD_SLHDSA_PUBGEN);
+ slot->hwc.hcq.cmd_slhdsa_pubgen.parameter_set = param_set;
+ slot->hwc.hcq.cmd_slhdsa_pubgen.sk_len = sk_len;
+ slot->hwc.hcq.cmd_slhdsa_pubgen.sk = sk;
+ slot->hwc.hcq.cmd_slhdsa_pubgen.pk = pk;
+}
+
+/* -- LMS -- */
+
+/**
+ * vcq_add_hcq_lms_verify() - Build an LMS/HSS signature verify VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @lms_hss: LMS/HSS mode flag (0 = LMS, 1 = HSS)
+ * @pk_len: Length of public key buffer in bytes
+ * @sig_len: Length of signature buffer in bytes
+ * @dig_len: Length of digest buffer in bytes
+ * @pk: DMA address of public key buffer
+ * @sig: DMA address of signature buffer
+ * @dig: DMA address of digest buffer
+ */
+void vcq_add_hcq_lms_verify(struct vcq_cmd *slot, u32 core_id, u32 lms_hss,
+ u32 pk_len, u32 sig_len, u32 dig_len,
+ u64 pk, u64 sig, u64 dig)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HCQ_CMD_LMS_VERIFY);
+ slot->hwc.hcq.cmd_lms_verify.lms_hss = lms_hss;
+ slot->hwc.hcq.cmd_lms_verify.pk_len = pk_len;
+ slot->hwc.hcq.cmd_lms_verify.sig_len = sig_len;
+ slot->hwc.hcq.cmd_lms_verify.dig_len = dig_len;
+ slot->hwc.hcq.cmd_lms_verify.pk = pk;
+ slot->hwc.hcq.cmd_lms_verify.sig = sig;
+ slot->hwc.hcq.cmd_lms_verify.dig = dig;
+}
+
+/* -- XMSS -- */
+
+/**
+ * vcq_add_hcq_xmss_verify() - Build an XMSS/XMSS^MT signature verify VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @xmss_mt: XMSS/XMSS^MT mode flag (0 = XMSS, 1 = XMSS^MT)
+ * @pk_len: Length of public key buffer in bytes
+ * @sig_len: Length of signature buffer in bytes
+ * @dig_len: Length of digest buffer in bytes
+ * @pk: DMA address of public key buffer
+ * @sig: DMA address of signature buffer
+ * @dig: DMA address of digest buffer
+ */
+void vcq_add_hcq_xmss_verify(struct vcq_cmd *slot, u32 core_id, u32 xmss_mt,
+ u32 pk_len, u32 sig_len, u32 dig_len,
+ u64 pk, u64 sig, u64 dig)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HCQ_CMD_XMSS_VERIFY);
+ slot->hwc.hcq.cmd_xmss_verify.xmss_mt = xmss_mt;
+ slot->hwc.hcq.cmd_xmss_verify.pk_len = pk_len;
+ slot->hwc.hcq.cmd_xmss_verify.sig_len = sig_len;
+ slot->hwc.hcq.cmd_xmss_verify.dig_len = dig_len;
+ slot->hwc.hcq.cmd_xmss_verify.pk = pk;
+ slot->hwc.hcq.cmd_xmss_verify.sig = sig;
+ slot->hwc.hcq.cmd_xmss_verify.dig = dig;
+}
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index 74d2b3672d2a..3008d623eb04 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -343,6 +343,21 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_pqc_mldsa_register;
+ /* Register PQC SLH-DSA */
+ ret = cmh_pqc_slhdsa_register();
+ if (ret)
+ goto err_pqc_slhdsa_register;
+
+ /* Register PQC LMS */
+ ret = cmh_pqc_lms_register();
+ if (ret)
+ goto err_pqc_lms_register;
+
+ /* Register PQC XMSS */
+ ret = cmh_pqc_xmss_register();
+ if (ret)
+ goto err_pqc_xmss_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -355,6 +370,12 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_pqc_xmss_unregister();
+err_pqc_xmss_register:
+ cmh_pqc_lms_unregister();
+err_pqc_lms_register:
+ cmh_pqc_slhdsa_unregister();
+err_pqc_slhdsa_register:
cmh_pqc_mldsa_unregister();
err_pqc_mldsa_register:
cmh_pke_ecdh_unregister();
@@ -419,6 +440,9 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_pqc_xmss_unregister();
+ cmh_pqc_lms_unregister();
+ cmh_pqc_slhdsa_unregister();
cmh_pqc_mldsa_unregister();
cmh_pke_ecdh_unregister();
cmh_pke_ecdsa_unregister();
diff --git a/drivers/crypto/cmh/cmh_pqc_lms.c b/drivers/crypto/cmh/cmh_pqc_lms.c
new file mode 100644
index 000000000000..3a17ca478660
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pqc_lms.c
@@ -0,0 +1,238 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- LMS/HSS Signature Driver (verify-only, sig_alg, synchronous)
+ *
+ * Registers "lms" and "lms-hss" sig algorithms with verify-only
+ * callbacks. Sign is not supported (stateful signature -- key
+ * management must happen externally).
+ *
+ * Verify: src = raw signature, digest = message bytes
+ * Public key: raw pk bytes (variable length, set via set_pub_key)
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <crypto/sig.h>
+#include <crypto/internal/sig.h>
+
+#include "cmh_sys.h"
+#include "cmh_hcq_abi.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_pqc.h"
+
+#define LMS_VCQ_CMDS 3 /* header + cmd + flush */
+
+struct cmh_lms_tfm_ctx {
+ u8 *pub_key;
+ u32 pub_key_len;
+ u32 lms_hss; /* 0 = LMS, 1 = LMS-HSS */
+};
+
+static inline struct cmh_lms_tfm_ctx *cmh_lms_ctx(struct crypto_sig *tfm)
+{
+ return crypto_sig_ctx(tfm);
+}
+
+/*
+ * LMS/HSS verify (synchronous sig_alg)
+ *
+ * @src: raw signature
+ * @slen: signature length
+ * @digest: message bytes
+ * @dlen: message length
+ *
+ * Returns 0 on successful verification, negative errno on failure.
+ */
+static int cmh_lms_verify(struct crypto_sig *tfm,
+ const void *src, unsigned int slen,
+ const void *digest, unsigned int dlen)
+{
+ struct cmh_lms_tfm_ctx *ctx = cmh_lms_ctx(tfm);
+ struct core_dispatch d = cmh_core_select_instance(CMH_CORE_HCQ);
+ struct vcq_cmd vcq[LMS_VCQ_CMDS];
+ u8 *sig_buf = NULL, *m_buf = NULL, *pk_buf = NULL;
+ dma_addr_t sig_dma = DMA_MAPPING_ERROR;
+ dma_addr_t m_dma = DMA_MAPPING_ERROR;
+ dma_addr_t pk_dma = DMA_MAPPING_ERROR;
+ int ret;
+
+ if (!ctx->pub_key)
+ return -EINVAL;
+ if (!slen || slen > LMS_MAX_SIG_LEN)
+ return -EINVAL;
+ if (!dlen || dlen > LMS_MAX_MSG_LEN)
+ return -EINVAL;
+
+ sig_buf = kmemdup(src, slen, GFP_KERNEL);
+ m_buf = kmemdup(digest, dlen, GFP_KERNEL);
+ pk_buf = kmemdup(ctx->pub_key, ctx->pub_key_len, GFP_KERNEL);
+ if (!sig_buf || !m_buf || !pk_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ sig_dma = cmh_dma_map_single(sig_buf, slen, DMA_TO_DEVICE);
+ m_dma = cmh_dma_map_single(m_buf, dlen, DMA_TO_DEVICE);
+ pk_dma = cmh_dma_map_single(pk_buf, ctx->pub_key_len, DMA_TO_DEVICE);
+
+ if (cmh_dma_map_error(sig_dma) || cmh_dma_map_error(m_dma) ||
+ cmh_dma_map_error(pk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], LMS_VCQ_CMDS);
+ vcq_add_hcq_lms_verify(&vcq[1], d.core_id, ctx->lms_hss,
+ ctx->pub_key_len, slen, dlen,
+ pk_dma, sig_dma, m_dma);
+ vcq_add_hcq_flush(&vcq[2], d.core_id);
+
+ /* LMS verify traverses Merkle hash chains -- inherently slow */
+ ret = cmh_tm_submit_sync_tmo(vcq, LMS_VCQ_CMDS, 1, d.mbx_idx,
+ cmh_tm_slow_op_timeout_jiffies());
+
+out_unmap:
+ if (!cmh_dma_map_error(pk_dma))
+ cmh_dma_unmap_single(pk_dma, ctx->pub_key_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, dlen, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, slen, DMA_TO_DEVICE);
+
+out_free:
+ kfree(pk_buf);
+ kfree(m_buf);
+ kfree(sig_buf);
+ return ret;
+}
+
+static int cmh_lms_set_pub_key(struct crypto_sig *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct cmh_lms_tfm_ctx *ctx = cmh_lms_ctx(tfm);
+
+ if (!keylen || keylen > LMS_MAX_PK_LEN)
+ return -EINVAL;
+
+ kfree(ctx->pub_key);
+ ctx->pub_key = NULL;
+ ctx->pub_key_len = 0;
+
+ ctx->pub_key = kmemdup(key, keylen, GFP_KERNEL);
+ if (!ctx->pub_key)
+ return -ENOMEM;
+
+ ctx->pub_key_len = keylen;
+ return 0;
+}
+
+static unsigned int cmh_lms_key_size(struct crypto_sig *tfm)
+{
+ struct cmh_lms_tfm_ctx *ctx = cmh_lms_ctx(tfm);
+
+ return ctx->pub_key_len * 8;
+}
+
+static int cmh_lms_init(struct crypto_sig *tfm)
+{
+ struct cmh_lms_tfm_ctx *ctx = cmh_lms_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ return 0;
+}
+
+static int cmh_lms_hss_init(struct crypto_sig *tfm)
+{
+ struct cmh_lms_tfm_ctx *ctx = cmh_lms_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->lms_hss = 1;
+ return 0;
+}
+
+static void cmh_lms_exit(struct crypto_sig *tfm)
+{
+ struct cmh_lms_tfm_ctx *ctx = cmh_lms_ctx(tfm);
+
+ kfree(ctx->pub_key);
+ ctx->pub_key = NULL;
+}
+
+static unsigned int cmh_lms_max_size(struct crypto_sig *tfm)
+{
+ /* Verify-only; report the max signature length for API parity. */
+ return LMS_MAX_SIG_LEN;
+}
+
+static struct sig_alg cmh_lms_algs[] = {
+ {
+ .verify = cmh_lms_verify,
+ .set_pub_key = cmh_lms_set_pub_key,
+ .key_size = cmh_lms_key_size,
+ .max_size = cmh_lms_max_size,
+ .init = cmh_lms_init,
+ .exit = cmh_lms_exit,
+ .base = {
+ .cra_name = "lms",
+ .cra_driver_name = "rambus-cmh-lms",
+ .cra_priority = 300,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_lms_tfm_ctx),
+ },
+ },
+ {
+ .verify = cmh_lms_verify,
+ .set_pub_key = cmh_lms_set_pub_key,
+ .key_size = cmh_lms_key_size,
+ .max_size = cmh_lms_max_size,
+ .init = cmh_lms_hss_init,
+ .exit = cmh_lms_exit,
+ .base = {
+ .cra_name = "lms-hss",
+ .cra_driver_name = "rambus-cmh-lms-hss",
+ .cra_priority = 300,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_lms_tfm_ctx),
+ },
+ },
+};
+
+/**
+ * cmh_pqc_lms_register() - Register LMS/LMS-HSS sig algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_pqc_lms_register(void)
+{
+ int ret, i;
+
+ for (i = 0; i < ARRAY_SIZE(cmh_lms_algs); i++) {
+ ret = crypto_register_sig(&cmh_lms_algs[i]);
+ if (ret) {
+ dev_err(cmh_dev(), "cmh: failed to register %s (%d)\n",
+ cmh_lms_algs[i].base.cra_name, ret);
+ goto err_unregister;
+ }
+ }
+
+ return 0;
+
+err_unregister:
+ while (i--)
+ crypto_unregister_sig(&cmh_lms_algs[i]);
+ return ret;
+}
+
+/**
+ * cmh_pqc_lms_unregister() - Unregister LMS/LMS-HSS sig algorithms from the crypto framework
+ */
+void cmh_pqc_lms_unregister(void)
+{
+ int i = ARRAY_SIZE(cmh_lms_algs);
+
+ while (i--)
+ crypto_unregister_sig(&cmh_lms_algs[i]);
+}
diff --git a/drivers/crypto/cmh/cmh_pqc_slhdsa.c b/drivers/crypto/cmh/cmh_pqc_slhdsa.c
new file mode 100644
index 000000000000..4bec4278cd9c
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pqc_slhdsa.c
@@ -0,0 +1,377 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- SLH-DSA Signature Driver (sig_alg, synchronous)
+ *
+ * Registers SLH-DSA sig algorithms for all 12 parameter sets
+ * (SHAKE/SHA2 x 128/192/256 x s/f) with sign and verify callbacks.
+ *
+ * Key format:
+ * Public key = raw pk bytes (2*n bytes)
+ * Private key = raw sk bytes (4*n)
+ *
+ * Sign: src = message, dst = raw signature
+ * Verify: src = raw signature, digest = message bytes
+ *
+ * Private keys are raw (written to SYS_REF_TEMP per-operation).
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/unaligned.h>
+#include <crypto/sig.h>
+#include <crypto/internal/sig.h>
+
+#include "cmh_sys.h"
+#include "cmh_qse_abi.h"
+#include "cmh_hcq_abi.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+#include "cmh_pqc.h"
+
+struct cmh_slhdsa_tfm_ctx {
+ struct cmh_key_ctx key; /* private key (raw sk bytes) */
+ u8 *pub_key;
+ u32 pub_key_len;
+ u32 param_set; /* HCQ_SLHDSA_SHAKE_128S .. SHA2_256F */
+};
+
+static inline struct cmh_slhdsa_tfm_ctx *cmh_slhdsa_ctx(struct crypto_sig *tfm)
+{
+ return crypto_sig_ctx(tfm);
+}
+
+/*
+ * SLH-DSA sign (synchronous sig_alg)
+ *
+ * @src: message bytes
+ * @slen: message length
+ * @dst: signature output buffer
+ * @dlen: output buffer length
+ *
+ * Returns signature length on success, negative errno on failure.
+ * Uses raw private keys written to SYS_REF_TEMP per-operation.
+ */
+static int cmh_slhdsa_sign(struct crypto_sig *tfm,
+ const void *src, unsigned int slen,
+ void *dst, unsigned int dlen)
+{
+ struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm);
+ u32 sig_sz = slhdsa_get_sig_size(ctx->param_set);
+ u32 sk_sz = slhdsa_sk_size(ctx->param_set);
+ struct vcq_cmd vcq[HCQ_VCQ_CMDS_MAX]; /* raw: hdr+write+sign+flush */
+ u32 vcq_count;
+ u8 *m_buf = NULL, *sig_buf = NULL, *sk_buf = NULL;
+ dma_addr_t m_dma = DMA_MAPPING_ERROR;
+ dma_addr_t sig_dma = DMA_MAPPING_ERROR;
+ dma_addr_t sk_dma = DMA_MAPPING_ERROR;
+ int ret, idx;
+
+ if (ctx->key.mode == CMH_KEY_NONE)
+ return -EINVAL;
+ if (dlen < sig_sz)
+ return -EINVAL;
+ if (!slen || slen > SLHDSA_MAX_MSG_LEN)
+ return -EINVAL;
+
+ m_buf = kmemdup(src, slen, GFP_KERNEL);
+ sig_buf = kzalloc(sig_sz, GFP_KERNEL);
+ if (!m_buf || !sig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ m_dma = cmh_dma_map_single(m_buf, slen, DMA_TO_DEVICE);
+ sig_dma = cmh_dma_map_single(sig_buf, sig_sz, DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(m_dma) || cmh_dma_map_error(sig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ sk_dma = DMA_MAPPING_ERROR;
+ idx = 0;
+
+ struct core_dispatch d;
+
+ d = cmh_core_select_instance(CMH_CORE_HCQ);
+
+ if (ctx->key.raw.len != sk_sz) {
+ ret = -EINVAL;
+ goto out_unmap;
+ }
+ sk_buf = kmemdup(ctx->key.raw.data, ctx->key.raw.len,
+ GFP_KERNEL);
+ if (!sk_buf) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+ sk_dma = cmh_dma_map_single(sk_buf, sk_sz, DMA_TO_DEVICE);
+ if (cmh_dma_map_error(sk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_count = HCQ_VCQ_CMDS_MIN + 1;
+ vcq_set_header(&vcq[idx++], vcq_count);
+ vcq_add_sys_write(&vcq[idx++], SYS_REF_TEMP, sk_dma,
+ SYS_REF_NONE, sk_sz,
+ ctx->key.raw.sys_type);
+ vcq_add_hcq_slhdsa_sign_internal(&vcq[idx++], d.core_id,
+ ctx->param_set,
+ slen, 0,
+ m_dma, SYS_REF_TEMP,
+ sig_dma);
+ vcq_add_hcq_flush(&vcq[idx++], d.core_id);
+
+ ret = cmh_tm_submit_sync_tmo(vcq, vcq_count, 1, d.mbx_idx,
+ cmh_tm_slow_op_timeout_jiffies());
+
+ if (!ret) {
+ /* Sync bounce buffer so CPU sees the DMA-written signature */
+ cmh_dma_sync_for_cpu(sig_dma, sig_sz, DMA_FROM_DEVICE);
+ memcpy(dst, sig_buf, sig_sz);
+ ret = sig_sz;
+ }
+
+out_unmap:
+ if (sk_buf) {
+ if (!cmh_dma_map_error(sk_dma))
+ cmh_dma_unmap_single(sk_dma, sk_sz, DMA_TO_DEVICE);
+ kfree_sensitive(sk_buf);
+ }
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_sz, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, slen, DMA_TO_DEVICE);
+
+out_free:
+ kfree(sig_buf);
+ kfree(m_buf);
+ return ret;
+}
+
+/*
+ * SLH-DSA verify (synchronous sig_alg)
+ *
+ * @src: raw signature
+ * @slen: signature length
+ * @digest: message bytes
+ * @dlen: message length
+ *
+ * Returns 0 on successful verification, negative errno on failure.
+ */
+static int cmh_slhdsa_verify(struct crypto_sig *tfm,
+ const void *src, unsigned int slen,
+ const void *digest, unsigned int dlen)
+{
+ struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm);
+ u32 sig_sz = slhdsa_get_sig_size(ctx->param_set);
+ u32 pk_sz = slhdsa_pk_size(ctx->param_set);
+ struct core_dispatch d = cmh_core_select_instance(CMH_CORE_HCQ);
+ struct vcq_cmd vcq[HCQ_VCQ_CMDS_MIN];
+ u8 *sig_buf = NULL, *m_buf = NULL, *pk_buf = NULL;
+ dma_addr_t sig_dma = DMA_MAPPING_ERROR;
+ dma_addr_t m_dma = DMA_MAPPING_ERROR;
+ dma_addr_t pk_dma = DMA_MAPPING_ERROR;
+ int ret;
+
+ if (!ctx->pub_key)
+ return -EINVAL;
+ if (slen != sig_sz)
+ return -EINVAL;
+ if (!dlen || dlen > SLHDSA_MAX_MSG_LEN)
+ return -EINVAL;
+
+ sig_buf = kmemdup(src, slen, GFP_KERNEL);
+ m_buf = kmemdup(digest, dlen, GFP_KERNEL);
+ pk_buf = kmemdup(ctx->pub_key, pk_sz, GFP_KERNEL);
+ if (!sig_buf || !m_buf || !pk_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ sig_dma = cmh_dma_map_single(sig_buf, sig_sz, DMA_TO_DEVICE);
+ m_dma = cmh_dma_map_single(m_buf, dlen, DMA_TO_DEVICE);
+ pk_dma = cmh_dma_map_single(pk_buf, pk_sz, DMA_TO_DEVICE);
+
+ if (cmh_dma_map_error(sig_dma) || cmh_dma_map_error(m_dma) ||
+ cmh_dma_map_error(pk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], HCQ_VCQ_CMDS_MIN);
+ vcq_add_hcq_slhdsa_verify_internal(&vcq[1], d.core_id, ctx->param_set,
+ dlen, m_dma, pk_dma, sig_dma);
+ vcq_add_hcq_flush(&vcq[2], d.core_id);
+
+ /* SLH-DSA verify recomputes hyper-tree hashes -- inherently slow */
+ ret = cmh_tm_submit_sync_tmo(vcq, HCQ_VCQ_CMDS_MIN, 1, d.mbx_idx,
+ cmh_tm_slow_op_timeout_jiffies());
+
+out_unmap:
+ if (!cmh_dma_map_error(pk_dma))
+ cmh_dma_unmap_single(pk_dma, pk_sz, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, dlen, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_sz, DMA_TO_DEVICE);
+
+out_free:
+ kfree(pk_buf);
+ kfree(m_buf);
+ kfree(sig_buf);
+ return ret;
+}
+
+static int cmh_slhdsa_set_pub_key(struct crypto_sig *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm);
+ u32 expected = slhdsa_pk_size(ctx->param_set);
+
+ if (keylen != expected)
+ return -EINVAL;
+
+ kfree(ctx->pub_key);
+ ctx->pub_key = NULL;
+ ctx->pub_key_len = 0;
+
+ ctx->pub_key = kmemdup(key, keylen, GFP_KERNEL);
+ if (!ctx->pub_key)
+ return -ENOMEM;
+
+ ctx->pub_key_len = keylen;
+ return 0;
+}
+
+static int cmh_slhdsa_set_priv_key(struct crypto_sig *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm);
+
+ /* Raw sk (4*n bytes) */
+ if (keylen != slhdsa_sk_size(ctx->param_set))
+ return -EINVAL;
+
+ return cmh_key_setkey_raw(&ctx->key, key, keylen, CORE_ID_HCQ);
+}
+
+static unsigned int cmh_slhdsa_key_size(struct crypto_sig *tfm)
+{
+ struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm);
+
+ /* crypto_sig_keysize() returns bits, not bytes */
+ return slhdsa_pk_size(ctx->param_set) * 8;
+}
+
+static unsigned int cmh_slhdsa_max_size(struct crypto_sig *tfm)
+{
+ struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm);
+
+ return slhdsa_get_sig_size(ctx->param_set);
+}
+
+static void cmh_slhdsa_exit(struct crypto_sig *tfm)
+{
+ struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm);
+
+ cmh_key_destroy(&ctx->key);
+ kfree(ctx->pub_key);
+ ctx->pub_key = NULL;
+}
+
+/* Generate init functions for all 12 parameter sets */
+#define SLHDSA_INIT(ps_val) \
+ static int cmh_slhdsa_init_##ps_val(struct crypto_sig *tfm) \
+ { \
+ struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm); \
+ memset(ctx, 0, sizeof(*ctx)); \
+ ctx->param_set = ps_val; \
+ return 0; \
+ }
+
+SLHDSA_INIT(HCQ_SLHDSA_SHAKE_128S)
+SLHDSA_INIT(HCQ_SLHDSA_SHAKE_128F)
+SLHDSA_INIT(HCQ_SLHDSA_SHAKE_192S)
+SLHDSA_INIT(HCQ_SLHDSA_SHAKE_192F)
+SLHDSA_INIT(HCQ_SLHDSA_SHAKE_256S)
+SLHDSA_INIT(HCQ_SLHDSA_SHAKE_256F)
+SLHDSA_INIT(HCQ_SLHDSA_SHA2_128S)
+SLHDSA_INIT(HCQ_SLHDSA_SHA2_128F)
+SLHDSA_INIT(HCQ_SLHDSA_SHA2_192S)
+SLHDSA_INIT(HCQ_SLHDSA_SHA2_192F)
+SLHDSA_INIT(HCQ_SLHDSA_SHA2_256S)
+SLHDSA_INIT(HCQ_SLHDSA_SHA2_256F)
+
+#define SLHDSA_ALG(name, drv, ps_val) { \
+ .sign = cmh_slhdsa_sign, \
+ .verify = cmh_slhdsa_verify, \
+ .set_pub_key = cmh_slhdsa_set_pub_key, \
+ .set_priv_key = cmh_slhdsa_set_priv_key, \
+ .key_size = cmh_slhdsa_key_size, \
+ .max_size = cmh_slhdsa_max_size, \
+ .init = cmh_slhdsa_init_##ps_val, \
+ .exit = cmh_slhdsa_exit, \
+ .base = { \
+ .cra_name = name, \
+ .cra_driver_name = drv, \
+ .cra_priority = 300, \
+ .cra_module = THIS_MODULE, \
+ .cra_ctxsize = sizeof(struct cmh_slhdsa_tfm_ctx), \
+ }, \
+ }
+
+static struct sig_alg cmh_slhdsa_algs[] = {
+ SLHDSA_ALG("slh-dsa-shake-128s", "rambus-cmh-slh-dsa-shake-128s", HCQ_SLHDSA_SHAKE_128S),
+ SLHDSA_ALG("slh-dsa-shake-128f", "rambus-cmh-slh-dsa-shake-128f", HCQ_SLHDSA_SHAKE_128F),
+ SLHDSA_ALG("slh-dsa-shake-192s", "rambus-cmh-slh-dsa-shake-192s", HCQ_SLHDSA_SHAKE_192S),
+ SLHDSA_ALG("slh-dsa-shake-192f", "rambus-cmh-slh-dsa-shake-192f", HCQ_SLHDSA_SHAKE_192F),
+ SLHDSA_ALG("slh-dsa-shake-256s", "rambus-cmh-slh-dsa-shake-256s", HCQ_SLHDSA_SHAKE_256S),
+ SLHDSA_ALG("slh-dsa-shake-256f", "rambus-cmh-slh-dsa-shake-256f", HCQ_SLHDSA_SHAKE_256F),
+ SLHDSA_ALG("slh-dsa-sha2-128s", "rambus-cmh-slh-dsa-sha2-128s", HCQ_SLHDSA_SHA2_128S),
+ SLHDSA_ALG("slh-dsa-sha2-128f", "rambus-cmh-slh-dsa-sha2-128f", HCQ_SLHDSA_SHA2_128F),
+ SLHDSA_ALG("slh-dsa-sha2-192s", "rambus-cmh-slh-dsa-sha2-192s", HCQ_SLHDSA_SHA2_192S),
+ SLHDSA_ALG("slh-dsa-sha2-192f", "rambus-cmh-slh-dsa-sha2-192f", HCQ_SLHDSA_SHA2_192F),
+ SLHDSA_ALG("slh-dsa-sha2-256s", "rambus-cmh-slh-dsa-sha2-256s", HCQ_SLHDSA_SHA2_256S),
+ SLHDSA_ALG("slh-dsa-sha2-256f", "rambus-cmh-slh-dsa-sha2-256f", HCQ_SLHDSA_SHA2_256F),
+};
+
+/**
+ * cmh_pqc_slhdsa_register() - Register SLH-DSA akcipher algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_pqc_slhdsa_register(void)
+{
+ int ret, i;
+
+ for (i = 0; i < ARRAY_SIZE(cmh_slhdsa_algs); i++) {
+ ret = crypto_register_sig(&cmh_slhdsa_algs[i]);
+ if (ret) {
+ dev_err(cmh_dev(), "cmh: failed to register %s (%d)\n",
+ cmh_slhdsa_algs[i].base.cra_name, ret);
+ goto err_unregister;
+ }
+ }
+
+ return 0;
+
+err_unregister:
+ while (i--)
+ crypto_unregister_sig(&cmh_slhdsa_algs[i]);
+ return ret;
+}
+
+/**
+ * cmh_pqc_slhdsa_unregister() - Unregister SLH-DSA akcipher algorithms from the crypto framework
+ */
+void cmh_pqc_slhdsa_unregister(void)
+{
+ int i = ARRAY_SIZE(cmh_slhdsa_algs);
+
+ while (i--)
+ crypto_unregister_sig(&cmh_slhdsa_algs[i]);
+}
diff --git a/drivers/crypto/cmh/cmh_pqc_xmss.c b/drivers/crypto/cmh/cmh_pqc_xmss.c
new file mode 100644
index 000000000000..50de1ef0f488
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pqc_xmss.c
@@ -0,0 +1,238 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- XMSS/XMSS-MT Signature Driver (verify-only, sig_alg, synchronous)
+ *
+ * Registers "xmss" and "xmss-mt" sig algorithms with verify-only
+ * callbacks. Sign is not supported (stateful signature -- key
+ * management must happen externally).
+ *
+ * Verify: src = raw signature, digest = message bytes
+ * Public key: raw pk bytes (variable length, set via set_pub_key)
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <crypto/sig.h>
+#include <crypto/internal/sig.h>
+
+#include "cmh_sys.h"
+#include "cmh_hcq_abi.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_pqc.h"
+
+#define XMSS_VCQ_CMDS 3 /* header + cmd + flush */
+
+struct cmh_xmss_tfm_ctx {
+ u8 *pub_key;
+ u32 pub_key_len;
+ u32 xmss_mt; /* 0 = XMSS, 1 = XMSS-MT */
+};
+
+static inline struct cmh_xmss_tfm_ctx *cmh_xmss_ctx(struct crypto_sig *tfm)
+{
+ return crypto_sig_ctx(tfm);
+}
+
+/*
+ * XMSS/XMSS-MT verify (synchronous sig_alg)
+ *
+ * @src: raw signature
+ * @slen: signature length
+ * @digest: message bytes
+ * @dlen: message length
+ *
+ * Returns 0 on successful verification, negative errno on failure.
+ */
+static int cmh_xmss_verify(struct crypto_sig *tfm,
+ const void *src, unsigned int slen,
+ const void *digest, unsigned int dlen)
+{
+ struct cmh_xmss_tfm_ctx *ctx = cmh_xmss_ctx(tfm);
+ struct core_dispatch d = cmh_core_select_instance(CMH_CORE_HCQ);
+ struct vcq_cmd vcq[XMSS_VCQ_CMDS];
+ u8 *sig_buf = NULL, *m_buf = NULL, *pk_buf = NULL;
+ dma_addr_t sig_dma = DMA_MAPPING_ERROR;
+ dma_addr_t m_dma = DMA_MAPPING_ERROR;
+ dma_addr_t pk_dma = DMA_MAPPING_ERROR;
+ int ret;
+
+ if (!ctx->pub_key)
+ return -EINVAL;
+ if (!slen || slen > XMSS_MAX_SIG_LEN)
+ return -EINVAL;
+ if (!dlen || dlen > XMSS_MAX_MSG_LEN)
+ return -EINVAL;
+
+ sig_buf = kmemdup(src, slen, GFP_KERNEL);
+ m_buf = kmemdup(digest, dlen, GFP_KERNEL);
+ pk_buf = kmemdup(ctx->pub_key, ctx->pub_key_len, GFP_KERNEL);
+ if (!sig_buf || !m_buf || !pk_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ sig_dma = cmh_dma_map_single(sig_buf, slen, DMA_TO_DEVICE);
+ m_dma = cmh_dma_map_single(m_buf, dlen, DMA_TO_DEVICE);
+ pk_dma = cmh_dma_map_single(pk_buf, ctx->pub_key_len, DMA_TO_DEVICE);
+
+ if (cmh_dma_map_error(sig_dma) || cmh_dma_map_error(m_dma) ||
+ cmh_dma_map_error(pk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], XMSS_VCQ_CMDS);
+ vcq_add_hcq_xmss_verify(&vcq[1], d.core_id, ctx->xmss_mt,
+ ctx->pub_key_len, slen, dlen,
+ pk_dma, sig_dma, m_dma);
+ vcq_add_hcq_flush(&vcq[2], d.core_id);
+
+ /* XMSS verify traverses Merkle hash chains -- inherently slow */
+ ret = cmh_tm_submit_sync_tmo(vcq, XMSS_VCQ_CMDS, 1, d.mbx_idx,
+ cmh_tm_slow_op_timeout_jiffies());
+
+out_unmap:
+ if (!cmh_dma_map_error(pk_dma))
+ cmh_dma_unmap_single(pk_dma, ctx->pub_key_len, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, dlen, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, slen, DMA_TO_DEVICE);
+
+out_free:
+ kfree(pk_buf);
+ kfree(m_buf);
+ kfree(sig_buf);
+ return ret;
+}
+
+static int cmh_xmss_set_pub_key(struct crypto_sig *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct cmh_xmss_tfm_ctx *ctx = cmh_xmss_ctx(tfm);
+
+ if (!keylen || keylen > XMSS_MAX_PK_LEN)
+ return -EINVAL;
+
+ kfree(ctx->pub_key);
+ ctx->pub_key = NULL;
+ ctx->pub_key_len = 0;
+
+ ctx->pub_key = kmemdup(key, keylen, GFP_KERNEL);
+ if (!ctx->pub_key)
+ return -ENOMEM;
+
+ ctx->pub_key_len = keylen;
+ return 0;
+}
+
+static unsigned int cmh_xmss_key_size(struct crypto_sig *tfm)
+{
+ struct cmh_xmss_tfm_ctx *ctx = cmh_xmss_ctx(tfm);
+
+ return ctx->pub_key_len * 8;
+}
+
+static int cmh_xmss_init(struct crypto_sig *tfm)
+{
+ struct cmh_xmss_tfm_ctx *ctx = cmh_xmss_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ return 0;
+}
+
+static int cmh_xmss_mt_init(struct crypto_sig *tfm)
+{
+ struct cmh_xmss_tfm_ctx *ctx = cmh_xmss_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->xmss_mt = 1;
+ return 0;
+}
+
+static void cmh_xmss_exit(struct crypto_sig *tfm)
+{
+ struct cmh_xmss_tfm_ctx *ctx = cmh_xmss_ctx(tfm);
+
+ kfree(ctx->pub_key);
+ ctx->pub_key = NULL;
+}
+
+static unsigned int cmh_xmss_max_size(struct crypto_sig *tfm)
+{
+ /* Verify-only; report the max signature length for API parity. */
+ return XMSS_MAX_SIG_LEN;
+}
+
+static struct sig_alg cmh_xmss_algs[] = {
+ {
+ .verify = cmh_xmss_verify,
+ .set_pub_key = cmh_xmss_set_pub_key,
+ .key_size = cmh_xmss_key_size,
+ .max_size = cmh_xmss_max_size,
+ .init = cmh_xmss_init,
+ .exit = cmh_xmss_exit,
+ .base = {
+ .cra_name = "xmss",
+ .cra_driver_name = "rambus-cmh-xmss",
+ .cra_priority = 300,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_xmss_tfm_ctx),
+ },
+ },
+ {
+ .verify = cmh_xmss_verify,
+ .set_pub_key = cmh_xmss_set_pub_key,
+ .key_size = cmh_xmss_key_size,
+ .max_size = cmh_xmss_max_size,
+ .init = cmh_xmss_mt_init,
+ .exit = cmh_xmss_exit,
+ .base = {
+ .cra_name = "xmss-mt",
+ .cra_driver_name = "rambus-cmh-xmss-mt",
+ .cra_priority = 300,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_xmss_tfm_ctx),
+ },
+ },
+};
+
+/**
+ * cmh_pqc_xmss_register() - Register XMSS/XMSS-MT sig algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_pqc_xmss_register(void)
+{
+ int ret, i;
+
+ for (i = 0; i < ARRAY_SIZE(cmh_xmss_algs); i++) {
+ ret = crypto_register_sig(&cmh_xmss_algs[i]);
+ if (ret) {
+ dev_err(cmh_dev(), "cmh: failed to register %s (%d)\n",
+ cmh_xmss_algs[i].base.cra_name, ret);
+ goto err_unregister;
+ }
+ }
+
+ return 0;
+
+err_unregister:
+ while (i--)
+ crypto_unregister_sig(&cmh_xmss_algs[i]);
+ return ret;
+}
+
+/**
+ * cmh_pqc_xmss_unregister() - Unregister XMSS/XMSS-MT sig algorithms from the crypto framework
+ */
+void cmh_pqc_xmss_unregister(void)
+{
+ int i = ARRAY_SIZE(cmh_xmss_algs);
+
+ while (i--)
+ crypto_unregister_sig(&cmh_xmss_algs[i]);
+}
--
2.43.7
^ permalink raw reply related
* [PATCH v3 04/19] crypto: cmh - add SHA-2/SHA-3/SHAKE ahash
From: Saravanakrishnan Krishnamoorthy @ 2026-08-06 19:55 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register ahash algorithms for SHA-224, SHA-256, SHA-384, SHA-512,
SHA3-224, SHA3-256, SHA3-384, SHA3-512, SHAKE128, and SHAKE256
using the CMH hash core (core ID 0x02).
Supports incremental update/finup/final, init/export/import for
request cloning, and the CRYPTO_AHASH_REQ_VIRT flag for zero-copy
from virtual buffers.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 3 +-
drivers/crypto/cmh/cmh_hash.c | 888 ++++++++++++++++++++++++++
drivers/crypto/cmh/cmh_main.c | 9 +
drivers/crypto/cmh/include/cmh_hash.h | 26 +
4 files changed, 925 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_hash.c
create mode 100644 drivers/crypto/cmh/include/cmh_hash.h
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index bb7772555d6a..79c94d87e9ee 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -14,7 +14,8 @@ cmh-y := \
cmh_dma.o \
cmh_sysfs.o \
cmh_key.o \
- cmh_sys.o
+ cmh_sys.o \
+ cmh_hash.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_hash.c b/drivers/crypto/cmh/cmh_hash.c
new file mode 100644
index 000000000000..2a9a3c6a6985
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_hash.c
@@ -0,0 +1,888 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API Hash Driver
+ *
+ * Registers asynchronous hash (ahash) algorithms with the Linux crypto
+ * subsystem. Implements SHA-2 (224/256/384/512), SHA-3
+ * (224/256/384/512), and SHAKE (128/256) families using the CMH Hash
+ * Core (HC).
+ *
+ * Incremental HW update model -- each .update() with enough data for
+ * at least one full block submits a self-contained VCQ transaction:
+ *
+ * .init() -> software-only: zero per-request context
+ * .update() -> buffer data in holdback; when >= block_size bytes:
+ * INIT [+ RESTORE] + UPDATE(full blocks) + SAVE + FLUSH
+ * -> return -EINPROGRESS (else return 0, data in holdback)
+ * .final() -> INIT [+ RESTORE] [+ UPDATE(residual)] + FINAL + FLUSH
+ * .finup() -> linearise holdback + new data, then final path
+ * .digest() -> INIT + UPDATE + FINAL + FLUSH (single-shot, zero-copy)
+ * .export() -> software-only: copy checkpoint + holdback to out
+ * .import() -> software-only: restore checkpoint + holdback from in
+ *
+ * The FLUSH after each .update() releases the HC core, so no lockout.
+ * Two hash sessions interleave fine on the same MBX -- each saves its
+ * own state via SAVE and restores via RESTORE on the next call.
+ *
+ * Export/import is purely software (no HW interaction), enabling
+ * crypto API transform clone for all plain-hash algorithms.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <crypto/internal/hash.h>
+#include <crypto/scatterwalk.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "cmh_hash.h"
+#include "cmh_vcq.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+
+/* Algorithm Table */
+
+struct cmh_hash_alg_info {
+ u32 hc_algo; /* HC_ALGO_* (SHA2, SHA3, SHAKE) */
+ u32 digest_size; /* bytes */
+ u32 block_size; /* cra_blocksize for Linux crypto API */
+ const char *alg_name; /* Linux crypto name: "sha256" */
+ const char *drv_name; /* driver name: "rambus-cmh-sha256" */
+};
+
+static const struct cmh_hash_alg_info cmh_hash_algs_info[] = {
+ /* SHA-2 family */
+ {
+ .hc_algo = HC_ALGO_SHA2_224,
+ .digest_size = CMH_SHA224_DIGEST_SIZE,
+ .block_size = 64,
+ .alg_name = "sha224",
+ .drv_name = "rambus-cmh-sha224",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA2_256,
+ .digest_size = CMH_SHA256_DIGEST_SIZE,
+ .block_size = 64,
+ .alg_name = "sha256",
+ .drv_name = "rambus-cmh-sha256",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA2_384,
+ .digest_size = CMH_SHA384_DIGEST_SIZE,
+ .block_size = 128,
+ .alg_name = "sha384",
+ .drv_name = "rambus-cmh-sha384",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA2_512,
+ .digest_size = CMH_SHA512_DIGEST_SIZE,
+ .block_size = 128,
+ .alg_name = "sha512",
+ .drv_name = "rambus-cmh-sha512",
+ },
+ /* SHA-3 family */
+ {
+ .hc_algo = HC_ALGO_SHA3_224,
+ .digest_size = CMH_SHA3_224_DIGEST_SIZE,
+ .block_size = 144, /* rate = 1600/8 - 2*224/8 = 144 */
+ .alg_name = "sha3-224",
+ .drv_name = "rambus-cmh-sha3-224",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA3_256,
+ .digest_size = CMH_SHA3_256_DIGEST_SIZE,
+ .block_size = 136, /* rate = 1600/8 - 2*256/8 = 136 */
+ .alg_name = "sha3-256",
+ .drv_name = "rambus-cmh-sha3-256",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA3_384,
+ .digest_size = CMH_SHA3_384_DIGEST_SIZE,
+ .block_size = 104, /* rate = 1600/8 - 2*384/8 = 104 */
+ .alg_name = "sha3-384",
+ .drv_name = "rambus-cmh-sha3-384",
+ },
+ {
+ .hc_algo = HC_ALGO_SHA3_512,
+ .digest_size = CMH_SHA3_512_DIGEST_SIZE,
+ .block_size = 72, /* rate = 1600/8 - 2*512/8 = 72 */
+ .alg_name = "sha3-512",
+ .drv_name = "rambus-cmh-sha3-512",
+ },
+ /*
+ * SHAKE (XOF) family -- fixed-output ahash registration.
+ *
+ * cra_blocksize = 1: SHAKE is a sponge/XOF, not Merkle-Damgaard.
+ * The Keccak rate (168 for SHAKE-128, 136 for SHAKE-256) exceeds
+ * MAX_ALGAPI_BLOCKSIZE (160) on Linux <=6.7. Using 1 signals
+ * "byte-oriented" which is correct for XOF consumers. The kernel
+ * raised the limit to 208 in 6.8 (commit 2f3a22704889).
+ */
+ {
+ .hc_algo = HC_ALGO_SHAKE128,
+ .digest_size = CMH_SHAKE128_DIGEST_SIZE,
+ .block_size = 1, /* XOF: no meaningful block for crypto API */
+ .alg_name = "shake128",
+ .drv_name = "rambus-cmh-shake128",
+ },
+ {
+ .hc_algo = HC_ALGO_SHAKE256,
+ .digest_size = CMH_SHAKE256_DIGEST_SIZE,
+ .block_size = 1, /* XOF: no meaningful block for crypto API */
+ .alg_name = "shake256",
+ .drv_name = "rambus-cmh-shake256",
+ },
+};
+
+#define CMH_HASH_ALG_COUNT ARRAY_SIZE(cmh_hash_algs_info)
+
+/* Per-Request State */
+
+/* Maximum cra_blocksize across all registered algorithms (SHA3-224) */
+#define CMH_HASH_MAX_BLOCK 144
+
+/*
+ * Exported hash state -- serialised by .export(), deserialised by
+ * .import(). This is what statesize advertises to the crypto subsystem.
+ */
+struct cmh_hash_export_state {
+ u8 checkpoint[HC_CONTEXT_SIZE]; /* HC context from last SAVE */
+ u8 buf[CMH_HASH_MAX_BLOCK]; /* holdback buffer */
+ u32 buf_len; /* valid bytes in buf[] */
+ u32 hw_started; /* non-zero if checkpoint valid */
+};
+
+/*
+ * Maximum payload commands any hash transaction can produce:
+ * INIT + RESTORE + UPDATE + SAVE/FINAL + FLUSH = 5
+ * Worst-case packed output (stride=7, 1 payload per VCQ):
+ * 5 VCQs x 2 entries = 10
+ */
+#define CMH_HASH_MAX_PAYLOAD 5
+#define CMH_HASH_MAX_PACKED (CMH_HASH_MAX_PAYLOAD * 2)
+
+/*
+ * Stored in ahash_request_ctx(). Tracks the algorithm, a holdback
+ * buffer for partial blocks, an HC context checkpoint from the last
+ * SAVE, and DMA state for the current in-flight async operation.
+ *
+ * The checkpoint is embedded inline rather than heap-allocated because
+ * the kernel ahash API has no per-request destructor. If a request is
+ * abandoned without .final() (e.g. transform freed early), a heap
+ * checkpoint would leak unconditionally.
+ */
+struct cmh_hash_reqctx {
+ const struct cmh_hash_alg_info *info;
+ int error;
+ u32 hw_started; /* non-zero after first HW submission */
+ u32 buf_len; /* bytes in holdback buf[] */
+ u32 has_checkpoint; /* non-zero if checkpoint[] valid */
+ /* DMA state for current async operation */
+ dma_addr_t ckpt_dma; /* RESTORE input */
+ dma_addr_t save_dma; /* SAVE output (update only) */
+ dma_addr_t data_dma; /* UPDATE input */
+ dma_addr_t digest_dma; /* FINAL output (final/digest only) */
+ u8 *save_buf; /* SAVE output buffer */
+ u8 *data_buf; /* linearised data for DMA */
+ u32 data_len; /* bytes in data_buf */
+ u8 *digest_buf; /* digest output buffer */
+ u8 buf[CMH_HASH_MAX_BLOCK]; /* holdback for partial block */
+ u8 checkpoint[HC_CONTEXT_SIZE]; /* HC context from last SAVE */
+ struct vcq_cmd packed[CMH_HASH_MAX_PACKED];
+};
+
+/* VCQ Builders (HC-specific; shared builders in cmh_hc_abi.h / cmh_vcq.h) */
+
+/* Add an HC_CMD_UPDATE entry */
+static void vcq_add_hc_update(struct vcq_cmd *slot, u32 core_id, u64 input_phys, u32 len)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_UPDATE);
+ slot->hwc.hc.cmd_update.input = input_phys;
+ slot->hwc.hc.cmd_update.inlen = len;
+}
+
+/* Add an HC_CMD_SAVE entry */
+static void vcq_add_hc_save(struct vcq_cmd *slot, u32 core_id, u64 output_phys, u32 outlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_SAVE);
+ slot->hwc.hc.cmd_save.output = output_phys;
+ slot->hwc.hc.cmd_save.outlen = outlen;
+}
+
+/* Add an HC_CMD_RESTORE entry */
+static void vcq_add_hc_restore(struct vcq_cmd *slot, u32 core_id, u64 input_phys, u32 inlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_RESTORE);
+ slot->hwc.hc.cmd_restore.input = input_phys;
+ slot->hwc.hc.cmd_restore.inlen = inlen;
+}
+
+/* Request Context Cleanup */
+
+static void cmh_hash_free_reqctx(struct cmh_hash_reqctx *rctx)
+{
+ rctx->has_checkpoint = 0;
+}
+
+/* VCQ Packing + Submit */
+
+/* ahash Operations */
+
+/*
+ * Wrapper struct: embeds ahash_alg + a pointer to our alg_info table
+ * entry so we can recover it in the tfm callbacks.
+ */
+struct cmh_hash_alg_drv {
+ struct ahash_alg alg;
+ const struct cmh_hash_alg_info *info;
+};
+
+/*
+ * Find the cmh_hash_alg_info from the crypto_ahash (embedded in our
+ * registered template). We stash the info pointer in the algorithm's
+ * driver-private area at registration time (see cmh_hash_register).
+ */
+static const struct cmh_hash_alg_info *
+cmh_hash_get_info(struct crypto_ahash *tfm)
+{
+ struct ahash_alg *alg = crypto_ahash_alg(tfm);
+
+ return container_of(alg, struct cmh_hash_alg_drv, alg)->info;
+}
+
+static int cmh_hash_init(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->info = cmh_hash_get_info(tfm);
+ return 0;
+}
+
+/*
+ * Update completion -- called from threaded IRQ after SAVE completes.
+ * Takes ownership of save_buf as the new checkpoint.
+ */
+static void cmh_hash_update_complete(void *data, int error)
+{
+ struct ahash_request *req = data;
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ /* Unmap DMA buffers */
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->save_dma, HC_CONTEXT_SIZE,
+ DMA_FROM_DEVICE);
+ cmh_dma_unmap_single(rctx->data_dma, rctx->data_len,
+ DMA_TO_DEVICE);
+
+ if (!error) {
+ memcpy(rctx->checkpoint, rctx->save_buf, HC_CONTEXT_SIZE);
+ rctx->has_checkpoint = 1;
+ kfree(rctx->save_buf);
+ rctx->save_buf = NULL;
+ rctx->hw_started = 1;
+ } else {
+ kfree(rctx->save_buf);
+ rctx->save_buf = NULL;
+ rctx->error = error;
+ }
+
+ kfree(rctx->data_buf);
+ rctx->data_buf = NULL;
+ rctx->data_len = 0;
+
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * .update -- buffer incoming data, submit full blocks to HW.
+ *
+ * Maintains a partial-block holdback buffer in rctx->buf[]. When
+ * enough data is available for at least one full block, the full
+ * blocks are linearised and submitted as:
+ * INIT [+ RESTORE] + UPDATE(full_blocks) + SAVE + FLUSH
+ *
+ * The tail (< block_size) stays in the holdback for the next call.
+ * Returns -EINPROGRESS on HW submission, 0 if only buffering.
+ */
+static int cmh_hash_update(struct ahash_request *req)
+{
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_hash_alg_info *info = rctx->info;
+ struct vcq_cmd cmds[CMH_HASH_MAX_PAYLOAD];
+ struct core_dispatch d;
+ u32 block_size = info->block_size;
+ u32 total_avail, full_len, tail_len, from_src;
+ u32 idx;
+ int ret;
+ gfp_t gfp;
+
+ if (rctx->error)
+ return rctx->error;
+
+ if (!req->nbytes)
+ return 0;
+
+ /*
+ * Each update() linearises its data into a single kmalloc buffer.
+ * Guard against rctx->buf_len + req->nbytes overflowing u32, which
+ * would mis-select the "buffer only" path below and overrun the
+ * holdback. Such an update is far too large to service anyway.
+ */
+ if (req->nbytes > U32_MAX - rctx->buf_len)
+ return -EMSGSIZE;
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ total_avail = rctx->buf_len + req->nbytes;
+
+ /* Not enough for a full block -- just buffer */
+ if (total_avail < block_size) {
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(rctx->buf + rctx->buf_len,
+ req->svirt, req->nbytes);
+ else
+ scatterwalk_map_and_copy(rctx->buf + rctx->buf_len,
+ req->src, 0,
+ req->nbytes, 0);
+ rctx->buf_len = total_avail;
+ return 0;
+ }
+
+ /* Have at least one full block -- submit to HW */
+ full_len = total_avail - total_avail % block_size;
+ tail_len = total_avail - full_len;
+ from_src = full_len - rctx->buf_len;
+
+ /*
+ * Linearise: holdback prefix + full blocks from scatterlist.
+ * full_len is user-controlled and can exceed KMALLOC_MAX_SIZE for a
+ * single large update; __GFP_NOWARN avoids splatting the page
+ * allocator on an oversized request.
+ */
+ rctx->data_buf = kmalloc(full_len, gfp | __GFP_NOWARN);
+ if (!rctx->data_buf)
+ return -ENOMEM;
+
+ if (rctx->buf_len > 0)
+ memcpy(rctx->data_buf, rctx->buf, rctx->buf_len);
+
+ if (from_src > 0) {
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(rctx->data_buf + rctx->buf_len,
+ req->svirt, from_src);
+ else
+ scatterwalk_map_and_copy(rctx->data_buf + rctx->buf_len,
+ req->src, 0,
+ from_src, 0);
+ }
+
+ /* Move tail to holdback */
+ if (tail_len > 0) {
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(rctx->buf, req->svirt + from_src,
+ tail_len);
+ else
+ scatterwalk_map_and_copy(rctx->buf, req->src,
+ from_src, tail_len,
+ 0);
+ }
+ rctx->buf_len = tail_len;
+ rctx->data_len = full_len;
+
+ /* Allocate SAVE output buffer */
+ rctx->save_buf = kzalloc(HC_CONTEXT_SIZE, gfp);
+ if (!rctx->save_buf) {
+ ret = -ENOMEM;
+ goto err_free;
+ }
+
+ /* DMA map data, save output, and checkpoint */
+ rctx->data_dma = cmh_dma_map_single(rctx->data_buf, full_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->data_dma)) {
+ ret = -ENOMEM;
+ goto err_free;
+ }
+
+ rctx->save_dma = cmh_dma_map_single(rctx->save_buf, HC_CONTEXT_SIZE,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->save_dma)) {
+ ret = -ENOMEM;
+ goto err_unmap_data;
+ }
+
+ rctx->ckpt_dma = DMA_MAPPING_ERROR;
+ if (rctx->has_checkpoint) {
+ rctx->ckpt_dma = cmh_dma_map_single(rctx->checkpoint,
+ HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->ckpt_dma)) {
+ ret = -ENOMEM;
+ goto err_unmap_save;
+ }
+ }
+
+ /* Build VCQ: INIT [+ RESTORE] + UPDATE + SAVE + FLUSH */
+ d = cmh_core_select_instance(CMH_CORE_HC);
+ idx = 0;
+
+ vcq_add_hc_init(&cmds[idx++], d.core_id, info->hc_algo);
+
+ if (rctx->has_checkpoint)
+ vcq_add_hc_restore(&cmds[idx++], d.core_id,
+ (u64)rctx->ckpt_dma, HC_CONTEXT_SIZE);
+
+ vcq_add_hc_update(&cmds[idx++], d.core_id,
+ (u64)rctx->data_dma, full_len);
+
+ vcq_add_hc_save(&cmds[idx++], d.core_id,
+ (u64)rctx->save_dma, HC_CONTEXT_SIZE);
+
+ vcq_add_flush(&cmds[idx++], d.core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_HASH_MAX_PACKED,
+ d.mbx_idx,
+ cmh_hash_update_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto err_unmap_ckpt;
+
+ return -EINPROGRESS;
+
+err_unmap_ckpt:
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+err_unmap_save:
+ cmh_dma_unmap_single(rctx->save_dma, HC_CONTEXT_SIZE,
+ DMA_FROM_DEVICE);
+err_unmap_data:
+ cmh_dma_unmap_single(rctx->data_dma, full_len, DMA_TO_DEVICE);
+err_free:
+ kfree(rctx->save_buf);
+ rctx->save_buf = NULL;
+ kfree(rctx->data_buf);
+ rctx->data_buf = NULL;
+ rctx->data_len = 0;
+ return ret;
+}
+
+/*
+ * Final completion -- unmap all DMA, copy digest, signal done.
+ */
+static void cmh_hash_final_complete(void *data, int error)
+{
+ struct ahash_request *req = data;
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+
+ if (error == -EINPROGRESS) {
+ cmh_complete(&req->base, error);
+ return;
+ }
+
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+ if (rctx->data_buf)
+ cmh_dma_unmap_single(rctx->data_dma, rctx->data_len,
+ DMA_TO_DEVICE);
+ cmh_dma_unmap_single(rctx->digest_dma, rctx->info->digest_size,
+ DMA_FROM_DEVICE);
+
+ if (!error)
+ memcpy(req->result, rctx->digest_buf,
+ rctx->info->digest_size);
+
+ kfree(rctx->digest_buf);
+ rctx->digest_buf = NULL;
+ kfree(rctx->data_buf);
+ rctx->data_buf = NULL;
+ cmh_hash_free_reqctx(rctx);
+ cmh_complete(&req->base, error);
+}
+
+/*
+ * Submit the final VCQ transaction:
+ * INIT [+ RESTORE] [+ UPDATE(residual)] + FINAL + FLUSH
+ *
+ * @data_buf: linearised residual data, or NULL for empty-hash.
+ * Ownership transferred -- callback frees it.
+ * @data_len: bytes in data_buf.
+ */
+static int cmh_hash_submit_final(struct ahash_request *req,
+ u8 *data_buf, u32 data_len)
+{
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_hash_alg_info *info = rctx->info;
+ struct vcq_cmd cmds[CMH_HASH_MAX_PAYLOAD];
+ struct core_dispatch d;
+ u32 idx;
+ int ret;
+ gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ rctx->data_buf = data_buf;
+ rctx->data_len = data_len;
+
+ /* Allocate digest output buffer */
+ rctx->digest_buf = kzalloc(info->digest_size, gfp);
+ if (!rctx->digest_buf) {
+ ret = -ENOMEM;
+ goto err_free_data;
+ }
+
+ rctx->digest_dma = cmh_dma_map_single(rctx->digest_buf,
+ info->digest_size,
+ DMA_FROM_DEVICE);
+ if (cmh_dma_map_error(rctx->digest_dma)) {
+ ret = -ENOMEM;
+ goto err_free_digest;
+ }
+
+ /* Map residual data for UPDATE */
+ rctx->data_dma = DMA_MAPPING_ERROR;
+ if (data_buf && data_len > 0) {
+ rctx->data_dma = cmh_dma_map_single(data_buf, data_len,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->data_dma)) {
+ ret = -ENOMEM;
+ goto err_unmap_digest;
+ }
+ }
+
+ /* Map checkpoint for RESTORE */
+ rctx->ckpt_dma = DMA_MAPPING_ERROR;
+ if (rctx->has_checkpoint) {
+ rctx->ckpt_dma = cmh_dma_map_single(rctx->checkpoint,
+ HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+ if (cmh_dma_map_error(rctx->ckpt_dma)) {
+ ret = -ENOMEM;
+ goto err_unmap_data;
+ }
+ }
+
+ /* Build VCQ: INIT [+ RESTORE] [+ UPDATE] + FINAL + FLUSH */
+ d = cmh_core_select_instance(CMH_CORE_HC);
+ idx = 0;
+
+ vcq_add_hc_init(&cmds[idx++], d.core_id, info->hc_algo);
+
+ if (rctx->has_checkpoint)
+ vcq_add_hc_restore(&cmds[idx++], d.core_id,
+ (u64)rctx->ckpt_dma, HC_CONTEXT_SIZE);
+
+ if (data_buf && data_len > 0)
+ vcq_add_hc_update(&cmds[idx++], d.core_id,
+ (u64)rctx->data_dma, data_len);
+
+ vcq_add_hc_final(&cmds[idx++], d.core_id,
+ (u64)rctx->digest_dma, info->digest_size);
+
+ vcq_add_flush(&cmds[idx++], d.core_id);
+
+ ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed,
+ CMH_HASH_MAX_PACKED,
+ d.mbx_idx,
+ cmh_hash_final_complete, req,
+ !!(req->base.flags &
+ CRYPTO_TFM_REQ_MAY_BACKLOG),
+ cmh_tm_async_timeout_jiffies());
+ if (ret == -EBUSY)
+ return -EBUSY;
+ if (ret)
+ goto err_unmap_ckpt;
+
+ return -EINPROGRESS;
+
+err_unmap_ckpt:
+ if (rctx->has_checkpoint)
+ cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE,
+ DMA_TO_DEVICE);
+err_unmap_data:
+ if (data_buf && data_len > 0)
+ cmh_dma_unmap_single(rctx->data_dma, data_len,
+ DMA_TO_DEVICE);
+err_unmap_digest:
+ cmh_dma_unmap_single(rctx->digest_dma, info->digest_size,
+ DMA_FROM_DEVICE);
+err_free_digest:
+ kfree(rctx->digest_buf);
+ rctx->digest_buf = NULL;
+err_free_data:
+ kfree(data_buf);
+ rctx->data_buf = NULL;
+ cmh_hash_free_reqctx(rctx);
+ return ret;
+}
+
+static int cmh_hash_final(struct ahash_request *req)
+{
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+ u8 *data_buf = NULL;
+ u32 data_len = 0;
+ gfp_t gfp;
+
+ if (rctx->error)
+ return rctx->error;
+
+ if (rctx->buf_len > 0) {
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+ data_buf = kmalloc(rctx->buf_len, gfp);
+ if (!data_buf)
+ return -ENOMEM;
+ memcpy(data_buf, rctx->buf, rctx->buf_len);
+ data_len = rctx->buf_len;
+ rctx->buf_len = 0;
+ }
+
+ return cmh_hash_submit_final(req, data_buf, data_len);
+}
+
+static int cmh_hash_finup(struct ahash_request *req);
+
+/*
+ * One-shot digest -- delegates to init + finup so that all data is
+ * linearised and mapped through cmh_dma_map_single(), which is the
+ * only DMA mapping path aware of all supported DMA backends.
+ */
+static int cmh_hash_digest(struct ahash_request *req)
+{
+ int ret;
+
+ ret = cmh_hash_init(req);
+ if (ret)
+ return ret;
+ return cmh_hash_finup(req);
+}
+
+/*
+ * .finup -- update + final combined into a single transaction.
+ *
+ * Linearises the holdback buffer + new data and submits everything
+ * through the final path. Avoids the kernel's ahash_def_finup()
+ * which would allocate a subrequest and clone via export/import.
+ */
+static int cmh_hash_finup(struct ahash_request *req)
+{
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+ u32 data_len;
+ u8 *data_buf;
+ gfp_t gfp;
+
+ if (rctx->error)
+ return rctx->error;
+
+ data_len = rctx->buf_len + req->nbytes;
+
+ if (data_len == 0)
+ return cmh_hash_submit_final(req, NULL, 0);
+
+ gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ data_buf = kmalloc(data_len, gfp | __GFP_NOWARN);
+ if (!data_buf)
+ return -ENOMEM;
+
+ if (rctx->buf_len > 0)
+ memcpy(data_buf, rctx->buf, rctx->buf_len);
+
+ if (req->nbytes > 0) {
+ if (req->base.flags & CRYPTO_AHASH_REQ_VIRT)
+ memcpy(data_buf + rctx->buf_len,
+ req->svirt, req->nbytes);
+ else
+ scatterwalk_map_and_copy(data_buf + rctx->buf_len,
+ req->src, 0,
+ req->nbytes, 0);
+ }
+
+ rctx->buf_len = 0;
+ return cmh_hash_submit_final(req, data_buf, data_len);
+}
+
+/*
+ * Export -- purely software.
+ *
+ * Serialise the HC checkpoint (if any) and holdback buffer into the
+ * export state structure. No HW interaction needed because the
+ * incremental model keeps checkpoint up-to-date after each .update().
+ */
+static int cmh_hash_export(struct ahash_request *req, void *out)
+{
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+ struct cmh_hash_export_state *state = out;
+
+ /*
+ * Zero the whole exported state first: only rctx->buf_len bytes of
+ * buf[] are populated below and the struct may carry padding, so
+ * without this the unused tail/padding would leak kernel memory to
+ * user space through the ahash export (e.g. algif_hash).
+ */
+ memset(state, 0, sizeof(*state));
+
+ if (rctx->hw_started && rctx->has_checkpoint)
+ memcpy(state->checkpoint, rctx->checkpoint, HC_CONTEXT_SIZE);
+
+ if (rctx->buf_len > 0)
+ memcpy(state->buf, rctx->buf, rctx->buf_len);
+
+ state->buf_len = rctx->buf_len;
+ state->hw_started = rctx->hw_started;
+
+ return 0;
+}
+
+/*
+ * Import -- purely software.
+ *
+ * Restore checkpoint and holdback from a previously exported state.
+ * The next .update() or .final() will RESTORE the checkpoint into HW.
+ */
+static int cmh_hash_import(struct ahash_request *req, const void *in)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cmh_hash_reqctx *rctx = ahash_request_ctx(req);
+ const struct cmh_hash_export_state *state = in;
+
+ memset(rctx, 0, sizeof(*rctx));
+ rctx->info = cmh_hash_get_info(tfm);
+
+ /*
+ * Reject a corrupt or malicious state. The holdback invariant is
+ * buf_len < block_size (update() never leaves a full block in the
+ * holdback); a larger value would underflow from_src and overrun
+ * the linearisation buffer in the next update(). block_size never
+ * exceeds the buf[] size (CMH_HASH_MAX_BLOCK), so this also bounds
+ * the memcpy below.
+ */
+ if (state->buf_len >= rctx->info->block_size)
+ return -EINVAL;
+
+ rctx->hw_started = state->hw_started;
+ rctx->buf_len = state->buf_len;
+ memcpy(rctx->buf, state->buf, state->buf_len);
+
+ if (state->hw_started) {
+ memcpy(rctx->checkpoint, state->checkpoint, HC_CONTEXT_SIZE);
+ rctx->has_checkpoint = 1;
+ }
+
+ return 0;
+}
+
+/* Transform init (cra_init) -- set per-request context size */
+
+static int cmh_hash_cra_init(struct crypto_tfm *tfm)
+{
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct cmh_hash_reqctx));
+ return 0;
+}
+
+/* Registration */
+
+static struct cmh_hash_alg_drv cmh_hash_drvs[CMH_HASH_ALG_COUNT];
+
+/**
+ * cmh_hash_register() - Register SHA-256/384/512/3-256/3-384/3-512 hash algorithms
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_hash_register(void)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < CMH_HASH_ALG_COUNT; i++) {
+ const struct cmh_hash_alg_info *info = &cmh_hash_algs_info[i];
+ struct cmh_hash_alg_drv *drv = &cmh_hash_drvs[i];
+ struct ahash_alg *alg = &drv->alg;
+
+ drv->info = info;
+
+ alg->init = cmh_hash_init;
+ alg->update = cmh_hash_update;
+ alg->final = cmh_hash_final;
+ alg->finup = cmh_hash_finup;
+ alg->digest = cmh_hash_digest;
+ alg->export = cmh_hash_export;
+ alg->import = cmh_hash_import;
+
+ alg->halg.digestsize = info->digest_size;
+ alg->halg.statesize = sizeof(struct cmh_hash_export_state);
+
+ strscpy(alg->halg.base.cra_name, info->alg_name,
+ CRYPTO_MAX_ALG_NAME);
+ strscpy(alg->halg.base.cra_driver_name, info->drv_name,
+ CRYPTO_MAX_ALG_NAME);
+ alg->halg.base.cra_priority = 300;
+ alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NO_FALLBACK |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_REQ_VIRT;
+ alg->halg.base.cra_blocksize = info->block_size;
+ alg->halg.base.cra_ctxsize = 0;
+ alg->halg.base.cra_init = cmh_hash_cra_init;
+ alg->halg.base.cra_module = THIS_MODULE;
+
+ ret = crypto_register_ahash(alg);
+ if (ret) {
+ dev_err(cmh_dev(), "hash: failed to register %s (rc=%d)\n",
+ info->drv_name, ret);
+ /* Unregister any already-registered algorithms */
+ while (i--)
+ crypto_unregister_ahash(&cmh_hash_drvs[i].alg);
+ return ret;
+ }
+
+ dev_dbg(cmh_dev(), "hash: registered %s (priority 300)\n",
+ info->drv_name);
+ }
+
+ dev_info(cmh_dev(), "hash: %zu algorithm(s) registered\n",
+ CMH_HASH_ALG_COUNT);
+ return 0;
+}
+
+/**
+ * cmh_hash_unregister() - Unregister SHA hash algorithms from the crypto framework
+ */
+void cmh_hash_unregister(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < CMH_HASH_ALG_COUNT; i++) {
+ crypto_unregister_ahash(&cmh_hash_drvs[i].alg);
+ dev_dbg(cmh_dev(), "hash: unregistered %s\n",
+ cmh_hash_algs_info[i].drv_name);
+ }
+
+ dev_info(cmh_dev(), "hash: cleaned up\n");
+}
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index 576a0c02e6f9..92ed9bc0ff2d 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -31,6 +31,7 @@
#include "cmh_mqi.h"
#include "cmh_txn.h"
#include "cmh_rh.h"
+#include "cmh_hash.h"
#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
@@ -237,6 +238,11 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_rh_init;
+ /* Register hash algorithms with the kernel crypto API */
+ ret = cmh_hash_register();
+ if (ret)
+ goto err_hash_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -249,6 +255,8 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_hash_unregister();
+err_hash_register:
cmh_rh_cleanup(cfg);
err_rh_init:
cmh_tm_cleanup();
@@ -275,6 +283,7 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_hash_unregister();
cmh_rh_cleanup(cfg);
cmh_tm_cleanup();
cmh_mqi_cleanup(cfg);
diff --git a/drivers/crypto/cmh/include/cmh_hash.h b/drivers/crypto/cmh/include/cmh_hash.h
new file mode 100644
index 000000000000..bf17d3af7787
--- /dev/null
+++ b/drivers/crypto/cmh/include/cmh_hash.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- Kernel Crypto API Hash Driver
+ *
+ * Registers ahash algorithms (SHA-2, SHA-3, and SHAKE families) with the
+ * Linux crypto subsystem. Uses an incremental HW update model:
+ *
+ * .init() -> software-only: zero per-request context
+ * .update() -> holdback partial blocks; submit full blocks via
+ * INIT [+ RESTORE] + UPDATE + SAVE + FLUSH
+ * .final() -> INIT [+ RESTORE] [+ UPDATE(residual)] + FINAL + FLUSH
+ * .digest() -> INIT + UPDATE + FINAL + FLUSH (single-shot)
+ * .export() -> software-only: copy checkpoint + holdback
+ * .import() -> software-only: restore checkpoint + holdback
+ */
+
+#ifndef CMH_HASH_H
+#define CMH_HASH_H
+
+#include "cmh_config.h"
+
+int cmh_hash_register(void);
+void cmh_hash_unregister(void);
+
+#endif /* CMH_HASH_H */
--
2.43.7
^ permalink raw reply related
* [PATCH v3 15/19] crypto: cmh - add ML-KEM/ML-DSA (QSE)
From: Saravanakrishnan Krishnamoorthy @ 2026-08-06 19:55 UTC (permalink / raw)
To: Albert Ou, Alex Ousherovitch, Conor Dooley, David S. Miller,
Herbert Xu, Jonathan Corbet, Krzysztof Kozlowski, Palmer Dabbelt,
Paul Walmsley, Rob Herring, Saravanakrishnan Krishnamoorthy,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-1-skrishnamoorthy@rambus.com>
From: Alex Ousherovitch <aousherovitch@rambus.com>
Register ML-KEM (Kyber) and ML-DSA (Dilithium) algorithms using
the CMH QSE core (core ID 0x09). ML-KEM is ioctl-only (keygen,
encaps, decaps). ML-DSA is registered as a sig algorithm with
priority 5001 to override the kernel's verify-only mldsa
implementation at priority 5000. This follows the established
pattern where hardware drivers override software-only fallbacks
(e.g. ccp at 300 over generic AES at 100, qat similarly). The
CMH driver provides full HW-accelerated sign + verify vs the
kernel's verify-only software implementation.
Includes cmh_pqc_sizes.c with compile-time tables of PQC key and
signature sizes for all supported parameter sets.
Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
Reviewed-by: Thi Nguyen <thin@rambus.com>
---
drivers/crypto/cmh/Makefile | 5 +-
drivers/crypto/cmh/cmh_main.c | 9 +
drivers/crypto/cmh/cmh_pqc_mldsa.c | 383 +++++++++++++++++++++++++++++
drivers/crypto/cmh/cmh_pqc_sizes.c | 39 +++
drivers/crypto/cmh/cmh_qse.c | 211 ++++++++++++++++
5 files changed, 646 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cmh/cmh_pqc_mldsa.c
create mode 100644 drivers/crypto/cmh/cmh_pqc_sizes.c
create mode 100644 drivers/crypto/cmh/cmh_qse.c
diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile
index da420d0ec1fe..7196d992487e 100644
--- a/drivers/crypto/cmh/Makefile
+++ b/drivers/crypto/cmh/Makefile
@@ -33,7 +33,10 @@ cmh-y := \
cmh_pke_common.o \
cmh_pke_rsa.o \
cmh_pke_ecdsa.o \
- cmh_pke_ecdh.o
+ cmh_pke_ecdh.o \
+ cmh_qse.o \
+ cmh_pqc_mldsa.o \
+ cmh_pqc_sizes.o
# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls.
cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \
diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c
index 302d975256e8..74d2b3672d2a 100644
--- a/drivers/crypto/cmh/cmh_main.c
+++ b/drivers/crypto/cmh/cmh_main.c
@@ -41,6 +41,7 @@
#include "cmh_sm4.h"
#include "cmh_ccp.h"
#include "cmh_pke.h"
+#include "cmh_pqc.h"
#include "cmh_mgmt.h"
#include "cmh_registers.h"
#include "cmh_debugfs.h"
@@ -337,6 +338,11 @@ static int cmh_probe(struct platform_device *pdev)
if (ret)
goto err_pke_ecdh_register;
+ /* Register PQC ML-KEM/ML-DSA */
+ ret = cmh_pqc_mldsa_register();
+ if (ret)
+ goto err_pqc_mldsa_register;
+
/* Register key management device (/dev/cmh_mgmt) */
ret = cmh_mgmt_register();
if (ret)
@@ -349,6 +355,8 @@ static int cmh_probe(struct platform_device *pdev)
return 0;
err_mgmt_register:
+ cmh_pqc_mldsa_unregister();
+err_pqc_mldsa_register:
cmh_pke_ecdh_unregister();
err_pke_ecdh_register:
cmh_pke_ecdsa_unregister();
@@ -411,6 +419,7 @@ static void cmh_remove(struct platform_device *pdev)
cfg = &dev->config;
cmh_mgmt_unregister();
+ cmh_pqc_mldsa_unregister();
cmh_pke_ecdh_unregister();
cmh_pke_ecdsa_unregister();
cmh_pke_rsa_unregister();
diff --git a/drivers/crypto/cmh/cmh_pqc_mldsa.c b/drivers/crypto/cmh/cmh_pqc_mldsa.c
new file mode 100644
index 000000000000..8f9bdf67aea2
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pqc_mldsa.c
@@ -0,0 +1,383 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- ML-DSA Signature Driver (sig_alg, synchronous)
+ *
+ * Registers "mldsa44", "mldsa65", "mldsa87" sig algorithms
+ * with sign, verify, set_pub_key, and set_priv_key callbacks.
+ *
+ * Key format:
+ * Public key = raw pk bytes (1312 / 1952 / 2592 bytes)
+ * Private key = raw sk bytes (2560 / 4032 / 4896 bytes)
+ *
+ * Sign: src = message bytes (up to 10240 bytes), dst = raw signature
+ * Verify: src = raw signature, digest = message bytes
+ *
+ * Non-masked mode only for sig_alg API.
+ * Masked mode available through /dev/cmh_mgmt ioctl.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <crypto/sig.h>
+#include <crypto/internal/sig.h>
+
+#include "cmh_sys.h"
+#include "cmh_qse_abi.h"
+#include "cmh_txn.h"
+#include "cmh_dma.h"
+#include "cmh_key.h"
+#include "cmh_pqc.h"
+
+struct cmh_mldsa_tfm_ctx {
+ struct cmh_key_ctx key; /* private key (raw only) */
+ u8 *pub_key;
+ u32 pub_key_len;
+ u32 mode; /* ML_DSA_MODE_44/65/87 */
+ int mode_idx; /* index into size tables */
+};
+
+static inline struct cmh_mldsa_tfm_ctx *cmh_mldsa_ctx(struct crypto_sig *tfm)
+{
+ return crypto_sig_ctx(tfm);
+}
+
+/*
+ * ML-DSA sign (synchronous sig_alg)
+ *
+ * @src: message bytes
+ * @slen: message length
+ * @dst: signature output buffer
+ * @dlen: output buffer length
+ *
+ * Returns signature length on success, negative errno on failure.
+ */
+static int cmh_mldsa_sign(struct crypto_sig *tfm,
+ const void *src, unsigned int slen,
+ void *dst, unsigned int dlen)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+ int mi = ctx->mode_idx;
+ u32 sig_size = ml_dsa_sig_size[mi];
+ u32 sk_size = ml_dsa_sk_size[mi];
+ struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN];
+ struct core_dispatch dd;
+ u8 *m_buf = NULL, *sig_buf = NULL;
+ dma_addr_t m_dma = DMA_MAPPING_ERROR;
+ dma_addr_t sig_dma = DMA_MAPPING_ERROR;
+ int ret, idx;
+
+ if (ctx->key.mode != CMH_KEY_RAW)
+ return -EINVAL;
+ if (dlen < sig_size)
+ return -EINVAL;
+ if (!slen || slen > ML_DSA_MAX_MLEN)
+ return -EINVAL;
+
+ m_buf = kmemdup(src, slen, GFP_KERNEL);
+ sig_buf = kzalloc(sig_size, GFP_KERNEL);
+ if (!m_buf || !sig_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ if (ctx->key.raw.len != sk_size) {
+ ret = -EINVAL;
+ goto out_free;
+ }
+
+ m_dma = cmh_dma_map_single(m_buf, slen, DMA_TO_DEVICE);
+ sig_dma = cmh_dma_map_single(sig_buf, sig_size, DMA_FROM_DEVICE);
+
+ if (cmh_dma_map_error(m_dma) || cmh_dma_map_error(sig_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ dd = cmh_core_select_instance(CMH_CORE_QSE);
+
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ idx = 1;
+ vcq_add_qse_ml_dsa_sign(&vcq[idx++], dd.core_id, ctx->mode,
+ QSE_FLAG_USE_RNG,
+ 0, m_dma, ctx->key.raw.dma, sig_dma, slen,
+ false);
+ vcq_add_qse_flush(&vcq[idx++], dd.core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, 1,
+ dd.mbx_idx);
+ if (!ret) {
+ /* Sync bounce buffer so CPU sees the DMA-written signature */
+ cmh_dma_sync_for_cpu(sig_dma, sig_size, DMA_FROM_DEVICE);
+ memcpy(dst, sig_buf, sig_size);
+ ret = sig_size;
+ }
+
+out_unmap:
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_size, DMA_FROM_DEVICE);
+ if (!cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, slen, DMA_TO_DEVICE);
+
+out_free:
+ kfree(sig_buf);
+ kfree(m_buf);
+ return ret;
+}
+
+/*
+ * ML-DSA verify (synchronous sig_alg)
+ *
+ * @src: raw signature
+ * @slen: signature length
+ * @digest: message bytes
+ * @dlen: message length
+ *
+ * Returns 0 on successful verification, negative errno on failure.
+ */
+static int cmh_mldsa_verify(struct crypto_sig *tfm,
+ const void *src, unsigned int slen,
+ const void *digest, unsigned int dlen)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+ int mi = ctx->mode_idx;
+ u32 sig_size = ml_dsa_sig_size[mi];
+ u32 pk_size = ml_dsa_pk_size[mi];
+ struct core_dispatch d = cmh_core_select_instance(CMH_CORE_QSE);
+ struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN];
+ u8 *sig_buf = NULL, *m_buf = NULL, *pk_buf = NULL;
+ dma_addr_t sig_dma = DMA_MAPPING_ERROR;
+ dma_addr_t m_dma = DMA_MAPPING_ERROR;
+ dma_addr_t pk_dma = DMA_MAPPING_ERROR;
+ int ret;
+
+ if (!ctx->pub_key)
+ return -EINVAL;
+ if (slen != sig_size)
+ return -EINVAL;
+ if (!dlen || dlen > ML_DSA_MAX_MLEN)
+ return -EINVAL;
+
+ sig_buf = kmemdup(src, slen, GFP_KERNEL);
+ m_buf = kmemdup(digest, dlen, GFP_KERNEL);
+ pk_buf = kmemdup(ctx->pub_key, pk_size, GFP_KERNEL);
+ if (!sig_buf || !m_buf || !pk_buf) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ sig_dma = cmh_dma_map_single(sig_buf, sig_size, DMA_TO_DEVICE);
+ m_dma = cmh_dma_map_single(m_buf, dlen, DMA_TO_DEVICE);
+ pk_dma = cmh_dma_map_single(pk_buf, pk_size, DMA_TO_DEVICE);
+
+ if (cmh_dma_map_error(sig_dma) || cmh_dma_map_error(m_dma) ||
+ cmh_dma_map_error(pk_dma)) {
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN);
+ vcq_add_qse_ml_dsa_verify(&vcq[1], d.core_id, ctx->mode, 0,
+ m_dma, pk_dma, sig_dma, dlen);
+ vcq_add_qse_flush(&vcq[2], d.core_id);
+
+ ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, 1, d.mbx_idx);
+
+out_unmap:
+ if (!cmh_dma_map_error(pk_dma))
+ cmh_dma_unmap_single(pk_dma, pk_size, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(m_dma))
+ cmh_dma_unmap_single(m_dma, dlen, DMA_TO_DEVICE);
+ if (!cmh_dma_map_error(sig_dma))
+ cmh_dma_unmap_single(sig_dma, sig_size, DMA_TO_DEVICE);
+
+out_free:
+ kfree(pk_buf);
+ kfree(m_buf);
+ kfree(sig_buf);
+ return ret;
+}
+
+static int cmh_mldsa_set_pub_key(struct crypto_sig *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+ u32 expected = ml_dsa_pk_size[ctx->mode_idx];
+
+ if (keylen != expected)
+ return -EINVAL;
+
+ kfree(ctx->pub_key);
+ ctx->pub_key = NULL;
+ ctx->pub_key_len = 0;
+
+ ctx->pub_key = kmemdup(key, keylen, GFP_KERNEL);
+ if (!ctx->pub_key)
+ return -ENOMEM;
+
+ ctx->pub_key_len = keylen;
+ return 0;
+}
+
+static int cmh_mldsa_set_priv_key(struct crypto_sig *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+ u32 expected = ml_dsa_sk_size[ctx->mode_idx];
+
+ if (keylen != expected)
+ return -EINVAL;
+
+ return cmh_key_setkey_raw(&ctx->key, key, keylen, CORE_ID_QSE);
+}
+
+static unsigned int cmh_mldsa_key_size(struct crypto_sig *tfm)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+
+ /* crypto_sig_keysize() returns bits, not bytes */
+ return ml_dsa_pk_size[ctx->mode_idx] * 8;
+}
+
+static unsigned int cmh_mldsa_max_size(struct crypto_sig *tfm)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+
+ return ml_dsa_sig_size[ctx->mode_idx];
+}
+
+static int cmh_mldsa_44_init(struct crypto_sig *tfm)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->mode = ML_DSA_MODE_44;
+ ctx->mode_idx = 0;
+ return 0;
+}
+
+static int cmh_mldsa_65_init(struct crypto_sig *tfm)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->mode = ML_DSA_MODE_65;
+ ctx->mode_idx = 1;
+ return 0;
+}
+
+static int cmh_mldsa_87_init(struct crypto_sig *tfm)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->mode = ML_DSA_MODE_87;
+ ctx->mode_idx = 2;
+ return 0;
+}
+
+static void cmh_mldsa_exit(struct crypto_sig *tfm)
+{
+ struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm);
+
+ cmh_key_destroy(&ctx->key);
+ kfree(ctx->pub_key);
+ ctx->pub_key = NULL;
+}
+
+/*
+ * Priority 5001: the kernel's software ML-DSA (crypto/mldsa.c) registers
+ * at priority 5000 but only implements verify -- sign returns -EOPNOTSUPP.
+ * We provide full HW-accelerated sign + verify, so we must override.
+ */
+static struct sig_alg cmh_mldsa_algs[] = {
+ {
+ .sign = cmh_mldsa_sign,
+ .verify = cmh_mldsa_verify,
+ .set_pub_key = cmh_mldsa_set_pub_key,
+ .set_priv_key = cmh_mldsa_set_priv_key,
+ .key_size = cmh_mldsa_key_size,
+ .max_size = cmh_mldsa_max_size,
+ .init = cmh_mldsa_44_init,
+ .exit = cmh_mldsa_exit,
+ .base = {
+ .cra_name = "mldsa44",
+ .cra_driver_name = "rambus-cmh-mldsa44",
+ .cra_priority = 5001,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_mldsa_tfm_ctx),
+ },
+ },
+ {
+ .sign = cmh_mldsa_sign,
+ .verify = cmh_mldsa_verify,
+ .set_pub_key = cmh_mldsa_set_pub_key,
+ .set_priv_key = cmh_mldsa_set_priv_key,
+ .key_size = cmh_mldsa_key_size,
+ .max_size = cmh_mldsa_max_size,
+ .init = cmh_mldsa_65_init,
+ .exit = cmh_mldsa_exit,
+ .base = {
+ .cra_name = "mldsa65",
+ .cra_driver_name = "rambus-cmh-mldsa65",
+ .cra_priority = 5001,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_mldsa_tfm_ctx),
+ },
+ },
+ {
+ .sign = cmh_mldsa_sign,
+ .verify = cmh_mldsa_verify,
+ .set_pub_key = cmh_mldsa_set_pub_key,
+ .set_priv_key = cmh_mldsa_set_priv_key,
+ .key_size = cmh_mldsa_key_size,
+ .max_size = cmh_mldsa_max_size,
+ .init = cmh_mldsa_87_init,
+ .exit = cmh_mldsa_exit,
+ .base = {
+ .cra_name = "mldsa87",
+ .cra_driver_name = "rambus-cmh-mldsa87",
+ .cra_priority = 5001,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct cmh_mldsa_tfm_ctx),
+ },
+ },
+};
+
+/**
+ * cmh_pqc_mldsa_register() - Register ML-DSA akcipher algorithms with the crypto framework
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int cmh_pqc_mldsa_register(void)
+{
+ int ret, i;
+
+ for (i = 0; i < ARRAY_SIZE(cmh_mldsa_algs); i++) {
+ ret = crypto_register_sig(&cmh_mldsa_algs[i]);
+ if (ret) {
+ dev_err(cmh_dev(), "cmh: failed to register %s (%d)\n",
+ cmh_mldsa_algs[i].base.cra_name, ret);
+ goto err_unregister;
+ }
+ }
+
+ return 0;
+
+err_unregister:
+ while (i--)
+ crypto_unregister_sig(&cmh_mldsa_algs[i]);
+ return ret;
+}
+
+/**
+ * cmh_pqc_mldsa_unregister() - Unregister ML-DSA akcipher algorithms from the crypto framework
+ */
+void cmh_pqc_mldsa_unregister(void)
+{
+ int i = ARRAY_SIZE(cmh_mldsa_algs);
+
+ while (i--)
+ crypto_unregister_sig(&cmh_mldsa_algs[i]);
+}
diff --git a/drivers/crypto/cmh/cmh_pqc_sizes.c b/drivers/crypto/cmh/cmh_pqc_sizes.c
new file mode 100644
index 000000000000..39e3d56f4312
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_pqc_sizes.c
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- PQC Algorithm Size Tables
+ *
+ * Centralised ML-DSA and SLH-DSA parameter-size arrays. Declared
+ * extern in cmh_qse_abi.h / cmh_hcq_abi.h, defined here once to
+ * avoid per-TU duplication.
+ */
+
+#include <linux/build_bug.h>
+#include <linux/kernel.h>
+#include <linux/types.h>
+
+#include "cmh_qse_abi.h"
+#include "cmh_hcq_abi.h"
+
+/* ML-DSA size tables (indexed by ml_dsa_mode_idx()) */
+const u32 ml_dsa_pk_size[3] = { 1312U, 1952U, 2592U };
+const u32 ml_dsa_sk_size[3] = { 2560U, 4032U, 4896U };
+const u32 ml_dsa_sk_size_masked[3] = { 3360U, 5472U, 6368U };
+const u32 ml_dsa_sig_size[3] = { 2420U, 3309U, 4627U };
+
+static_assert(ARRAY_SIZE(ml_dsa_pk_size) == ARRAY_SIZE(ml_dsa_sk_size));
+static_assert(ARRAY_SIZE(ml_dsa_pk_size) == ARRAY_SIZE(ml_dsa_sk_size_masked));
+static_assert(ARRAY_SIZE(ml_dsa_pk_size) == ARRAY_SIZE(ml_dsa_sig_size));
+
+/* SLH-DSA n-values and signature sizes (indexed by param_set - 1) */
+const u32 slhdsa_n[12] = {
+ 16, 16, 24, 24, 32, 32, /* SHAKE 128s/f, 192s/f, 256s/f */
+ 16, 16, 24, 24, 32, 32, /* SHA2 128s/f, 192s/f, 256s/f */
+};
+
+const u32 slhdsa_sig_size[12] = {
+ 7856, 17088, 16224, 35664, 29792, 49856, /* SHAKE */
+ 7856, 17088, 16224, 35664, 29792, 49856, /* SHA2 */
+};
+
+static_assert(ARRAY_SIZE(slhdsa_n) == ARRAY_SIZE(slhdsa_sig_size));
diff --git a/drivers/crypto/cmh/cmh_qse.c b/drivers/crypto/cmh/cmh_qse.c
new file mode 100644
index 000000000000..257dc3ee29a8
--- /dev/null
+++ b/drivers/crypto/cmh/cmh_qse.c
@@ -0,0 +1,211 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Cryptography Research, Inc. (CRI).
+ * CMH LKM -- QSE Core VCQ Builders
+ *
+ * VCQ builder functions for ML-KEM and ML-DSA commands (plain and masked).
+ * Each function populates a single vcq_cmd slot. Callers assemble
+ * complete VCQs with header + command(s) + flush, then submit via
+ * cmh_tm_submit_sync().
+ */
+
+#include <linux/string.h>
+
+#include "cmh_sys.h"
+
+/* -- QSE flush -- */
+
+/**
+ * vcq_add_qse_flush() - Build a QSE flush VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ */
+void vcq_add_qse_flush(struct vcq_cmd *slot, u32 core_id)
+{
+ vcq_add_flush(slot, core_id);
+}
+
+/* -- ML-KEM -- */
+
+/**
+ * vcq_add_qse_ml_kem_keygen() - Build an ML-KEM key generation VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @k: ML-KEM security parameter (k = 2, 3, or 4)
+ * @flags: Command flags
+ * @seed: DMA address of seed input buffer
+ * @z: DMA address of implicit rejection value buffer
+ * @ek: DMA address of encapsulation key output buffer
+ * @dk: DMA address of decapsulation key output buffer
+ * @dk_type: Decapsulation key datastore type
+ * @masked: Use masked (side-channel protected) variant
+ */
+void vcq_add_qse_ml_kem_keygen(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags,
+ u64 seed, u64 z, u64 ek, u64 dk, u32 dk_type,
+ bool masked)
+{
+ u32 cmd_id = masked ? QSE_CMD_ML_KEM_KEYGEN_MASKED
+ : QSE_CMD_ML_KEM_KEYGEN;
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id);
+ slot->hwc.qse.cmd_ml_kem_keygen.k = k;
+ slot->hwc.qse.cmd_ml_kem_keygen.flags = flags;
+ slot->hwc.qse.cmd_ml_kem_keygen.seed = seed;
+ slot->hwc.qse.cmd_ml_kem_keygen.z = z;
+ slot->hwc.qse.cmd_ml_kem_keygen.ek = ek;
+ slot->hwc.qse.cmd_ml_kem_keygen.dk = dk;
+ slot->hwc.qse.cmd_ml_kem_keygen.dk_type = dk_type;
+}
+
+/**
+ * vcq_add_qse_ml_kem_enc() - Build an ML-KEM encapsulation VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @k: ML-KEM security parameter (k = 2, 3, or 4)
+ * @flags: Command flags
+ * @coin: DMA address of encapsulation coin/randomness buffer
+ * @ek: DMA address of encapsulation key input buffer
+ * @ct: DMA address of ciphertext output buffer
+ * @ss: DMA address of shared secret output buffer
+ * @ss_type: Shared secret datastore type
+ * @masked: Use masked (side-channel protected) variant
+ */
+void vcq_add_qse_ml_kem_enc(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags,
+ u64 coin, u64 ek, u64 ct, u64 ss, u32 ss_type,
+ bool masked)
+{
+ u32 cmd_id = masked ? QSE_CMD_ML_KEM_ENC_MASKED
+ : QSE_CMD_ML_KEM_ENC;
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id);
+ slot->hwc.qse.cmd_ml_kem_enc.k = k;
+ slot->hwc.qse.cmd_ml_kem_enc.flags = flags;
+ slot->hwc.qse.cmd_ml_kem_enc.coin = coin;
+ slot->hwc.qse.cmd_ml_kem_enc.ek = ek;
+ slot->hwc.qse.cmd_ml_kem_enc.ct = ct;
+ slot->hwc.qse.cmd_ml_kem_enc.ss = ss;
+ slot->hwc.qse.cmd_ml_kem_enc.ss_type = ss_type;
+}
+
+/**
+ * vcq_add_qse_ml_kem_dec() - Build an ML-KEM decapsulation VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @k: ML-KEM security parameter (k = 2, 3, or 4)
+ * @flags: Command flags
+ * @ct: DMA address of ciphertext input buffer
+ * @dk: DMA address of decapsulation key input buffer
+ * @ss: DMA address of shared secret output buffer
+ * @ss_type: Shared secret datastore type
+ * @masked: Use masked (side-channel protected) variant
+ */
+void vcq_add_qse_ml_kem_dec(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags,
+ u64 ct, u64 dk, u64 ss, u32 ss_type,
+ bool masked)
+{
+ u32 cmd_id = masked ? QSE_CMD_ML_KEM_DEC_MASKED
+ : QSE_CMD_ML_KEM_DEC;
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id);
+ slot->hwc.qse.cmd_ml_kem_dec.k = k;
+ slot->hwc.qse.cmd_ml_kem_dec.flags = flags;
+ slot->hwc.qse.cmd_ml_kem_dec.ct = ct;
+ slot->hwc.qse.cmd_ml_kem_dec.dk = dk;
+ slot->hwc.qse.cmd_ml_kem_dec.ss = ss;
+ slot->hwc.qse.cmd_ml_kem_dec.ss_type = ss_type;
+}
+
+/* -- ML-DSA -- */
+
+/**
+ * vcq_add_qse_ml_dsa_keygen() - Build an ML-DSA key generation VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @mode: ML-DSA mode (44, 65, or 87)
+ * @flags: Command flags
+ * @seed: DMA address of seed input buffer
+ * @pk: DMA address of public key output buffer
+ * @sk: DMA address of secret key output buffer
+ * @sk_type: Secret key datastore type
+ * @masked: Use masked (side-channel protected) variant
+ */
+void vcq_add_qse_ml_dsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags,
+ u64 seed, u64 pk, u64 sk, u32 sk_type,
+ bool masked)
+{
+ u32 cmd_id = masked ? QSE_CMD_ML_DSA_KEYGEN_MASKED
+ : QSE_CMD_ML_DSA_KEYGEN;
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id);
+ slot->hwc.qse.cmd_ml_dsa_keygen.mode = mode;
+ slot->hwc.qse.cmd_ml_dsa_keygen.flags = flags;
+ slot->hwc.qse.cmd_ml_dsa_keygen.seed = seed;
+ slot->hwc.qse.cmd_ml_dsa_keygen.pk = pk;
+ slot->hwc.qse.cmd_ml_dsa_keygen.sk = sk;
+ slot->hwc.qse.cmd_ml_dsa_keygen.sk_type = sk_type;
+}
+
+/**
+ * vcq_add_qse_ml_dsa_sign() - Build an ML-DSA signing VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @mode: ML-DSA mode (44, 65, or 87)
+ * @flags: Command flags
+ * @rnd: DMA address of signing randomness buffer
+ * @m: DMA address of message buffer
+ * @sk: DMA address of secret key buffer
+ * @sig: DMA address of signature output buffer
+ * @mlen: Length of message in bytes
+ * @masked: Use masked (side-channel protected) variant
+ */
+void vcq_add_qse_ml_dsa_sign(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags,
+ u64 rnd, u64 m, u64 sk, u64 sig, u32 mlen,
+ bool masked)
+{
+ u32 cmd_id = masked ? QSE_CMD_ML_DSA_SIGN_MASKED
+ : QSE_CMD_ML_DSA_SIGN;
+
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id);
+ slot->hwc.qse.cmd_ml_dsa_sign.mode = mode;
+ slot->hwc.qse.cmd_ml_dsa_sign.flags = flags;
+ slot->hwc.qse.cmd_ml_dsa_sign.rnd = rnd;
+ slot->hwc.qse.cmd_ml_dsa_sign.m = m;
+ slot->hwc.qse.cmd_ml_dsa_sign.sk = sk;
+ slot->hwc.qse.cmd_ml_dsa_sign.sig = sig;
+ slot->hwc.qse.cmd_ml_dsa_sign.mlen = mlen;
+}
+
+/**
+ * vcq_add_qse_ml_dsa_verify() - Build an ML-DSA signature verify VCQ command
+ * @slot: VCQ command slot to populate
+ * @core_id: Hardware core ID for dispatch
+ * @mode: ML-DSA mode (44, 65, or 87)
+ * @flags: Command flags
+ * @m: DMA address of message buffer
+ * @pk: DMA address of public key buffer
+ * @sig: DMA address of signature buffer to verify
+ * @mlen: Length of message in bytes
+ */
+void vcq_add_qse_ml_dsa_verify(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags,
+ u64 m, u64 pk, u64 sig, u32 mlen)
+{
+ memset(slot, 0, sizeof(*slot));
+ slot->magic = VCQ_CMD_MAGIC;
+ slot->id = VCQ_CMD_ID(core_id, 0, 1, QSE_CMD_ML_DSA_VERIFY);
+ slot->hwc.qse.cmd_ml_dsa_verify.mode = mode;
+ slot->hwc.qse.cmd_ml_dsa_verify.flags = flags;
+ slot->hwc.qse.cmd_ml_dsa_verify.m = m;
+ slot->hwc.qse.cmd_ml_dsa_verify.pk = pk;
+ slot->hwc.qse.cmd_ml_dsa_verify.sig = sig;
+ slot->hwc.qse.cmd_ml_dsa_verify.mlen = mlen;
+}
--
2.43.7
^ permalink raw reply related
* Re: [PATCH] Documentation: sysfs-class-power: Update Long_Life description
From: Hans de Goede @ 2026-08-08 13:37 UTC (permalink / raw)
To: Derek J. Clark, Sebastian Reichel
Cc: Pierre-Loup A . Griffais, linux-pm, linux-kernel, linux-api
In-Reply-To: <20260803210720.23844-1-derekjohn.clark@gmail.com>
Hi Derek,
On 3-Aug-26 23:07, Derek J. Clark wrote:
> While adding charge limiting support to the Lenovo WMI drivers, there
> was some back and forth about whether charge_types or
> charge_control_end_threshold was the appropriate attribute to expose a
> battery charge limiting toggle that is fixed in the BIOS. The confusion
> arose because the charge_control_end_threshold description closely
> matches the functional change the hardware is making, while the
> charge_types functionality better suits the actual an on/off toggle that
> occurs in the BIOS. This specific scenario is not explicitly enumerated
> in the documentation, though it is fairly common.
>
> Given that the original intention was to use it this way[1],[2], and that
> the samsung-laptop[3], ideapad-laptop[4], and lenovo-wmi-other[5] drivers
> all use the convention of charge_types with an exposed Long_Life and
> Standard value for this, codify it in the Documentation to avoid confusion
> in the future.
>
> [1] https://lore.kernel.org/linux-pm/49993a42-aa91-46bf-acef-4a089db4c2db@redhat.com/
> [2] https://lore.kernel.org/platform-driver-x86/20241209204051.8786-1-hdegoede@redhat.com/
> [3] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=de2884c6cdd3d133704ce37393590dd1c761500c
> [4] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=da8f2708f9b69707f4efeb432a18395e46b4666f
> [5] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=9ca8fc065b88b327acbfdc33454efea391639716
>
> Suggested-by: Hans de Goede <hansg@kernel.org>
> Signed-off-by: Derek J. Clark <derekjohn.clark@gmail.com>
Thank you for updating the docs, patch looks good to me:
Reviewed-by: Hans de Goede <johannes.goede@oss.qualcomm.com>
Regards,
Hans
> ---
> Documentation/ABI/testing/sysfs-class-power | 14 +++++++++-----
> 1 file changed, 9 insertions(+), 5 deletions(-)
>
> diff --git a/Documentation/ABI/testing/sysfs-class-power b/Documentation/ABI/testing/sysfs-class-power
> index 5641f1fd5fd6..98b389845d1e 100644
> --- a/Documentation/ABI/testing/sysfs-class-power
> +++ b/Documentation/ABI/testing/sysfs-class-power
> @@ -365,9 +365,12 @@ Contact: linux-pm@vger.kernel.org
> Description:
> Represents a battery percentage level, above which charging will
> stop. Not all hardware is capable of setting this to an arbitrary
> - percentage. Drivers will round written values to the nearest
> - supported value. Reading back the value will show the actual
> - threshold set by the driver.
> + value, instead providing different minimum, maximum, or step
> + values. Drivers will round written values to the nearest supported
> + value. Reading back the value will show the actual threshold set
> + by the driver. For hardware that only supports a single fixed
> + value, use charge_types with a value of "Long Life" (vs "Standard")
> + instead'
>
> Access: Read, Write
>
> @@ -398,8 +401,9 @@ Description:
> when to start and stop charging. Advanced users
> can use this to drastically extend battery life.
> Long Life:
> - The charger reduces its charging rate in order to
> - prolong the battery health.
> + The charger firmware reduces its charging rate and/or
> + maximum charging percentage to a hardware specified
> + fixed limit in order to prolong the battery health.
> Bypass:
> The charger bypasses the charging path around the
> integrated converter allowing for a "smart" wall
^ permalink raw reply
* Re: [PATCH v3 01/19] dt-bindings: crypto: add Rambus CryptoManager Hub
From: Krzysztof Kozlowski @ 2026-08-10 8:16 UTC (permalink / raw)
To: Saravanakrishnan Krishnamoorthy, Albert Ou, Alex Ousherovitch,
Conor Dooley, David S. Miller, Herbert Xu, Jonathan Corbet,
Krzysztof Kozlowski, Palmer Dabbelt, Paul Walmsley, Rob Herring,
Shuah Khan
Cc: Alexandre Ghiti, devicetree, Joel Wittenauer, linux-api,
linux-crypto, linux-doc, linux-kernel, linux-kselftest,
linux-riscv, Shuah Khan, Thi Nguyen
In-Reply-To: <20260806195519.2703224-2-skrishnamoorthy@rambus.com>
On 06/08/2026 21:55, Saravanakrishnan Krishnamoorthy wrote:
> From: Alex Ousherovitch <aousherovitch@rambus.com>
>
> Add device tree binding schema for the Rambus CryptoManager Hub (CMH)
> hardware crypto accelerator. The binding describes the parent
> SoC-level node with its SIC register region and one mailbox@N child
> node per mailbox the host owns, each carrying a reg (mailbox instance
> index), an optional interrupt, VCQ ring geometry (rambus,slots-log2 /
> rambus,strides-log2) and a rambus,cores affinity list. Which crypto cores
> are present is discovered from the SIC CORE_ENABLE register at probe,
> not described in the device tree.
>
> Register the 'rambus' vendor prefix for Rambus Inc.
>
> Co-developed-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
> Signed-off-by: Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
Completely messed order of tags.
> Signed-off-by: Alex Ousherovitch <aousherovitch@rambus.com>
How Alex could sign after you?
> Reviewed-by: Joel Wittenauer <Joel.Wittenauer@cryptography.com>
> Reviewed-by: Thi Nguyen <thin@rambus.com>
Your SoB is the last.
Are you sure these people reviewed THIS code instead of blanket-review
for everything?
Did they find any issues in the schema which you fixed?
> ---
> .../bindings/crypto/rambus,cmh.yaml | 207 ++++++++++++++++++
> .../devicetree/bindings/vendor-prefixes.yaml | 2 +
> 2 files changed, 209 insertions(+)
> create mode 100644 Documentation/devicetree/bindings/crypto/rambus,cmh.yaml
>
> diff --git a/Documentation/devicetree/bindings/crypto/rambus,cmh.yaml b/Documentation/devicetree/bindings/crypto/rambus,cmh.yaml
> new file mode 100644
> index 000000000000..2709e20fb76f
> --- /dev/null
> +++ b/Documentation/devicetree/bindings/crypto/rambus,cmh.yaml
> @@ -0,0 +1,207 @@
> +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
> +%YAML 1.2
> +---
> +$id: http://devicetree.org/schemas/crypto/rambus,cmh.yaml#
> +$schema: http://devicetree.org/meta-schemas/core.yaml#
> +
> +title: Rambus CryptoManager Hub (CMH) Hardware Crypto Accelerator
> +
> +maintainers:
> + - Alex Ousherovitch <aousherovitch@rambus.com>
> + - Saravanakrishnan Krishnamoorthy <skrishnamoorthy@rambus.com>
> + - Joel Wittenauer <Joel.Wittenauer@cryptography.com>
> +
> +description: |
> + The Rambus CryptoManager Hub (CMH) is a hardware cryptographic accelerator
> + accessed via a mailbox-based VCQ (Virtual Command Queue) interface. The
> + host writes VCQ command sequences into per-mailbox DMA queue buffers and
> + rings a doorbell; the CMH eSW processes them and signals completion via
> + interrupt.
> +
> + The management host statically partitions the hardware mailboxes across
> + the SoC's host interfaces at integration time; the set of mailboxes a
> + given host owns is therefore fixed and not runtime-discoverable (a
> + mailbox locked to a host reads as unavailable in the SIC availability
> + register). Each owned mailbox is described by a child node. Which
> + crypto cores are present, by contrast, is a fixed silicon-build property
> + the driver reads from the hardware at probe (the SIC CORE_ENABLE
> + register), so cores are not described in the device tree.
> +
> + CMH identifies the host that owns a mailbox by a hardware HOST ID
> + presented on the bus with every access, and permits only that HOST ID
> + to access a mailbox while it is locked. An integration must therefore
> + present a single, stable HOST ID for all accesses to a given mailbox,
> + independent of which CPU issues them. On SMP hosts whose interconnect
> + encodes the issuing CPU in the HOST ID, the integration must provide a
> + single consistent HOST ID for the set of CPUs that share a mailbox; the
> + driver neither selects nor can override the HOST ID presented by the
> + hardware.
> +
> + Supported algorithm families: SHA-2, SHA-3, SM3, AES, SM4,
> + ChaCha20-Poly1305, RSA, ECDSA, EdDSA, ECDH, SM2, ML-KEM, ML-DSA,
> + SLH-DSA, LMS, XMSS, DRBG.
> +
> +properties:
> + compatible:
> + const: rambus,cmh-v1030
> + description:
> + Identifies the CryptoManager Hub v1.030 IP revision (register map and
> + command ABI). A specific SoC integration should list its own
> + "<vendor>,<soc>-cmh" compatible first with "rambus,cmh-v1030" as a
> + fallback; that extended form will be added when such a platform is
> + upstreamed.
Considering how variable the binding looks like, this needs real SoC
compatibles.
> +> + reg:
> + maxItems: 1
> + description:
> + SIC (System Interface Controller) MMIO region. The registers of
> + mailbox instance N are at offset N * 0x1000 within this region.
> +
> + clocks:
> + minItems: 1
> + maxItems: 3
> + description:
> + Functional clocks driving the CryptoManager Hub. The block gates its
> + clocks internally (integrated clock-gating cell); the host does not
> + gate them, so these describe the input pins only. The names, in
> + order, are "core" (the main functional clock), "core-div2" (a
> + half-rate clock present only on configurations with
> + side-channel-protected cores) and "rt" (the real-time tick clock
Don't repeat constraints in free form text.
> + for the internal timer). Absent on integrations where a separate
> + management/power controller owns the clocks and Linux has no clock
> + handle.
List the items instead with description. minItems stays.
> +
> + clock-names:
> + minItems: 1
> + items:
> + - const: core
> + - const: core-div2
> + - const: rt
> +
> + reset-gpios:
> + maxItems: 1
> + description:
> + Optional host-driven reset for the CryptoManager Hub. The hub has
> + two external, active-low reset inputs -- a power-on reset and a hard
> + reset; where a board routes one of them to a host-controlled GPIO,
> + that line is described here. Present only on integrations where the
> + Linux host is the management host; where a separate management
> + controller owns reset (the usual case) it is absent and Linux does
> + not drive reset.
> +
> + "#address-cells":
> + const: 1
> +
> + "#size-cells":
> + const: 0
> +
> +patternProperties:
> + "^mailbox@[0-9a-f]+$":
> + type: object
> + description:
> + One node per hardware mailbox owned by this host. The mailboxes a
> + host owns are assigned by the management host at integration time
So they are SoC-deducible.
> + and cannot be probed, so they are enumerated here.
Where are mbox-cells?
> + properties:
> + reg:
> + maxItems: 1
> + description:
> + 0-based mailbox instance index. The instance's registers are
> + at reg * 0x1000 within the SIC region.
> +
> + interrupts:
> + maxItems: 1
> + description:
> + Completion/error interrupt for this mailbox. Optional; when no
Don't repeat constraints in free form text.
The schema tells if something is optional or not.
> + mailbox has an interrupt the driver falls back to polling.
Driver is usually irrelevant here and that internal review which
supposedly happen for this code should tell you this.
> +
> + rambus,slots-log2:
> + $ref: /schemas/types.yaml#/definitions/uint32
> + minimum: 1
> + maximum: 15
> + description:
> + log2 of the number of VCQ ring slots the driver provisions for
Driver stuff does not belong to DT.
> + this mailbox in host DMA memory. This is a per-board,
> + per-mailbox host-memory layout choice programmed into the
> + mailbox QUEUE/SLOTS registers, not a silicon constant --
> + boards built around the same SoC (hence the same compatible)
> + may provision different ring geometry, so it is described per
> + mailbox here rather than derived from the compatible. The
> + upper bound is imposed by the hardware. Optional; defaults to
> + 6 (64 slots).
Missing default. Anyway, I do not understand why standard number you
express as log. slots-num is a number. Number is 1, 2 or 4 etc. Not
logarithm of something. Do you say: I have log-8 dollars in my wallet?
> +
> + rambus,strides-log2:
> + $ref: /schemas/types.yaml#/definitions/uint32
> + minimum: 7
> + maximum: 10
> + description:
> + log2 of the per-slot stride in bytes for this mailbox's VCQ
> + ring. Like rambus,slots-log2 this is a per-board layout
> + choice, not derived from the compatible. The range is imposed
> + by the hardware. Optional; defaults to 9 (512 bytes per slot).
> +
Best regards,
Krzysztof
^ permalink raw reply
* [PATCH v5] Input: synaptics - add pass-through mode for TrackPoint
From: Evan Lawrence @ 2026-08-10 14:25 UTC (permalink / raw)
To: linux-input; +Cc: linux-api, dmitry.torokhov, miroslav.bendik, Evan Lawrence
Synaptics touchpads with a pass-through port (PS/2 guest) default to
encapsulating guest data in the touchpad's own packets, limiting the
guest device (typically a TrackPoint) to the touchpad's low poll rate.
Enabling transparent pass-through mode tells the touchpad to stop
generating its own packets and relay the raw byte stream from the guest
directly, letting the TrackPoint run at its full native speed while the
touchpad is effectively disabled.
This reworks an earlier implementation by Miroslav Bendík from 2022
that was never merged. It keeps the same hardware-mode approach but
commits the mode flag only after a successful mode command, moves the
reconnect handling into a generic psmouse hook instead of protocol-
specific checks in psmouse-interrupt, and documents the interface.
The mode is exposed via a sysfs transparent_mode attribute that allows
userspace to switch between full-rate TrackPoint and full-rate touchpad
operation at runtime.
If the system warm-boots while transparent mode is active the touchpad
stays in that mode and does not acknowledge command bytes, so
ps2_command() hangs and the device fails to be detected. Escape
transparent mode by writing SETSCALE21 + SETSCALE11 directly to the wire
with serio_write() before attempting a reset or probe, both during
initial detection and on reconnect.
Link: https://lore.kernel.org/all/6932d599-2625-0376-d9c6-58cbb8879ff4@gmail.com/
Suggested-by: Miroslav Bendík <miroslav.bendik@gmail.com>
Assisted-by: DeepSeek:deepseek-v4-flash,deepseek-v4-pro
Signed-off-by: Evan Lawrence <development@laserology.net>
---
Notes:
Changes in v2 (addressing review feedback):
- Only consult the parent serio in psmouse_receive_byte() when it is a
genuine psmouse pass-through port; a psmouse attached to a ps2mult
child port has a parent whose drvdata is not a struct psmouse, so the
unconditional cast could read a bogus function pointer.
- Guard the guest state reset in synaptics_set_transparent_mode() on the
pass-through port actually being bound to a psmouse, and commit the
protocol handler/pktsize update with RX paused to avoid a torn update.
- Exit transparent mode on the hardware as well as in software when the
pass-through guest port is removed while the mode is active.
- Re-run the full initialization (query, identity check, mode setup)
before re-entering transparent mode during reconnect, so the device is
in a known configuration after a reset.
- Use psmouse_set_state(PSMOUSE_ACTIVATED) to drop stale guest data
instead of the bare pktcnt = 0 assignment, which could leave the
psmouse state machine (flags, state) out of sync with the packet
counter.
- Move the dev-attr creation ahead of the pass-through port registration
so that an attribute creation failure cannot leak a registered child
port whose callbacks reference freed synaptics_data.
Changes in v3 (addressing review feedback):
- Scope the serio_pause_rx guard in synaptics_set_transparent_mode() to
the protocol-handler commit only; calling psmouse_set_state() inside
it re-enables interrupts via serio_continue_rx() while the parent lock
is held, which can deadlock.
- In synaptics_pt_stop(), exit hardware transparent mode before updating
the software protocol handler, so guest data is never misparsed as
host touchpad packets.
Changes in v4 (addressing review feedback):
- Drop the pt_psmouse field and the child-state reset in
synaptics_set_transparent_mode(): storing drvdata as a psmouse pointer
without verifying the driver type creates a type-confusion risk.
The PS/2 psmouse handler already resynchronizes after stale bytes,
so dropping the explicit reset is safe.
- Remove the pass-through serio unregistration from
synaptics_disconnect(): calling serio_unregister_port() from inside
the parent disconnect callback deadlocks on serio_mutex, and the
child port is already destroyed by serio_disconnect_port() before
the parent disconnect runs, making the call a use-after-free.
- Guard the two-command write sequence in synaptics_pt_write() with a
per-device mutex and hold it across both the transparent fast-path
and the hardware mode-switch commands in
synaptics_set_transparent_mode(), so a concurrent pt_write cannot
inject a byte onto the PS/2 bus mid-command and corrupt the
passthrough transaction.
- Add defensive id.type == SERIO_PS_PSTHRU checks before the
psmouse_from_serio() casts in synaptics_pass_pt_packet() and
synaptics_pt_activate().
- Issue SETSCALE21/SETSCALE11 commands after the initial reset in
__synaptics_init() so that a touchpad left in transparent mode from
a previous session can still be detected and initialized after a
warm reboot.
- Skip the BAT (0xAA) detection in psmouse_receive_byte() while
transparent mode is active, via a pt_bypass_bat flag on struct
psmouse. Every byte on the parent serio belongs to the
pass-through guest; a TrackPoint can legitimately send 0xAA 0x00
as motion data, and consuming the 0xAA byte would silently drop it
before it reaches the guest handler.
- Clear pt_bypass_bat in synaptics_pt_stop() when the guest port is
removed so that BAT detection resumes for the touchpad itself.
Changes in v5 (addressing review feedback):
- In synaptics_pt_stop(), sample the transparent_mode flag inside the
pt_mutex guard instead of before it. A concurrent
synaptics_set_transparent_mode() could otherwise change the flag
between the snapshot and the lock acquire, causing the function to
either skip hardware exit when it should not, or double-exit on an
already-disabled device.
- In synaptics_set_transparent_mode(), check pt_port for NULL inside
the pt_mutex guard rather than before it, so a concurrent
synaptics_pt_stop() cannot clear pt_port after the check and before
the mode-switch commands are sent.
- In synaptics_set_rate(), bail out early when transparent mode is
active. The set_rate callback sends a mode command via
synaptics_mode_cmd() without holding pt_mutex, while
synaptics_pt_write() in transparent fast-path mode holds pt_mutex
but bypasses ps2_cmd_mutex, so the two byte streams can interleave
on the PS/2 bus and corrupt the command.
- In synaptics_reconnect(), hold pt_mutex across the
synaptics_enter_transparent_mode() call that re-enters the mode
after resume, and re-read pt_port inside the mutex, saving it to a
local variable so that the subsequent serio_reconnect() uses a
consistent pointer.
- Use READ_ONCE() when reading pt_port and pt_port_open in interrupt
and serio callback contexts to prevent loads from being torn or
reordered across the pt_mutex boundary.
- Move the serio_reconnect() call in synaptics_reconnect() inside the
pt_mutex guard so that pt_port remains valid for its duration.
- After a warm reboot while transparent mode is active the touchpad
does not acknowledge command bytes, so ps2_command() hangs and the
device is not detected. Escape transparent mode by writing
SETSCALE21 + SETSCALE11 directly to the wire with serio_write()
before attempting a reset or probe, both during initial detection
and on reconnect.
.../ABI/testing/sysfs-driver-synaptics | 28 ++
MAINTAINERS | 1 +
drivers/input/mouse/psmouse-base.c | 20 +-
drivers/input/mouse/psmouse.h | 15 +
drivers/input/mouse/synaptics.c | 373 +++++++++++++++++-
drivers/input/mouse/synaptics.h | 6 +
6 files changed, 420 insertions(+), 23 deletions(-)
create mode 100644 Documentation/ABI/testing/sysfs-driver-synaptics
diff --git a/Documentation/ABI/testing/sysfs-driver-synaptics b/Documentation/ABI/testing/sysfs-driver-synaptics
new file mode 100644
index 000000000000..bec715677e3e
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-driver-synaptics
@@ -0,0 +1,28 @@
+What: /sys/bus/serio/devices/serioX/transparent_mode
+Date: August 2026
+Contact: linux-input@vger.kernel.org
+Description:
+ Controls the transparent pass-through mode of Synaptics
+ touchpads that feature a pass-through (PS/2 guest) port, such as
+ those used for a TrackPoint.
+
+ In normal operation the touchpad serves as the PS/2 host for
+ the pass-through guest: it scans the guest during its own scan
+ cycle and encapsulates the guest's data into its own packets,
+ which limits the guest to the touchpad's low poll rate. When
+ transparent mode is enabled the touchpad stops generating its
+ own packets and simply relays the raw byte stream of the
+ pass-through guest, letting the guest run at its high poll rate
+ while the touchpad is effectively disabled.
+
+ Write 1 to enable the mode, or 0 to disable it. Reading the
+ attribute returns the current state ("0" or "1").
+
+ Note that both devices cannot be serviced at full rate at the
+ same time on this hardware; enabling the mode disables the
+ touchpad until it is disabled again. The attribute is present
+ only on devices that report the pass-through capability.
+
+Users: Userspace that arbitrates between full-rate TrackPoint and
+ full-rate touchpad operation. No consumer exists yet; the
+ interface is provided so that one can be developed.
diff --git a/MAINTAINERS b/MAINTAINERS
index 8014b9f8253e..312c13690c91 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -12817,6 +12817,7 @@ L: linux-input@vger.kernel.org
S: Maintained
Q: http://patchwork.kernel.org/project/linux-input/list/
T: git git://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git
+F: Documentation/ABI/testing/sysfs-driver-synaptics
F: Documentation/devicetree/bindings/input/
F: Documentation/devicetree/bindings/serio/
F: Documentation/input/
diff --git a/drivers/input/mouse/psmouse-base.c b/drivers/input/mouse/psmouse-base.c
index 6ab5f1d96eae..b1be88899e65 100644
--- a/drivers/input/mouse/psmouse-base.c
+++ b/drivers/input/mouse/psmouse-base.c
@@ -378,15 +378,31 @@ static void psmouse_receive_byte(struct ps2dev *ps2dev, u8 data)
psmouse->packet[psmouse->pktcnt++] = data;
/* Check if this is a new device announcement (0xAA 0x00) */
- if (unlikely(psmouse->packet[0] == PSMOUSE_RET_BAT && psmouse->pktcnt <= 2)) {
+ if (unlikely(!READ_ONCE(psmouse->pt_bypass_bat) &&
+ psmouse->packet[0] == PSMOUSE_RET_BAT && psmouse->pktcnt <= 2)) {
if (psmouse->pktcnt == 1) {
psmouse->last = jiffies;
return;
}
if (psmouse->packet[1] == PSMOUSE_RET_ID) {
+ struct serio *serio = ps2dev->serio;
+ struct psmouse *parent_psmouse = NULL;
+
__psmouse_set_state(psmouse, PSMOUSE_IGNORE);
- serio_reconnect(ps2dev->serio);
+ /*
+ * Some devices need parent to be reconnected instead.
+ * Only consult the parent if this is a genuine psmouse
+ * pass-through port; other serio children may have a
+ * parent whose drvdata is not a struct psmouse.
+ */
+ if (serio->parent && serio->id.type == SERIO_PS_PSTHRU)
+ parent_psmouse = psmouse_from_serio(serio->parent);
+ if (parent_psmouse && parent_psmouse->pt_reconnect_parent &&
+ parent_psmouse->pt_reconnect_parent(parent_psmouse))
+ serio_reconnect(serio->parent);
+ else
+ serio_reconnect(serio);
return;
}
diff --git a/drivers/input/mouse/psmouse.h b/drivers/input/mouse/psmouse.h
index 90ed8cd15d85..9391002fedc9 100644
--- a/drivers/input/mouse/psmouse.h
+++ b/drivers/input/mouse/psmouse.h
@@ -128,6 +128,21 @@ struct psmouse {
void (*pt_activate)(struct psmouse *psmouse);
void (*pt_deactivate)(struct psmouse *psmouse);
+
+ /*
+ * Called on the parent device when a pass-through guest reports a new
+ * device announcement (0xAA 0x00). Return true if the parent, and not
+ * the guest, should be reconnected.
+ */
+ bool (*pt_reconnect_parent)(struct psmouse *psmouse);
+
+ /*
+ * When true the parent is in transparent mode and the standard
+ * BAT (0xAA) detection in psmouse_receive_byte must be skipped:
+ * every byte arriving on the parent serio belongs to the
+ * pass-through guest and is not a device announcement.
+ */
+ bool pt_bypass_bat;
};
struct psmouse *psmouse_from_serio(struct serio *serio);
diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c
index c70502e24031..aaf66b73bcfe 100644
--- a/drivers/input/mouse/synaptics.c
+++ b/drivers/input/mouse/synaptics.c
@@ -98,6 +98,17 @@ int synaptics_detect(struct psmouse *psmouse, bool set_properties)
struct ps2dev *ps2dev = &psmouse->ps2dev;
u8 param[4] = { 0 };
+ /*
+ * If the device was left in transparent mode after a warm
+ * reboot it will not acknowledge any command byte. Exit
+ * the mode first with serio_write(), which places the
+ * escape sequence directly on the wire without waiting
+ * for an acknowledgement.
+ */
+ serio_write(ps2dev->serio, PSMOUSE_CMD_SETSCALE21);
+ serio_write(ps2dev->serio, PSMOUSE_CMD_SETSCALE11);
+ mdelay(5);
+
ps2_command(ps2dev, param, PSMOUSE_CMD_SETRES);
ps2_command(ps2dev, param, PSMOUSE_CMD_SETRES);
ps2_command(ps2dev, param, PSMOUSE_CMD_SETRES);
@@ -622,6 +633,9 @@ static void synaptics_set_rate(struct psmouse *psmouse, unsigned int rate)
{
struct synaptics_data *priv = psmouse->private;
+ if (READ_ONCE(priv->transparent_mode))
+ return;
+
if (rate >= 80) {
priv->mode |= SYN_BIT_HIGH_RATE;
psmouse->rate = 80;
@@ -636,12 +650,90 @@ static void synaptics_set_rate(struct psmouse *psmouse, unsigned int rate)
/*****************************************************************************
* Synaptics pass-through PS/2 port support
****************************************************************************/
+static psmouse_ret_t synaptics_process_byte(struct psmouse *psmouse);
+
+static psmouse_ret_t transparent_process_byte(struct psmouse *psmouse)
+{
+ struct synaptics_data *priv = psmouse->private;
+ struct serio *pt_port = READ_ONCE(priv->pt_port);
+
+ if (!pt_port)
+ return PSMOUSE_BAD_DATA;
+
+ serio_interrupt(pt_port, psmouse->packet[psmouse->pktcnt - 1], 0);
+ return PSMOUSE_FULL_PACKET;
+}
+
+static void synaptics_update_protocol_handler(struct psmouse *psmouse)
+{
+ struct synaptics_data *priv = psmouse->private;
+ struct serio *pt_port = READ_ONCE(priv->pt_port);
+ bool absolute_mode = priv->absolute_mode;
+ bool transparent_mode = READ_ONCE(priv->transparent_mode);
+
+ if (transparent_mode && pt_port) {
+ psmouse->protocol_handler = transparent_process_byte;
+ } else {
+ if (absolute_mode) {
+ psmouse->protocol_handler = synaptics_process_byte;
+ psmouse->pktsize = 6;
+ } else {
+ /* Relative mode follows standard PS/2 mouse protocol */
+ psmouse->protocol_handler = psmouse_process_byte;
+ psmouse->pktsize = 3;
+ }
+ }
+}
+
+static int synaptics_enter_transparent_mode(struct psmouse *psmouse)
+{
+ struct synaptics_data *priv = psmouse->private;
+ int error;
+
+ error = synaptics_mode_cmd(psmouse, priv->mode | SYN_BIT_TRANSPARENT_MODE);
+ if (error)
+ return error;
+
+ priv->mode |= SYN_BIT_TRANSPARENT_MODE;
+
+ return 0;
+}
+
+static int synaptics_exit_transparent_mode(struct psmouse *psmouse)
+{
+ struct synaptics_data *priv = psmouse->private;
+ int error;
+
+ /* Send scaling 2:1, 1:1 to exit transparent mode */
+ error = ps2_command(&psmouse->ps2dev, NULL, PSMOUSE_CMD_SETSCALE21);
+ if (error)
+ return error;
+ error = ps2_command(&psmouse->ps2dev, NULL, PSMOUSE_CMD_SETSCALE11);
+ if (error)
+ return error;
+
+ /* Re-enter the regular operating mode of the touchpad */
+ error = synaptics_mode_cmd(psmouse, priv->mode & ~SYN_BIT_TRANSPARENT_MODE);
+ if (error)
+ return error;
+
+ priv->mode &= ~SYN_BIT_TRANSPARENT_MODE;
+
+ return 0;
+}
+
static int synaptics_pt_write(struct serio *serio, u8 c)
{
struct psmouse *parent = psmouse_from_serio(serio->parent);
+ struct synaptics_data *priv = parent->private;
u8 rate_param = SYN_PS_CLIENT_CMD; /* indicates that we want pass-through port */
int error;
+ guard(mutex)(&priv->pt_mutex);
+
+ if (READ_ONCE(priv->transparent_mode))
+ return parent->ps2dev.serio->write(parent->ps2dev.serio, c);
+
error = ps2_sliced_command(&parent->ps2dev, c);
if (error)
return error;
@@ -661,6 +753,8 @@ static int synaptics_pt_start(struct serio *serio)
guard(serio_pause_rx)(parent->ps2dev.serio);
priv->pt_port = serio;
+ synaptics_update_protocol_handler(parent);
+
return 0;
}
@@ -669,8 +763,28 @@ static void synaptics_pt_stop(struct serio *serio)
struct psmouse *parent = psmouse_from_serio(serio->parent);
struct synaptics_data *priv = parent->private;
- guard(serio_pause_rx)(parent->ps2dev.serio);
- priv->pt_port = NULL;
+ /*
+ * The guest is going away. Hold the pass-through mutex across
+ * the hardware exit and the software state update so a
+ * concurrent synaptics_pt_write() cannot observe the old
+ * transparent_mode after the hardware has already been
+ * switched back to normal operation.
+ */
+ guard(mutex)(&priv->pt_mutex);
+
+ if (READ_ONCE(priv->transparent_mode)) {
+ if (synaptics_exit_transparent_mode(parent))
+ psmouse_warn(parent,
+ "failed to exit transparent mode after pass-through port removal\n");
+ }
+
+ {
+ guard(serio_pause_rx)(parent->ps2dev.serio);
+ priv->pt_port = NULL;
+ WRITE_ONCE(priv->transparent_mode, false);
+ WRITE_ONCE(parent->pt_bypass_bat, false);
+ synaptics_update_protocol_handler(parent);
+ }
}
static int synaptics_pt_open(struct serio *serio)
@@ -681,6 +795,19 @@ static int synaptics_pt_open(struct serio *serio)
guard(serio_pause_rx)(parent->ps2dev.serio);
priv->pt_port_open = true;
+ /*
+ * In transparent mode every byte on the parent serio belongs to
+ * the pass-through guest and is not a BAT announcement. Skip
+ * BAT detection on the child as well so that guest motion data
+ * that happens to look like 0xAA 0x00 does not trigger a
+ * spurious parent reconnect.
+ */
+ if (READ_ONCE(priv->transparent_mode) &&
+ serio->id.type == SERIO_PS_PSTHRU) {
+ struct psmouse *child = psmouse_from_serio(serio);
+ WRITE_ONCE(child->pt_bypass_bat, true);
+ }
+
return 0;
}
@@ -691,6 +818,24 @@ static void synaptics_pt_close(struct serio *serio)
guard(serio_pause_rx)(parent->ps2dev.serio);
priv->pt_port_open = false;
+
+ /* Restore BAT detection when the port is closed */
+ if (serio->id.type == SERIO_PS_PSTHRU) {
+ struct psmouse *child = psmouse_from_serio(serio);
+ WRITE_ONCE(child->pt_bypass_bat, false);
+ }
+}
+
+/*
+ * When the touchpad is in transparent mode resetting the guest alone is not
+ * enough, the host needs to be reconnected so that transparent mode is
+ * re-established.
+ */
+static bool synaptics_pt_reconnect_parent(struct psmouse *psmouse)
+{
+ struct synaptics_data *priv = psmouse->private;
+
+ return READ_ONCE(priv->transparent_mode);
}
static int synaptics_is_pt_packet(u8 *buf)
@@ -702,13 +847,13 @@ static void synaptics_pass_pt_packet(struct synaptics_data *priv, u8 *packet)
{
struct serio *ptport;
- ptport = priv->pt_port;
+ ptport = READ_ONCE(priv->pt_port);
if (!ptport)
return;
serio_interrupt(ptport, packet[1], 0);
- if (priv->pt_port_open) {
+ if (READ_ONCE(priv->pt_port_open) && ptport->id.type == SERIO_PS_PSTHRU) {
struct psmouse *child = psmouse_from_serio(ptport);
if (child->state == PSMOUSE_ACTIVATED) {
@@ -723,7 +868,15 @@ static void synaptics_pass_pt_packet(struct synaptics_data *priv, u8 *packet)
static void synaptics_pt_activate(struct psmouse *psmouse)
{
struct synaptics_data *priv = psmouse->private;
- struct psmouse *child = psmouse_from_serio(priv->pt_port);
+ struct psmouse *child = NULL;
+ struct serio *pt_port = READ_ONCE(priv->pt_port);
+
+ if (pt_port && pt_port->id.type == SERIO_PS_PSTHRU)
+ child = psmouse_from_serio(pt_port);
+
+ /* no mode change needed when transparent mode is active */
+ if (READ_ONCE(priv->transparent_mode))
+ return;
/* adjust the touchpad to child's choice of protocol */
if (child) {
@@ -760,6 +913,7 @@ static void synaptics_pt_create(struct psmouse *psmouse)
serio->parent = psmouse->ps2dev.serio;
psmouse->pt_activate = synaptics_pt_activate;
+ psmouse->pt_reconnect_parent = synaptics_pt_reconnect_parent;
psmouse_info(psmouse, "serio: %s port at %s\n",
serio->name, psmouse->phys);
@@ -987,17 +1141,21 @@ static void synaptics_report_ext_buttons(struct psmouse *psmouse,
* physically wired to the touchpad. Re-route them through
* the pass-through interface.
*/
- if (priv->pt_port) {
+ {
+ struct serio *pt_port = READ_ONCE(priv->pt_port);
u8 pt_buttons;
+ if (!pt_port)
+ return;
+
/* The trackstick expects at most 3 buttons */
pt_buttons = SYN_EXT_BUTTON_STICK_L(hw->ext_buttons) |
SYN_EXT_BUTTON_STICK_R(hw->ext_buttons) << 1 |
SYN_EXT_BUTTON_STICK_M(hw->ext_buttons) << 2;
- serio_interrupt(priv->pt_port,
+ serio_interrupt(pt_port,
PSMOUSE_OOB_EXTRA_BTNS, SERIO_OOB_DATA);
- serio_interrupt(priv->pt_port, pt_buttons, SERIO_OOB_DATA);
+ serio_interrupt(pt_port, pt_buttons, SERIO_OOB_DATA);
}
}
@@ -1437,6 +1595,93 @@ PSMOUSE_DEFINE_ATTR(disable_gesture, S_IWUSR | S_IRUGO, NULL,
synaptics_show_disable_gesture,
synaptics_set_disable_gesture);
+static ssize_t synaptics_show_transparent_mode(struct psmouse *psmouse,
+ void *data, char *buf)
+{
+ struct synaptics_data *priv = psmouse->private;
+
+ return sysfs_emit(buf, "%c\n",
+ READ_ONCE(priv->transparent_mode) ? '1' : '0');
+}
+
+static ssize_t synaptics_set_transparent_mode(struct psmouse *psmouse,
+ void *data, const char *buf,
+ size_t len)
+{
+ struct synaptics_data *priv = psmouse->private;
+ unsigned int value;
+ int err;
+
+ err = kstrtouint(buf, 10, &value);
+ if (err)
+ return err;
+
+ if (value > 1)
+ return -EINVAL;
+
+ if (value == READ_ONCE(priv->transparent_mode))
+ return len;
+
+ /*
+ * Switch the hardware first and only commit the new mode and protocol
+ * handler once it succeeded, so a failing command cannot leave the
+ * driver with bookkeeping that does not match the device. The touchpad
+ * is deactivated for the duration of this write (see
+ * psmouse_attr_set_helper), so no live data can be misparsed in
+ * between. Hold the pass-through mutex so that a concurrent
+ * synaptics_pt_write() cannot inject a byte onto the bus in the
+ * middle of the mode-switch command sequence.
+ */
+ guard(mutex)(&priv->pt_mutex);
+
+ /* Transparent mode only makes sense while the guest is attached */
+ if (value && !READ_ONCE(priv->pt_port))
+ return -ENODEV;
+
+ if (value) {
+ err = synaptics_enter_transparent_mode(psmouse);
+ if (err)
+ return err;
+ } else {
+ err = synaptics_exit_transparent_mode(psmouse);
+ if (err)
+ return err;
+ }
+
+ /*
+ * Commit the software state with RX paused so the interrupt handler
+ * never observes a torn (protocol_handler, pktsize) pair while the
+ * pass-through guest keeps streaming.
+ */
+ {
+ guard(serio_pause_rx)(psmouse->ps2dev.serio);
+ WRITE_ONCE(priv->transparent_mode, value);
+ synaptics_update_protocol_handler(psmouse);
+ WRITE_ONCE(psmouse->pt_bypass_bat, value);
+
+ /*
+ * Mirror the BAT-bypass flag on the child so that
+ * guest motion data resembling a reset announcement
+ * (0xAA 0x00) does not trigger a spurious parent
+ * reconnect through the child's bat-detection path.
+ */
+ {
+ struct serio *pt_port = READ_ONCE(priv->pt_port);
+
+ if (pt_port && pt_port->id.type == SERIO_PS_PSTHRU) {
+ struct psmouse *child = psmouse_from_serio(pt_port);
+ WRITE_ONCE(child->pt_bypass_bat, value);
+ }
+ }
+ }
+
+ return len;
+}
+
+PSMOUSE_DEFINE_ATTR(transparent_mode, 0644, NULL,
+ synaptics_show_transparent_mode,
+ synaptics_set_transparent_mode);
+
static void synaptics_disconnect(struct psmouse *psmouse)
{
struct synaptics_data *priv = psmouse->private;
@@ -1447,10 +1692,20 @@ static void synaptics_disconnect(struct psmouse *psmouse)
*/
psmouse_smbus_cleanup(psmouse);
+ if (READ_ONCE(priv->transparent_mode)) {
+ guard(mutex)(&priv->pt_mutex);
+ synaptics_exit_transparent_mode(psmouse);
+ WRITE_ONCE(priv->transparent_mode, false);
+ WRITE_ONCE(psmouse->pt_bypass_bat, false);
+ }
+
if (!priv->absolute_mode &&
SYN_ID_DISGEST_SUPPORTED(priv->info.identity))
device_remove_file(&psmouse->ps2dev.serio->dev,
&psmouse_attr_disable_gesture.dattr);
+ if (SYN_CAP_PASS_THROUGH(priv->info.capabilities))
+ device_remove_file(&psmouse->ps2dev.serio->dev,
+ &psmouse_attr_transparent_mode.dattr);
synaptics_reset(psmouse);
kfree(priv);
@@ -1465,6 +1720,16 @@ static int synaptics_reconnect(struct psmouse *psmouse)
int retry = 0;
int error;
+ /*
+ * If the touchpad is in transparent mode it will not ACK
+ * any command byte, so we place the escape sequence directly
+ * on the wire with serio_write() before attempting a reset
+ * or a probe.
+ */
+ serio_write(psmouse->ps2dev.serio, PSMOUSE_CMD_SETSCALE21);
+ serio_write(psmouse->ps2dev.serio, PSMOUSE_CMD_SETSCALE11);
+ mdelay(5);
+
do {
psmouse_reset(psmouse);
if (retry) {
@@ -1477,8 +1742,42 @@ static int synaptics_reconnect(struct psmouse *psmouse)
*/
ssleep(1);
}
- ps2_command(&psmouse->ps2dev, param, PSMOUSE_CMD_GETID);
- error = synaptics_detect(psmouse, 0);
+ if (READ_ONCE(priv->transparent_mode) && READ_ONCE(priv->pt_port)) {
+ struct serio *pt_port;
+ /*
+ * psmouse_reset() above put the touchpad back into its
+ * power-on state, so run the full initialization before
+ * re-entering transparent mode. Otherwise the device
+ * would come back missing the configuration set up by
+ * synaptics_set_mode() (e.g. Advanced Gesture Mode) and
+ * would stay uninitialized once transparent mode is
+ * disabled again.
+ */
+ error = synaptics_query_hardware(psmouse, &info);
+ if (!error &&
+ (info.identity != priv->info.identity ||
+ info.model_id != priv->info.model_id ||
+ info.capabilities != priv->info.capabilities ||
+ info.ext_cap != priv->info.ext_cap))
+ error = -ENXIO;
+ if (!error)
+ error = synaptics_set_mode(psmouse);
+ if (!error) {
+ guard(mutex)(&priv->pt_mutex);
+ pt_port = READ_ONCE(priv->pt_port);
+ if (pt_port)
+ error = synaptics_enter_transparent_mode(psmouse);
+ else
+ error = -ENODEV;
+ if (!error) {
+ serio_reconnect(pt_port);
+ return 0;
+ }
+ }
+ } else {
+ ps2_command(&psmouse->ps2dev, param, PSMOUSE_CMD_GETID);
+ error = synaptics_detect(psmouse, 0);
+ }
} while (error && ++retry < 3);
if (error)
@@ -1602,6 +1901,8 @@ static int synaptics_init_ps2(struct psmouse *psmouse,
if (!priv)
return -ENOMEM;
+ mutex_init(&priv->pt_mutex);
+
priv->info = *info;
priv->absolute_mode = absolute_mode;
if (SYN_ID_DISGEST_SUPPORTED(info->identity))
@@ -1647,14 +1948,7 @@ static int synaptics_init_ps2(struct psmouse *psmouse,
psmouse->model = ((info->model_id & 0x00ff0000) >> 8) |
(info->model_id & 0x000000ff);
- if (absolute_mode) {
- psmouse->protocol_handler = synaptics_process_byte;
- psmouse->pktsize = 6;
- } else {
- /* Relative mode follows standard PS/2 mouse protocol */
- psmouse->protocol_handler = psmouse_process_byte;
- psmouse->pktsize = 3;
- }
+ synaptics_update_protocol_handler(psmouse);
psmouse->set_rate = synaptics_set_rate;
psmouse->disconnect = synaptics_disconnect;
@@ -1664,9 +1958,6 @@ static int synaptics_init_ps2(struct psmouse *psmouse,
/* Synaptics can usually stay in sync without extra help */
psmouse->resync_time = 0;
- if (SYN_CAP_PASS_THROUGH(info->capabilities))
- synaptics_pt_create(psmouse);
-
/*
* Toshiba's KBC seems to have trouble handling data from
* Synaptics at full rate. Switch to a lower rate (roughly
@@ -1690,9 +1981,35 @@ static int synaptics_init_ps2(struct psmouse *psmouse,
}
}
+ if (SYN_CAP_PASS_THROUGH(info->capabilities)) {
+ err = device_create_file(&psmouse->ps2dev.serio->dev,
+ &psmouse_attr_transparent_mode.dattr);
+ if (err) {
+ psmouse_err(psmouse,
+ "Failed to create transparent_mode attribute (%d)",
+ err);
+ goto init_fail;
+ }
+ }
+
+ /*
+ * Register the pass-through port only once all initialization steps
+ * that can fail have succeeded, so that a failure in one of the steps
+ * above cannot leak a registered child port whose callbacks would
+ * dereference the freed synaptics_data.
+ */
+ if (SYN_CAP_PASS_THROUGH(info->capabilities))
+ synaptics_pt_create(psmouse);
+
return 0;
init_fail:
+ if (!priv->absolute_mode && SYN_ID_DISGEST_SUPPORTED(priv->info.identity))
+ device_remove_file(&psmouse->ps2dev.serio->dev,
+ &psmouse_attr_disable_gesture.dattr);
+ if (SYN_CAP_PASS_THROUGH(priv->info.capabilities))
+ device_remove_file(&psmouse->ps2dev.serio->dev,
+ &psmouse_attr_transparent_mode.dattr);
kfree(priv);
return err;
}
@@ -1702,6 +2019,20 @@ static int __synaptics_init(struct psmouse *psmouse, bool absolute_mode)
struct synaptics_device_info info;
int error;
+ /*
+ * After a warm reboot the touchpad may still be in transparent
+ * mode, silently relaying the pass-through guest's byte stream
+ * and responding to neither a reset nor the Synaptics probe
+ * queries. Exit transparent mode here, before the reset,
+ * because once the touchpad enters this mode it never ACKs
+ * a command byte. serio_write() places the bytes directly on
+ * the wire without waiting for an acknowledgement, so we can
+ * reach the device even when it is deaf.
+ */
+ serio_write(psmouse->ps2dev.serio, PSMOUSE_CMD_SETSCALE21);
+ serio_write(psmouse->ps2dev.serio, PSMOUSE_CMD_SETSCALE11);
+ mdelay(5);
+
psmouse_reset(psmouse);
error = synaptics_query_hardware(psmouse, &info);
diff --git a/drivers/input/mouse/synaptics.h b/drivers/input/mouse/synaptics.h
index 3853165b6b3a..dc2031851cfd 100644
--- a/drivers/input/mouse/synaptics.h
+++ b/drivers/input/mouse/synaptics.h
@@ -6,6 +6,8 @@
#ifndef _SYNAPTICS_H
#define _SYNAPTICS_H
+#include <linux/mutex.h>
+
/* synaptics queries */
#define SYN_QUE_IDENTIFY 0x00
#define SYN_QUE_MODES 0x01
@@ -24,6 +26,7 @@
/* synaptics modes */
#define SYN_BIT_ABSOLUTE_MODE BIT(7)
#define SYN_BIT_HIGH_RATE BIT(6)
+#define SYN_BIT_TRANSPARENT_MODE BIT(5)
#define SYN_BIT_SLEEP_MODE BIT(3)
#define SYN_BIT_DISABLE_GESTURE BIT(2)
#define SYN_BIT_FOUR_BYTE_CLIENT BIT(1)
@@ -186,10 +189,13 @@ struct synaptics_data {
bool absolute_mode; /* run in Absolute mode */
bool disable_gesture; /* disable gestures */
+ bool transparent_mode; /* pass packets directly from guest */
struct serio *pt_port; /* Pass-through serio port */
bool pt_port_open;
+ struct mutex pt_mutex; /* Serializes pt_write commands */
+
/*
* Last received Advanced Gesture Mode (AGM) packet. An AGM packet
* contains position data for a second contact, at half resolution.
base-commit: db2ddb87143519e20a95aa36c60b36107b736a58
--
2.55.0
^ permalink raw reply related
page: | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox