[PATCH v11 8/8] modsign: Enable RSASSA-PSS module signing

[David Howells <dhowells@redhat.com>]

Add support for RSASSA-PSS signatures (RFC8017) for use with module signing
and other public key cryptography done by the kernel.

Note that only signature verification is supported by the kernel.

Note further that this alters some of the same code as the MLDSA support,
so that needs to be applied first to avoid conflicts.

Signed-off-by: David Howells <dhowells@redhat.com>
cc: Lukas Wunner <lukas@wunner.de>
cc: Ignat Korchagin <ignat@cloudflare.com>
cc: Herbert Xu <herbert@gondor.apana.org.au>
cc: keyrings@vger.kernel.org
cc: linux-crypto@vger.kernel.org
---
 certs/Kconfig       |  6 ++++++
 certs/Makefile      |  1 +
 scripts/sign-file.c | 39 +++++++++++++++++++++++++++++++++++++--
 3 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/certs/Kconfig b/certs/Kconfig
index 94b086684d07..beb8991ad761 100644
--- a/certs/Kconfig
+++ b/certs/Kconfig
@@ -27,6 +27,12 @@ config MODULE_SIG_KEY_TYPE_RSA
 	help
 	 Use an RSA key for module signing.
 
+config MODULE_SIG_KEY_TYPE_RSASSA_PSS
+	bool "RSASSA-PSS"
+	select CRYPTO_RSA
+	help
+	 Use an RSASSA-PSS key for module signing.
+
 config MODULE_SIG_KEY_TYPE_ECDSA
 	bool "ECDSA"
 	select CRYPTO_ECDSA
diff --git a/certs/Makefile b/certs/Makefile
index 3ee1960f9f4a..3b5a3a303f4c 100644
--- a/certs/Makefile
+++ b/certs/Makefile
@@ -42,6 +42,7 @@ targets += x509_certificate_list
 # boolean option and we unfortunately can't make it depend on !RANDCONFIG.
 ifeq ($(CONFIG_MODULE_SIG_KEY),certs/signing_key.pem)
 
+keytype-$(CONFIG_MODULE_SIG_KEY_TYPE_RSASSA_PSS) := -newkey rsassa-pss
 keytype-$(CONFIG_MODULE_SIG_KEY_TYPE_ECDSA) := -newkey ec -pkeyopt ec_paramgen_curve:secp384r1
 keytype-$(CONFIG_MODULE_SIG_KEY_TYPE_MLDSA_44) := -newkey ml-dsa-44
 keytype-$(CONFIG_MODULE_SIG_KEY_TYPE_MLDSA_65) := -newkey ml-dsa-65
diff --git a/scripts/sign-file.c b/scripts/sign-file.c
index b726581075f9..ca605095194e 100644
--- a/scripts/sign-file.c
+++ b/scripts/sign-file.c
@@ -233,6 +233,7 @@ int main(int argc, char **argv)
 	EVP_PKEY *private_key;
 #ifndef USE_PKCS7
 	CMS_ContentInfo *cms = NULL;
+	CMS_SignerInfo *signer;
 	unsigned int use_keyid = 0;
 #else
 	PKCS7 *pkcs7 = NULL;
@@ -329,13 +330,47 @@ int main(int argc, char **argv)
 		    !EVP_PKEY_is_a(private_key, "ML-DSA-65") &&
 		    !EVP_PKEY_is_a(private_key, "ML-DSA-87"))
 			flags |= use_signed_attrs;
+		if (EVP_PKEY_is_a(private_key, "RSASSA-PSS"))
+			flags |= CMS_KEY_PARAM;
+	if (EVP_PKEY_is_a(private_key, "RSASSA-PSS")) {
+			EVP_PKEY_CTX *pkctx;
+			char mdname[1024] = {};
+
+			pkctx = EVP_PKEY_CTX_new(private_key, NULL);
+
+			ERR(!EVP_PKEY_sign_init(pkctx), "EVP_PKEY_sign_init");
+			ERR(!EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_PSS_PADDING),
+			    "EVP_PKEY_CTX_set_rsa_padding");
+			ERR(!EVP_PKEY_CTX_set_rsa_mgf1_md_name(pkctx, hash_algo, NULL),
+			    "EVP_PKEY_CTX_set_rsa_mgf1_md_name");
+
+			ERR(!EVP_PKEY_CTX_get_rsa_mgf1_md_name(pkctx, mdname, sizeof(mdname)),
+			    "EVP_PKEY_CTX_get_rsa_mgf1_md_name");
+			printf("RSASSA-PSS %s\n", mdname);
+		}
 
 		/* Load the signature message from the digest buffer. */
 		cms = CMS_sign(NULL, NULL, NULL, NULL, flags);
 		ERR(!cms, "CMS_sign");
 
-		ERR(!CMS_add1_signer(cms, x509, private_key, digest_algo, flags),
-		    "CMS_add1_signer");
+		signer = CMS_add1_signer(cms, x509, private_key, digest_algo, flags);
+		ERR(!signer, "CMS_add1_signer");
+
+		if (EVP_PKEY_is_a(private_key, "RSASSA-PSS")) {
+			EVP_PKEY_CTX *pkctx;
+			char mdname[1024] = {};
+
+			pkctx = CMS_SignerInfo_get0_pkey_ctx(signer);
+			ERR(!EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_PSS_PADDING),
+			    "EVP_PKEY_CTX_set_rsa_padding");
+			ERR(!EVP_PKEY_CTX_set_rsa_mgf1_md_name(pkctx, hash_algo, NULL),
+			    "EVP_PKEY_CTX_set_rsa_mgf1_md_name");
+
+			ERR(!EVP_PKEY_CTX_get_rsa_mgf1_md_name(pkctx, mdname, sizeof(mdname)),
+			    "EVP_PKEY_CTX_get_rsa_mgf1_md_name");
+			printf("RSASSA-PSS %s\n", mdname);
+		}
+
 		ERR(CMS_final(cms, bm, NULL, flags) != 1,
 		    "CMS_final");