From: Richard Henderson <richard.henderson@linaro.org>
To: qemu-devel@nongnu.org
Cc: berrange@redhat.com, qemu-ppc@nongnu.org, qemu-arm@nongnu.org,
qemu-riscv@nongnu.org,
Daniel Henrique Barboza <danielhb413@gmail.com>
Subject: [PATCH v3 37/37] host/include/ppc: Implement aes-round.h
Date: Tue, 20 Jun 2023 13:07:58 +0200 [thread overview]
Message-ID: <20230620110758.787479-38-richard.henderson@linaro.org> (raw)
In-Reply-To: <20230620110758.787479-1-richard.henderson@linaro.org>
Detect CRYPTO in cpuinfo; implement the accel hooks.
Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
host/include/ppc/host/cpuinfo.h | 1 +
host/include/ppc/host/crypto/aes-round.h | 182 +++++++++++++++++++++
host/include/ppc64/host/crypto/aes-round.h | 1 +
util/cpuinfo-ppc.c | 8 +
4 files changed, 192 insertions(+)
create mode 100644 host/include/ppc/host/crypto/aes-round.h
create mode 100644 host/include/ppc64/host/crypto/aes-round.h
diff --git a/host/include/ppc/host/cpuinfo.h b/host/include/ppc/host/cpuinfo.h
index df11e8d417..29ee7f9ef8 100644
--- a/host/include/ppc/host/cpuinfo.h
+++ b/host/include/ppc/host/cpuinfo.h
@@ -16,6 +16,7 @@
#define CPUINFO_ISEL (1u << 5)
#define CPUINFO_ALTIVEC (1u << 6)
#define CPUINFO_VSX (1u << 7)
+#define CPUINFO_CRYPTO (1u << 8)
/* Initialized with a constructor. */
extern unsigned cpuinfo;
diff --git a/host/include/ppc/host/crypto/aes-round.h b/host/include/ppc/host/crypto/aes-round.h
new file mode 100644
index 0000000000..8062d2a537
--- /dev/null
+++ b/host/include/ppc/host/crypto/aes-round.h
@@ -0,0 +1,182 @@
+/*
+ * Power v2.07 specific aes acceleration.
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#ifndef PPC_HOST_CRYPTO_AES_ROUND_H
+#define PPC_HOST_CRYPTO_AES_ROUND_H
+
+#ifdef __ALTIVEC__
+#include "host/cpuinfo.h"
+
+#ifdef __CRYPTO__
+# define HAVE_AES_ACCEL true
+#else
+# define HAVE_AES_ACCEL likely(cpuinfo & CPUINFO_CRYPTO)
+#endif
+#define ATTR_AES_ACCEL
+
+/*
+ * While there is <altivec.h>, both gcc and clang "aid" with the
+ * endianness issues in different ways. Just use inline asm instead.
+ */
+
+/* Bytes in memory are host-endian; bytes in register are @be. */
+static inline AESStateVec aes_accel_ld(const AESState *p, bool be)
+{
+ AESStateVec r;
+
+ if (be) {
+ asm("lvx %0, 0, %1" : "=v"(r) : "r"(p), "m"(*p));
+ } else if (HOST_BIG_ENDIAN) {
+ AESStateVec rev = {
+ 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
+ };
+ asm("lvx %0, 0, %1\n\t"
+ "vperm %0, %0, %0, %2"
+ : "=v"(r) : "r"(p), "v"(rev), "m"(*p));
+ } else {
+#ifdef __POWER9_VECTOR__
+ asm("lxvb16x %x0, 0, %1" : "=v"(r) : "r"(p), "m"(*p));
+#else
+ asm("lxvd2x %x0, 0, %1\n\t"
+ "xxpermdi %x0, %x0, %x0, 2"
+ : "=v"(r) : "r"(p), "m"(*p));
+#endif
+ }
+ return r;
+}
+
+static void aes_accel_st(AESState *p, AESStateVec r, bool be)
+{
+ if (be) {
+ asm("stvx %1, 0, %2" : "=m"(*p) : "v"(r), "r"(p));
+ } else if (HOST_BIG_ENDIAN) {
+ AESStateVec rev = {
+ 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
+ };
+ asm("vperm %1, %1, %1, %2\n\t"
+ "stvx %1, 0, %3"
+ : "=m"(*p), "+v"(r) : "v"(rev), "r"(p));
+ } else {
+#ifdef __POWER9_VECTOR__
+ asm("stxvb16x %x1, 0, %2" : "=m"(*p) : "v"(r), "r"(p));
+#else
+ asm("xxpermdi %x1, %x1, %x1, 2\n\t"
+ "stxvd2x %x1, 0, %2"
+ : "=m"(*p), "+v"(r) : "r"(p));
+#endif
+ }
+}
+
+static inline AESStateVec aes_accel_vcipher(AESStateVec d, AESStateVec k)
+{
+ asm("vcipher %0, %0, %1" : "+v"(d) : "v"(k));
+ return d;
+}
+
+static inline AESStateVec aes_accel_vncipher(AESStateVec d, AESStateVec k)
+{
+ asm("vncipher %0, %0, %1" : "+v"(d) : "v"(k));
+ return d;
+}
+
+static inline AESStateVec aes_accel_vcipherlast(AESStateVec d, AESStateVec k)
+{
+ asm("vcipherlast %0, %0, %1" : "+v"(d) : "v"(k));
+ return d;
+}
+
+static inline AESStateVec aes_accel_vncipherlast(AESStateVec d, AESStateVec k)
+{
+ asm("vncipherlast %0, %0, %1" : "+v"(d) : "v"(k));
+ return d;
+}
+
+static inline void
+aesenc_MC_accel(AESState *ret, const AESState *st, bool be)
+{
+ AESStateVec t, z = { };
+
+ t = aes_accel_ld(st, be);
+ t = aes_accel_vncipherlast(t, z);
+ t = aes_accel_vcipher(t, z);
+ aes_accel_st(ret, t, be);
+}
+
+static inline void
+aesenc_SB_SR_AK_accel(AESState *ret, const AESState *st,
+ const AESState *rk, bool be)
+{
+ AESStateVec t, k;
+
+ t = aes_accel_ld(st, be);
+ k = aes_accel_ld(rk, be);
+ t = aes_accel_vcipherlast(t, k);
+ aes_accel_st(ret, t, be);
+}
+
+static inline void
+aesenc_SB_SR_MC_AK_accel(AESState *ret, const AESState *st,
+ const AESState *rk, bool be)
+{
+ AESStateVec t, k;
+
+ t = aes_accel_ld(st, be);
+ k = aes_accel_ld(rk, be);
+ t = aes_accel_vcipher(t, k);
+ aes_accel_st(ret, t, be);
+}
+
+static inline void
+aesdec_IMC_accel(AESState *ret, const AESState *st, bool be)
+{
+ AESStateVec t, z = { };
+
+ t = aes_accel_ld(st, be);
+ t = aes_accel_vcipherlast(t, z);
+ t = aes_accel_vncipher(t, z);
+ aes_accel_st(ret, t, be);
+}
+
+static inline void
+aesdec_ISB_ISR_AK_accel(AESState *ret, const AESState *st,
+ const AESState *rk, bool be)
+{
+ AESStateVec t, k;
+
+ t = aes_accel_ld(st, be);
+ k = aes_accel_ld(rk, be);
+ t = aes_accel_vncipherlast(t, k);
+ aes_accel_st(ret, t, be);
+}
+
+static inline void
+aesdec_ISB_ISR_AK_IMC_accel(AESState *ret, const AESState *st,
+ const AESState *rk, bool be)
+{
+ AESStateVec t, k;
+
+ t = aes_accel_ld(st, be);
+ k = aes_accel_ld(rk, be);
+ t = aes_accel_vncipher(t, k);
+ aes_accel_st(ret, t, be);
+}
+
+static inline void
+aesdec_ISB_ISR_IMC_AK_accel(AESState *ret, const AESState *st,
+ const AESState *rk, bool be)
+{
+ AESStateVec t, k, z = { };
+
+ t = aes_accel_ld(st, be);
+ k = aes_accel_ld(rk, be);
+ t = aes_accel_vncipher(t, z);
+ aes_accel_st(ret, t ^ k, be);
+}
+#else
+/* Without ALTIVEC, we can't even write inline assembly. */
+#include "host/include/generic/host/crypto/aes-round.h"
+#endif
+
+#endif /* PPC_HOST_CRYPTO_AES_ROUND_H */
diff --git a/host/include/ppc64/host/crypto/aes-round.h b/host/include/ppc64/host/crypto/aes-round.h
new file mode 100644
index 0000000000..5eeba6dcb7
--- /dev/null
+++ b/host/include/ppc64/host/crypto/aes-round.h
@@ -0,0 +1 @@
+#include "host/include/ppc/host/crypto/aes-round.h"
diff --git a/util/cpuinfo-ppc.c b/util/cpuinfo-ppc.c
index d95adc8ccd..7212afa45d 100644
--- a/util/cpuinfo-ppc.c
+++ b/util/cpuinfo-ppc.c
@@ -48,6 +48,14 @@ unsigned __attribute__((constructor)) cpuinfo_init(void)
/* We only care about the portion of VSX that overlaps Altivec. */
if (hwcap & PPC_FEATURE_HAS_VSX) {
info |= CPUINFO_VSX;
+ /*
+ * We use VSX especially for little-endian, but we should
+ * always have both anyway, since VSX came with Power7
+ * and crypto came with Power8.
+ */
+ if (hwcap2 & PPC_FEATURE2_HAS_VEC_CRYPTO) {
+ info |= CPUINFO_CRYPTO;
+ }
}
}
--
2.34.1
next prev parent reply other threads:[~2023-06-20 11:15 UTC|newest]
Thread overview: 52+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-06-20 11:07 [PATCH v3 00/37] crypto: Provide aes-round.h and host accel Richard Henderson
2023-06-20 11:07 ` [PATCH v3 01/37] util: Add cpuinfo-ppc.c Richard Henderson
2023-06-20 11:07 ` [PATCH v3 02/37] tests/multiarch: Add test-aes Richard Henderson
2023-06-20 11:07 ` [PATCH v3 03/37] target/arm: Move aesmc and aesimc tables to crypto/aes.c Richard Henderson
2023-06-20 11:07 ` [PATCH v3 04/37] crypto/aes: Add constants for ShiftRows, InvShiftRows Richard Henderson
2023-06-20 11:07 ` [PATCH v3 05/37] crypto: Add aesenc_SB_SR_AK Richard Henderson
2023-06-21 16:38 ` Daniel P. Berrangé
2023-06-29 15:45 ` Max Chou
2023-06-29 16:02 ` Richard Henderson
2023-06-20 11:07 ` [PATCH v3 06/37] target/i386: Use aesenc_SB_SR_AK Richard Henderson
2023-06-20 11:07 ` [PATCH v3 07/37] target/arm: Demultiplex AESE and AESMC Richard Henderson
2023-06-20 11:07 ` [PATCH v3 08/37] target/arm: Use aesenc_SB_SR_AK Richard Henderson
2023-06-20 11:07 ` [PATCH v3 09/37] target/ppc: " Richard Henderson
2023-06-20 11:07 ` [PATCH v3 10/37] target/riscv: " Richard Henderson
2023-06-20 11:07 ` [PATCH v3 11/37] crypto: Add aesdec_ISB_ISR_AK Richard Henderson
2023-06-21 16:38 ` Daniel P. Berrangé
2023-06-20 11:07 ` [PATCH v3 12/37] target/i386: Use aesdec_ISB_ISR_AK Richard Henderson
2023-06-20 11:07 ` [PATCH v3 13/37] target/arm: " Richard Henderson
2023-06-20 11:07 ` [PATCH v3 14/37] target/ppc: " Richard Henderson
2023-06-20 11:07 ` [PATCH v3 15/37] target/riscv: " Richard Henderson
2023-06-20 11:07 ` [PATCH v3 16/37] crypto: Add aesenc_MC Richard Henderson
2023-06-21 16:39 ` Daniel P. Berrangé
2023-06-20 11:07 ` [PATCH v3 17/37] target/arm: Use aesenc_MC Richard Henderson
2023-06-20 11:07 ` [PATCH v3 18/37] crypto: Add aesdec_IMC Richard Henderson
2023-06-21 16:40 ` Daniel P. Berrangé
2023-06-20 11:07 ` [PATCH v3 19/37] target/i386: Use aesdec_IMC Richard Henderson
2023-06-20 11:07 ` [PATCH v3 20/37] target/arm: " Richard Henderson
2023-06-20 11:07 ` [PATCH v3 21/37] target/riscv: " Richard Henderson
2023-06-20 11:07 ` [PATCH v3 22/37] crypto: Add aesenc_SB_SR_MC_AK Richard Henderson
2023-06-21 16:40 ` Daniel P. Berrangé
2023-06-20 11:07 ` [PATCH v3 23/37] target/i386: Use aesenc_SB_SR_MC_AK Richard Henderson
2023-06-20 11:07 ` [PATCH v3 24/37] target/ppc: " Richard Henderson
2023-06-20 11:07 ` [PATCH v3 25/37] target/riscv: " Richard Henderson
2023-06-20 11:07 ` [PATCH v3 26/37] crypto: Add aesdec_ISB_ISR_IMC_AK Richard Henderson
2023-06-21 16:40 ` Daniel P. Berrangé
2023-06-20 11:07 ` [PATCH v3 27/37] target/i386: Use aesdec_ISB_ISR_IMC_AK Richard Henderson
2023-06-20 11:07 ` [PATCH v3 28/37] target/riscv: " Richard Henderson
2023-06-20 11:07 ` [PATCH v3 29/37] crypto: Add aesdec_ISB_ISR_AK_IMC Richard Henderson
2023-06-21 16:41 ` Daniel P. Berrangé
2023-06-20 11:07 ` [PATCH v3 30/37] target/ppc: Use aesdec_ISB_ISR_AK_IMC Richard Henderson
2023-06-20 11:07 ` [PATCH v3 31/37] crypto: Remove AES_shifts, AES_ishifts Richard Henderson
2023-06-21 16:41 ` Daniel P. Berrangé
2023-06-20 11:07 ` [PATCH v3 32/37] crypto: Implement aesdec_IMC with AES_imc_rot Richard Henderson
2023-06-21 16:42 ` Daniel P. Berrangé
2023-06-20 11:07 ` [PATCH v3 33/37] crypto: Remove AES_imc Richard Henderson
2023-06-21 16:42 ` Daniel P. Berrangé
2023-06-20 11:07 ` [PATCH v3 34/37] crypto: Unexport AES_*_rot, AES_TeN, AES_TdN Richard Henderson
2023-06-21 16:42 ` Daniel P. Berrangé
2023-06-20 11:07 ` [PATCH v3 35/37] host/include/i386: Implement aes-round.h Richard Henderson
2023-06-20 11:07 ` [PATCH v3 36/37] host/include/aarch64: " Richard Henderson
2023-06-20 11:07 ` Richard Henderson [this message]
2023-06-22 4:52 ` [PATCH v3 00/37] crypto: Provide aes-round.h and host accel Richard Henderson
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230620110758.787479-38-richard.henderson@linaro.org \
--to=richard.henderson@linaro.org \
--cc=berrange@redhat.com \
--cc=danielhb413@gmail.com \
--cc=qemu-arm@nongnu.org \
--cc=qemu-devel@nongnu.org \
--cc=qemu-ppc@nongnu.org \
--cc=qemu-riscv@nongnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).