From: Richard Henderson <richard.henderson@linaro.org>
To: qemu-devel@nongnu.org
Cc: qemu-arm@nongnu.org, qemu-riscv@nongnu.org, pbonzini@redhat.com,
eduardo@habkost.net, alistair.francis@wdc.com,
danielhb413@gmail.com
Subject: [PATCH v4 12/37] host/include/i386: Implement aes-round.h
Date: Mon, 3 Jul 2023 12:04:55 +0200 [thread overview]
Message-ID: <20230703100520.68224-13-richard.henderson@linaro.org> (raw)
In-Reply-To: <20230703100520.68224-1-richard.henderson@linaro.org>
Detect AES in cpuinfo; implement the accel hooks.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
host/include/i386/host/cpuinfo.h | 1 +
host/include/i386/host/crypto/aes-round.h | 152 ++++++++++++++++++++
host/include/x86_64/host/crypto/aes-round.h | 1 +
util/cpuinfo-i386.c | 3 +
4 files changed, 157 insertions(+)
create mode 100644 host/include/i386/host/crypto/aes-round.h
create mode 100644 host/include/x86_64/host/crypto/aes-round.h
diff --git a/host/include/i386/host/cpuinfo.h b/host/include/i386/host/cpuinfo.h
index a6537123cf..073d0a426f 100644
--- a/host/include/i386/host/cpuinfo.h
+++ b/host/include/i386/host/cpuinfo.h
@@ -26,6 +26,7 @@
#define CPUINFO_AVX512VBMI2 (1u << 15)
#define CPUINFO_ATOMIC_VMOVDQA (1u << 16)
#define CPUINFO_ATOMIC_VMOVDQU (1u << 17)
+#define CPUINFO_AES (1u << 18)
/* Initialized with a constructor. */
extern unsigned cpuinfo;
diff --git a/host/include/i386/host/crypto/aes-round.h b/host/include/i386/host/crypto/aes-round.h
new file mode 100644
index 0000000000..59a64130f7
--- /dev/null
+++ b/host/include/i386/host/crypto/aes-round.h
@@ -0,0 +1,152 @@
+/*
+ * x86 specific aes acceleration.
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#ifndef X86_HOST_CRYPTO_AES_ROUND_H
+#define X86_HOST_CRYPTO_AES_ROUND_H
+
+#include "host/cpuinfo.h"
+#include <immintrin.h>
+
+#if defined(__AES__) && defined(__SSSE3__)
+# define HAVE_AES_ACCEL true
+# define ATTR_AES_ACCEL
+#else
+# define HAVE_AES_ACCEL likely(cpuinfo & CPUINFO_AES)
+# define ATTR_AES_ACCEL __attribute__((target("aes,ssse3")))
+#endif
+
+static inline __m128i ATTR_AES_ACCEL
+aes_accel_bswap(__m128i x)
+{
+ return _mm_shuffle_epi8(x, _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8,
+ 9, 10, 11, 12, 13, 14, 15));
+}
+
+static inline void ATTR_AES_ACCEL
+aesenc_MC_accel(AESState *ret, const AESState *st, bool be)
+{
+ __m128i t = (__m128i)st->v;
+ __m128i z = _mm_setzero_si128();
+
+ if (be) {
+ t = aes_accel_bswap(t);
+ t = _mm_aesdeclast_si128(t, z);
+ t = _mm_aesenc_si128(t, z);
+ t = aes_accel_bswap(t);
+ } else {
+ t = _mm_aesdeclast_si128(t, z);
+ t = _mm_aesenc_si128(t, z);
+ }
+ ret->v = (AESStateVec)t;
+}
+
+static inline void ATTR_AES_ACCEL
+aesenc_SB_SR_AK_accel(AESState *ret, const AESState *st,
+ const AESState *rk, bool be)
+{
+ __m128i t = (__m128i)st->v;
+ __m128i k = (__m128i)rk->v;
+
+ if (be) {
+ t = aes_accel_bswap(t);
+ k = aes_accel_bswap(k);
+ t = _mm_aesenclast_si128(t, k);
+ t = aes_accel_bswap(t);
+ } else {
+ t = _mm_aesenclast_si128(t, k);
+ }
+ ret->v = (AESStateVec)t;
+}
+
+static inline void ATTR_AES_ACCEL
+aesenc_SB_SR_MC_AK_accel(AESState *ret, const AESState *st,
+ const AESState *rk, bool be)
+{
+ __m128i t = (__m128i)st->v;
+ __m128i k = (__m128i)rk->v;
+
+ if (be) {
+ t = aes_accel_bswap(t);
+ k = aes_accel_bswap(k);
+ t = _mm_aesenc_si128(t, k);
+ t = aes_accel_bswap(t);
+ } else {
+ t = _mm_aesenc_si128(t, k);
+ }
+ ret->v = (AESStateVec)t;
+}
+
+static inline void ATTR_AES_ACCEL
+aesdec_IMC_accel(AESState *ret, const AESState *st, bool be)
+{
+ __m128i t = (__m128i)st->v;
+
+ if (be) {
+ t = aes_accel_bswap(t);
+ t = _mm_aesimc_si128(t);
+ t = aes_accel_bswap(t);
+ } else {
+ t = _mm_aesimc_si128(t);
+ }
+ ret->v = (AESStateVec)t;
+}
+
+static inline void ATTR_AES_ACCEL
+aesdec_ISB_ISR_AK_accel(AESState *ret, const AESState *st,
+ const AESState *rk, bool be)
+{
+ __m128i t = (__m128i)st->v;
+ __m128i k = (__m128i)rk->v;
+
+ if (be) {
+ t = aes_accel_bswap(t);
+ k = aes_accel_bswap(k);
+ t = _mm_aesdeclast_si128(t, k);
+ t = aes_accel_bswap(t);
+ } else {
+ t = _mm_aesdeclast_si128(t, k);
+ }
+ ret->v = (AESStateVec)t;
+}
+
+static inline void ATTR_AES_ACCEL
+aesdec_ISB_ISR_AK_IMC_accel(AESState *ret, const AESState *st,
+ const AESState *rk, bool be)
+{
+ __m128i t = (__m128i)st->v;
+ __m128i k = (__m128i)rk->v;
+
+ if (be) {
+ t = aes_accel_bswap(t);
+ k = aes_accel_bswap(k);
+ t = _mm_aesdeclast_si128(t, k);
+ t = _mm_aesimc_si128(t);
+ t = aes_accel_bswap(t);
+ } else {
+ t = _mm_aesdeclast_si128(t, k);
+ t = _mm_aesimc_si128(t);
+ }
+ ret->v = (AESStateVec)t;
+}
+
+static inline void ATTR_AES_ACCEL
+aesdec_ISB_ISR_IMC_AK_accel(AESState *ret, const AESState *st,
+ const AESState *rk, bool be)
+{
+ __m128i t = (__m128i)st->v;
+ __m128i k = (__m128i)rk->v;
+
+ if (be) {
+ t = aes_accel_bswap(t);
+ k = aes_accel_bswap(k);
+ t = _mm_aesdec_si128(t, k);
+ t = aes_accel_bswap(t);
+ } else {
+ t = _mm_aesdec_si128(t, k);
+ }
+ ret->v = (AESStateVec)t;
+}
+
+#endif /* X86_HOST_CRYPTO_AES_ROUND_H */
diff --git a/host/include/x86_64/host/crypto/aes-round.h b/host/include/x86_64/host/crypto/aes-round.h
new file mode 100644
index 0000000000..2773cc9f10
--- /dev/null
+++ b/host/include/x86_64/host/crypto/aes-round.h
@@ -0,0 +1 @@
+#include "host/include/i386/host/crypto/aes-round.h"
diff --git a/util/cpuinfo-i386.c b/util/cpuinfo-i386.c
index ab6143d9e7..3a7b7e0ad1 100644
--- a/util/cpuinfo-i386.c
+++ b/util/cpuinfo-i386.c
@@ -40,6 +40,9 @@ unsigned __attribute__((constructor)) cpuinfo_init(void)
info |= (c & bit_MOVBE ? CPUINFO_MOVBE : 0);
info |= (c & bit_POPCNT ? CPUINFO_POPCNT : 0);
+ /* Our AES support requires PSHUFB as well. */
+ info |= ((c & bit_AES) && (c & bit_SSSE3) ? CPUINFO_AES : 0);
+
/* For AVX features, we must check available and usable. */
if ((c & bit_AVX) && (c & bit_OSXSAVE)) {
unsigned bv = xgetbv_low(0);
--
2.34.1
next prev parent reply other threads:[~2023-07-03 10:12 UTC|newest]
Thread overview: 56+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-07-03 10:04 [PATCH v4 00/37] crypto: Provide aes-round.h and host accel Richard Henderson
2023-07-03 10:04 ` [PATCH v4 01/37] util: Add cpuinfo-ppc.c Richard Henderson
2023-07-03 10:04 ` [PATCH v4 02/37] tests/multiarch: Add test-aes Richard Henderson
2023-07-03 12:08 ` Christoph Müllner
2023-07-05 14:28 ` Richard Henderson
2023-07-03 10:04 ` [PATCH v4 03/37] target/arm: Move aesmc and aesimc tables to crypto/aes.c Richard Henderson
2023-07-03 10:04 ` [PATCH v4 04/37] crypto/aes: Add AES_SH, AES_ISH macros Richard Henderson
2023-07-03 10:04 ` [PATCH v4 05/37] crypto: Add aesenc_SB_SR_AK Richard Henderson
2023-07-03 10:04 ` [PATCH v4 06/37] crypto: Add aesdec_ISB_ISR_AK Richard Henderson
2023-07-03 10:04 ` [PATCH v4 07/37] crypto: Add aesenc_MC Richard Henderson
2023-07-03 10:04 ` [PATCH v4 08/37] crypto: Add aesdec_IMC Richard Henderson
2023-07-03 10:04 ` [PATCH v4 09/37] crypto: Add aesenc_SB_SR_MC_AK Richard Henderson
2023-07-03 10:04 ` [PATCH v4 10/37] crypto: Add aesdec_ISB_ISR_IMC_AK Richard Henderson
2023-07-03 10:04 ` [PATCH v4 11/37] crypto: Add aesdec_ISB_ISR_AK_IMC Richard Henderson
2023-07-03 10:04 ` Richard Henderson [this message]
2023-07-03 10:04 ` [PATCH v4 13/37] host/include/aarch64: Implement aes-round.h Richard Henderson
2023-07-08 17:35 ` Philippe Mathieu-Daudé
2023-07-03 10:04 ` [PATCH v4 14/37] host/include/ppc: " Richard Henderson
2023-07-03 10:04 ` [PATCH v4 15/37] target/ppc: Use aesenc_SB_SR_AK Richard Henderson
2023-07-03 10:04 ` [PATCH v4 16/37] target/ppc: Use aesdec_ISB_ISR_AK Richard Henderson
2023-07-03 10:05 ` [PATCH v4 17/37] target/ppc: Use aesenc_SB_SR_MC_AK Richard Henderson
2023-07-07 19:50 ` Philippe Mathieu-Daudé
2023-07-03 10:05 ` [PATCH v4 18/37] target/ppc: Use aesdec_ISB_ISR_AK_IMC Richard Henderson
2023-07-03 10:05 ` [PATCH v4 19/37] target/i386: Use aesenc_SB_SR_AK Richard Henderson
2023-07-03 10:05 ` [PATCH v4 20/37] target/i386: Use aesdec_ISB_ISR_AK Richard Henderson
2023-07-03 10:05 ` [PATCH v4 21/37] target/i386: Use aesdec_IMC Richard Henderson
2023-07-07 19:48 ` Philippe Mathieu-Daudé
2023-07-03 10:05 ` [PATCH v4 22/37] target/i386: Use aesenc_SB_SR_MC_AK Richard Henderson
2023-07-07 19:50 ` Philippe Mathieu-Daudé
2023-07-03 10:05 ` [PATCH v4 23/37] target/i386: Use aesdec_ISB_ISR_IMC_AK Richard Henderson
2023-07-07 20:34 ` Philippe Mathieu-Daudé
2023-07-03 10:05 ` [PATCH v4 24/37] target/arm: Demultiplex AESE and AESMC Richard Henderson
2023-07-03 10:05 ` [PATCH v4 25/37] target/arm: Use aesenc_SB_SR_AK Richard Henderson
2023-07-08 17:18 ` Philippe Mathieu-Daudé
2023-07-03 10:05 ` [PATCH v4 26/37] target/arm: Use aesdec_ISB_ISR_AK Richard Henderson
2023-07-08 17:19 ` Philippe Mathieu-Daudé
2023-07-03 10:05 ` [PATCH v4 27/37] target/arm: Use aesenc_MC Richard Henderson
2023-07-08 17:20 ` Philippe Mathieu-Daudé
2023-07-03 10:05 ` [PATCH v4 28/37] target/arm: Use aesdec_IMC Richard Henderson
2023-07-08 17:20 ` Philippe Mathieu-Daudé
2023-07-03 10:05 ` [PATCH v4 29/37] target/riscv: Use aesenc_SB_SR_AK Richard Henderson
2023-07-07 20:38 ` Philippe Mathieu-Daudé
2023-07-03 10:05 ` [PATCH v4 30/37] target/riscv: Use aesdec_ISB_ISR_AK Richard Henderson
2023-07-07 20:32 ` Philippe Mathieu-Daudé
2023-07-03 10:05 ` [PATCH v4 31/37] target/riscv: Use aesdec_IMC Richard Henderson
2023-07-08 17:22 ` Philippe Mathieu-Daudé
2023-07-03 10:05 ` [PATCH v4 32/37] target/riscv: Use aesenc_SB_SR_MC_AK Richard Henderson
2023-07-07 20:26 ` Philippe Mathieu-Daudé
2023-07-03 10:05 ` [PATCH v4 33/37] target/riscv: Use aesdec_ISB_ISR_IMC_AK Richard Henderson
2023-07-08 17:33 ` Philippe Mathieu-Daudé
2023-07-03 10:05 ` [PATCH v4 34/37] crypto: Remove AES_shifts, AES_ishifts Richard Henderson
2023-07-03 10:05 ` [PATCH v4 35/37] crypto: Implement aesdec_IMC with AES_imc_rot Richard Henderson
2023-07-03 10:05 ` [PATCH v4 36/37] crypto: Remove AES_imc Richard Henderson
2023-07-03 10:05 ` [PATCH v4 37/37] crypto: Unexport AES_*_rot, AES_TeN, AES_TdN Richard Henderson
2023-07-07 17:30 ` [PATCH v4 00/37] crypto: Provide aes-round.h and host accel Daniel Henrique Barboza
2023-07-08 17:38 ` Philippe Mathieu-Daudé
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230703100520.68224-13-richard.henderson@linaro.org \
--to=richard.henderson@linaro.org \
--cc=alistair.francis@wdc.com \
--cc=danielhb413@gmail.com \
--cc=eduardo@habkost.net \
--cc=pbonzini@redhat.com \
--cc=qemu-arm@nongnu.org \
--cc=qemu-devel@nongnu.org \
--cc=qemu-riscv@nongnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).