From: Richard Henderson <richard.henderson@linaro.org>
To: qemu-devel@nongnu.org
Cc: qemu-arm@nongnu.org, Peter Maydell <peter.maydell@linaro.org>
Subject: [PATCH v4 27/45] target/arm: Implement SME integer outer product
Date: Tue, 28 Jun 2022 09:50:59 +0530 [thread overview]
Message-ID: <20220628042117.368549-28-richard.henderson@linaro.org> (raw)
In-Reply-To: <20220628042117.368549-1-richard.henderson@linaro.org>
This is SMOPA, SUMOPA, USMOPA_s, UMOPA, for both Int8 and Int16.
Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sme.h | 16 ++++++++
target/arm/sme.decode | 10 +++++
target/arm/sme_helper.c | 82 ++++++++++++++++++++++++++++++++++++++
target/arm/translate-sme.c | 10 +++++
4 files changed, 118 insertions(+)
diff --git a/target/arm/helper-sme.h b/target/arm/helper-sme.h
index 4d5d05db3a..d2d544a696 100644
--- a/target/arm/helper-sme.h
+++ b/target/arm/helper-sme.h
@@ -129,3 +129,19 @@ DEF_HELPER_FLAGS_7(sme_fmopa_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sme_bfmopa, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_smopa_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_umopa_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_sumopa_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_usmopa_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_smopa_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_umopa_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_sumopa_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_usmopa_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sme.decode b/target/arm/sme.decode
index e8d27fd8a0..628804e37a 100644
--- a/target/arm/sme.decode
+++ b/target/arm/sme.decode
@@ -76,3 +76,13 @@ FMOPA_d 10000000 110 ..... ... ... ..... . 0 ... @op_64
BFMOPA 10000001 100 ..... ... ... ..... . 00 .. @op_32
FMOPA_h 10000001 101 ..... ... ... ..... . 00 .. @op_32
+
+SMOPA_s 1010000 0 10 0 ..... ... ... ..... . 00 .. @op_32
+SUMOPA_s 1010000 0 10 1 ..... ... ... ..... . 00 .. @op_32
+USMOPA_s 1010000 1 10 0 ..... ... ... ..... . 00 .. @op_32
+UMOPA_s 1010000 1 10 1 ..... ... ... ..... . 00 .. @op_32
+
+SMOPA_d 1010000 0 11 0 ..... ... ... ..... . 0 ... @op_64
+SUMOPA_d 1010000 0 11 1 ..... ... ... ..... . 0 ... @op_64
+USMOPA_d 1010000 1 11 0 ..... ... ... ..... . 0 ... @op_64
+UMOPA_d 1010000 1 11 1 ..... ... ... ..... . 0 ... @op_64
diff --git a/target/arm/sme_helper.c b/target/arm/sme_helper.c
index 39d630a91c..e6204ab236 100644
--- a/target/arm/sme_helper.c
+++ b/target/arm/sme_helper.c
@@ -1081,3 +1081,85 @@ void HELPER(sme_bfmopa)(void *vza, void *vzn, void *vzm, void *vpn,
} while (row & 15);
}
}
+
+typedef uint64_t IMOPFn(uint64_t, uint64_t, uint64_t, uint8_t, bool);
+
+static inline void do_imopa(uint64_t *za, uint64_t *zn, uint64_t *zm,
+ uint8_t *pn, uint8_t *pm,
+ uint32_t desc, IMOPFn *fn)
+{
+ intptr_t row, col, oprsz = simd_oprsz(desc) / 8;
+ bool neg = simd_data(desc);
+
+ for (row = 0; row < oprsz; ++row) {
+ uint8_t pa = pn[H1(row)];
+ uint64_t *za_row = &za[row * sizeof(ARMVectorReg)];
+ uint64_t n = zn[row];
+
+ for (col = 0; col < oprsz; ++col) {
+ uint8_t pb = pm[H1(col)];
+ uint64_t *a = &za_row[col];
+
+ *a = fn(n, zm[col], *a, pa & pb, neg);
+ }
+ }
+}
+
+#define DEF_IMOP_32(NAME, NTYPE, MTYPE) \
+static uint64_t NAME(uint64_t n, uint64_t m, uint64_t a, uint8_t p, bool neg) \
+{ \
+ uint32_t sum0 = 0, sum1 = 0; \
+ /* Apply P to N as a mask, making the inactive elements 0. */ \
+ n &= expand_pred_b(p); \
+ sum0 += (NTYPE)(n >> 0) * (MTYPE)(m >> 0); \
+ sum0 += (NTYPE)(n >> 8) * (MTYPE)(m >> 8); \
+ sum0 += (NTYPE)(n >> 16) * (MTYPE)(m >> 16); \
+ sum0 += (NTYPE)(n >> 24) * (MTYPE)(m >> 24); \
+ sum1 += (NTYPE)(n >> 32) * (MTYPE)(m >> 32); \
+ sum1 += (NTYPE)(n >> 40) * (MTYPE)(m >> 40); \
+ sum1 += (NTYPE)(n >> 48) * (MTYPE)(m >> 48); \
+ sum1 += (NTYPE)(n >> 56) * (MTYPE)(m >> 56); \
+ if (neg) { \
+ sum0 = (uint32_t)a - sum0, sum1 = (uint32_t)(a >> 32) - sum1; \
+ } else { \
+ sum0 = (uint32_t)a + sum0, sum1 = (uint32_t)(a >> 32) + sum1; \
+ } \
+ return ((uint64_t)sum1 << 32) | sum0; \
+}
+
+#define DEF_IMOP_64(NAME, NTYPE, MTYPE) \
+static uint64_t NAME(uint64_t n, uint64_t m, uint64_t a, uint8_t p, bool neg) \
+{ \
+ uint64_t sum = 0; \
+ /* Apply P to N as a mask, making the inactive elements 0. */ \
+ n &= expand_pred_h(p); \
+ sum += (NTYPE)(n >> 0) * (MTYPE)(m >> 0); \
+ sum += (NTYPE)(n >> 16) * (MTYPE)(m >> 16); \
+ sum += (NTYPE)(n >> 32) * (MTYPE)(m >> 32); \
+ sum += (NTYPE)(n >> 48) * (MTYPE)(m >> 48); \
+ return neg ? a - sum : a + sum; \
+}
+
+DEF_IMOP_32(smopa_s, int8_t, int8_t)
+DEF_IMOP_32(umopa_s, uint8_t, uint8_t)
+DEF_IMOP_32(sumopa_s, int8_t, uint8_t)
+DEF_IMOP_32(usmopa_s, uint8_t, int8_t)
+
+DEF_IMOP_64(smopa_d, int16_t, int16_t)
+DEF_IMOP_64(umopa_d, uint16_t, uint16_t)
+DEF_IMOP_64(sumopa_d, int16_t, uint16_t)
+DEF_IMOP_64(usmopa_d, uint16_t, int16_t)
+
+#define DEF_IMOPH(NAME) \
+ void HELPER(sme_##NAME)(void *vza, void *vzn, void *vzm, void *vpn, \
+ void *vpm, uint32_t desc) \
+ { do_imopa(vza, vzn, vzm, vpn, vpm, desc, NAME); }
+
+DEF_IMOPH(smopa_s)
+DEF_IMOPH(umopa_s)
+DEF_IMOPH(sumopa_s)
+DEF_IMOPH(usmopa_s)
+DEF_IMOPH(smopa_d)
+DEF_IMOPH(umopa_d)
+DEF_IMOPH(sumopa_d)
+DEF_IMOPH(usmopa_d)
diff --git a/target/arm/translate-sme.c b/target/arm/translate-sme.c
index 0fcb33cb3f..876184e8bd 100644
--- a/target/arm/translate-sme.c
+++ b/target/arm/translate-sme.c
@@ -364,3 +364,13 @@ TRANS_FEAT(FMOPA_d, aa64_sme_f64f64, do_outprod_fpst, a, MO_64, gen_helper_sme_f
/* TODO: FEAT_EBF16 */
TRANS_FEAT(BFMOPA, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_bfmopa)
+
+TRANS_FEAT(SMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_smopa_s)
+TRANS_FEAT(UMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_umopa_s)
+TRANS_FEAT(SUMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_sumopa_s)
+TRANS_FEAT(USMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_usmopa_s)
+
+TRANS_FEAT(SMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_smopa_d)
+TRANS_FEAT(UMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_umopa_d)
+TRANS_FEAT(SUMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_sumopa_d)
+TRANS_FEAT(USMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_usmopa_d)
--
2.34.1
next prev parent reply other threads:[~2022-06-28 4:39 UTC|newest]
Thread overview: 93+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-06-28 4:20 [PATCH v4 00/45] target/arm: Scalable Matrix Extension Richard Henderson
2022-06-28 4:20 ` [PATCH v4 01/45] target/arm: Handle SME in aarch64_cpu_dump_state Richard Henderson
2022-07-01 10:11 ` Peter Maydell
2022-07-03 8:43 ` Richard Henderson
2022-06-28 4:20 ` [PATCH v4 02/45] target/arm: Add infrastructure for disas_sme Richard Henderson
2022-06-28 4:20 ` [PATCH v4 03/45] target/arm: Trap non-streaming usage when Streaming SVE is active Richard Henderson
2022-07-01 11:06 ` Peter Maydell
2022-07-04 8:28 ` Richard Henderson
2022-07-04 8:33 ` Peter Maydell
2022-06-28 4:20 ` [PATCH v4 04/45] target/arm: Mark ADR as non-streaming Richard Henderson
2022-07-01 11:11 ` Peter Maydell
2022-06-28 4:20 ` [PATCH v4 05/45] target/arm: Mark RDFFR, WRFFR, SETFFR " Richard Henderson
2022-07-01 11:15 ` Peter Maydell
2022-06-28 4:20 ` [PATCH v4 06/45] target/arm: Mark BDEP, BEXT, BGRP, COMPACT, FEXPA, FTSSEL " Richard Henderson
2022-07-01 12:14 ` Peter Maydell
2022-06-28 4:20 ` [PATCH v4 07/45] target/arm: Mark PMULL, FMMLA " Richard Henderson
2022-07-01 12:18 ` Peter Maydell
2022-07-04 8:48 ` Richard Henderson
2022-06-28 4:20 ` [PATCH v4 08/45] target/arm: Mark FTSMUL, FTMAD, FADDA " Richard Henderson
2022-07-01 12:21 ` Peter Maydell
2022-06-28 4:20 ` [PATCH v4 09/45] target/arm: Mark SMMLA, UMMLA, USMMLA " Richard Henderson
2022-07-01 12:22 ` Peter Maydell
2022-06-28 4:20 ` [PATCH v4 10/45] target/arm: Mark string/histo/crypto " Richard Henderson
2022-07-01 12:25 ` Peter Maydell
2022-06-28 4:20 ` [PATCH v4 11/45] target/arm: Mark gather/scatter load/store " Richard Henderson
2022-07-01 12:29 ` Peter Maydell
2022-06-28 4:20 ` [PATCH v4 12/45] target/arm: Mark gather prefetch " Richard Henderson
2022-07-01 12:31 ` Peter Maydell
2022-06-28 4:20 ` [PATCH v4 13/45] target/arm: Mark LDFF1 and LDNF1 " Richard Henderson
2022-07-01 12:33 ` Peter Maydell
2022-06-28 4:20 ` [PATCH v4 14/45] target/arm: Mark LD1RO " Richard Henderson
2022-07-01 13:00 ` Peter Maydell
2022-06-28 4:20 ` [PATCH v4 15/45] target/arm: Add SME enablement checks Richard Henderson
2022-07-01 13:05 ` Peter Maydell
2022-06-28 4:20 ` [PATCH v4 16/45] target/arm: Handle SME in sve_access_check Richard Henderson
2022-07-01 13:07 ` Peter Maydell
2022-06-28 4:20 ` [PATCH v4 17/45] target/arm: Implement SME RDSVL, ADDSVL, ADDSPL Richard Henderson
2022-06-28 4:20 ` [PATCH v4 18/45] target/arm: Implement SME ZERO Richard Henderson
2022-06-28 4:20 ` [PATCH v4 19/45] target/arm: Implement SME MOVA Richard Henderson
2022-07-01 16:19 ` Peter Maydell
2022-07-04 9:08 ` Richard Henderson
2022-07-04 9:31 ` Peter Maydell
2022-07-04 9:43 ` Richard Henderson
2022-06-28 4:20 ` [PATCH v4 20/45] target/arm: Implement SME LD1, ST1 Richard Henderson
2022-07-04 10:39 ` Peter Maydell
2022-07-05 1:49 ` Richard Henderson
2022-07-05 10:48 ` Peter Maydell
2022-07-05 11:21 ` Richard Henderson
2022-06-28 4:20 ` [PATCH v4 21/45] target/arm: Export unpredicated ld/st from translate-sve.c Richard Henderson
2022-06-28 4:20 ` [PATCH v4 22/45] target/arm: Implement SME LDR, STR Richard Henderson
2022-06-28 4:20 ` [PATCH v4 23/45] target/arm: Implement SME ADDHA, ADDVA Richard Henderson
2022-07-04 10:50 ` Peter Maydell
2022-07-05 2:05 ` Richard Henderson
2022-06-28 4:20 ` [PATCH v4 24/45] target/arm: Implement FMOPA, FMOPS (non-widening) Richard Henderson
2022-06-28 4:20 ` [PATCH v4 25/45] target/arm: Implement BFMOPA, BFMOPS Richard Henderson
2022-06-28 4:20 ` [PATCH v4 26/45] target/arm: Implement FMOPA, FMOPS (widening) Richard Henderson
2022-06-28 4:20 ` Richard Henderson [this message]
2022-06-28 4:21 ` [PATCH v4 28/45] target/arm: Implement PSEL Richard Henderson
2022-06-28 4:21 ` [PATCH v4 29/45] target/arm: Implement REVD Richard Henderson
2022-06-28 4:21 ` [PATCH v4 30/45] target/arm: Implement SCLAMP, UCLAMP Richard Henderson
2022-06-28 4:21 ` [PATCH v4 31/45] target/arm: Reset streaming sve state on exception boundaries Richard Henderson
2022-06-28 4:21 ` [PATCH v4 32/45] target/arm: Enable SME for -cpu max Richard Henderson
2022-06-28 4:21 ` [PATCH v4 33/45] linux-user/aarch64: Clear tpidr2_el0 if CLONE_SETTLS Richard Henderson
2022-07-04 11:45 ` Peter Maydell
2022-06-28 4:21 ` [PATCH v4 34/45] linux-user/aarch64: Reset PSTATE.SM on syscalls Richard Henderson
2022-07-04 11:50 ` Peter Maydell
2022-06-28 4:21 ` [PATCH v4 35/45] linux-user/aarch64: Add SM bit to SVE signal context Richard Henderson
2022-07-04 12:02 ` Peter Maydell
2022-07-05 3:24 ` Richard Henderson
2022-06-28 4:21 ` [PATCH v4 36/45] linux-user/aarch64: Tidy target_restore_sigframe error return Richard Henderson
2022-07-04 12:04 ` Peter Maydell
2022-06-28 4:21 ` [PATCH v4 37/45] linux-user/aarch64: Do not allow duplicate or short sve records Richard Henderson
2022-07-04 12:08 ` Peter Maydell
2022-07-05 3:27 ` Richard Henderson
2022-07-05 3:30 ` Richard Henderson
2022-07-05 3:32 ` Richard Henderson
2022-06-28 4:21 ` [PATCH v4 38/45] linux-user/aarch64: Verify extra record lock succeeded Richard Henderson
2022-07-04 12:11 ` Peter Maydell
2022-06-28 4:21 ` [PATCH v4 39/45] linux-user/aarch64: Move sve record checks into restore Richard Henderson
2022-07-04 12:15 ` Peter Maydell
2022-06-28 4:21 ` [PATCH v4 40/45] linux-user/aarch64: Implement SME signal handling Richard Henderson
2022-07-04 13:05 ` Peter Maydell
2022-06-28 4:21 ` [PATCH v4 41/45] linux-user: Rename sve prctls Richard Henderson
2022-07-04 12:16 ` Peter Maydell
2022-06-28 4:21 ` [PATCH v4 42/45] linux-user/aarch64: Implement PR_SME_GET_VL, PR_SME_SET_VL Richard Henderson
2022-07-04 12:20 ` Peter Maydell
2022-06-28 4:21 ` [PATCH v4 43/45] target/arm: Only set ZEN in reset if SVE present Richard Henderson
2022-07-04 12:21 ` Peter Maydell
2022-06-28 4:21 ` [PATCH v4 44/45] target/arm: Enable SME for user-only Richard Henderson
2022-07-04 12:22 ` Peter Maydell
2022-06-28 4:21 ` [PATCH v4 45/45] linux-user/aarch64: Add SME related hwcap entries Richard Henderson
2022-07-04 12:24 ` Peter Maydell
2022-07-04 13:09 ` [PATCH v4 00/45] target/arm: Scalable Matrix Extension Peter Maydell
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220628042117.368549-28-richard.henderson@linaro.org \
--to=richard.henderson@linaro.org \
--cc=peter.maydell@linaro.org \
--cc=qemu-arm@nongnu.org \
--cc=qemu-devel@nongnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).