* [PATCH v3 01/81] target/arm: Fix sve_uzp_p vs odd vector lengths
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 02/81] target/arm: Fix sve_zip_p " Richard Henderson
` (81 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: Laurent Desnogues, peter.maydell, alex.bennee
Missed out on compressing the second half of a predicate
with length vl % 512 > 256.
Adjust all of the x + (y << s) to x | (y << s) as a
general style fix. Drop the extract64 because the input
uint64_t are known to be already zero-extended from the
current size of the predicate.
Reported-by: Laurent Desnogues <laurent.desnogues@gmail.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve_helper.c | 30 +++++++++++++++++++++---------
1 file changed, 21 insertions(+), 9 deletions(-)
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 4758d46f34..fcb46f150f 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1938,7 +1938,7 @@ void HELPER(sve_uzp_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
if (oprsz <= 8) {
l = compress_bits(n[0] >> odd, esz);
h = compress_bits(m[0] >> odd, esz);
- d[0] = extract64(l + (h << (4 * oprsz)), 0, 8 * oprsz);
+ d[0] = l | (h << (4 * oprsz));
} else {
ARMPredicateReg tmp_m;
intptr_t oprsz_16 = oprsz / 16;
@@ -1952,23 +1952,35 @@ void HELPER(sve_uzp_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
h = n[2 * i + 1];
l = compress_bits(l >> odd, esz);
h = compress_bits(h >> odd, esz);
- d[i] = l + (h << 32);
+ d[i] = l | (h << 32);
}
- /* For VL which is not a power of 2, the results from M do not
- align nicely with the uint64_t for D. Put the aligned results
- from M into TMP_M and then copy it into place afterward. */
+ /*
+ * For VL which is not a multiple of 512, the results from M do not
+ * align nicely with the uint64_t for D. Put the aligned results
+ * from M into TMP_M and then copy it into place afterward.
+ */
if (oprsz & 15) {
- d[i] = compress_bits(n[2 * i] >> odd, esz);
+ int final_shift = (oprsz & 15) * 2;
+
+ l = n[2 * i + 0];
+ h = n[2 * i + 1];
+ l = compress_bits(l >> odd, esz);
+ h = compress_bits(h >> odd, esz);
+ d[i] = l | (h << final_shift);
for (i = 0; i < oprsz_16; i++) {
l = m[2 * i + 0];
h = m[2 * i + 1];
l = compress_bits(l >> odd, esz);
h = compress_bits(h >> odd, esz);
- tmp_m.p[i] = l + (h << 32);
+ tmp_m.p[i] = l | (h << 32);
}
- tmp_m.p[i] = compress_bits(m[2 * i] >> odd, esz);
+ l = m[2 * i + 0];
+ h = m[2 * i + 1];
+ l = compress_bits(l >> odd, esz);
+ h = compress_bits(h >> odd, esz);
+ tmp_m.p[i] = l | (h << final_shift);
swap_memmove(vd + oprsz / 2, &tmp_m, oprsz / 2);
} else {
@@ -1977,7 +1989,7 @@ void HELPER(sve_uzp_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
h = m[2 * i + 1];
l = compress_bits(l >> odd, esz);
h = compress_bits(h >> odd, esz);
- d[oprsz_16 + i] = l + (h << 32);
+ d[oprsz_16 + i] = l | (h << 32);
}
}
}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 02/81] target/arm: Fix sve_zip_p vs odd vector lengths
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
2020-09-18 18:36 ` [PATCH v3 01/81] target/arm: Fix sve_uzp_p vs odd vector lengths Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 03/81] target/arm: Fix sve_punpk_p " Richard Henderson
` (80 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: Laurent Desnogues, peter.maydell, alex.bennee
Wrote too much with low-half zip (zip1) with vl % 512 != 0.
Adjust all of the x + (y << s) to x | (y << s) as a style fix.
We only ever have exact overlap between D, M, and N. Therefore
we only need a single temporary, and we do not need to check for
partial overlap.
Reported-by: Laurent Desnogues <laurent.desnogues@gmail.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve_helper.c | 25 ++++++++++++++-----------
1 file changed, 14 insertions(+), 11 deletions(-)
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index fcb46f150f..b8651ae173 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1870,6 +1870,7 @@ void HELPER(sve_zip_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
int esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
intptr_t high = extract32(pred_desc, SIMD_DATA_SHIFT + 2, 1);
+ int esize = 1 << esz;
uint64_t *d = vd;
intptr_t i;
@@ -1882,33 +1883,35 @@ void HELPER(sve_zip_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
mm = extract64(mm, high * half, half);
nn = expand_bits(nn, esz);
mm = expand_bits(mm, esz);
- d[0] = nn + (mm << (1 << esz));
+ d[0] = nn | (mm << esize);
} else {
- ARMPredicateReg tmp_n, tmp_m;
+ ARMPredicateReg tmp;
/* We produce output faster than we consume input.
Therefore we must be mindful of possible overlap. */
- if ((vn - vd) < (uintptr_t)oprsz) {
- vn = memcpy(&tmp_n, vn, oprsz);
- }
- if ((vm - vd) < (uintptr_t)oprsz) {
- vm = memcpy(&tmp_m, vm, oprsz);
+ if (vd == vn) {
+ vn = memcpy(&tmp, vn, oprsz);
+ if (vd == vm) {
+ vm = vn;
+ }
+ } else if (vd == vm) {
+ vm = memcpy(&tmp, vm, oprsz);
}
if (high) {
high = oprsz >> 1;
}
- if ((high & 3) == 0) {
+ if ((oprsz & 7) == 0) {
uint32_t *n = vn, *m = vm;
high >>= 2;
- for (i = 0; i < DIV_ROUND_UP(oprsz, 8); i++) {
+ for (i = 0; i < oprsz / 8; i++) {
uint64_t nn = n[H4(high + i)];
uint64_t mm = m[H4(high + i)];
nn = expand_bits(nn, esz);
mm = expand_bits(mm, esz);
- d[i] = nn + (mm << (1 << esz));
+ d[i] = nn | (mm << esize);
}
} else {
uint8_t *n = vn, *m = vm;
@@ -1920,7 +1923,7 @@ void HELPER(sve_zip_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
nn = expand_bits(nn, esz);
mm = expand_bits(mm, esz);
- d16[H2(i)] = nn + (mm << (1 << esz));
+ d16[H2(i)] = nn | (mm << esize);
}
}
}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 03/81] target/arm: Fix sve_punpk_p vs odd vector lengths
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
2020-09-18 18:36 ` [PATCH v3 01/81] target/arm: Fix sve_uzp_p vs odd vector lengths Richard Henderson
2020-09-18 18:36 ` [PATCH v3 02/81] target/arm: Fix sve_zip_p " Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 04/81] target/arm: Add ID_AA64ZFR0 fields and isar_feature_aa64_sve2 Richard Henderson
` (79 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: Laurent Desnogues, peter.maydell, alex.bennee
Wrote too much with punpk1 with vl % 512 != 0.
Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
Reported-by: Laurent Desnogues <laurent.desnogues@gmail.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve_helper.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index b8651ae173..c983cd4356 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -2104,11 +2104,11 @@ void HELPER(sve_punpk_p)(void *vd, void *vn, uint32_t pred_desc)
high = oprsz >> 1;
}
- if ((high & 3) == 0) {
+ if ((oprsz & 7) == 0) {
uint32_t *n = vn;
high >>= 2;
- for (i = 0; i < DIV_ROUND_UP(oprsz, 8); i++) {
+ for (i = 0; i < oprsz / 8; i++) {
uint64_t nn = n[H4(high + i)];
d[i] = expand_bits(nn, 0);
}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 04/81] target/arm: Add ID_AA64ZFR0 fields and isar_feature_aa64_sve2
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (2 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 03/81] target/arm: Fix sve_punpk_p " Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 05/81] target/arm: Implement SVE2 Integer Multiply - Unpredicated Richard Henderson
` (78 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Will be used for SVE2 isa subset enablement.
Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Do not read zfr0 from kvm unless sve is available.
---
target/arm/cpu.h | 16 ++++++++++++++++
target/arm/helper.c | 3 +--
target/arm/kvm64.c | 11 +++++++++++
3 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index 6036f61d60..6f80a85fd0 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -920,6 +920,7 @@ struct ARMCPU {
uint64_t id_aa64mmfr2;
uint64_t id_aa64dfr0;
uint64_t id_aa64dfr1;
+ uint64_t id_aa64zfr0;
} isar;
uint64_t midr;
uint32_t revidr;
@@ -1880,6 +1881,16 @@ FIELD(ID_AA64DFR0, PMSVER, 32, 4)
FIELD(ID_AA64DFR0, DOUBLELOCK, 36, 4)
FIELD(ID_AA64DFR0, TRACEFILT, 40, 4)
+FIELD(ID_AA64ZFR0, SVEVER, 0, 4)
+FIELD(ID_AA64ZFR0, AES, 4, 4)
+FIELD(ID_AA64ZFR0, BITPERM, 16, 4)
+FIELD(ID_AA64ZFR0, BFLOAT16, 20, 4)
+FIELD(ID_AA64ZFR0, SHA3, 32, 4)
+FIELD(ID_AA64ZFR0, SM4, 40, 4)
+FIELD(ID_AA64ZFR0, I8MM, 44, 4)
+FIELD(ID_AA64ZFR0, F32MM, 52, 4)
+FIELD(ID_AA64ZFR0, F64MM, 56, 4)
+
FIELD(ID_DFR0, COPDBG, 0, 4)
FIELD(ID_DFR0, COPSDBG, 4, 4)
FIELD(ID_DFR0, MMAPDBG, 8, 4)
@@ -3886,6 +3897,11 @@ static inline bool isar_feature_aa64_tts2uxn(const ARMISARegisters *id)
return FIELD_EX64(id->id_aa64mmfr1, ID_AA64MMFR1, XNX) != 0;
}
+static inline bool isar_feature_aa64_sve2(const ARMISARegisters *id)
+{
+ return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SVEVER) != 0;
+}
+
/*
* Feature tests for "does this exist in either 32-bit or 64-bit?"
*/
diff --git a/target/arm/helper.c b/target/arm/helper.c
index 88bd9dd35d..c0b84f9b99 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -7414,8 +7414,7 @@ void register_cp_regs_for_features(ARMCPU *cpu)
.opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 4,
.access = PL1_R, .type = ARM_CP_CONST,
.accessfn = access_aa64_tid3,
- /* At present, only SVEver == 0 is defined anyway. */
- .resetvalue = 0 },
+ .resetvalue = cpu->isar.id_aa64zfr0 },
{ .name = "ID_AA64PFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 5,
.access = PL1_R, .type = ARM_CP_CONST,
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index 987b35e33f..904d97bdd6 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -548,6 +548,17 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr2,
ARM64_SYS_REG(3, 0, 0, 7, 2));
+ /*
+ * Before v5.1, KVM did not support SVE and did not expose
+ * ID_AA64ZFR0_EL1 even as RAZ. Afterward, KVM still does
+ * not expose the register to "user" requests like this
+ * unless the host supports SVE.
+ */
+ if (isar_feature_aa64_sve(&ahcf->isar)) {
+ err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64zfr0,
+ ARM64_SYS_REG(3, 0, 0, 4, 4));
+ }
+
/*
* Note that if AArch32 support is not present in the host,
* the AArch32 sysregs are present to be read, but will
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 05/81] target/arm: Implement SVE2 Integer Multiply - Unpredicated
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (3 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 04/81] target/arm: Add ID_AA64ZFR0 fields and isar_feature_aa64_sve2 Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 06/81] target/arm: Implement SVE2 integer pairwise add and accumulate long Richard Henderson
` (77 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
For MUL, we can rely on generic support. For SMULH and UMULH,
create some trivial helpers. For PMUL, back in a21bb78e5817,
we organized helper_gvec_pmul_b in preparation for this use.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper.h | 10 ++++
target/arm/sve.decode | 10 ++++
target/arm/translate-sve.c | 50 ++++++++++++++++++++
target/arm/vec_helper.c | 96 ++++++++++++++++++++++++++++++++++++++
4 files changed, 166 insertions(+)
diff --git a/target/arm/helper.h b/target/arm/helper.h
index 8defd7c801..94658169b2 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -815,6 +815,16 @@ DEF_HELPER_FLAGS_3(gvec_cgt0_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(gvec_cge0_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(gvec_cge0_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_smulh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_smulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_smulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_smulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(gvec_umulh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_umulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_umulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_umulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_4(gvec_sshl_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_sshl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_ushl_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 6425396ac1..2c71d9e446 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1090,3 +1090,13 @@ ST1_zprz 1110010 .. 00 ..... 100 ... ..... ..... \
@rprr_scatter_store xs=0 esz=3 scale=0
ST1_zprz 1110010 .. 00 ..... 110 ... ..... ..... \
@rprr_scatter_store xs=1 esz=3 scale=0
+
+#### SVE2 Support
+
+### SVE2 Integer Multiply - Unpredicated
+
+# SVE2 integer multiply vectors (unpredicated)
+MUL_zzz 00000100 .. 1 ..... 0110 00 ..... ..... @rd_rn_rm
+SMULH_zzz 00000100 .. 1 ..... 0110 10 ..... ..... @rd_rn_rm
+UMULH_zzz 00000100 .. 1 ..... 0110 11 ..... ..... @rd_rn_rm
+PMUL_zzz 00000100 00 1 ..... 0110 01 ..... ..... @rd_rn_rm_e0
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 9095586fc9..04c5a2c8bd 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -5805,3 +5805,53 @@ static bool trans_MOVPRFX_z(DisasContext *s, arg_rpr_esz *a)
{
return do_movz_zpz(s, a->rd, a->rn, a->pg, a->esz, false);
}
+
+/*
+ * SVE2 Integer Multiply - Unpredicated
+ */
+
+static bool trans_MUL_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_fn_zzz(s, tcg_gen_gvec_mul, a->esz, a->rd, a->rn, a->rm);
+ }
+ return true;
+}
+
+static bool do_sve2_zzz_ool(DisasContext *s, arg_rrr_esz *a,
+ gen_helper_gvec_3 *fn)
+{
+ if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, 0);
+ }
+ return true;
+}
+
+static bool trans_SMULH_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_gvec_smulh_b, gen_helper_gvec_smulh_h,
+ gen_helper_gvec_smulh_s, gen_helper_gvec_smulh_d,
+ };
+ return do_sve2_zzz_ool(s, a, fns[a->esz]);
+}
+
+static bool trans_UMULH_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_gvec_umulh_b, gen_helper_gvec_umulh_h,
+ gen_helper_gvec_umulh_s, gen_helper_gvec_umulh_d,
+ };
+ return do_sve2_zzz_ool(s, a, fns[a->esz]);
+}
+
+static bool trans_PMUL_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_sve2_zzz_ool(s, a, gen_helper_gvec_pmul_b);
+}
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index a973454e4f..84e1ae3933 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -1937,3 +1937,99 @@ DO_VRINT_RMODE(gvec_vrint_rm_h, helper_rinth, uint16_t)
DO_VRINT_RMODE(gvec_vrint_rm_s, helper_rints, uint32_t)
#undef DO_VRINT_RMODE
+
+/*
+ * NxN -> N highpart multiply
+ *
+ * TODO: expose this as a generic vector operation.
+ */
+
+void HELPER(gvec_smulh_b)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int8_t *d = vd, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz; ++i) {
+ d[i] = ((int32_t)n[i] * m[i]) >> 8;
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_smulh_h)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int16_t *d = vd, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz / 2; ++i) {
+ d[i] = ((int32_t)n[i] * m[i]) >> 16;
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_smulh_s)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int32_t *d = vd, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz / 4; ++i) {
+ d[i] = ((int64_t)n[i] * m[i]) >> 32;
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_smulh_d)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ uint64_t *d = vd, *n = vn, *m = vm;
+ uint64_t discard;
+
+ for (i = 0; i < opr_sz / 8; ++i) {
+ muls64(&discard, &d[i], n[i], m[i]);
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_umulh_b)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ uint8_t *d = vd, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz; ++i) {
+ d[i] = ((uint32_t)n[i] * m[i]) >> 8;
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_umulh_h)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ uint16_t *d = vd, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz / 2; ++i) {
+ d[i] = ((uint32_t)n[i] * m[i]) >> 16;
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_umulh_s)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ uint32_t *d = vd, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz / 4; ++i) {
+ d[i] = ((uint64_t)n[i] * m[i]) >> 32;
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_umulh_d)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ uint64_t *d = vd, *n = vn, *m = vm;
+ uint64_t discard;
+
+ for (i = 0; i < opr_sz / 8; ++i) {
+ mulu64(&discard, &d[i], n[i], m[i]);
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 06/81] target/arm: Implement SVE2 integer pairwise add and accumulate long
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (4 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 05/81] target/arm: Implement SVE2 Integer Multiply - Unpredicated Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 07/81] target/arm: Implement SVE2 integer unary operations (predicated) Richard Henderson
` (76 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 14 ++++++++++++
target/arm/sve.decode | 5 +++++
target/arm/sve_helper.c | 44 ++++++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 39 +++++++++++++++++++++++++++++++++
4 files changed, 102 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 4411c47120..e185405cdc 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -158,6 +158,20 @@ DEF_HELPER_FLAGS_5(sve_umulh_zpzz_s, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(sve_umulh_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sadalp_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sadalp_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sadalp_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_5(sve_sdiv_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_sdiv_zpzz_d, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 2c71d9e446..4f54a30538 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1100,3 +1100,8 @@ MUL_zzz 00000100 .. 1 ..... 0110 00 ..... ..... @rd_rn_rm
SMULH_zzz 00000100 .. 1 ..... 0110 10 ..... ..... @rd_rn_rm
UMULH_zzz 00000100 .. 1 ..... 0110 11 ..... ..... @rd_rn_rm
PMUL_zzz 00000100 00 1 ..... 0110 01 ..... ..... @rd_rn_rm_e0
+
+### SVE2 Integer - Predicated
+
+SADALP_zpzz 01000100 .. 000 100 101 ... ..... ..... @rdm_pg_rn
+UADALP_zpzz 01000100 .. 000 101 101 ... ..... ..... @rdm_pg_rn
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index c983cd4356..4705722b71 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -517,6 +517,50 @@ DO_ZPZZ_D(sve_asr_zpzz_d, int64_t, DO_ASR)
DO_ZPZZ_D(sve_lsr_zpzz_d, uint64_t, DO_LSR)
DO_ZPZZ_D(sve_lsl_zpzz_d, uint64_t, DO_LSL)
+static inline uint16_t do_sadalp_h(uint16_t n, uint16_t m)
+{
+ int8_t n1 = n, n2 = n >> 8;
+ return m + n1 + n2;
+}
+
+static inline uint32_t do_sadalp_s(uint32_t n, uint32_t m)
+{
+ int16_t n1 = n, n2 = n >> 16;
+ return m + n1 + n2;
+}
+
+static inline uint64_t do_sadalp_d(uint64_t n, uint64_t m)
+{
+ int32_t n1 = n, n2 = n >> 32;
+ return m + n1 + n2;
+}
+
+DO_ZPZZ(sve2_sadalp_zpzz_h, int16_t, H1_2, do_sadalp_h)
+DO_ZPZZ(sve2_sadalp_zpzz_s, int32_t, H1_4, do_sadalp_s)
+DO_ZPZZ_D(sve2_sadalp_zpzz_d, uint64_t, do_sadalp_d)
+
+static inline uint16_t do_uadalp_h(uint16_t n, uint16_t m)
+{
+ uint8_t n1 = n, n2 = n >> 8;
+ return m + n1 + n2;
+}
+
+static inline uint32_t do_uadalp_s(uint32_t n, uint32_t m)
+{
+ uint16_t n1 = n, n2 = n >> 16;
+ return m + n1 + n2;
+}
+
+static inline uint64_t do_uadalp_d(uint64_t n, uint64_t m)
+{
+ uint32_t n1 = n, n2 = n >> 32;
+ return m + n1 + n2;
+}
+
+DO_ZPZZ(sve2_uadalp_zpzz_h, int16_t, H1_2, do_uadalp_h)
+DO_ZPZZ(sve2_uadalp_zpzz_s, int32_t, H1_4, do_uadalp_s)
+DO_ZPZZ_D(sve2_uadalp_zpzz_d, uint64_t, do_uadalp_d)
+
#undef DO_ZPZZ
#undef DO_ZPZZ_D
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 04c5a2c8bd..56e9e60a89 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -5855,3 +5855,42 @@ static bool trans_PMUL_zzz(DisasContext *s, arg_rrr_esz *a)
{
return do_sve2_zzz_ool(s, a, gen_helper_gvec_pmul_b);
}
+
+/*
+ * SVE2 Integer - Predicated
+ */
+
+static bool do_sve2_zpzz_ool(DisasContext *s, arg_rprr_esz *a,
+ gen_helper_gvec_4 *fn)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_zpzz_ool(s, a, fn);
+}
+
+static bool trans_SADALP_zpzz(DisasContext *s, arg_rprr_esz *a)
+{
+ static gen_helper_gvec_4 * const fns[3] = {
+ gen_helper_sve2_sadalp_zpzz_h,
+ gen_helper_sve2_sadalp_zpzz_s,
+ gen_helper_sve2_sadalp_zpzz_d,
+ };
+ if (a->esz == 0) {
+ return false;
+ }
+ return do_sve2_zpzz_ool(s, a, fns[a->esz - 1]);
+}
+
+static bool trans_UADALP_zpzz(DisasContext *s, arg_rprr_esz *a)
+{
+ static gen_helper_gvec_4 * const fns[3] = {
+ gen_helper_sve2_uadalp_zpzz_h,
+ gen_helper_sve2_uadalp_zpzz_s,
+ gen_helper_sve2_uadalp_zpzz_d,
+ };
+ if (a->esz == 0) {
+ return false;
+ }
+ return do_sve2_zpzz_ool(s, a, fns[a->esz - 1]);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 07/81] target/arm: Implement SVE2 integer unary operations (predicated)
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (5 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 06/81] target/arm: Implement SVE2 integer pairwise add and accumulate long Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 08/81] target/arm: Split out saturating/rounding shifts from neon Richard Henderson
` (75 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Fix sqabs, sqneg (laurent desnogues)
---
target/arm/helper-sve.h | 13 +++++++++++
target/arm/sve.decode | 7 ++++++
target/arm/sve_helper.c | 29 +++++++++++++++++++----
target/arm/translate-sve.c | 47 ++++++++++++++++++++++++++++++++++++++
4 files changed, 92 insertions(+), 4 deletions(-)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index e185405cdc..938c50f2c9 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -502,6 +502,19 @@ DEF_HELPER_FLAGS_4(sve_rbit_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_rbit_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_rbit_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqabs_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqabs_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqabs_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqabs_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_sqneg_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqneg_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqneg_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqneg_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_urecpe_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_ursqrte_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_5(sve_splice, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_cmpeq_ppzz_b, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 4f54a30538..2ab463c722 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1105,3 +1105,10 @@ PMUL_zzz 00000100 00 1 ..... 0110 01 ..... ..... @rd_rn_rm_e0
SADALP_zpzz 01000100 .. 000 100 101 ... ..... ..... @rdm_pg_rn
UADALP_zpzz 01000100 .. 000 101 101 ... ..... ..... @rdm_pg_rn
+
+### SVE2 integer unary operations (predicated)
+
+URECPE 01000100 .. 000 000 101 ... ..... ..... @rd_pg_rn
+URSQRTE 01000100 .. 000 001 101 ... ..... ..... @rd_pg_rn
+SQABS 01000100 .. 001 000 101 ... ..... ..... @rd_pg_rn
+SQNEG 01000100 .. 001 001 101 ... ..... ..... @rd_pg_rn
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 4705722b71..b1c89f77d4 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -535,8 +535,8 @@ static inline uint64_t do_sadalp_d(uint64_t n, uint64_t m)
return m + n1 + n2;
}
-DO_ZPZZ(sve2_sadalp_zpzz_h, int16_t, H1_2, do_sadalp_h)
-DO_ZPZZ(sve2_sadalp_zpzz_s, int32_t, H1_4, do_sadalp_s)
+DO_ZPZZ(sve2_sadalp_zpzz_h, uint16_t, H1_2, do_sadalp_h)
+DO_ZPZZ(sve2_sadalp_zpzz_s, uint32_t, H1_4, do_sadalp_s)
DO_ZPZZ_D(sve2_sadalp_zpzz_d, uint64_t, do_sadalp_d)
static inline uint16_t do_uadalp_h(uint16_t n, uint16_t m)
@@ -557,8 +557,8 @@ static inline uint64_t do_uadalp_d(uint64_t n, uint64_t m)
return m + n1 + n2;
}
-DO_ZPZZ(sve2_uadalp_zpzz_h, int16_t, H1_2, do_uadalp_h)
-DO_ZPZZ(sve2_uadalp_zpzz_s, int32_t, H1_4, do_uadalp_s)
+DO_ZPZZ(sve2_uadalp_zpzz_h, uint16_t, H1_2, do_uadalp_h)
+DO_ZPZZ(sve2_uadalp_zpzz_s, uint32_t, H1_4, do_uadalp_s)
DO_ZPZZ_D(sve2_uadalp_zpzz_d, uint64_t, do_uadalp_d)
#undef DO_ZPZZ
@@ -728,6 +728,27 @@ DO_ZPZ(sve_rbit_h, uint16_t, H1_2, revbit16)
DO_ZPZ(sve_rbit_s, uint32_t, H1_4, revbit32)
DO_ZPZ_D(sve_rbit_d, uint64_t, revbit64)
+#define DO_SQABS(X) \
+ ({ __typeof(X) x_ = (X), min_ = 1ull << (sizeof(X) * 8 - 1); \
+ x_ >= 0 ? x_ : x_ == min_ ? -min_ - 1 : -x_; })
+
+DO_ZPZ(sve2_sqabs_b, int8_t, H1, DO_SQABS)
+DO_ZPZ(sve2_sqabs_h, int16_t, H1_2, DO_SQABS)
+DO_ZPZ(sve2_sqabs_s, int32_t, H1_4, DO_SQABS)
+DO_ZPZ_D(sve2_sqabs_d, int64_t, DO_SQABS)
+
+#define DO_SQNEG(X) \
+ ({ __typeof(X) x_ = (X), min_ = 1ull << (sizeof(X) * 8 - 1); \
+ x_ == min_ ? -min_ - 1 : -x_; })
+
+DO_ZPZ(sve2_sqneg_b, uint8_t, H1, DO_SQNEG)
+DO_ZPZ(sve2_sqneg_h, uint16_t, H1_2, DO_SQNEG)
+DO_ZPZ(sve2_sqneg_s, uint32_t, H1_4, DO_SQNEG)
+DO_ZPZ_D(sve2_sqneg_d, uint64_t, DO_SQNEG)
+
+DO_ZPZ(sve2_urecpe_s, uint32_t, H1_4, helper_recpe_u32)
+DO_ZPZ(sve2_ursqrte_s, uint32_t, H1_4, helper_rsqrte_u32)
+
/* Three-operand expander, unpredicated, in which the third operand is "wide".
*/
#define DO_ZZW(NAME, TYPE, TYPEW, H, OP) \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 56e9e60a89..1dacb3fbc4 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -5894,3 +5894,50 @@ static bool trans_UADALP_zpzz(DisasContext *s, arg_rprr_esz *a)
}
return do_sve2_zpzz_ool(s, a, fns[a->esz - 1]);
}
+
+/*
+ * SVE2 integer unary operations (predicated)
+ */
+
+static bool do_sve2_zpz_ool(DisasContext *s, arg_rpr_esz *a,
+ gen_helper_gvec_3 *fn)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_zpz_ool(s, a, fn);
+}
+
+static bool trans_URECPE(DisasContext *s, arg_rpr_esz *a)
+{
+ if (a->esz != 2) {
+ return false;
+ }
+ return do_sve2_zpz_ool(s, a, gen_helper_sve2_urecpe_s);
+}
+
+static bool trans_URSQRTE(DisasContext *s, arg_rpr_esz *a)
+{
+ if (a->esz != 2) {
+ return false;
+ }
+ return do_sve2_zpz_ool(s, a, gen_helper_sve2_ursqrte_s);
+}
+
+static bool trans_SQABS(DisasContext *s, arg_rpr_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve2_sqabs_b, gen_helper_sve2_sqabs_h,
+ gen_helper_sve2_sqabs_s, gen_helper_sve2_sqabs_d,
+ };
+ return do_sve2_zpz_ool(s, a, fns[a->esz]);
+}
+
+static bool trans_SQNEG(DisasContext *s, arg_rpr_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve2_sqneg_b, gen_helper_sve2_sqneg_h,
+ gen_helper_sve2_sqneg_s, gen_helper_sve2_sqneg_d,
+ };
+ return do_sve2_zpz_ool(s, a, fns[a->esz]);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 08/81] target/arm: Split out saturating/rounding shifts from neon
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (6 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 07/81] target/arm: Implement SVE2 integer unary operations (predicated) Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 09/81] target/arm: Implement SVE2 saturating/rounding bitwise shift left (predicated) Richard Henderson
` (74 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Split these operations out into a header that can be shared
between neon and sve. The "sat" pointer acts both as a boolean
for control of saturating behavior and controls the difference
in behavior between neon and sve -- QC bit or no QC bit.
Widen the shift operand in the new helpers, as the SVE2 insns treat
the whole input element as significant. For the neon uses, truncate
the shift to int8_t while passing the parameter.
Implement right-shift rounding as
tmp = src >> (shift - 1);
dst = (tmp >> 1) + (tmp & 1);
This is the same number of instructions as the current
tmp = 1 << (shift - 1);
dst = (src + tmp) >> shift;
without any possibility of intermediate overflow.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Widen the shift operand (laurent desnouges)
---
target/arm/vec_internal.h | 138 +++++++++++
target/arm/neon_helper.c | 507 +++++++-------------------------------
2 files changed, 221 insertions(+), 424 deletions(-)
diff --git a/target/arm/vec_internal.h b/target/arm/vec_internal.h
index 00a8277765..372fe76523 100644
--- a/target/arm/vec_internal.h
+++ b/target/arm/vec_internal.h
@@ -30,4 +30,142 @@ static inline void clear_tail(void *vd, uintptr_t opr_sz, uintptr_t max_sz)
}
}
+static inline int32_t do_sqrshl_bhs(int32_t src, int32_t shift, int bits,
+ bool round, uint32_t *sat)
+{
+ if (shift <= -bits) {
+ /* Rounding the sign bit always produces 0. */
+ if (round) {
+ return 0;
+ }
+ return src >> 31;
+ } else if (shift < 0) {
+ if (round) {
+ src >>= -shift - 1;
+ return (src >> 1) + (src & 1);
+ }
+ return src >> -shift;
+ } else if (shift < bits) {
+ int32_t val = src << shift;
+ if (bits == 32) {
+ if (!sat || val >> shift == src) {
+ return val;
+ }
+ } else {
+ int32_t extval = sextract32(val, 0, bits);
+ if (!sat || val == extval) {
+ return extval;
+ }
+ }
+ } else if (!sat || src == 0) {
+ return 0;
+ }
+
+ *sat = 1;
+ return (1u << (bits - 1)) - (src >= 0);
+}
+
+static inline uint32_t do_uqrshl_bhs(uint32_t src, int32_t shift, int bits,
+ bool round, uint32_t *sat)
+{
+ if (shift <= -(bits + round)) {
+ return 0;
+ } else if (shift < 0) {
+ if (round) {
+ src >>= -shift - 1;
+ return (src >> 1) + (src & 1);
+ }
+ return src >> -shift;
+ } else if (shift < bits) {
+ uint32_t val = src << shift;
+ if (bits == 32) {
+ if (!sat || val >> shift == src) {
+ return val;
+ }
+ } else {
+ uint32_t extval = extract32(val, 0, bits);
+ if (!sat || val == extval) {
+ return extval;
+ }
+ }
+ } else if (!sat || src == 0) {
+ return 0;
+ }
+
+ *sat = 1;
+ return MAKE_64BIT_MASK(0, bits);
+}
+
+static inline int32_t do_suqrshl_bhs(int32_t src, int32_t shift, int bits,
+ bool round, uint32_t *sat)
+{
+ if (src < 0) {
+ *sat = 1;
+ return 0;
+ }
+ return do_uqrshl_bhs(src, shift, bits, round, sat);
+}
+
+static inline int64_t do_sqrshl_d(int64_t src, int64_t shift,
+ bool round, uint32_t *sat)
+{
+ if (shift <= -64) {
+ /* Rounding the sign bit always produces 0. */
+ if (round) {
+ return 0;
+ }
+ return src >> 63;
+ } else if (shift < 0) {
+ if (round) {
+ src >>= -shift - 1;
+ return (src >> 1) + (src & 1);
+ }
+ return src >> -shift;
+ } else if (shift < 64) {
+ int64_t val = src << shift;
+ if (!sat || val >> shift == src) {
+ return val;
+ }
+ } else if (!sat || src == 0) {
+ return 0;
+ }
+
+ *sat = 1;
+ return src < 0 ? INT64_MIN : INT64_MAX;
+}
+
+static inline uint64_t do_uqrshl_d(uint64_t src, int64_t shift,
+ bool round, uint32_t *sat)
+{
+ if (shift <= -(64 + round)) {
+ return 0;
+ } else if (shift < 0) {
+ if (round) {
+ src >>= -shift - 1;
+ return (src >> 1) + (src & 1);
+ }
+ return src >> -shift;
+ } else if (shift < 64) {
+ uint64_t val = src << shift;
+ if (!sat || val >> shift == src) {
+ return val;
+ }
+ } else if (!sat || src == 0) {
+ return 0;
+ }
+
+ *sat = 1;
+ return UINT64_MAX;
+}
+
+static inline int64_t do_suqrshl_d(int64_t src, int64_t shift,
+ bool round, uint32_t *sat)
+{
+ if (src < 0) {
+ *sat = 1;
+ return 0;
+ }
+ return do_uqrshl_d(src, shift, round, sat);
+}
+
#endif /* TARGET_ARM_VEC_INTERNALS_H */
diff --git a/target/arm/neon_helper.c b/target/arm/neon_helper.c
index b637265691..338b9189d5 100644
--- a/target/arm/neon_helper.c
+++ b/target/arm/neon_helper.c
@@ -11,6 +11,7 @@
#include "cpu.h"
#include "exec/helper-proto.h"
#include "fpu/softfloat.h"
+#include "vec_internal.h"
#define SIGNBIT (uint32_t)0x80000000
#define SIGNBIT64 ((uint64_t)1 << 63)
@@ -576,496 +577,154 @@ NEON_POP(pmax_s16, neon_s16, 2)
NEON_POP(pmax_u16, neon_u16, 2)
#undef NEON_FN
-#define NEON_FN(dest, src1, src2) do { \
- int8_t tmp; \
- tmp = (int8_t)src2; \
- if (tmp >= (ssize_t)sizeof(src1) * 8 || \
- tmp <= -(ssize_t)sizeof(src1) * 8) { \
- dest = 0; \
- } else if (tmp < 0) { \
- dest = src1 >> -tmp; \
- } else { \
- dest = src1 << tmp; \
- }} while (0)
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, false, NULL))
NEON_VOP(shl_u16, neon_u16, 2)
#undef NEON_FN
-#define NEON_FN(dest, src1, src2) do { \
- int8_t tmp; \
- tmp = (int8_t)src2; \
- if (tmp >= (ssize_t)sizeof(src1) * 8) { \
- dest = 0; \
- } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
- dest = src1 >> (sizeof(src1) * 8 - 1); \
- } else if (tmp < 0) { \
- dest = src1 >> -tmp; \
- } else { \
- dest = src1 << tmp; \
- }} while (0)
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, false, NULL))
NEON_VOP(shl_s16, neon_s16, 2)
#undef NEON_FN
-#define NEON_FN(dest, src1, src2) do { \
- int8_t tmp; \
- tmp = (int8_t)src2; \
- if ((tmp >= (ssize_t)sizeof(src1) * 8) \
- || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
- dest = 0; \
- } else if (tmp < 0) { \
- dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
- } else { \
- dest = src1 << tmp; \
- }} while (0)
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_sqrshl_bhs(src1, (int8_t)src2, 8, true, NULL))
NEON_VOP(rshl_s8, neon_s8, 4)
+#undef NEON_FN
+
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, true, NULL))
NEON_VOP(rshl_s16, neon_s16, 2)
#undef NEON_FN
-/* The addition of the rounding constant may overflow, so we use an
- * intermediate 64 bit accumulator. */
-uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
+uint32_t HELPER(neon_rshl_s32)(uint32_t val, uint32_t shift)
{
- int32_t dest;
- int32_t val = (int32_t)valop;
- int8_t shift = (int8_t)shiftop;
- if ((shift >= 32) || (shift <= -32)) {
- dest = 0;
- } else if (shift < 0) {
- int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
- dest = big_dest >> -shift;
- } else {
- dest = val << shift;
- }
- return dest;
+ return do_sqrshl_bhs(val, (int8_t)shift, 32, true, NULL);
}
-/* Handling addition overflow with 64 bit input values is more
- * tricky than with 32 bit values. */
-uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
+uint64_t HELPER(neon_rshl_s64)(uint64_t val, uint64_t shift)
{
- int8_t shift = (int8_t)shiftop;
- int64_t val = valop;
- if ((shift >= 64) || (shift <= -64)) {
- val = 0;
- } else if (shift < 0) {
- val >>= (-shift - 1);
- if (val == INT64_MAX) {
- /* In this case, it means that the rounding constant is 1,
- * and the addition would overflow. Return the actual
- * result directly. */
- val = 0x4000000000000000LL;
- } else {
- val++;
- val >>= 1;
- }
- } else {
- val <<= shift;
- }
- return val;
+ return do_sqrshl_d(val, (int8_t)shift, true, NULL);
}
-#define NEON_FN(dest, src1, src2) do { \
- int8_t tmp; \
- tmp = (int8_t)src2; \
- if (tmp >= (ssize_t)sizeof(src1) * 8 || \
- tmp < -(ssize_t)sizeof(src1) * 8) { \
- dest = 0; \
- } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
- dest = src1 >> (-tmp - 1); \
- } else if (tmp < 0) { \
- dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
- } else { \
- dest = src1 << tmp; \
- }} while (0)
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_uqrshl_bhs(src1, (int8_t)src2, 8, true, NULL))
NEON_VOP(rshl_u8, neon_u8, 4)
+#undef NEON_FN
+
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, true, NULL))
NEON_VOP(rshl_u16, neon_u16, 2)
#undef NEON_FN
-/* The addition of the rounding constant may overflow, so we use an
- * intermediate 64 bit accumulator. */
-uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
+uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shift)
{
- uint32_t dest;
- int8_t shift = (int8_t)shiftop;
- if (shift >= 32 || shift < -32) {
- dest = 0;
- } else if (shift == -32) {
- dest = val >> 31;
- } else if (shift < 0) {
- uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
- dest = big_dest >> -shift;
- } else {
- dest = val << shift;
- }
- return dest;
+ return do_uqrshl_bhs(val, (int8_t)shift, 32, true, NULL);
}
-/* Handling addition overflow with 64 bit input values is more
- * tricky than with 32 bit values. */
-uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
+uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shift)
{
- int8_t shift = (uint8_t)shiftop;
- if (shift >= 64 || shift < -64) {
- val = 0;
- } else if (shift == -64) {
- /* Rounding a 1-bit result just preserves that bit. */
- val >>= 63;
- } else if (shift < 0) {
- val >>= (-shift - 1);
- if (val == UINT64_MAX) {
- /* In this case, it means that the rounding constant is 1,
- * and the addition would overflow. Return the actual
- * result directly. */
- val = 0x8000000000000000ULL;
- } else {
- val++;
- val >>= 1;
- }
- } else {
- val <<= shift;
- }
- return val;
+ return do_uqrshl_d(val, (int8_t)shift, true, NULL);
}
-#define NEON_FN(dest, src1, src2) do { \
- int8_t tmp; \
- tmp = (int8_t)src2; \
- if (tmp >= (ssize_t)sizeof(src1) * 8) { \
- if (src1) { \
- SET_QC(); \
- dest = ~0; \
- } else { \
- dest = 0; \
- } \
- } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
- dest = 0; \
- } else if (tmp < 0) { \
- dest = src1 >> -tmp; \
- } else { \
- dest = src1 << tmp; \
- if ((dest >> tmp) != src1) { \
- SET_QC(); \
- dest = ~0; \
- } \
- }} while (0)
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_uqrshl_bhs(src1, (int8_t)src2, 8, false, env->vfp.qc))
NEON_VOP_ENV(qshl_u8, neon_u8, 4)
+#undef NEON_FN
+
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, false, env->vfp.qc))
NEON_VOP_ENV(qshl_u16, neon_u16, 2)
-NEON_VOP_ENV(qshl_u32, neon_u32, 1)
#undef NEON_FN
-uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
+uint32_t HELPER(neon_qshl_u32)(CPUARMState *env, uint32_t val, uint32_t shift)
{
- int8_t shift = (int8_t)shiftop;
- if (shift >= 64) {
- if (val) {
- val = ~(uint64_t)0;
- SET_QC();
- }
- } else if (shift <= -64) {
- val = 0;
- } else if (shift < 0) {
- val >>= -shift;
- } else {
- uint64_t tmp = val;
- val <<= shift;
- if ((val >> shift) != tmp) {
- SET_QC();
- val = ~(uint64_t)0;
- }
- }
- return val;
+ return do_uqrshl_bhs(val, (int8_t)shift, 32, false, env->vfp.qc);
}
-#define NEON_FN(dest, src1, src2) do { \
- int8_t tmp; \
- tmp = (int8_t)src2; \
- if (tmp >= (ssize_t)sizeof(src1) * 8) { \
- if (src1) { \
- SET_QC(); \
- dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
- if (src1 > 0) { \
- dest--; \
- } \
- } else { \
- dest = src1; \
- } \
- } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
- dest = src1 >> 31; \
- } else if (tmp < 0) { \
- dest = src1 >> -tmp; \
- } else { \
- dest = src1 << tmp; \
- if ((dest >> tmp) != src1) { \
- SET_QC(); \
- dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
- if (src1 > 0) { \
- dest--; \
- } \
- } \
- }} while (0)
+uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shift)
+{
+ return do_uqrshl_d(val, (int8_t)shift, false, env->vfp.qc);
+}
+
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_sqrshl_bhs(src1, (int8_t)src2, 8, false, env->vfp.qc))
NEON_VOP_ENV(qshl_s8, neon_s8, 4)
+#undef NEON_FN
+
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, false, env->vfp.qc))
NEON_VOP_ENV(qshl_s16, neon_s16, 2)
-NEON_VOP_ENV(qshl_s32, neon_s32, 1)
#undef NEON_FN
-uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
+uint32_t HELPER(neon_qshl_s32)(CPUARMState *env, uint32_t val, uint32_t shift)
{
- int8_t shift = (uint8_t)shiftop;
- int64_t val = valop;
- if (shift >= 64) {
- if (val) {
- SET_QC();
- val = (val >> 63) ^ ~SIGNBIT64;
- }
- } else if (shift <= -64) {
- val >>= 63;
- } else if (shift < 0) {
- val >>= -shift;
- } else {
- int64_t tmp = val;
- val <<= shift;
- if ((val >> shift) != tmp) {
- SET_QC();
- val = (tmp >> 63) ^ ~SIGNBIT64;
- }
- }
- return val;
+ return do_sqrshl_bhs(val, (int8_t)shift, 32, false, env->vfp.qc);
}
-#define NEON_FN(dest, src1, src2) do { \
- if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
- SET_QC(); \
- dest = 0; \
- } else { \
- int8_t tmp; \
- tmp = (int8_t)src2; \
- if (tmp >= (ssize_t)sizeof(src1) * 8) { \
- if (src1) { \
- SET_QC(); \
- dest = ~0; \
- } else { \
- dest = 0; \
- } \
- } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
- dest = 0; \
- } else if (tmp < 0) { \
- dest = src1 >> -tmp; \
- } else { \
- dest = src1 << tmp; \
- if ((dest >> tmp) != src1) { \
- SET_QC(); \
- dest = ~0; \
- } \
- } \
- }} while (0)
-NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
-NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
+uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t val, uint64_t shift)
+{
+ return do_sqrshl_d(val, (int8_t)shift, false, env->vfp.qc);
+}
+
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_suqrshl_bhs(src1, (int8_t)src2, 8, false, env->vfp.qc))
+NEON_VOP_ENV(qshlu_s8, neon_s8, 4)
#undef NEON_FN
-uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_suqrshl_bhs(src1, (int8_t)src2, 16, false, env->vfp.qc))
+NEON_VOP_ENV(qshlu_s16, neon_s16, 2)
+#undef NEON_FN
+
+uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t val, uint32_t shift)
{
- if ((int32_t)valop < 0) {
- SET_QC();
- return 0;
- }
- return helper_neon_qshl_u32(env, valop, shiftop);
+ return do_suqrshl_bhs(val, (int8_t)shift, 32, false, env->vfp.qc);
}
-uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
+uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t val, uint64_t shift)
{
- if ((int64_t)valop < 0) {
- SET_QC();
- return 0;
- }
- return helper_neon_qshl_u64(env, valop, shiftop);
+ return do_suqrshl_d(val, (int8_t)shift, false, env->vfp.qc);
}
-#define NEON_FN(dest, src1, src2) do { \
- int8_t tmp; \
- tmp = (int8_t)src2; \
- if (tmp >= (ssize_t)sizeof(src1) * 8) { \
- if (src1) { \
- SET_QC(); \
- dest = ~0; \
- } else { \
- dest = 0; \
- } \
- } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
- dest = 0; \
- } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
- dest = src1 >> (sizeof(src1) * 8 - 1); \
- } else if (tmp < 0) { \
- dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
- } else { \
- dest = src1 << tmp; \
- if ((dest >> tmp) != src1) { \
- SET_QC(); \
- dest = ~0; \
- } \
- }} while (0)
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_uqrshl_bhs(src1, (int8_t)src2, 8, true, env->vfp.qc))
NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
+#undef NEON_FN
+
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, true, env->vfp.qc))
NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
#undef NEON_FN
-/* The addition of the rounding constant may overflow, so we use an
- * intermediate 64 bit accumulator. */
-uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shiftop)
+uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shift)
{
- uint32_t dest;
- int8_t shift = (int8_t)shiftop;
- if (shift >= 32) {
- if (val) {
- SET_QC();
- dest = ~0;
- } else {
- dest = 0;
- }
- } else if (shift < -32) {
- dest = 0;
- } else if (shift == -32) {
- dest = val >> 31;
- } else if (shift < 0) {
- uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
- dest = big_dest >> -shift;
- } else {
- dest = val << shift;
- if ((dest >> shift) != val) {
- SET_QC();
- dest = ~0;
- }
- }
- return dest;
+ return do_uqrshl_bhs(val, (int8_t)shift, 32, true, env->vfp.qc);
}
-/* Handling addition overflow with 64 bit input values is more
- * tricky than with 32 bit values. */
-uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
+uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shift)
{
- int8_t shift = (int8_t)shiftop;
- if (shift >= 64) {
- if (val) {
- SET_QC();
- val = ~0;
- }
- } else if (shift < -64) {
- val = 0;
- } else if (shift == -64) {
- val >>= 63;
- } else if (shift < 0) {
- val >>= (-shift - 1);
- if (val == UINT64_MAX) {
- /* In this case, it means that the rounding constant is 1,
- * and the addition would overflow. Return the actual
- * result directly. */
- val = 0x8000000000000000ULL;
- } else {
- val++;
- val >>= 1;
- }
- } else { \
- uint64_t tmp = val;
- val <<= shift;
- if ((val >> shift) != tmp) {
- SET_QC();
- val = ~0;
- }
- }
- return val;
+ return do_uqrshl_d(val, (int8_t)shift, true, env->vfp.qc);
}
-#define NEON_FN(dest, src1, src2) do { \
- int8_t tmp; \
- tmp = (int8_t)src2; \
- if (tmp >= (ssize_t)sizeof(src1) * 8) { \
- if (src1) { \
- SET_QC(); \
- dest = (typeof(dest))(1 << (sizeof(src1) * 8 - 1)); \
- if (src1 > 0) { \
- dest--; \
- } \
- } else { \
- dest = 0; \
- } \
- } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
- dest = 0; \
- } else if (tmp < 0) { \
- dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
- } else { \
- dest = src1 << tmp; \
- if ((dest >> tmp) != src1) { \
- SET_QC(); \
- dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
- if (src1 > 0) { \
- dest--; \
- } \
- } \
- }} while (0)
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_sqrshl_bhs(src1, (int8_t)src2, 8, true, env->vfp.qc))
NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
+#undef NEON_FN
+
+#define NEON_FN(dest, src1, src2) \
+ (dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, true, env->vfp.qc))
NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
#undef NEON_FN
-/* The addition of the rounding constant may overflow, so we use an
- * intermediate 64 bit accumulator. */
-uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
+uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t val, uint32_t shift)
{
- int32_t dest;
- int32_t val = (int32_t)valop;
- int8_t shift = (int8_t)shiftop;
- if (shift >= 32) {
- if (val) {
- SET_QC();
- dest = (val >> 31) ^ ~SIGNBIT;
- } else {
- dest = 0;
- }
- } else if (shift <= -32) {
- dest = 0;
- } else if (shift < 0) {
- int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
- dest = big_dest >> -shift;
- } else {
- dest = val << shift;
- if ((dest >> shift) != val) {
- SET_QC();
- dest = (val >> 31) ^ ~SIGNBIT;
- }
- }
- return dest;
+ return do_sqrshl_bhs(val, (int8_t)shift, 32, true, env->vfp.qc);
}
-/* Handling addition overflow with 64 bit input values is more
- * tricky than with 32 bit values. */
-uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
+uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t val, uint64_t shift)
{
- int8_t shift = (uint8_t)shiftop;
- int64_t val = valop;
-
- if (shift >= 64) {
- if (val) {
- SET_QC();
- val = (val >> 63) ^ ~SIGNBIT64;
- }
- } else if (shift <= -64) {
- val = 0;
- } else if (shift < 0) {
- val >>= (-shift - 1);
- if (val == INT64_MAX) {
- /* In this case, it means that the rounding constant is 1,
- * and the addition would overflow. Return the actual
- * result directly. */
- val = 0x4000000000000000ULL;
- } else {
- val++;
- val >>= 1;
- }
- } else {
- int64_t tmp = val;
- val <<= shift;
- if ((val >> shift) != tmp) {
- SET_QC();
- val = (tmp >> 63) ^ ~SIGNBIT64;
- }
- }
- return val;
+ return do_sqrshl_d(val, (int8_t)shift, true, env->vfp.qc);
}
uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 09/81] target/arm: Implement SVE2 saturating/rounding bitwise shift left (predicated)
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (7 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 08/81] target/arm: Split out saturating/rounding shifts from neon Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 10/81] target/arm: Implement SVE2 integer halving add/subtract (predicated) Richard Henderson
` (73 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Shift values are always signed (laurent desnogues).
---
target/arm/helper-sve.h | 54 ++++++++++++++++++++++++++
target/arm/sve.decode | 17 +++++++++
target/arm/sve_helper.c | 78 ++++++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 18 +++++++++
4 files changed, 167 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 938c50f2c9..3c87b1ce29 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -172,6 +172,60 @@ DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_s, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_5(sve_sdiv_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_sdiv_zpzz_d, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 2ab463c722..7c940e45c6 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1112,3 +1112,20 @@ URECPE 01000100 .. 000 000 101 ... ..... ..... @rd_pg_rn
URSQRTE 01000100 .. 000 001 101 ... ..... ..... @rd_pg_rn
SQABS 01000100 .. 001 000 101 ... ..... ..... @rd_pg_rn
SQNEG 01000100 .. 001 001 101 ... ..... ..... @rd_pg_rn
+
+### SVE2 saturating/rounding bitwise shift left (predicated)
+
+SRSHL 01000100 .. 000 010 100 ... ..... ..... @rdn_pg_rm
+URSHL 01000100 .. 000 011 100 ... ..... ..... @rdn_pg_rm
+SRSHL 01000100 .. 000 110 100 ... ..... ..... @rdm_pg_rn # SRSHLR
+URSHL 01000100 .. 000 111 100 ... ..... ..... @rdm_pg_rn # URSHLR
+
+SQSHL 01000100 .. 001 000 100 ... ..... ..... @rdn_pg_rm
+UQSHL 01000100 .. 001 001 100 ... ..... ..... @rdn_pg_rm
+SQSHL 01000100 .. 001 100 100 ... ..... ..... @rdm_pg_rn # SQSHLR
+UQSHL 01000100 .. 001 101 100 ... ..... ..... @rdm_pg_rn # UQSHLR
+
+SQRSHL 01000100 .. 001 010 100 ... ..... ..... @rdn_pg_rm
+UQRSHL 01000100 .. 001 011 100 ... ..... ..... @rdn_pg_rm
+SQRSHL 01000100 .. 001 110 100 ... ..... ..... @rdm_pg_rn # SQRSHLR
+UQRSHL 01000100 .. 001 111 100 ... ..... ..... @rdm_pg_rn # UQRSHLR
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index b1c89f77d4..40e1b5df30 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -26,6 +26,7 @@
#include "tcg/tcg-gvec-desc.h"
#include "fpu/softfloat.h"
#include "tcg/tcg.h"
+#include "vec_internal.h"
/* Note that vector data is stored in host-endian 64-bit chunks,
@@ -561,6 +562,83 @@ DO_ZPZZ(sve2_uadalp_zpzz_h, uint16_t, H1_2, do_uadalp_h)
DO_ZPZZ(sve2_uadalp_zpzz_s, uint32_t, H1_4, do_uadalp_s)
DO_ZPZZ_D(sve2_uadalp_zpzz_d, uint64_t, do_uadalp_d)
+#define do_srshl_b(n, m) do_sqrshl_bhs(n, m, 8, true, NULL)
+#define do_srshl_h(n, m) do_sqrshl_bhs(n, m, 16, true, NULL)
+#define do_srshl_s(n, m) do_sqrshl_bhs(n, m, 32, true, NULL)
+#define do_srshl_d(n, m) do_sqrshl_d(n, m, true, NULL)
+
+DO_ZPZZ(sve2_srshl_zpzz_b, int8_t, H1_2, do_srshl_b)
+DO_ZPZZ(sve2_srshl_zpzz_h, int16_t, H1_2, do_srshl_h)
+DO_ZPZZ(sve2_srshl_zpzz_s, int32_t, H1_4, do_srshl_s)
+DO_ZPZZ_D(sve2_srshl_zpzz_d, int64_t, do_srshl_d)
+
+#define do_urshl_b(n, m) do_uqrshl_bhs(n, (int8_t)m, 8, true, NULL)
+#define do_urshl_h(n, m) do_uqrshl_bhs(n, (int16_t)m, 16, true, NULL)
+#define do_urshl_s(n, m) do_uqrshl_bhs(n, m, 32, true, NULL)
+#define do_urshl_d(n, m) do_uqrshl_d(n, m, true, NULL)
+
+DO_ZPZZ(sve2_urshl_zpzz_b, uint8_t, H1_2, do_urshl_b)
+DO_ZPZZ(sve2_urshl_zpzz_h, uint16_t, H1_2, do_urshl_h)
+DO_ZPZZ(sve2_urshl_zpzz_s, uint32_t, H1_4, do_urshl_s)
+DO_ZPZZ_D(sve2_urshl_zpzz_d, uint64_t, do_urshl_d)
+
+/* Unlike the NEON and AdvSIMD versions, there is no QC bit to set. */
+#define do_sqshl_b(n, m) \
+ ({ uint32_t discard; do_sqrshl_bhs(n, m, 8, false, &discard); })
+#define do_sqshl_h(n, m) \
+ ({ uint32_t discard; do_sqrshl_bhs(n, m, 16, false, &discard); })
+#define do_sqshl_s(n, m) \
+ ({ uint32_t discard; do_sqrshl_bhs(n, m, 32, false, &discard); })
+#define do_sqshl_d(n, m) \
+ ({ uint32_t discard; do_sqrshl_d(n, m, false, &discard); })
+
+DO_ZPZZ(sve2_sqshl_zpzz_b, int8_t, H1_2, do_sqshl_b)
+DO_ZPZZ(sve2_sqshl_zpzz_h, int16_t, H1_2, do_sqshl_h)
+DO_ZPZZ(sve2_sqshl_zpzz_s, int32_t, H1_4, do_sqshl_s)
+DO_ZPZZ_D(sve2_sqshl_zpzz_d, int64_t, do_sqshl_d)
+
+#define do_uqshl_b(n, m) \
+ ({ uint32_t discard; do_uqrshl_bhs(n, (int8_t)m, 8, false, &discard); })
+#define do_uqshl_h(n, m) \
+ ({ uint32_t discard; do_uqrshl_bhs(n, (int16_t)m, 16, false, &discard); })
+#define do_uqshl_s(n, m) \
+ ({ uint32_t discard; do_uqrshl_bhs(n, m, 32, false, &discard); })
+#define do_uqshl_d(n, m) \
+ ({ uint32_t discard; do_uqrshl_d(n, m, false, &discard); })
+
+DO_ZPZZ(sve2_uqshl_zpzz_b, uint8_t, H1_2, do_uqshl_b)
+DO_ZPZZ(sve2_uqshl_zpzz_h, uint16_t, H1_2, do_uqshl_h)
+DO_ZPZZ(sve2_uqshl_zpzz_s, uint32_t, H1_4, do_uqshl_s)
+DO_ZPZZ_D(sve2_uqshl_zpzz_d, uint64_t, do_uqshl_d)
+
+#define do_sqrshl_b(n, m) \
+ ({ uint32_t discard; do_sqrshl_bhs(n, m, 8, true, &discard); })
+#define do_sqrshl_h(n, m) \
+ ({ uint32_t discard; do_sqrshl_bhs(n, m, 16, true, &discard); })
+#define do_sqrshl_s(n, m) \
+ ({ uint32_t discard; do_sqrshl_bhs(n, m, 32, true, &discard); })
+#define do_sqrshl_d(n, m) \
+ ({ uint32_t discard; do_sqrshl_d(n, m, true, &discard); })
+
+DO_ZPZZ(sve2_sqrshl_zpzz_b, int8_t, H1_2, do_sqrshl_b)
+DO_ZPZZ(sve2_sqrshl_zpzz_h, int16_t, H1_2, do_sqrshl_h)
+DO_ZPZZ(sve2_sqrshl_zpzz_s, int32_t, H1_4, do_sqrshl_s)
+DO_ZPZZ_D(sve2_sqrshl_zpzz_d, int64_t, do_sqrshl_d)
+
+#define do_uqrshl_b(n, m) \
+ ({ uint32_t discard; do_uqrshl_bhs(n, (int8_t)m, 8, true, &discard); })
+#define do_uqrshl_h(n, m) \
+ ({ uint32_t discard; do_uqrshl_bhs(n, (int16_t)m, 16, true, &discard); })
+#define do_uqrshl_s(n, m) \
+ ({ uint32_t discard; do_uqrshl_bhs(n, m, 32, true, &discard); })
+#define do_uqrshl_d(n, m) \
+ ({ uint32_t discard; do_uqrshl_d(n, m, true, &discard); })
+
+DO_ZPZZ(sve2_uqrshl_zpzz_b, uint8_t, H1_2, do_uqrshl_b)
+DO_ZPZZ(sve2_uqrshl_zpzz_h, uint16_t, H1_2, do_uqrshl_h)
+DO_ZPZZ(sve2_uqrshl_zpzz_s, uint32_t, H1_4, do_uqrshl_s)
+DO_ZPZZ_D(sve2_uqrshl_zpzz_d, uint64_t, do_uqrshl_d)
+
#undef DO_ZPZZ
#undef DO_ZPZZ_D
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 1dacb3fbc4..1aa8db673f 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -5941,3 +5941,21 @@ static bool trans_SQNEG(DisasContext *s, arg_rpr_esz *a)
};
return do_sve2_zpz_ool(s, a, fns[a->esz]);
}
+
+#define DO_SVE2_ZPZZ(NAME, name) \
+static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
+{ \
+ static gen_helper_gvec_4 * const fns[4] = { \
+ gen_helper_sve2_##name##_zpzz_b, gen_helper_sve2_##name##_zpzz_h, \
+ gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d, \
+ }; \
+ return do_sve2_zpzz_ool(s, a, fns[a->esz]); \
+}
+
+DO_SVE2_ZPZZ(SQSHL, sqshl)
+DO_SVE2_ZPZZ(SQRSHL, sqrshl)
+DO_SVE2_ZPZZ(SRSHL, srshl)
+
+DO_SVE2_ZPZZ(UQSHL, uqshl)
+DO_SVE2_ZPZZ(UQRSHL, uqrshl)
+DO_SVE2_ZPZZ(URSHL, urshl)
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 10/81] target/arm: Implement SVE2 integer halving add/subtract (predicated)
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (8 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 09/81] target/arm: Implement SVE2 saturating/rounding bitwise shift left (predicated) Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 11/81] target/arm: Implement SVE2 integer pairwise arithmetic Richard Henderson
` (72 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 54 ++++++++++++++++++++++++++++++++++++++
target/arm/sve.decode | 11 ++++++++
target/arm/sve_helper.c | 39 +++++++++++++++++++++++++++
target/arm/translate-sve.c | 8 ++++++
4 files changed, 112 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 3c87b1ce29..d2acd46cf9 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -226,6 +226,60 @@ DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_s, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_5(sve_sdiv_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_sdiv_zpzz_d, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 7c940e45c6..25508a6d69 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1129,3 +1129,14 @@ SQRSHL 01000100 .. 001 010 100 ... ..... ..... @rdn_pg_rm
UQRSHL 01000100 .. 001 011 100 ... ..... ..... @rdn_pg_rm
SQRSHL 01000100 .. 001 110 100 ... ..... ..... @rdm_pg_rn # SQRSHLR
UQRSHL 01000100 .. 001 111 100 ... ..... ..... @rdm_pg_rn # UQRSHLR
+
+### SVE2 integer halving add/subtract (predicated)
+
+SHADD 01000100 .. 010 000 100 ... ..... ..... @rdn_pg_rm
+UHADD 01000100 .. 010 001 100 ... ..... ..... @rdn_pg_rm
+SHSUB 01000100 .. 010 010 100 ... ..... ..... @rdn_pg_rm
+UHSUB 01000100 .. 010 011 100 ... ..... ..... @rdn_pg_rm
+SRHADD 01000100 .. 010 100 100 ... ..... ..... @rdn_pg_rm
+URHADD 01000100 .. 010 101 100 ... ..... ..... @rdn_pg_rm
+SHSUB 01000100 .. 010 110 100 ... ..... ..... @rdm_pg_rn # SHSUBR
+UHSUB 01000100 .. 010 111 100 ... ..... ..... @rdm_pg_rn # UHSUBR
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 40e1b5df30..bbd2666e3a 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -639,6 +639,45 @@ DO_ZPZZ(sve2_uqrshl_zpzz_h, uint16_t, H1_2, do_uqrshl_h)
DO_ZPZZ(sve2_uqrshl_zpzz_s, uint32_t, H1_4, do_uqrshl_s)
DO_ZPZZ_D(sve2_uqrshl_zpzz_d, uint64_t, do_uqrshl_d)
+#define DO_HADD_BHS(n, m) (((int64_t)n + m) >> 1)
+#define DO_HADD_D(n, m) ((n >> 1) + (m >> 1) + (n & m & 1))
+
+DO_ZPZZ(sve2_shadd_zpzz_b, int8_t, H1_2, DO_HADD_BHS)
+DO_ZPZZ(sve2_shadd_zpzz_h, int16_t, H1_2, DO_HADD_BHS)
+DO_ZPZZ(sve2_shadd_zpzz_s, int32_t, H1_4, DO_HADD_BHS)
+DO_ZPZZ_D(sve2_shadd_zpzz_d, int64_t, DO_HADD_D)
+
+DO_ZPZZ(sve2_uhadd_zpzz_b, uint8_t, H1_2, DO_HADD_BHS)
+DO_ZPZZ(sve2_uhadd_zpzz_h, uint16_t, H1_2, DO_HADD_BHS)
+DO_ZPZZ(sve2_uhadd_zpzz_s, uint32_t, H1_4, DO_HADD_BHS)
+DO_ZPZZ_D(sve2_uhadd_zpzz_d, uint64_t, DO_HADD_D)
+
+#define DO_RHADD_BHS(n, m) (((int64_t)n + m + 1) >> 1)
+#define DO_RHADD_D(n, m) ((n >> 1) + (m >> 1) + ((n | m) & 1))
+
+DO_ZPZZ(sve2_srhadd_zpzz_b, int8_t, H1_2, DO_RHADD_BHS)
+DO_ZPZZ(sve2_srhadd_zpzz_h, int16_t, H1_2, DO_RHADD_BHS)
+DO_ZPZZ(sve2_srhadd_zpzz_s, int32_t, H1_4, DO_RHADD_BHS)
+DO_ZPZZ_D(sve2_srhadd_zpzz_d, int64_t, DO_RHADD_D)
+
+DO_ZPZZ(sve2_urhadd_zpzz_b, uint8_t, H1_2, DO_RHADD_BHS)
+DO_ZPZZ(sve2_urhadd_zpzz_h, uint16_t, H1_2, DO_RHADD_BHS)
+DO_ZPZZ(sve2_urhadd_zpzz_s, uint32_t, H1_4, DO_RHADD_BHS)
+DO_ZPZZ_D(sve2_urhadd_zpzz_d, uint64_t, DO_RHADD_D)
+
+#define DO_HSUB_BHS(n, m) (((int64_t)n - m) >> 1)
+#define DO_HSUB_D(n, m) ((n >> 1) - (m >> 1) - (~n & m & 1))
+
+DO_ZPZZ(sve2_shsub_zpzz_b, int8_t, H1_2, DO_HSUB_BHS)
+DO_ZPZZ(sve2_shsub_zpzz_h, int16_t, H1_2, DO_HSUB_BHS)
+DO_ZPZZ(sve2_shsub_zpzz_s, int32_t, H1_4, DO_HSUB_BHS)
+DO_ZPZZ_D(sve2_shsub_zpzz_d, int64_t, DO_HSUB_D)
+
+DO_ZPZZ(sve2_uhsub_zpzz_b, uint8_t, H1_2, DO_HSUB_BHS)
+DO_ZPZZ(sve2_uhsub_zpzz_h, uint16_t, H1_2, DO_HSUB_BHS)
+DO_ZPZZ(sve2_uhsub_zpzz_s, uint32_t, H1_4, DO_HSUB_BHS)
+DO_ZPZZ_D(sve2_uhsub_zpzz_d, uint64_t, DO_HSUB_D)
+
#undef DO_ZPZZ
#undef DO_ZPZZ_D
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 1aa8db673f..9703400849 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -5959,3 +5959,11 @@ DO_SVE2_ZPZZ(SRSHL, srshl)
DO_SVE2_ZPZZ(UQSHL, uqshl)
DO_SVE2_ZPZZ(UQRSHL, uqrshl)
DO_SVE2_ZPZZ(URSHL, urshl)
+
+DO_SVE2_ZPZZ(SHADD, shadd)
+DO_SVE2_ZPZZ(SRHADD, srhadd)
+DO_SVE2_ZPZZ(SHSUB, shsub)
+
+DO_SVE2_ZPZZ(UHADD, uhadd)
+DO_SVE2_ZPZZ(URHADD, urhadd)
+DO_SVE2_ZPZZ(UHSUB, uhsub)
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 11/81] target/arm: Implement SVE2 integer pairwise arithmetic
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (9 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 10/81] target/arm: Implement SVE2 integer halving add/subtract (predicated) Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 12/81] target/arm: Implement SVE2 saturating add/subtract (predicated) Richard Henderson
` (71 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Load all inputs before writing any output (laurent desnogues)
---
target/arm/helper-sve.h | 45 ++++++++++++++++++++++
target/arm/sve.decode | 8 ++++
target/arm/sve_helper.c | 76 ++++++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 6 +++
4 files changed, 135 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index d2acd46cf9..63d35ddd4f 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -326,6 +326,51 @@ DEF_HELPER_FLAGS_5(sve_sel_zpzz_s, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(sve_sel_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_addp_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_addp_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_addp_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_addp_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_5(sve_asr_zpzw_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_asr_zpzw_h, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 25508a6d69..859137df34 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1140,3 +1140,11 @@ SRHADD 01000100 .. 010 100 100 ... ..... ..... @rdn_pg_rm
URHADD 01000100 .. 010 101 100 ... ..... ..... @rdn_pg_rm
SHSUB 01000100 .. 010 110 100 ... ..... ..... @rdm_pg_rn # SHSUBR
UHSUB 01000100 .. 010 111 100 ... ..... ..... @rdm_pg_rn # UHSUBR
+
+### SVE2 integer pairwise arithmetic
+
+ADDP 01000100 .. 010 001 101 ... ..... ..... @rdn_pg_rm
+SMAXP 01000100 .. 010 100 101 ... ..... ..... @rdn_pg_rm
+UMAXP 01000100 .. 010 101 101 ... ..... ..... @rdn_pg_rm
+SMINP 01000100 .. 010 110 101 ... ..... ..... @rdn_pg_rm
+UMINP 01000100 .. 010 111 101 ... ..... ..... @rdn_pg_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index bbd2666e3a..f861ba0504 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -681,6 +681,82 @@ DO_ZPZZ_D(sve2_uhsub_zpzz_d, uint64_t, DO_HSUB_D)
#undef DO_ZPZZ
#undef DO_ZPZZ_D
+/*
+ * Three operand expander, operating on element pairs.
+ * If the slot I is even, the elements from from VN {I, I+1}.
+ * If the slot I is odd, the elements from from VM {I-1, I}.
+ * Load all of the input elements in each pair before overwriting output.
+ */
+#define DO_ZPZZ_PAIR(NAME, TYPE, H, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ for (i = 0; i < opr_sz; ) { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ TYPE n0 = *(TYPE *)(vn + H(i)); \
+ TYPE m0 = *(TYPE *)(vm + H(i)); \
+ TYPE n1 = *(TYPE *)(vn + H(i + sizeof(TYPE))); \
+ TYPE m1 = *(TYPE *)(vm + H(i + sizeof(TYPE))); \
+ if (pg & 1) { \
+ *(TYPE *)(vd + H(i)) = OP(n0, n1); \
+ } \
+ i += sizeof(TYPE), pg >>= sizeof(TYPE); \
+ if (pg & 1) { \
+ *(TYPE *)(vd + H(i)) = OP(m0, m1); \
+ } \
+ i += sizeof(TYPE), pg >>= sizeof(TYPE); \
+ } while (i & 15); \
+ } \
+}
+
+/* Similarly, specialized for 64-bit operands. */
+#define DO_ZPZZ_PAIR_D(NAME, TYPE, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc) / 8; \
+ TYPE *d = vd, *n = vn, *m = vm; \
+ uint8_t *pg = vg; \
+ for (i = 0; i < opr_sz; i += 2) { \
+ TYPE n0 = n[i], n1 = n[i + 1]; \
+ TYPE m0 = m[i], m1 = m[i + 1]; \
+ if (pg[H1(i)] & 1) { \
+ d[i] = OP(n0, n1); \
+ } \
+ if (pg[H1(i + 1)] & 1) { \
+ d[i + 1] = OP(m0, m1); \
+ } \
+ } \
+}
+
+DO_ZPZZ_PAIR(sve2_addp_zpzz_b, uint8_t, H1_2, DO_ADD)
+DO_ZPZZ_PAIR(sve2_addp_zpzz_h, uint16_t, H1_2, DO_ADD)
+DO_ZPZZ_PAIR(sve2_addp_zpzz_s, uint32_t, H1_4, DO_ADD)
+DO_ZPZZ_PAIR_D(sve2_addp_zpzz_d, uint64_t, DO_ADD)
+
+DO_ZPZZ_PAIR(sve2_umaxp_zpzz_b, uint8_t, H1_2, DO_MAX)
+DO_ZPZZ_PAIR(sve2_umaxp_zpzz_h, uint16_t, H1_2, DO_MAX)
+DO_ZPZZ_PAIR(sve2_umaxp_zpzz_s, uint32_t, H1_4, DO_MAX)
+DO_ZPZZ_PAIR_D(sve2_umaxp_zpzz_d, uint64_t, DO_MAX)
+
+DO_ZPZZ_PAIR(sve2_uminp_zpzz_b, uint8_t, H1_2, DO_MIN)
+DO_ZPZZ_PAIR(sve2_uminp_zpzz_h, uint16_t, H1_2, DO_MIN)
+DO_ZPZZ_PAIR(sve2_uminp_zpzz_s, uint32_t, H1_4, DO_MIN)
+DO_ZPZZ_PAIR_D(sve2_uminp_zpzz_d, uint64_t, DO_MIN)
+
+DO_ZPZZ_PAIR(sve2_smaxp_zpzz_b, int8_t, H1_2, DO_MAX)
+DO_ZPZZ_PAIR(sve2_smaxp_zpzz_h, int16_t, H1_2, DO_MAX)
+DO_ZPZZ_PAIR(sve2_smaxp_zpzz_s, int32_t, H1_4, DO_MAX)
+DO_ZPZZ_PAIR_D(sve2_smaxp_zpzz_d, int64_t, DO_MAX)
+
+DO_ZPZZ_PAIR(sve2_sminp_zpzz_b, int8_t, H1_2, DO_MIN)
+DO_ZPZZ_PAIR(sve2_sminp_zpzz_h, int16_t, H1_2, DO_MIN)
+DO_ZPZZ_PAIR(sve2_sminp_zpzz_s, int32_t, H1_4, DO_MIN)
+DO_ZPZZ_PAIR_D(sve2_sminp_zpzz_d, int64_t, DO_MIN)
+
+#undef DO_ZPZZ_PAIR
+#undef DO_ZPZZ_PAIR_D
+
/* Three-operand expander, controlled by a predicate, in which the
* third operand is "wide". That is, for D = N op M, the same 64-bit
* value of M is used with all of the narrower values of N.
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 9703400849..3d3b4d61d4 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -5967,3 +5967,9 @@ DO_SVE2_ZPZZ(SHSUB, shsub)
DO_SVE2_ZPZZ(UHADD, uhadd)
DO_SVE2_ZPZZ(URHADD, urhadd)
DO_SVE2_ZPZZ(UHSUB, uhsub)
+
+DO_SVE2_ZPZZ(ADDP, addp)
+DO_SVE2_ZPZZ(SMAXP, smaxp)
+DO_SVE2_ZPZZ(UMAXP, umaxp)
+DO_SVE2_ZPZZ(SMINP, sminp)
+DO_SVE2_ZPZZ(UMINP, uminp)
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 12/81] target/arm: Implement SVE2 saturating add/subtract (predicated)
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (10 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 11/81] target/arm: Implement SVE2 integer pairwise arithmetic Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 13/81] target/arm: Implement SVE2 integer add/subtract long Richard Henderson
` (70 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 54 +++++++++++
target/arm/sve.decode | 11 +++
target/arm/sve_helper.c | 194 ++++++++++++++++++++++++++-----------
target/arm/translate-sve.c | 7 ++
4 files changed, 210 insertions(+), 56 deletions(-)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 63d35ddd4f..ae06619d7d 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -371,6 +371,60 @@ DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_s, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_5(sve_asr_zpzw_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_asr_zpzw_h, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 859137df34..d018aef3dc 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1148,3 +1148,14 @@ SMAXP 01000100 .. 010 100 101 ... ..... ..... @rdn_pg_rm
UMAXP 01000100 .. 010 101 101 ... ..... ..... @rdn_pg_rm
SMINP 01000100 .. 010 110 101 ... ..... ..... @rdn_pg_rm
UMINP 01000100 .. 010 111 101 ... ..... ..... @rdn_pg_rm
+
+### SVE2 saturating add/subtract (predicated)
+
+SQADD_zpzz 01000100 .. 011 000 100 ... ..... ..... @rdn_pg_rm
+UQADD_zpzz 01000100 .. 011 001 100 ... ..... ..... @rdn_pg_rm
+SQSUB_zpzz 01000100 .. 011 010 100 ... ..... ..... @rdn_pg_rm
+UQSUB_zpzz 01000100 .. 011 011 100 ... ..... ..... @rdn_pg_rm
+SUQADD 01000100 .. 011 100 100 ... ..... ..... @rdn_pg_rm
+USQADD 01000100 .. 011 101 100 ... ..... ..... @rdn_pg_rm
+SQSUB_zpzz 01000100 .. 011 110 100 ... ..... ..... @rdm_pg_rn # SQSUBR
+UQSUB_zpzz 01000100 .. 011 111 100 ... ..... ..... @rdm_pg_rn # UQSUBR
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index f861ba0504..d3c7e0e03c 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -678,6 +678,135 @@ DO_ZPZZ(sve2_uhsub_zpzz_h, uint16_t, H1_2, DO_HSUB_BHS)
DO_ZPZZ(sve2_uhsub_zpzz_s, uint32_t, H1_4, DO_HSUB_BHS)
DO_ZPZZ_D(sve2_uhsub_zpzz_d, uint64_t, DO_HSUB_D)
+static inline int32_t do_sat_bhs(int64_t val, int64_t min, int64_t max)
+{
+ return val >= max ? max : val <= min ? min : val;
+}
+
+#define DO_SQADD_B(n, m) do_sat_bhs((int64_t)n + m, INT8_MIN, INT8_MAX)
+#define DO_SQADD_H(n, m) do_sat_bhs((int64_t)n + m, INT16_MIN, INT16_MAX)
+#define DO_SQADD_S(n, m) do_sat_bhs((int64_t)n + m, INT32_MIN, INT32_MAX)
+
+static inline int64_t do_sqadd_d(int64_t n, int64_t m)
+{
+ int64_t r = n + m;
+ if (((r ^ n) & ~(n ^ m)) < 0) {
+ /* Signed overflow. */
+ return r < 0 ? INT64_MAX : INT64_MIN;
+ }
+ return r;
+}
+
+DO_ZPZZ(sve2_sqadd_zpzz_b, int8_t, H1_2, DO_SQADD_B)
+DO_ZPZZ(sve2_sqadd_zpzz_h, int16_t, H1_2, DO_SQADD_H)
+DO_ZPZZ(sve2_sqadd_zpzz_s, int32_t, H1_4, DO_SQADD_S)
+DO_ZPZZ_D(sve2_sqadd_zpzz_d, int64_t, do_sqadd_d)
+
+#define DO_UQADD_B(n, m) do_sat_bhs((int64_t)n + m, 0, UINT8_MAX)
+#define DO_UQADD_H(n, m) do_sat_bhs((int64_t)n + m, 0, UINT16_MAX)
+#define DO_UQADD_S(n, m) do_sat_bhs((int64_t)n + m, 0, UINT32_MAX)
+
+static inline uint64_t do_uqadd_d(uint64_t n, uint64_t m)
+{
+ uint64_t r = n + m;
+ return r < n ? UINT64_MAX : r;
+}
+
+DO_ZPZZ(sve2_uqadd_zpzz_b, uint8_t, H1_2, DO_UQADD_B)
+DO_ZPZZ(sve2_uqadd_zpzz_h, uint16_t, H1_2, DO_UQADD_H)
+DO_ZPZZ(sve2_uqadd_zpzz_s, uint32_t, H1_4, DO_UQADD_S)
+DO_ZPZZ_D(sve2_uqadd_zpzz_d, uint64_t, do_uqadd_d)
+
+#define DO_SQSUB_B(n, m) do_sat_bhs((int64_t)n - m, INT8_MIN, INT8_MAX)
+#define DO_SQSUB_H(n, m) do_sat_bhs((int64_t)n - m, INT16_MIN, INT16_MAX)
+#define DO_SQSUB_S(n, m) do_sat_bhs((int64_t)n - m, INT32_MIN, INT32_MAX)
+
+static inline int64_t do_sqsub_d(int64_t n, int64_t m)
+{
+ int64_t r = n - m;
+ if (((r ^ n) & (n ^ m)) < 0) {
+ /* Signed overflow. */
+ return r < 0 ? INT64_MAX : INT64_MIN;
+ }
+ return r;
+}
+
+DO_ZPZZ(sve2_sqsub_zpzz_b, int8_t, H1_2, DO_SQSUB_B)
+DO_ZPZZ(sve2_sqsub_zpzz_h, int16_t, H1_2, DO_SQSUB_H)
+DO_ZPZZ(sve2_sqsub_zpzz_s, int32_t, H1_4, DO_SQSUB_S)
+DO_ZPZZ_D(sve2_sqsub_zpzz_d, int64_t, do_sqsub_d)
+
+#define DO_UQSUB_B(n, m) do_sat_bhs((int64_t)n - m, 0, UINT8_MAX)
+#define DO_UQSUB_H(n, m) do_sat_bhs((int64_t)n - m, 0, UINT16_MAX)
+#define DO_UQSUB_S(n, m) do_sat_bhs((int64_t)n - m, 0, UINT32_MAX)
+
+static inline uint64_t do_uqsub_d(uint64_t n, uint64_t m)
+{
+ return n > m ? n - m : 0;
+}
+
+DO_ZPZZ(sve2_uqsub_zpzz_b, uint8_t, H1_2, DO_UQSUB_B)
+DO_ZPZZ(sve2_uqsub_zpzz_h, uint16_t, H1_2, DO_UQSUB_H)
+DO_ZPZZ(sve2_uqsub_zpzz_s, uint32_t, H1_4, DO_UQSUB_S)
+DO_ZPZZ_D(sve2_uqsub_zpzz_d, uint64_t, do_uqsub_d)
+
+#define DO_SUQADD_B(n, m) \
+ do_sat_bhs((int64_t)(int8_t)n + m, INT8_MIN, INT8_MAX)
+#define DO_SUQADD_H(n, m) \
+ do_sat_bhs((int64_t)(int16_t)n + m, INT16_MIN, INT16_MAX)
+#define DO_SUQADD_S(n, m) \
+ do_sat_bhs((int64_t)(int32_t)n + m, INT32_MIN, INT32_MAX)
+
+static inline int64_t do_suqadd_d(int64_t n, uint64_t m)
+{
+ uint64_t r = n + m;
+
+ if (n < 0) {
+ /* Note that m - abs(n) cannot underflow. */
+ if (r > INT64_MAX) {
+ /* Result is either very large positive or negative. */
+ if (m > -n) {
+ /* m > abs(n), so r is a very large positive. */
+ return INT64_MAX;
+ }
+ /* Result is negative. */
+ }
+ } else {
+ /* Both inputs are positive: check for overflow. */
+ if (r < m || r > INT64_MAX) {
+ return INT64_MAX;
+ }
+ }
+ return r;
+}
+
+DO_ZPZZ(sve2_suqadd_zpzz_b, uint8_t, H1_2, DO_SUQADD_B)
+DO_ZPZZ(sve2_suqadd_zpzz_h, uint16_t, H1_2, DO_SUQADD_H)
+DO_ZPZZ(sve2_suqadd_zpzz_s, uint32_t, H1_4, DO_SUQADD_S)
+DO_ZPZZ_D(sve2_suqadd_zpzz_d, uint64_t, do_suqadd_d)
+
+#define DO_USQADD_B(n, m) \
+ do_sat_bhs((int64_t)n + (int8_t)m, 0, UINT8_MAX)
+#define DO_USQADD_H(n, m) \
+ do_sat_bhs((int64_t)n + (int16_t)m, 0, UINT16_MAX)
+#define DO_USQADD_S(n, m) \
+ do_sat_bhs((int64_t)n + (int32_t)m, 0, UINT32_MAX)
+
+static inline uint64_t do_usqadd_d(uint64_t n, int64_t m)
+{
+ uint64_t r = n + m;
+
+ if (m < 0) {
+ return n < -m ? 0 : r;
+ }
+ return r < n ? UINT64_MAX : r;
+}
+
+DO_ZPZZ(sve2_usqadd_zpzz_b, uint8_t, H1_2, DO_USQADD_B)
+DO_ZPZZ(sve2_usqadd_zpzz_h, uint16_t, H1_2, DO_USQADD_H)
+DO_ZPZZ(sve2_usqadd_zpzz_s, uint32_t, H1_4, DO_USQADD_S)
+DO_ZPZZ_D(sve2_usqadd_zpzz_d, uint64_t, do_usqadd_d)
+
#undef DO_ZPZZ
#undef DO_ZPZZ_D
@@ -1613,13 +1742,7 @@ void HELPER(sve_sqaddi_b)(void *d, void *a, int32_t b, uint32_t desc)
intptr_t i, oprsz = simd_oprsz(desc);
for (i = 0; i < oprsz; i += sizeof(int8_t)) {
- int r = *(int8_t *)(a + i) + b;
- if (r > INT8_MAX) {
- r = INT8_MAX;
- } else if (r < INT8_MIN) {
- r = INT8_MIN;
- }
- *(int8_t *)(d + i) = r;
+ *(int8_t *)(d + i) = DO_SQADD_B(b, *(int8_t *)(a + i));
}
}
@@ -1628,13 +1751,7 @@ void HELPER(sve_sqaddi_h)(void *d, void *a, int32_t b, uint32_t desc)
intptr_t i, oprsz = simd_oprsz(desc);
for (i = 0; i < oprsz; i += sizeof(int16_t)) {
- int r = *(int16_t *)(a + i) + b;
- if (r > INT16_MAX) {
- r = INT16_MAX;
- } else if (r < INT16_MIN) {
- r = INT16_MIN;
- }
- *(int16_t *)(d + i) = r;
+ *(int16_t *)(d + i) = DO_SQADD_H(b, *(int16_t *)(a + i));
}
}
@@ -1643,13 +1760,7 @@ void HELPER(sve_sqaddi_s)(void *d, void *a, int64_t b, uint32_t desc)
intptr_t i, oprsz = simd_oprsz(desc);
for (i = 0; i < oprsz; i += sizeof(int32_t)) {
- int64_t r = *(int32_t *)(a + i) + b;
- if (r > INT32_MAX) {
- r = INT32_MAX;
- } else if (r < INT32_MIN) {
- r = INT32_MIN;
- }
- *(int32_t *)(d + i) = r;
+ *(int32_t *)(d + i) = DO_SQADD_S(b, *(int32_t *)(a + i));
}
}
@@ -1658,13 +1769,7 @@ void HELPER(sve_sqaddi_d)(void *d, void *a, int64_t b, uint32_t desc)
intptr_t i, oprsz = simd_oprsz(desc);
for (i = 0; i < oprsz; i += sizeof(int64_t)) {
- int64_t ai = *(int64_t *)(a + i);
- int64_t r = ai + b;
- if (((r ^ ai) & ~(ai ^ b)) < 0) {
- /* Signed overflow. */
- r = (r < 0 ? INT64_MAX : INT64_MIN);
- }
- *(int64_t *)(d + i) = r;
+ *(int64_t *)(d + i) = do_sqadd_d(b, *(int64_t *)(a + i));
}
}
@@ -1677,13 +1782,7 @@ void HELPER(sve_uqaddi_b)(void *d, void *a, int32_t b, uint32_t desc)
intptr_t i, oprsz = simd_oprsz(desc);
for (i = 0; i < oprsz; i += sizeof(uint8_t)) {
- int r = *(uint8_t *)(a + i) + b;
- if (r > UINT8_MAX) {
- r = UINT8_MAX;
- } else if (r < 0) {
- r = 0;
- }
- *(uint8_t *)(d + i) = r;
+ *(uint8_t *)(d + i) = DO_UQADD_B(b, *(uint8_t *)(a + i));
}
}
@@ -1692,13 +1791,7 @@ void HELPER(sve_uqaddi_h)(void *d, void *a, int32_t b, uint32_t desc)
intptr_t i, oprsz = simd_oprsz(desc);
for (i = 0; i < oprsz; i += sizeof(uint16_t)) {
- int r = *(uint16_t *)(a + i) + b;
- if (r > UINT16_MAX) {
- r = UINT16_MAX;
- } else if (r < 0) {
- r = 0;
- }
- *(uint16_t *)(d + i) = r;
+ *(uint16_t *)(d + i) = DO_UQADD_H(b, *(uint16_t *)(a + i));
}
}
@@ -1707,13 +1800,7 @@ void HELPER(sve_uqaddi_s)(void *d, void *a, int64_t b, uint32_t desc)
intptr_t i, oprsz = simd_oprsz(desc);
for (i = 0; i < oprsz; i += sizeof(uint32_t)) {
- int64_t r = *(uint32_t *)(a + i) + b;
- if (r > UINT32_MAX) {
- r = UINT32_MAX;
- } else if (r < 0) {
- r = 0;
- }
- *(uint32_t *)(d + i) = r;
+ *(uint32_t *)(d + i) = DO_UQADD_S(b, *(uint32_t *)(a + i));
}
}
@@ -1722,11 +1809,7 @@ void HELPER(sve_uqaddi_d)(void *d, void *a, uint64_t b, uint32_t desc)
intptr_t i, oprsz = simd_oprsz(desc);
for (i = 0; i < oprsz; i += sizeof(uint64_t)) {
- uint64_t r = *(uint64_t *)(a + i) + b;
- if (r < b) {
- r = UINT64_MAX;
- }
- *(uint64_t *)(d + i) = r;
+ *(uint64_t *)(d + i) = do_uqadd_d(b, *(uint64_t *)(a + i));
}
}
@@ -1735,8 +1818,7 @@ void HELPER(sve_uqsubi_d)(void *d, void *a, uint64_t b, uint32_t desc)
intptr_t i, oprsz = simd_oprsz(desc);
for (i = 0; i < oprsz; i += sizeof(uint64_t)) {
- uint64_t ai = *(uint64_t *)(a + i);
- *(uint64_t *)(d + i) = (ai < b ? 0 : ai - b);
+ *(uint64_t *)(d + i) = do_uqsub_d(*(uint64_t *)(a + i), b);
}
}
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 3d3b4d61d4..f3e979bfb7 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -5973,3 +5973,10 @@ DO_SVE2_ZPZZ(SMAXP, smaxp)
DO_SVE2_ZPZZ(UMAXP, umaxp)
DO_SVE2_ZPZZ(SMINP, sminp)
DO_SVE2_ZPZZ(UMINP, uminp)
+
+DO_SVE2_ZPZZ(SQADD_zpzz, sqadd)
+DO_SVE2_ZPZZ(UQADD_zpzz, uqadd)
+DO_SVE2_ZPZZ(SQSUB_zpzz, sqsub)
+DO_SVE2_ZPZZ(UQSUB_zpzz, uqsub)
+DO_SVE2_ZPZZ(SUQADD, suqadd)
+DO_SVE2_ZPZZ(USQADD, usqadd)
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 13/81] target/arm: Implement SVE2 integer add/subtract long
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (11 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 12/81] target/arm: Implement SVE2 saturating add/subtract (predicated) Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 14/81] target/arm: Implement SVE2 integer add/subtract interleaved long Richard Henderson
` (69 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Fix select offsets (laurent desnogues).
---
target/arm/helper-sve.h | 24 ++++++++++++++++++++
target/arm/sve.decode | 19 ++++++++++++++++
target/arm/sve_helper.c | 43 +++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 46 ++++++++++++++++++++++++++++++++++++++
4 files changed, 132 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index ae06619d7d..676ddfbb2d 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -1367,6 +1367,30 @@ DEF_HELPER_FLAGS_5(sve_ftmad_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_ftmad_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_ftmad_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_saddl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_saddl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_saddl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_ssubl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_ssubl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_ssubl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_sabdl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sabdl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sabdl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_uaddl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uaddl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uaddl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_usubl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_usubl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_usubl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_uabdl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uabdl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uabdl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_4(sve_ld1bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
DEF_HELPER_FLAGS_4(sve_ld2bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
DEF_HELPER_FLAGS_4(sve_ld3bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index d018aef3dc..70712039e4 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1159,3 +1159,22 @@ SUQADD 01000100 .. 011 100 100 ... ..... ..... @rdn_pg_rm
USQADD 01000100 .. 011 101 100 ... ..... ..... @rdn_pg_rm
SQSUB_zpzz 01000100 .. 011 110 100 ... ..... ..... @rdm_pg_rn # SQSUBR
UQSUB_zpzz 01000100 .. 011 111 100 ... ..... ..... @rdm_pg_rn # UQSUBR
+
+#### SVE2 Widening Integer Arithmetic
+
+## SVE2 integer add/subtract long
+
+SADDLB 01000101 .. 0 ..... 00 0000 ..... ..... @rd_rn_rm
+SADDLT 01000101 .. 0 ..... 00 0001 ..... ..... @rd_rn_rm
+UADDLB 01000101 .. 0 ..... 00 0010 ..... ..... @rd_rn_rm
+UADDLT 01000101 .. 0 ..... 00 0011 ..... ..... @rd_rn_rm
+
+SSUBLB 01000101 .. 0 ..... 00 0100 ..... ..... @rd_rn_rm
+SSUBLT 01000101 .. 0 ..... 00 0101 ..... ..... @rd_rn_rm
+USUBLB 01000101 .. 0 ..... 00 0110 ..... ..... @rd_rn_rm
+USUBLT 01000101 .. 0 ..... 00 0111 ..... ..... @rd_rn_rm
+
+SABDLB 01000101 .. 0 ..... 00 1100 ..... ..... @rd_rn_rm
+SABDLT 01000101 .. 0 ..... 00 1101 ..... ..... @rd_rn_rm
+UABDLB 01000101 .. 0 ..... 00 1110 ..... ..... @rd_rn_rm
+UABDLT 01000101 .. 0 ..... 00 1111 ..... ..... @rd_rn_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index d3c7e0e03c..ca39b0f6f1 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1113,6 +1113,49 @@ DO_ZZW(sve_lsl_zzw_s, uint32_t, uint64_t, H1_4, DO_LSL)
#undef DO_ZPZ
#undef DO_ZPZ_D
+/*
+ * Three-operand expander, unpredicated, in which the two inputs are
+ * selected from the top or bottom half of the wide column.
+ */
+#define DO_ZZZ_TB(NAME, TYPEW, TYPEN, HW, HN, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ int sel1 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \
+ int sel2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(TYPEN); \
+ for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
+ TYPEW nn = *(TYPEN *)(vn + HN(i + sel1)); \
+ TYPEW mm = *(TYPEN *)(vm + HN(i + sel2)); \
+ *(TYPEW *)(vd + HW(i)) = OP(nn, mm); \
+ } \
+}
+
+DO_ZZZ_TB(sve2_saddl_h, int16_t, int8_t, H1_2, H1, DO_ADD)
+DO_ZZZ_TB(sve2_saddl_s, int32_t, int16_t, H1_4, H1_2, DO_ADD)
+DO_ZZZ_TB(sve2_saddl_d, int64_t, int32_t, , H1_4, DO_ADD)
+
+DO_ZZZ_TB(sve2_ssubl_h, int16_t, int8_t, H1_2, H1, DO_SUB)
+DO_ZZZ_TB(sve2_ssubl_s, int32_t, int16_t, H1_4, H1_2, DO_SUB)
+DO_ZZZ_TB(sve2_ssubl_d, int64_t, int32_t, , H1_4, DO_SUB)
+
+DO_ZZZ_TB(sve2_sabdl_h, int16_t, int8_t, H1_2, H1, DO_ABD)
+DO_ZZZ_TB(sve2_sabdl_s, int32_t, int16_t, H1_4, H1_2, DO_ABD)
+DO_ZZZ_TB(sve2_sabdl_d, int64_t, int32_t, , H1_4, DO_ABD)
+
+DO_ZZZ_TB(sve2_uaddl_h, uint16_t, uint8_t, H1_2, H1, DO_ADD)
+DO_ZZZ_TB(sve2_uaddl_s, uint32_t, uint16_t, H1_4, H1_2, DO_ADD)
+DO_ZZZ_TB(sve2_uaddl_d, uint64_t, uint32_t, , H1_4, DO_ADD)
+
+DO_ZZZ_TB(sve2_usubl_h, uint16_t, uint8_t, H1_2, H1, DO_SUB)
+DO_ZZZ_TB(sve2_usubl_s, uint32_t, uint16_t, H1_4, H1_2, DO_SUB)
+DO_ZZZ_TB(sve2_usubl_d, uint64_t, uint32_t, , H1_4, DO_SUB)
+
+DO_ZZZ_TB(sve2_uabdl_h, uint16_t, uint8_t, H1_2, H1, DO_ABD)
+DO_ZZZ_TB(sve2_uabdl_s, uint32_t, uint16_t, H1_4, H1_2, DO_ABD)
+DO_ZZZ_TB(sve2_uabdl_d, uint64_t, uint32_t, , H1_4, DO_ABD)
+
+#undef DO_ZZZ_TB
+
/* Two-operand reduction expander, controlled by a predicate.
* The difference between TYPERED and TYPERET has to do with
* sign-extension. E.g. for SMAX, TYPERED must be signed,
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index f3e979bfb7..cbbce6a3e2 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -5980,3 +5980,49 @@ DO_SVE2_ZPZZ(SQSUB_zpzz, sqsub)
DO_SVE2_ZPZZ(UQSUB_zpzz, uqsub)
DO_SVE2_ZPZZ(SUQADD, suqadd)
DO_SVE2_ZPZZ(USQADD, usqadd)
+
+/*
+ * SVE2 Widening Integer Arithmetic
+ */
+
+static bool do_sve2_zzw_ool(DisasContext *s, arg_rrr_esz *a,
+ gen_helper_gvec_3 *fn, int data)
+{
+ if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ vsz, vsz, data, fn);
+ }
+ return true;
+}
+
+#define DO_SVE2_ZZZ_TB(NAME, name, SEL1, SEL2) \
+static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
+{ \
+ static gen_helper_gvec_3 * const fns[4] = { \
+ NULL, gen_helper_sve2_##name##_h, \
+ gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
+ }; \
+ return do_sve2_zzw_ool(s, a, fns[a->esz], (SEL2 << 1) | SEL1); \
+}
+
+DO_SVE2_ZZZ_TB(SADDLB, saddl, false, false)
+DO_SVE2_ZZZ_TB(SSUBLB, ssubl, false, false)
+DO_SVE2_ZZZ_TB(SABDLB, sabdl, false, false)
+
+DO_SVE2_ZZZ_TB(UADDLB, uaddl, false, false)
+DO_SVE2_ZZZ_TB(USUBLB, usubl, false, false)
+DO_SVE2_ZZZ_TB(UABDLB, uabdl, false, false)
+
+DO_SVE2_ZZZ_TB(SADDLT, saddl, true, true)
+DO_SVE2_ZZZ_TB(SSUBLT, ssubl, true, true)
+DO_SVE2_ZZZ_TB(SABDLT, sabdl, true, true)
+
+DO_SVE2_ZZZ_TB(UADDLT, uaddl, true, true)
+DO_SVE2_ZZZ_TB(USUBLT, usubl, true, true)
+DO_SVE2_ZZZ_TB(UABDLT, uabdl, true, true)
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 14/81] target/arm: Implement SVE2 integer add/subtract interleaved long
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (12 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 13/81] target/arm: Implement SVE2 integer add/subtract long Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 15/81] target/arm: Implement SVE2 integer add/subtract wide Richard Henderson
` (68 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve.decode | 6 ++++++
target/arm/translate-sve.c | 4 ++++
2 files changed, 10 insertions(+)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 70712039e4..a28172c017 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1178,3 +1178,9 @@ SABDLB 01000101 .. 0 ..... 00 1100 ..... ..... @rd_rn_rm
SABDLT 01000101 .. 0 ..... 00 1101 ..... ..... @rd_rn_rm
UABDLB 01000101 .. 0 ..... 00 1110 ..... ..... @rd_rn_rm
UABDLT 01000101 .. 0 ..... 00 1111 ..... ..... @rd_rn_rm
+
+## SVE2 integer add/subtract interleaved long
+
+SADDLBT 01000101 .. 0 ..... 1000 00 ..... ..... @rd_rn_rm
+SSUBLBT 01000101 .. 0 ..... 1000 10 ..... ..... @rd_rn_rm
+SSUBLTB 01000101 .. 0 ..... 1000 11 ..... ..... @rd_rn_rm
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index cbbce6a3e2..02eba872f9 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6026,3 +6026,7 @@ DO_SVE2_ZZZ_TB(SABDLT, sabdl, true, true)
DO_SVE2_ZZZ_TB(UADDLT, uaddl, true, true)
DO_SVE2_ZZZ_TB(USUBLT, usubl, true, true)
DO_SVE2_ZZZ_TB(UABDLT, uabdl, true, true)
+
+DO_SVE2_ZZZ_TB(SADDLBT, saddl, false, true)
+DO_SVE2_ZZZ_TB(SSUBLBT, ssubl, false, true)
+DO_SVE2_ZZZ_TB(SSUBLTB, ssubl, true, false)
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 15/81] target/arm: Implement SVE2 integer add/subtract wide
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (13 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 14/81] target/arm: Implement SVE2 integer add/subtract interleaved long Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 16/81] target/arm: Implement SVE2 integer multiply long Richard Henderson
` (67 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Fix select offsets (laurent desnogues).
---
target/arm/helper-sve.h | 16 ++++++++++++++++
target/arm/sve.decode | 12 ++++++++++++
target/arm/sve_helper.c | 30 ++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 20 ++++++++++++++++++++
4 files changed, 78 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 676ddfbb2d..a3cbfe0ea5 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -1391,6 +1391,22 @@ DEF_HELPER_FLAGS_4(sve2_uabdl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uabdl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_uabdl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_saddw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_saddw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_saddw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_ssubw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_ssubw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_ssubw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_uaddw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uaddw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uaddw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_usubw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_usubw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_usubw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_4(sve_ld1bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
DEF_HELPER_FLAGS_4(sve_ld2bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
DEF_HELPER_FLAGS_4(sve_ld3bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index a28172c017..b9a34726d6 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1184,3 +1184,15 @@ UABDLT 01000101 .. 0 ..... 00 1111 ..... ..... @rd_rn_rm
SADDLBT 01000101 .. 0 ..... 1000 00 ..... ..... @rd_rn_rm
SSUBLBT 01000101 .. 0 ..... 1000 10 ..... ..... @rd_rn_rm
SSUBLTB 01000101 .. 0 ..... 1000 11 ..... ..... @rd_rn_rm
+
+## SVE2 integer add/subtract wide
+
+SADDWB 01000101 .. 0 ..... 010 000 ..... ..... @rd_rn_rm
+SADDWT 01000101 .. 0 ..... 010 001 ..... ..... @rd_rn_rm
+UADDWB 01000101 .. 0 ..... 010 010 ..... ..... @rd_rn_rm
+UADDWT 01000101 .. 0 ..... 010 011 ..... ..... @rd_rn_rm
+
+SSUBWB 01000101 .. 0 ..... 010 100 ..... ..... @rd_rn_rm
+SSUBWT 01000101 .. 0 ..... 010 101 ..... ..... @rd_rn_rm
+USUBWB 01000101 .. 0 ..... 010 110 ..... ..... @rd_rn_rm
+USUBWT 01000101 .. 0 ..... 010 111 ..... ..... @rd_rn_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index ca39b0f6f1..75b87b1070 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1156,6 +1156,36 @@ DO_ZZZ_TB(sve2_uabdl_d, uint64_t, uint32_t, , H1_4, DO_ABD)
#undef DO_ZZZ_TB
+#define DO_ZZZ_WTB(NAME, TYPEW, TYPEN, HW, HN, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ int sel2 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \
+ for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
+ TYPEW nn = *(TYPEW *)(vn + HW(i)); \
+ TYPEW mm = *(TYPEN *)(vm + HN(i + sel2)); \
+ *(TYPEW *)(vd + HW(i)) = OP(nn, mm); \
+ } \
+}
+
+DO_ZZZ_WTB(sve2_saddw_h, int16_t, int8_t, H1_2, H1, DO_ADD)
+DO_ZZZ_WTB(sve2_saddw_s, int32_t, int16_t, H1_4, H1_2, DO_ADD)
+DO_ZZZ_WTB(sve2_saddw_d, int64_t, int32_t, , H1_4, DO_ADD)
+
+DO_ZZZ_WTB(sve2_ssubw_h, int16_t, int8_t, H1_2, H1, DO_SUB)
+DO_ZZZ_WTB(sve2_ssubw_s, int32_t, int16_t, H1_4, H1_2, DO_SUB)
+DO_ZZZ_WTB(sve2_ssubw_d, int64_t, int32_t, , H1_4, DO_SUB)
+
+DO_ZZZ_WTB(sve2_uaddw_h, uint16_t, uint8_t, H1_2, H1, DO_ADD)
+DO_ZZZ_WTB(sve2_uaddw_s, uint32_t, uint16_t, H1_4, H1_2, DO_ADD)
+DO_ZZZ_WTB(sve2_uaddw_d, uint64_t, uint32_t, , H1_4, DO_ADD)
+
+DO_ZZZ_WTB(sve2_usubw_h, uint16_t, uint8_t, H1_2, H1, DO_SUB)
+DO_ZZZ_WTB(sve2_usubw_s, uint32_t, uint16_t, H1_4, H1_2, DO_SUB)
+DO_ZZZ_WTB(sve2_usubw_d, uint64_t, uint32_t, , H1_4, DO_SUB)
+
+#undef DO_ZZZ_WTB
+
/* Two-operand reduction expander, controlled by a predicate.
* The difference between TYPERED and TYPERET has to do with
* sign-extension. E.g. for SMAX, TYPERED must be signed,
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 02eba872f9..a3eba11bab 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6030,3 +6030,23 @@ DO_SVE2_ZZZ_TB(UABDLT, uabdl, true, true)
DO_SVE2_ZZZ_TB(SADDLBT, saddl, false, true)
DO_SVE2_ZZZ_TB(SSUBLBT, ssubl, false, true)
DO_SVE2_ZZZ_TB(SSUBLTB, ssubl, true, false)
+
+#define DO_SVE2_ZZZ_WTB(NAME, name, SEL2) \
+static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
+{ \
+ static gen_helper_gvec_3 * const fns[4] = { \
+ NULL, gen_helper_sve2_##name##_h, \
+ gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
+ }; \
+ return do_sve2_zzw_ool(s, a, fns[a->esz], SEL2); \
+}
+
+DO_SVE2_ZZZ_WTB(SADDWB, saddw, false)
+DO_SVE2_ZZZ_WTB(SADDWT, saddw, true)
+DO_SVE2_ZZZ_WTB(SSUBWB, ssubw, false)
+DO_SVE2_ZZZ_WTB(SSUBWT, ssubw, true)
+
+DO_SVE2_ZZZ_WTB(UADDWB, uaddw, false)
+DO_SVE2_ZZZ_WTB(UADDWT, uaddw, true)
+DO_SVE2_ZZZ_WTB(USUBWB, usubw, false)
+DO_SVE2_ZZZ_WTB(USUBWT, usubw, true)
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 16/81] target/arm: Implement SVE2 integer multiply long
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (14 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 15/81] target/arm: Implement SVE2 integer add/subtract wide Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 17/81] target/arm: Implement PMULLB and PMULLT Richard Henderson
` (66 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Exclude PMULL from this category for the moment.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 15 +++++++++++++++
target/arm/sve.decode | 9 +++++++++
target/arm/sve_helper.c | 31 +++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 9 +++++++++
4 files changed, 64 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index a3cbfe0ea5..fff7316292 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2347,4 +2347,19 @@ DEF_HELPER_FLAGS_6(sve_stdd_le_zd_mte, TCG_CALL_NO_WG,
DEF_HELPER_FLAGS_6(sve_stdd_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmull_zzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmull_zzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmull_zzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_smull_zzz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_smull_zzz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_smull_zzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_umull_zzz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_umull_zzz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_umull_zzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_4(sve2_pmull_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index b9a34726d6..f39ca29320 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1196,3 +1196,12 @@ SSUBWB 01000101 .. 0 ..... 010 100 ..... ..... @rd_rn_rm
SSUBWT 01000101 .. 0 ..... 010 101 ..... ..... @rd_rn_rm
USUBWB 01000101 .. 0 ..... 010 110 ..... ..... @rd_rn_rm
USUBWT 01000101 .. 0 ..... 010 111 ..... ..... @rd_rn_rm
+
+## SVE2 integer multiply long
+
+SQDMULLB_zzz 01000101 .. 0 ..... 011 000 ..... ..... @rd_rn_rm
+SQDMULLT_zzz 01000101 .. 0 ..... 011 001 ..... ..... @rd_rn_rm
+SMULLB_zzz 01000101 .. 0 ..... 011 100 ..... ..... @rd_rn_rm
+SMULLT_zzz 01000101 .. 0 ..... 011 101 ..... ..... @rd_rn_rm
+UMULLB_zzz 01000101 .. 0 ..... 011 110 ..... ..... @rd_rn_rm
+UMULLT_zzz 01000101 .. 0 ..... 011 111 ..... ..... @rd_rn_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 75b87b1070..fbaf1f2911 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1154,6 +1154,37 @@ DO_ZZZ_TB(sve2_uabdl_h, uint16_t, uint8_t, H1_2, H1, DO_ABD)
DO_ZZZ_TB(sve2_uabdl_s, uint32_t, uint16_t, H1_4, H1_2, DO_ABD)
DO_ZZZ_TB(sve2_uabdl_d, uint64_t, uint32_t, , H1_4, DO_ABD)
+DO_ZZZ_TB(sve2_smull_zzz_h, int16_t, int8_t, H1_2, H1, DO_MUL)
+DO_ZZZ_TB(sve2_smull_zzz_s, int32_t, int16_t, H1_4, H1_2, DO_MUL)
+DO_ZZZ_TB(sve2_smull_zzz_d, int64_t, int32_t, , H1_4, DO_MUL)
+
+DO_ZZZ_TB(sve2_umull_zzz_h, uint16_t, uint8_t, H1_2, H1, DO_MUL)
+DO_ZZZ_TB(sve2_umull_zzz_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL)
+DO_ZZZ_TB(sve2_umull_zzz_d, uint64_t, uint32_t, , H1_4, DO_MUL)
+
+/* Note that the multiply cannot overflow, but the doubling can. */
+static inline int16_t do_sqdmull_h(int16_t n, int16_t m)
+{
+ int16_t val = n * m;
+ return DO_SQADD_H(val, val);
+}
+
+static inline int32_t do_sqdmull_s(int32_t n, int32_t m)
+{
+ int32_t val = n * m;
+ return DO_SQADD_S(val, val);
+}
+
+static inline int64_t do_sqdmull_d(int64_t n, int64_t m)
+{
+ int64_t val = n * m;
+ return do_sqadd_d(val, val);
+}
+
+DO_ZZZ_TB(sve2_sqdmull_zzz_h, int16_t, int8_t, H1_2, H1, do_sqdmull_h)
+DO_ZZZ_TB(sve2_sqdmull_zzz_s, int32_t, int16_t, H1_4, H1_2, do_sqdmull_s)
+DO_ZZZ_TB(sve2_sqdmull_zzz_d, int64_t, int32_t, , H1_4, do_sqdmull_d)
+
#undef DO_ZZZ_TB
#define DO_ZZZ_WTB(NAME, TYPEW, TYPEN, HW, HN, OP) \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index a3eba11bab..71ea0cd0c9 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6031,6 +6031,15 @@ DO_SVE2_ZZZ_TB(SADDLBT, saddl, false, true)
DO_SVE2_ZZZ_TB(SSUBLBT, ssubl, false, true)
DO_SVE2_ZZZ_TB(SSUBLTB, ssubl, true, false)
+DO_SVE2_ZZZ_TB(SQDMULLB_zzz, sqdmull_zzz, false, false)
+DO_SVE2_ZZZ_TB(SQDMULLT_zzz, sqdmull_zzz, true, true)
+
+DO_SVE2_ZZZ_TB(SMULLB_zzz, smull_zzz, false, false)
+DO_SVE2_ZZZ_TB(SMULLT_zzz, smull_zzz, true, true)
+
+DO_SVE2_ZZZ_TB(UMULLB_zzz, umull_zzz, false, false)
+DO_SVE2_ZZZ_TB(UMULLT_zzz, umull_zzz, true, true)
+
#define DO_SVE2_ZZZ_WTB(NAME, name, SEL2) \
static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
{ \
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 17/81] target/arm: Implement PMULLB and PMULLT
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (15 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 16/81] target/arm: Implement SVE2 integer multiply long Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 18/81] target/arm: Implement SVE2 bitwise shift left long Richard Henderson
` (65 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/cpu.h | 10 ++++++++++
target/arm/helper-sve.h | 1 +
target/arm/sve.decode | 2 ++
target/arm/translate-sve.c | 22 ++++++++++++++++++++++
target/arm/vec_helper.c | 24 ++++++++++++++++++++++++
5 files changed, 59 insertions(+)
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index 6f80a85fd0..59415184db 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -3902,6 +3902,16 @@ static inline bool isar_feature_aa64_sve2(const ARMISARegisters *id)
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SVEVER) != 0;
}
+static inline bool isar_feature_aa64_sve2_aes(const ARMISARegisters *id)
+{
+ return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, AES) != 0;
+}
+
+static inline bool isar_feature_aa64_sve2_pmull128(const ARMISARegisters *id)
+{
+ return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, AES) >= 2;
+}
+
/*
* Feature tests for "does this exist in either 32-bit or 64-bit?"
*/
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index fff7316292..fd1ebe180f 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2363,3 +2363,4 @@ DEF_HELPER_FLAGS_4(sve2_umull_zzz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_umull_zzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_pmull_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_pmull_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index f39ca29320..417b11fdd5 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1201,6 +1201,8 @@ USUBWT 01000101 .. 0 ..... 010 111 ..... ..... @rd_rn_rm
SQDMULLB_zzz 01000101 .. 0 ..... 011 000 ..... ..... @rd_rn_rm
SQDMULLT_zzz 01000101 .. 0 ..... 011 001 ..... ..... @rd_rn_rm
+PMULLB 01000101 .. 0 ..... 011 010 ..... ..... @rd_rn_rm
+PMULLT 01000101 .. 0 ..... 011 011 ..... ..... @rd_rn_rm
SMULLB_zzz 01000101 .. 0 ..... 011 100 ..... ..... @rd_rn_rm
SMULLT_zzz 01000101 .. 0 ..... 011 101 ..... ..... @rd_rn_rm
UMULLB_zzz 01000101 .. 0 ..... 011 110 ..... ..... @rd_rn_rm
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 71ea0cd0c9..748730e1a8 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6040,6 +6040,28 @@ DO_SVE2_ZZZ_TB(SMULLT_zzz, smull_zzz, true, true)
DO_SVE2_ZZZ_TB(UMULLB_zzz, umull_zzz, false, false)
DO_SVE2_ZZZ_TB(UMULLT_zzz, umull_zzz, true, true)
+static bool do_trans_pmull(DisasContext *s, arg_rrr_esz *a, bool sel)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_gvec_pmull_q, gen_helper_sve2_pmull_h,
+ NULL, gen_helper_sve2_pmull_d,
+ };
+ if (a->esz == 0 && !dc_isar_feature(aa64_sve2_pmull128, s)) {
+ return false;
+ }
+ return do_sve2_zzw_ool(s, a, fns[a->esz], sel);
+}
+
+static bool trans_PMULLB(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_trans_pmull(s, a, false);
+}
+
+static bool trans_PMULLT(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_trans_pmull(s, a, true);
+}
+
#define DO_SVE2_ZZZ_WTB(NAME, name, SEL2) \
static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
{ \
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index 84e1ae3933..dcaf3926e1 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -1750,6 +1750,30 @@ void HELPER(sve2_pmull_h)(void *vd, void *vn, void *vm, uint32_t desc)
d[i] = pmull_h(nn, mm);
}
}
+
+static uint64_t pmull_d(uint64_t op1, uint64_t op2)
+{
+ uint64_t result = 0;
+ int i;
+
+ for (i = 0; i < 32; ++i) {
+ uint64_t mask = -((op1 >> i) & 1);
+ result ^= (op2 << i) & mask;
+ }
+ return result;
+}
+
+void HELPER(sve2_pmull_d)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t sel = H4(simd_data(desc));
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ uint32_t *n = vn, *m = vm;
+ uint64_t *d = vd;
+
+ for (i = 0; i < opr_sz / 8; ++i) {
+ d[i] = pmull_d(n[2 * i + sel], m[2 * i + sel]);
+ }
+}
#endif
#define DO_CMP0(NAME, TYPE, OP) \
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 18/81] target/arm: Implement SVE2 bitwise shift left long
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (16 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 17/81] target/arm: Implement PMULLB and PMULLT Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 19/81] target/arm: Implement SVE2 bitwise exclusive-or interleaved Richard Henderson
` (64 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 8 ++
target/arm/sve.decode | 8 ++
target/arm/sve_helper.c | 26 ++++++
target/arm/translate-sve.c | 159 +++++++++++++++++++++++++++++++++++++
4 files changed, 201 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index fd1ebe180f..fd7abbf3ff 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2364,3 +2364,11 @@ DEF_HELPER_FLAGS_4(sve2_umull_zzz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_pmull_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_pmull_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_sshll_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sshll_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sshll_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_ushll_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_ushll_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_ushll_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 417b11fdd5..851b336c7b 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1207,3 +1207,11 @@ SMULLB_zzz 01000101 .. 0 ..... 011 100 ..... ..... @rd_rn_rm
SMULLT_zzz 01000101 .. 0 ..... 011 101 ..... ..... @rd_rn_rm
UMULLB_zzz 01000101 .. 0 ..... 011 110 ..... ..... @rd_rn_rm
UMULLT_zzz 01000101 .. 0 ..... 011 111 ..... ..... @rd_rn_rm
+
+## SVE2 bitwise shift left long
+
+# Note bit23 == 0 is handled by esz > 0 in do_sve2_shll_tb.
+SSHLLB 01000101 .. 0 ..... 1010 00 ..... ..... @rd_rn_tszimm_shl
+SSHLLT 01000101 .. 0 ..... 1010 01 ..... ..... @rd_rn_tszimm_shl
+USHLLB 01000101 .. 0 ..... 1010 10 ..... ..... @rd_rn_tszimm_shl
+USHLLT 01000101 .. 0 ..... 1010 11 ..... ..... @rd_rn_tszimm_shl
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index fbaf1f2911..5c0ed5208a 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -625,6 +625,8 @@ DO_ZPZZ(sve2_sqrshl_zpzz_h, int16_t, H1_2, do_sqrshl_h)
DO_ZPZZ(sve2_sqrshl_zpzz_s, int32_t, H1_4, do_sqrshl_s)
DO_ZPZZ_D(sve2_sqrshl_zpzz_d, int64_t, do_sqrshl_d)
+#undef do_sqrshl_d
+
#define do_uqrshl_b(n, m) \
({ uint32_t discard; do_uqrshl_bhs(n, (int8_t)m, 8, true, &discard); })
#define do_uqrshl_h(n, m) \
@@ -639,6 +641,8 @@ DO_ZPZZ(sve2_uqrshl_zpzz_h, uint16_t, H1_2, do_uqrshl_h)
DO_ZPZZ(sve2_uqrshl_zpzz_s, uint32_t, H1_4, do_uqrshl_s)
DO_ZPZZ_D(sve2_uqrshl_zpzz_d, uint64_t, do_uqrshl_d)
+#undef do_uqrshl_d
+
#define DO_HADD_BHS(n, m) (((int64_t)n + m) >> 1)
#define DO_HADD_D(n, m) ((n >> 1) + (m >> 1) + (n & m & 1))
@@ -1217,6 +1221,28 @@ DO_ZZZ_WTB(sve2_usubw_d, uint64_t, uint32_t, , H1_4, DO_SUB)
#undef DO_ZZZ_WTB
+#define DO_ZZI_SHLL(NAME, TYPEW, TYPEN, HW, HN) \
+void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ intptr_t sel = (simd_data(desc) & 1) * sizeof(TYPEN); \
+ int shift = simd_data(desc) >> 1; \
+ for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
+ TYPEW nn = *(TYPEN *)(vn + HN(i + sel)); \
+ *(TYPEW *)(vd + HW(i)) = nn << shift; \
+ } \
+}
+
+DO_ZZI_SHLL(sve2_sshll_h, int16_t, int8_t, H1_2, H1)
+DO_ZZI_SHLL(sve2_sshll_s, int32_t, int16_t, H1_4, H1_2)
+DO_ZZI_SHLL(sve2_sshll_d, int64_t, int32_t, , H1_4)
+
+DO_ZZI_SHLL(sve2_ushll_h, uint16_t, uint8_t, H1_2, H1)
+DO_ZZI_SHLL(sve2_ushll_s, uint32_t, uint16_t, H1_4, H1_2)
+DO_ZZI_SHLL(sve2_ushll_d, uint64_t, uint32_t, , H1_4)
+
+#undef DO_ZZI_SHLL
+
/* Two-operand reduction expander, controlled by a predicate.
* The difference between TYPERED and TYPERET has to do with
* sign-extension. E.g. for SMAX, TYPERED must be signed,
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 748730e1a8..4c3c197b5d 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6081,3 +6081,162 @@ DO_SVE2_ZZZ_WTB(UADDWB, uaddw, false)
DO_SVE2_ZZZ_WTB(UADDWT, uaddw, true)
DO_SVE2_ZZZ_WTB(USUBWB, usubw, false)
DO_SVE2_ZZZ_WTB(USUBWT, usubw, true)
+
+static void gen_sshll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm)
+{
+ int top = imm & 1;
+ int shl = imm >> 1;
+ int halfbits = 4 << vece;
+
+ if (top) {
+ if (shl == halfbits) {
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(halfbits, halfbits));
+ tcg_gen_and_vec(vece, d, n, t);
+ tcg_temp_free_vec(t);
+ } else {
+ tcg_gen_sari_vec(vece, d, n, halfbits);
+ tcg_gen_shli_vec(vece, d, d, shl);
+ }
+ } else {
+ tcg_gen_shli_vec(vece, d, n, halfbits);
+ tcg_gen_sari_vec(vece, d, d, halfbits - shl);
+ }
+}
+
+static void gen_ushll_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int imm)
+{
+ int halfbits = 4 << vece;
+ int top = imm & 1;
+ int shl = (imm >> 1);
+ int shift;
+ uint64_t mask;
+
+ mask = MAKE_64BIT_MASK(0, halfbits);
+ mask <<= shl;
+ mask = dup_const(vece, mask);
+
+ shift = shl - top * halfbits;
+ if (shift < 0) {
+ tcg_gen_shri_i64(d, n, -shift);
+ } else {
+ tcg_gen_shri_i64(d, n, shift);
+ }
+ tcg_gen_andi_i64(d, d, mask);
+}
+
+static void gen_ushll16_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm)
+{
+ gen_ushll_i64(MO_16, d, n, imm);
+}
+
+static void gen_ushll32_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm)
+{
+ gen_ushll_i64(MO_32, d, n, imm);
+}
+
+static void gen_ushll64_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm)
+{
+ gen_ushll_i64(MO_64, d, n, imm);
+}
+
+static void gen_ushll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm)
+{
+ int halfbits = 4 << vece;
+ int top = imm & 1;
+ int shl = imm >> 1;
+
+ if (top) {
+ if (shl == halfbits) {
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(halfbits, halfbits));
+ tcg_gen_and_vec(vece, d, n, t);
+ tcg_temp_free_vec(t);
+ } else {
+ tcg_gen_shri_vec(vece, d, n, halfbits);
+ tcg_gen_shli_vec(vece, d, d, shl);
+ }
+ } else {
+ if (shl == 0) {
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
+ tcg_gen_and_vec(vece, d, n, t);
+ tcg_temp_free_vec(t);
+ } else {
+ tcg_gen_shli_vec(vece, d, n, halfbits);
+ tcg_gen_shri_vec(vece, d, d, halfbits - shl);
+ }
+ }
+}
+
+static bool do_sve2_shll_tb(DisasContext *s, arg_rri_esz *a,
+ bool sel, bool uns)
+{
+ static const TCGOpcode sshll_list[] = {
+ INDEX_op_shli_vec, INDEX_op_sari_vec, 0
+ };
+ static const TCGOpcode ushll_list[] = {
+ INDEX_op_shli_vec, INDEX_op_shri_vec, 0
+ };
+ static const GVecGen2i ops[2][3] = {
+ { { .fniv = gen_sshll_vec,
+ .opt_opc = sshll_list,
+ .fno = gen_helper_sve2_sshll_h,
+ .vece = MO_16 },
+ { .fniv = gen_sshll_vec,
+ .opt_opc = sshll_list,
+ .fno = gen_helper_sve2_sshll_s,
+ .vece = MO_32 },
+ { .fniv = gen_sshll_vec,
+ .opt_opc = sshll_list,
+ .fno = gen_helper_sve2_sshll_d,
+ .vece = MO_64 } },
+ { { .fni8 = gen_ushll16_i64,
+ .fniv = gen_ushll_vec,
+ .opt_opc = ushll_list,
+ .fno = gen_helper_sve2_ushll_h,
+ .vece = MO_16 },
+ { .fni8 = gen_ushll32_i64,
+ .fniv = gen_ushll_vec,
+ .opt_opc = ushll_list,
+ .fno = gen_helper_sve2_ushll_s,
+ .vece = MO_32 },
+ { .fni8 = gen_ushll64_i64,
+ .fniv = gen_ushll_vec,
+ .opt_opc = ushll_list,
+ .fno = gen_helper_sve2_ushll_d,
+ .vece = MO_64 } },
+ };
+
+ if (a->esz < 0 || a->esz > 2 || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vsz, vsz, (a->imm << 1) | sel,
+ &ops[uns][a->esz]);
+ }
+ return true;
+}
+
+static bool trans_SSHLLB(DisasContext *s, arg_rri_esz *a)
+{
+ return do_sve2_shll_tb(s, a, false, false);
+}
+
+static bool trans_SSHLLT(DisasContext *s, arg_rri_esz *a)
+{
+ return do_sve2_shll_tb(s, a, true, false);
+}
+
+static bool trans_USHLLB(DisasContext *s, arg_rri_esz *a)
+{
+ return do_sve2_shll_tb(s, a, false, true);
+}
+
+static bool trans_USHLLT(DisasContext *s, arg_rri_esz *a)
+{
+ return do_sve2_shll_tb(s, a, true, true);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 19/81] target/arm: Implement SVE2 bitwise exclusive-or interleaved
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (17 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 18/81] target/arm: Implement SVE2 bitwise shift left long Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 20/81] target/arm: Implement SVE2 bitwise permute Richard Henderson
` (63 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 5 +++++
target/arm/sve.decode | 5 +++++
target/arm/sve_helper.c | 20 ++++++++++++++++++++
target/arm/translate-sve.c | 19 +++++++++++++++++++
4 files changed, 49 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index fd7abbf3ff..cdd3cab607 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2372,3 +2372,8 @@ DEF_HELPER_FLAGS_3(sve2_sshll_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_ushll_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_ushll_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_ushll_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_eoril_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_eoril_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_eoril_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_eoril_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 851b336c7b..79d915cf5b 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1215,3 +1215,8 @@ SSHLLB 01000101 .. 0 ..... 1010 00 ..... ..... @rd_rn_tszimm_shl
SSHLLT 01000101 .. 0 ..... 1010 01 ..... ..... @rd_rn_tszimm_shl
USHLLB 01000101 .. 0 ..... 1010 10 ..... ..... @rd_rn_tszimm_shl
USHLLT 01000101 .. 0 ..... 1010 11 ..... ..... @rd_rn_tszimm_shl
+
+## SVE2 bitwise exclusive-or interleaved
+
+EORBT 01000101 .. 0 ..... 10010 0 ..... ..... @rd_rn_rm
+EORTB 01000101 .. 0 ..... 10010 1 ..... ..... @rd_rn_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 5c0ed5208a..9133d0a58d 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1221,6 +1221,26 @@ DO_ZZZ_WTB(sve2_usubw_d, uint64_t, uint32_t, , H1_4, DO_SUB)
#undef DO_ZZZ_WTB
+#define DO_ZZZ_NTB(NAME, TYPE, H, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ intptr_t sel1 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPE); \
+ intptr_t sel2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(TYPE); \
+ for (i = 0; i < opr_sz; i += 2 * sizeof(TYPE)) { \
+ TYPE nn = *(TYPE *)(vn + H(i + sel1)); \
+ TYPE mm = *(TYPE *)(vm + H(i + sel2)); \
+ *(TYPE *)(vd + H(i + sel1)) = OP(nn, mm); \
+ } \
+}
+
+DO_ZZZ_NTB(sve2_eoril_b, uint8_t, H1, DO_EOR)
+DO_ZZZ_NTB(sve2_eoril_h, uint16_t, H1_2, DO_EOR)
+DO_ZZZ_NTB(sve2_eoril_s, uint32_t, H1_4, DO_EOR)
+DO_ZZZ_NTB(sve2_eoril_d, uint64_t, , DO_EOR)
+
+#undef DO_ZZZ_NTB
+
#define DO_ZZI_SHLL(NAME, TYPEW, TYPEN, HW, HN) \
void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
{ \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 4c3c197b5d..0a194a0e8c 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6040,6 +6040,25 @@ DO_SVE2_ZZZ_TB(SMULLT_zzz, smull_zzz, true, true)
DO_SVE2_ZZZ_TB(UMULLB_zzz, umull_zzz, false, false)
DO_SVE2_ZZZ_TB(UMULLT_zzz, umull_zzz, true, true)
+static bool do_eor_tb(DisasContext *s, arg_rrr_esz *a, bool sel1)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve2_eoril_b, gen_helper_sve2_eoril_h,
+ gen_helper_sve2_eoril_s, gen_helper_sve2_eoril_d,
+ };
+ return do_sve2_zzw_ool(s, a, fns[a->esz], (!sel1 << 1) | sel1);
+}
+
+static bool trans_EORBT(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_eor_tb(s, a, false);
+}
+
+static bool trans_EORTB(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_eor_tb(s, a, true);
+}
+
static bool do_trans_pmull(DisasContext *s, arg_rrr_esz *a, bool sel)
{
static gen_helper_gvec_3 * const fns[4] = {
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 20/81] target/arm: Implement SVE2 bitwise permute
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (18 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 19/81] target/arm: Implement SVE2 bitwise exclusive-or interleaved Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 21/81] target/arm: Implement SVE2 complex integer add Richard Henderson
` (62 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/cpu.h | 5 +++
target/arm/helper-sve.h | 15 ++++++++
target/arm/sve.decode | 6 ++++
target/arm/sve_helper.c | 73 ++++++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 36 +++++++++++++++++++
5 files changed, 135 insertions(+)
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index 59415184db..5ed2f2c65b 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -3912,6 +3912,11 @@ static inline bool isar_feature_aa64_sve2_pmull128(const ARMISARegisters *id)
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, AES) >= 2;
}
+static inline bool isar_feature_aa64_sve2_bitperm(const ARMISARegisters *id)
+{
+ return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, BITPERM) != 0;
+}
+
/*
* Feature tests for "does this exist in either 32-bit or 64-bit?"
*/
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index cdd3cab607..492079578d 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2377,3 +2377,18 @@ DEF_HELPER_FLAGS_4(sve2_eoril_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_eoril_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_eoril_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_eoril_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_bext_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bext_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bext_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bext_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_bdep_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bdep_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bdep_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bdep_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_bgrp_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bgrp_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bgrp_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bgrp_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 79d915cf5b..b316610bbb 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1220,3 +1220,9 @@ USHLLT 01000101 .. 0 ..... 1010 11 ..... ..... @rd_rn_tszimm_shl
EORBT 01000101 .. 0 ..... 10010 0 ..... ..... @rd_rn_rm
EORTB 01000101 .. 0 ..... 10010 1 ..... ..... @rd_rn_rm
+
+## SVE2 bitwise permute
+
+BEXT 01000101 .. 0 ..... 1011 00 ..... ..... @rd_rn_rm
+BDEP 01000101 .. 0 ..... 1011 01 ..... ..... @rd_rn_rm
+BGRP 01000101 .. 0 ..... 1011 10 ..... ..... @rd_rn_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 9133d0a58d..fffd3ca4a5 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1241,6 +1241,79 @@ DO_ZZZ_NTB(sve2_eoril_d, uint64_t, , DO_EOR)
#undef DO_ZZZ_NTB
+#define DO_BITPERM(NAME, TYPE, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ for (i = 0; i < opr_sz; i += sizeof(TYPE)) { \
+ TYPE nn = *(TYPE *)(vn + i); \
+ TYPE mm = *(TYPE *)(vm + i); \
+ *(TYPE *)(vd + i) = OP(nn, mm, sizeof(TYPE) * 8); \
+ } \
+}
+
+static uint64_t bitextract(uint64_t data, uint64_t mask, int n)
+{
+ uint64_t res = 0;
+ int db, rb = 0;
+
+ for (db = 0; db < n; ++db) {
+ if ((mask >> db) & 1) {
+ res |= ((data >> db) & 1) << rb;
+ ++rb;
+ }
+ }
+ return res;
+}
+
+DO_BITPERM(sve2_bext_b, uint8_t, bitextract)
+DO_BITPERM(sve2_bext_h, uint16_t, bitextract)
+DO_BITPERM(sve2_bext_s, uint32_t, bitextract)
+DO_BITPERM(sve2_bext_d, uint64_t, bitextract)
+
+static uint64_t bitdeposit(uint64_t data, uint64_t mask, int n)
+{
+ uint64_t res = 0;
+ int rb, db = 0;
+
+ for (rb = 0; rb < n; ++rb) {
+ if ((mask >> rb) & 1) {
+ res |= ((data >> db) & 1) << rb;
+ ++db;
+ }
+ }
+ return res;
+}
+
+DO_BITPERM(sve2_bdep_b, uint8_t, bitdeposit)
+DO_BITPERM(sve2_bdep_h, uint16_t, bitdeposit)
+DO_BITPERM(sve2_bdep_s, uint32_t, bitdeposit)
+DO_BITPERM(sve2_bdep_d, uint64_t, bitdeposit)
+
+static uint64_t bitgroup(uint64_t data, uint64_t mask, int n)
+{
+ uint64_t resm = 0, resu = 0;
+ int db, rbm = 0, rbu = 0;
+
+ for (db = 0; db < n; ++db) {
+ uint64_t val = (data >> db) & 1;
+ if ((mask >> db) & 1) {
+ resm |= val << rbm++;
+ } else {
+ resu |= val << rbu++;
+ }
+ }
+
+ return resm | (resu << rbm);
+}
+
+DO_BITPERM(sve2_bgrp_b, uint8_t, bitgroup)
+DO_BITPERM(sve2_bgrp_h, uint16_t, bitgroup)
+DO_BITPERM(sve2_bgrp_s, uint32_t, bitgroup)
+DO_BITPERM(sve2_bgrp_d, uint64_t, bitgroup)
+
+#undef DO_BITPERM
+
#define DO_ZZI_SHLL(NAME, TYPEW, TYPEN, HW, HN) \
void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
{ \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 0a194a0e8c..c2be092370 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6259,3 +6259,39 @@ static bool trans_USHLLT(DisasContext *s, arg_rri_esz *a)
{
return do_sve2_shll_tb(s, a, true, true);
}
+
+static bool trans_BEXT(DisasContext *s, arg_rrr_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve2_bext_b, gen_helper_sve2_bext_h,
+ gen_helper_sve2_bext_s, gen_helper_sve2_bext_d,
+ };
+ if (!dc_isar_feature(aa64_sve2_bitperm, s)) {
+ return false;
+ }
+ return do_sve2_zzw_ool(s, a, fns[a->esz], 0);
+}
+
+static bool trans_BDEP(DisasContext *s, arg_rrr_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve2_bdep_b, gen_helper_sve2_bdep_h,
+ gen_helper_sve2_bdep_s, gen_helper_sve2_bdep_d,
+ };
+ if (!dc_isar_feature(aa64_sve2_bitperm, s)) {
+ return false;
+ }
+ return do_sve2_zzw_ool(s, a, fns[a->esz], 0);
+}
+
+static bool trans_BGRP(DisasContext *s, arg_rrr_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve2_bgrp_b, gen_helper_sve2_bgrp_h,
+ gen_helper_sve2_bgrp_s, gen_helper_sve2_bgrp_d,
+ };
+ if (!dc_isar_feature(aa64_sve2_bitperm, s)) {
+ return false;
+ }
+ return do_sve2_zzw_ool(s, a, fns[a->esz], 0);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 21/81] target/arm: Implement SVE2 complex integer add
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (19 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 20/81] target/arm: Implement SVE2 bitwise permute Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 22/81] target/arm: Implement SVE2 integer absolute difference and accumulate long Richard Henderson
` (61 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Fix subtraction ordering (laurent desnogues).
---
target/arm/helper-sve.h | 10 +++++++++
target/arm/sve.decode | 9 ++++++++
target/arm/sve_helper.c | 42 ++++++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 31 ++++++++++++++++++++++++++++
4 files changed, 92 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 492079578d..b399fb2576 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2392,3 +2392,13 @@ DEF_HELPER_FLAGS_4(sve2_bgrp_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bgrp_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bgrp_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_bgrp_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_cadd_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_cadd_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_cadd_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_cadd_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_sqcadd_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqcadd_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqcadd_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqcadd_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index b316610bbb..655cb5c12f 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1226,3 +1226,12 @@ EORTB 01000101 .. 0 ..... 10010 1 ..... ..... @rd_rn_rm
BEXT 01000101 .. 0 ..... 1011 00 ..... ..... @rd_rn_rm
BDEP 01000101 .. 0 ..... 1011 01 ..... ..... @rd_rn_rm
BGRP 01000101 .. 0 ..... 1011 10 ..... ..... @rd_rn_rm
+
+#### SVE2 Accumulate
+
+## SVE2 complex integer add
+
+CADD_rot90 01000101 .. 00000 0 11011 0 ..... ..... @rdn_rm
+CADD_rot270 01000101 .. 00000 0 11011 1 ..... ..... @rdn_rm
+SQCADD_rot90 01000101 .. 00000 1 11011 0 ..... ..... @rdn_rm
+SQCADD_rot270 01000101 .. 00000 1 11011 1 ..... ..... @rdn_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index fffd3ca4a5..b8541168bf 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1314,6 +1314,48 @@ DO_BITPERM(sve2_bgrp_d, uint64_t, bitgroup)
#undef DO_BITPERM
+#define DO_CADD(NAME, TYPE, H, ADD_OP, SUB_OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ int sub_r = simd_data(desc); \
+ if (sub_r) { \
+ for (i = 0; i < opr_sz; i += 2 * sizeof(TYPE)) { \
+ TYPE acc_r = *(TYPE *)(vn + H(i)); \
+ TYPE acc_i = *(TYPE *)(vn + H(i + sizeof(TYPE))); \
+ TYPE el2_r = *(TYPE *)(vm + H(i)); \
+ TYPE el2_i = *(TYPE *)(vm + H(i + sizeof(TYPE))); \
+ acc_r = ADD_OP(acc_r, el2_i); \
+ acc_i = SUB_OP(acc_i, el2_r); \
+ *(TYPE *)(vd + H(i)) = acc_r; \
+ *(TYPE *)(vd + H(i + sizeof(TYPE))) = acc_i; \
+ } \
+ } else { \
+ for (i = 0; i < opr_sz; i += 2 * sizeof(TYPE)) { \
+ TYPE acc_r = *(TYPE *)(vn + H(i)); \
+ TYPE acc_i = *(TYPE *)(vn + H(i + sizeof(TYPE))); \
+ TYPE el2_r = *(TYPE *)(vm + H(i)); \
+ TYPE el2_i = *(TYPE *)(vm + H(i + sizeof(TYPE))); \
+ acc_r = SUB_OP(acc_r, el2_i); \
+ acc_i = ADD_OP(acc_i, el2_r); \
+ *(TYPE *)(vd + H(i)) = acc_r; \
+ *(TYPE *)(vd + H(i + sizeof(TYPE))) = acc_i; \
+ } \
+ } \
+}
+
+DO_CADD(sve2_cadd_b, int8_t, H1, DO_ADD, DO_SUB)
+DO_CADD(sve2_cadd_h, int16_t, H1_2, DO_ADD, DO_SUB)
+DO_CADD(sve2_cadd_s, int32_t, H1_4, DO_ADD, DO_SUB)
+DO_CADD(sve2_cadd_d, int64_t, , DO_ADD, DO_SUB)
+
+DO_CADD(sve2_sqcadd_b, int8_t, H1, DO_SQADD_B, DO_SQSUB_B)
+DO_CADD(sve2_sqcadd_h, int16_t, H1_2, DO_SQADD_H, DO_SQSUB_H)
+DO_CADD(sve2_sqcadd_s, int32_t, H1_4, DO_SQADD_S, DO_SQSUB_S)
+DO_CADD(sve2_sqcadd_d, int64_t, , do_sqadd_d, do_sqsub_d)
+
+#undef DO_CADD
+
#define DO_ZZI_SHLL(NAME, TYPEW, TYPEN, HW, HN) \
void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
{ \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index c2be092370..95a81eb101 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6295,3 +6295,34 @@ static bool trans_BGRP(DisasContext *s, arg_rrr_esz *a)
}
return do_sve2_zzw_ool(s, a, fns[a->esz], 0);
}
+
+static bool do_cadd(DisasContext *s, arg_rrr_esz *a, bool sq, bool rot)
+{
+ static gen_helper_gvec_3 * const fns[2][4] = {
+ { gen_helper_sve2_cadd_b, gen_helper_sve2_cadd_h,
+ gen_helper_sve2_cadd_s, gen_helper_sve2_cadd_d },
+ { gen_helper_sve2_sqcadd_b, gen_helper_sve2_sqcadd_h,
+ gen_helper_sve2_sqcadd_s, gen_helper_sve2_sqcadd_d },
+ };
+ return do_sve2_zzw_ool(s, a, fns[sq][a->esz], rot);
+}
+
+static bool trans_CADD_rot90(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_cadd(s, a, false, false);
+}
+
+static bool trans_CADD_rot270(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_cadd(s, a, false, true);
+}
+
+static bool trans_SQCADD_rot90(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_cadd(s, a, true, false);
+}
+
+static bool trans_SQCADD_rot270(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_cadd(s, a, true, true);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 22/81] target/arm: Implement SVE2 integer absolute difference and accumulate long
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (20 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 21/81] target/arm: Implement SVE2 complex integer add Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 23/81] target/arm: Implement SVE2 integer add/subtract long with carry Richard Henderson
` (60 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Fix select offsetting and argument order (laurent desnogues).
---
target/arm/helper-sve.h | 14 ++++++++++
target/arm/sve.decode | 12 +++++++++
target/arm/sve_helper.c | 23 ++++++++++++++++
target/arm/translate-sve.c | 55 ++++++++++++++++++++++++++++++++++++++
4 files changed, 104 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index b399fb2576..d5dfd4edea 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2402,3 +2402,17 @@ DEF_HELPER_FLAGS_4(sve2_sqcadd_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqcadd_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqcadd_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqcadd_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_sabal_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sabal_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sabal_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_uabal_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uabal_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uabal_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 655cb5c12f..6cf09847a0 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -70,6 +70,7 @@
&rpr_s rd pg rn s
&rprr_s rd pg rn rm s
&rprr_esz rd pg rn rm esz
+&rrrr_esz rd ra rn rm esz
&rprrr_esz rd pg rn rm ra esz
&rpri_esz rd pg rn imm esz
&ptrue rd esz pat s
@@ -119,6 +120,10 @@
@rdn_i8s ........ esz:2 ...... ... imm:s8 rd:5 \
&rri_esz rn=%reg_movprfx
+# Four operand, vector element size
+@rda_rn_rm ........ esz:2 . rm:5 ... ... rn:5 rd:5 \
+ &rrrr_esz ra=%reg_movprfx
+
# Three operand with "memory" size, aka immediate left shift
@rd_rn_msz_rm ........ ... rm:5 .... imm:2 rn:5 rd:5 &rrri
@@ -1235,3 +1240,10 @@ CADD_rot90 01000101 .. 00000 0 11011 0 ..... ..... @rdn_rm
CADD_rot270 01000101 .. 00000 0 11011 1 ..... ..... @rdn_rm
SQCADD_rot90 01000101 .. 00000 1 11011 0 ..... ..... @rdn_rm
SQCADD_rot270 01000101 .. 00000 1 11011 1 ..... ..... @rdn_rm
+
+## SVE2 integer absolute difference and accumulate long
+
+SABALB 01000101 .. 0 ..... 1100 00 ..... ..... @rda_rn_rm
+SABALT 01000101 .. 0 ..... 1100 01 ..... ..... @rda_rn_rm
+UABALB 01000101 .. 0 ..... 1100 10 ..... ..... @rda_rn_rm
+UABALT 01000101 .. 0 ..... 1100 11 ..... ..... @rda_rn_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index b8541168bf..cc8450c44e 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1241,6 +1241,29 @@ DO_ZZZ_NTB(sve2_eoril_d, uint64_t, , DO_EOR)
#undef DO_ZZZ_NTB
+#define DO_ZZZW_ACC(NAME, TYPEW, TYPEN, HW, HN, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ intptr_t sel1 = simd_data(desc) * sizeof(TYPEN); \
+ for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
+ TYPEW nn = *(TYPEN *)(vn + HN(i + sel1)); \
+ TYPEW mm = *(TYPEN *)(vm + HN(i + sel1)); \
+ TYPEW aa = *(TYPEW *)(va + HW(i)); \
+ *(TYPEW *)(vd + HW(i)) = OP(nn, mm) + aa; \
+ } \
+}
+
+DO_ZZZW_ACC(sve2_sabal_h, int16_t, int8_t, H1_2, H1, DO_ABD)
+DO_ZZZW_ACC(sve2_sabal_s, int32_t, int16_t, H1_4, H1_2, DO_ABD)
+DO_ZZZW_ACC(sve2_sabal_d, int64_t, int32_t, , H1_4, DO_ABD)
+
+DO_ZZZW_ACC(sve2_uabal_h, uint16_t, uint8_t, H1_2, H1, DO_ABD)
+DO_ZZZW_ACC(sve2_uabal_s, uint32_t, uint16_t, H1_4, H1_2, DO_ABD)
+DO_ZZZW_ACC(sve2_uabal_d, uint64_t, uint32_t, , H1_4, DO_ABD)
+
+#undef DO_ZZZW_ACC
+
#define DO_BITPERM(NAME, TYPE, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
{ \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 95a81eb101..7b3720e8ef 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -163,6 +163,18 @@ static void gen_gvec_ool_zzz(DisasContext *s, gen_helper_gvec_3 *fn,
vsz, vsz, data, fn);
}
+/* Invoke an out-of-line helper on 4 Zregs. */
+static void gen_gvec_ool_zzzz(DisasContext *s, gen_helper_gvec_4 *fn,
+ int rd, int rn, int rm, int ra, int data)
+{
+ unsigned vsz = vec_full_reg_size(s);
+ tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
+ vec_full_reg_offset(s, rn),
+ vec_full_reg_offset(s, rm),
+ vec_full_reg_offset(s, ra),
+ vsz, vsz, data, fn);
+}
+
/* Invoke an out-of-line helper on 2 Zregs and a predicate. */
static void gen_gvec_ool_zzp(DisasContext *s, gen_helper_gvec_3 *fn,
int rd, int rn, int pg, int data)
@@ -6326,3 +6338,46 @@ static bool trans_SQCADD_rot270(DisasContext *s, arg_rrr_esz *a)
{
return do_cadd(s, a, true, true);
}
+
+static bool do_sve2_zzzz_ool(DisasContext *s, arg_rrrr_esz *a,
+ gen_helper_gvec_4 *fn, int data)
+{
+ if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data);
+ }
+ return true;
+}
+
+static bool do_abal(DisasContext *s, arg_rrrr_esz *a, bool uns, bool sel)
+{
+ static gen_helper_gvec_4 * const fns[2][4] = {
+ { NULL, gen_helper_sve2_sabal_h,
+ gen_helper_sve2_sabal_s, gen_helper_sve2_sabal_d },
+ { NULL, gen_helper_sve2_uabal_h,
+ gen_helper_sve2_uabal_s, gen_helper_sve2_uabal_d },
+ };
+ return do_sve2_zzzz_ool(s, a, fns[uns][a->esz], sel);
+}
+
+static bool trans_SABALB(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_abal(s, a, false, false);
+}
+
+static bool trans_SABALT(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_abal(s, a, false, true);
+}
+
+static bool trans_UABALB(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_abal(s, a, true, false);
+}
+
+static bool trans_UABALT(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_abal(s, a, true, true);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 23/81] target/arm: Implement SVE2 integer add/subtract long with carry
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (21 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 22/81] target/arm: Implement SVE2 integer absolute difference and accumulate long Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 24/81] target/arm: Implement SVE2 bitwise shift right and accumulate Richard Henderson
` (59 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Fix sel indexing and argument order (laurent desnogues).
---
target/arm/helper-sve.h | 3 +++
target/arm/sve.decode | 6 ++++++
target/arm/sve_helper.c | 34 ++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 23 +++++++++++++++++++++++
4 files changed, 66 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index d5dfd4edea..a806973221 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2416,3 +2416,6 @@ DEF_HELPER_FLAGS_5(sve2_uabal_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_uabal_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_adcl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_adcl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 6cf09847a0..f4f0c2ade6 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1247,3 +1247,9 @@ SABALB 01000101 .. 0 ..... 1100 00 ..... ..... @rda_rn_rm
SABALT 01000101 .. 0 ..... 1100 01 ..... ..... @rda_rn_rm
UABALB 01000101 .. 0 ..... 1100 10 ..... ..... @rda_rn_rm
UABALT 01000101 .. 0 ..... 1100 11 ..... ..... @rda_rn_rm
+
+## SVE2 integer add/subtract long with carry
+
+# ADC and SBC decoded via size in helper dispatch.
+ADCLB 01000101 .. 0 ..... 11010 0 ..... ..... @rda_rn_rm
+ADCLT 01000101 .. 0 ..... 11010 1 ..... ..... @rda_rn_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index cc8450c44e..a7ba9e71d5 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1264,6 +1264,40 @@ DO_ZZZW_ACC(sve2_uabal_d, uint64_t, uint32_t, , H1_4, DO_ABD)
#undef DO_ZZZW_ACC
+void HELPER(sve2_adcl_s)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int sel = H4(extract32(desc, SIMD_DATA_SHIFT, 1));
+ uint32_t inv = -extract32(desc, SIMD_DATA_SHIFT + 1, 1);
+ uint32_t *a = va, *n = vn;
+ uint64_t *d = vd, *m = vm;
+
+ for (i = 0; i < opr_sz / 8; ++i) {
+ uint32_t e1 = a[2 * i + H4(0)];
+ uint32_t e2 = n[2 * i + sel] ^ inv;
+ uint64_t c = extract64(m[i], 32, 1);
+ /* Compute and store the entire 33-bit result at once. */
+ d[i] = c + e1 + e2;
+ }
+}
+
+void HELPER(sve2_adcl_d)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int sel = extract32(desc, SIMD_DATA_SHIFT, 1);
+ uint64_t inv = -(uint64_t)extract32(desc, SIMD_DATA_SHIFT + 1, 1);
+ uint64_t *d = vd, *a = va, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz / 8; i += 2) {
+ Int128 e1 = int128_make64(a[i]);
+ Int128 e2 = int128_make64(n[i + sel] ^ inv);
+ Int128 c = int128_make64(m[i + 1] & 1);
+ Int128 r = int128_add(int128_add(e1, e2), c);
+ d[i + 0] = int128_getlo(r);
+ d[i + 1] = int128_gethi(r);
+ }
+}
+
#define DO_BITPERM(NAME, TYPE, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
{ \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 7b3720e8ef..4e23b6cedd 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6381,3 +6381,26 @@ static bool trans_UABALT(DisasContext *s, arg_rrrr_esz *a)
{
return do_abal(s, a, true, true);
}
+
+static bool do_adcl(DisasContext *s, arg_rrrr_esz *a, bool sel)
+{
+ static gen_helper_gvec_4 * const fns[2] = {
+ gen_helper_sve2_adcl_s,
+ gen_helper_sve2_adcl_d,
+ };
+ /*
+ * Note that in this case the ESZ field encodes both size and sign.
+ * Split out 'subtract' into bit 1 of the data field for the helper.
+ */
+ return do_sve2_zzzz_ool(s, a, fns[a->esz & 1], (a->esz & 2) | sel);
+}
+
+static bool trans_ADCLB(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_adcl(s, a, false);
+}
+
+static bool trans_ADCLT(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_adcl(s, a, true);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 24/81] target/arm: Implement SVE2 bitwise shift right and accumulate
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (22 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 23/81] target/arm: Implement SVE2 integer add/subtract long with carry Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 25/81] target/arm: Implement SVE2 bitwise shift and insert Richard Henderson
` (58 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve.decode | 8 ++++++++
target/arm/translate-sve.c | 34 ++++++++++++++++++++++++++++++++++
2 files changed, 42 insertions(+)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index f4f0c2ade6..7783e9f0d3 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1253,3 +1253,11 @@ UABALT 01000101 .. 0 ..... 1100 11 ..... ..... @rda_rn_rm
# ADC and SBC decoded via size in helper dispatch.
ADCLB 01000101 .. 0 ..... 11010 0 ..... ..... @rda_rn_rm
ADCLT 01000101 .. 0 ..... 11010 1 ..... ..... @rda_rn_rm
+
+## SVE2 bitwise shift right and accumulate
+
+# TODO: Use @rda and %reg_movprfx here.
+SSRA 01000101 .. 0 ..... 1110 00 ..... ..... @rd_rn_tszimm_shr
+USRA 01000101 .. 0 ..... 1110 01 ..... ..... @rd_rn_tszimm_shr
+SRSRA 01000101 .. 0 ..... 1110 10 ..... ..... @rd_rn_tszimm_shr
+URSRA 01000101 .. 0 ..... 1110 11 ..... ..... @rd_rn_tszimm_shr
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 4e23b6cedd..312a47f4b9 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6404,3 +6404,37 @@ static bool trans_ADCLT(DisasContext *s, arg_rrrr_esz *a)
{
return do_adcl(s, a, true);
}
+
+static bool do_sve2_fn2i(DisasContext *s, arg_rri_esz *a, GVecGen2iFn *fn)
+{
+ if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ unsigned rd_ofs = vec_full_reg_offset(s, a->rd);
+ unsigned rn_ofs = vec_full_reg_offset(s, a->rn);
+ fn(a->esz, rd_ofs, rn_ofs, a->imm, vsz, vsz);
+ }
+ return true;
+}
+
+static bool trans_SSRA(DisasContext *s, arg_rri_esz *a)
+{
+ return do_sve2_fn2i(s, a, gen_gvec_ssra);
+}
+
+static bool trans_USRA(DisasContext *s, arg_rri_esz *a)
+{
+ return do_sve2_fn2i(s, a, gen_gvec_usra);
+}
+
+static bool trans_SRSRA(DisasContext *s, arg_rri_esz *a)
+{
+ return do_sve2_fn2i(s, a, gen_gvec_srsra);
+}
+
+static bool trans_URSRA(DisasContext *s, arg_rri_esz *a)
+{
+ return do_sve2_fn2i(s, a, gen_gvec_ursra);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 25/81] target/arm: Implement SVE2 bitwise shift and insert
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (23 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 24/81] target/arm: Implement SVE2 bitwise shift right and accumulate Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 26/81] target/arm: Implement SVE2 integer absolute difference and accumulate Richard Henderson
` (57 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve.decode | 5 +++++
target/arm/translate-sve.c | 10 ++++++++++
2 files changed, 15 insertions(+)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 7783e9f0d3..90a9d6552a 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1261,3 +1261,8 @@ SSRA 01000101 .. 0 ..... 1110 00 ..... ..... @rd_rn_tszimm_shr
USRA 01000101 .. 0 ..... 1110 01 ..... ..... @rd_rn_tszimm_shr
SRSRA 01000101 .. 0 ..... 1110 10 ..... ..... @rd_rn_tszimm_shr
URSRA 01000101 .. 0 ..... 1110 11 ..... ..... @rd_rn_tszimm_shr
+
+## SVE2 bitwise shift and insert
+
+SRI 01000101 .. 0 ..... 11110 0 ..... ..... @rd_rn_tszimm_shr
+SLI 01000101 .. 0 ..... 11110 1 ..... ..... @rd_rn_tszimm_shl
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 312a47f4b9..33e575ce72 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6438,3 +6438,13 @@ static bool trans_URSRA(DisasContext *s, arg_rri_esz *a)
{
return do_sve2_fn2i(s, a, gen_gvec_ursra);
}
+
+static bool trans_SRI(DisasContext *s, arg_rri_esz *a)
+{
+ return do_sve2_fn2i(s, a, gen_gvec_sri);
+}
+
+static bool trans_SLI(DisasContext *s, arg_rri_esz *a)
+{
+ return do_sve2_fn2i(s, a, gen_gvec_sli);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 26/81] target/arm: Implement SVE2 integer absolute difference and accumulate
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (24 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 25/81] target/arm: Implement SVE2 bitwise shift and insert Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 27/81] target/arm: Implement SVE2 saturating extract narrow Richard Henderson
` (56 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve.decode | 6 ++++++
target/arm/translate-sve.c | 21 +++++++++++++++++++++
2 files changed, 27 insertions(+)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 90a9d6552a..b5450b1d4d 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1266,3 +1266,9 @@ URSRA 01000101 .. 0 ..... 1110 11 ..... ..... @rd_rn_tszimm_shr
SRI 01000101 .. 0 ..... 11110 0 ..... ..... @rd_rn_tszimm_shr
SLI 01000101 .. 0 ..... 11110 1 ..... ..... @rd_rn_tszimm_shl
+
+## SVE2 integer absolute difference and accumulate
+
+# TODO: Use @rda and %reg_movprfx here.
+SABA 01000101 .. 0 ..... 11111 0 ..... ..... @rd_rn_rm
+UABA 01000101 .. 0 ..... 11111 1 ..... ..... @rd_rn_rm
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 33e575ce72..e21f078af7 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6448,3 +6448,24 @@ static bool trans_SLI(DisasContext *s, arg_rri_esz *a)
{
return do_sve2_fn2i(s, a, gen_gvec_sli);
}
+
+static bool do_sve2_fn_zzz(DisasContext *s, arg_rrr_esz *a, GVecGen3Fn *fn)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_fn_zzz(s, fn, a->esz, a->rd, a->rn, a->rm);
+ }
+ return true;
+}
+
+static bool trans_SABA(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_sve2_fn_zzz(s, a, gen_gvec_saba);
+}
+
+static bool trans_UABA(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_sve2_fn_zzz(s, a, gen_gvec_uaba);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 27/81] target/arm: Implement SVE2 saturating extract narrow
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (25 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 26/81] target/arm: Implement SVE2 integer absolute difference and accumulate Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 28/81] target/arm: Implement SVE2 floating-point pairwise Richard Henderson
` (55 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 24 ++++
target/arm/sve.decode | 12 ++
target/arm/sve_helper.c | 56 +++++++++
target/arm/translate-sve.c | 248 ++++++++++++++++++++++++++++++++++++-
4 files changed, 335 insertions(+), 5 deletions(-)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index a806973221..8abc06f358 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2419,3 +2419,27 @@ DEF_HELPER_FLAGS_5(sve2_uabal_d, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(sve2_adcl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_adcl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_sqxtnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_uqxtnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqxtnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqxtnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_sqxtunb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtunb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtunb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_sqxtnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_uqxtnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqxtnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqxtnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_sqxtunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index b5450b1d4d..733f9a3db4 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1272,3 +1272,15 @@ SLI 01000101 .. 0 ..... 11110 1 ..... ..... @rd_rn_tszimm_shl
# TODO: Use @rda and %reg_movprfx here.
SABA 01000101 .. 0 ..... 11111 0 ..... ..... @rd_rn_rm
UABA 01000101 .. 0 ..... 11111 1 ..... ..... @rd_rn_rm
+
+#### SVE2 Narrowing
+
+## SVE2 saturating extract narrow
+
+# Bits 23, 18-16 are zero, limited in the translator via esz < 3 & imm == 0.
+SQXTNB 01000101 .. 1 ..... 010 000 ..... ..... @rd_rn_tszimm_shl
+SQXTNT 01000101 .. 1 ..... 010 001 ..... ..... @rd_rn_tszimm_shl
+UQXTNB 01000101 .. 1 ..... 010 010 ..... ..... @rd_rn_tszimm_shl
+UQXTNT 01000101 .. 1 ..... 010 011 ..... ..... @rd_rn_tszimm_shl
+SQXTUNB 01000101 .. 1 ..... 010 100 ..... ..... @rd_rn_tszimm_shl
+SQXTUNT 01000101 .. 1 ..... 010 101 ..... ..... @rd_rn_tszimm_shl
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index a7ba9e71d5..07c3e8f02a 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1264,6 +1264,62 @@ DO_ZZZW_ACC(sve2_uabal_d, uint64_t, uint32_t, , H1_4, DO_ABD)
#undef DO_ZZZW_ACC
+#define DO_XTNB(NAME, TYPE, OP) \
+void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ for (i = 0; i < opr_sz; i += sizeof(TYPE)) { \
+ TYPE nn = *(TYPE *)(vn + i); \
+ nn = OP(nn) & MAKE_64BIT_MASK(0, sizeof(TYPE) * 4); \
+ *(TYPE *)(vd + i) = nn; \
+ } \
+}
+
+#define DO_XTNT(NAME, TYPE, TYPEN, H, OP) \
+void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc), odd = H(sizeof(TYPEN)); \
+ for (i = 0; i < opr_sz; i += sizeof(TYPE)) { \
+ TYPE nn = *(TYPE *)(vn + i); \
+ *(TYPEN *)(vd + i + odd) = OP(nn); \
+ } \
+}
+
+#define DO_SQXTN_H(n) do_sat_bhs(n, INT8_MIN, INT8_MAX)
+#define DO_SQXTN_S(n) do_sat_bhs(n, INT16_MIN, INT16_MAX)
+#define DO_SQXTN_D(n) do_sat_bhs(n, INT32_MIN, INT32_MAX)
+
+DO_XTNB(sve2_sqxtnb_h, int16_t, DO_SQXTN_H)
+DO_XTNB(sve2_sqxtnb_s, int32_t, DO_SQXTN_S)
+DO_XTNB(sve2_sqxtnb_d, int64_t, DO_SQXTN_D)
+
+DO_XTNT(sve2_sqxtnt_h, int16_t, int8_t, H1, DO_SQXTN_H)
+DO_XTNT(sve2_sqxtnt_s, int32_t, int16_t, H1_2, DO_SQXTN_S)
+DO_XTNT(sve2_sqxtnt_d, int64_t, int32_t, H1_4, DO_SQXTN_D)
+
+#define DO_UQXTN_H(n) do_sat_bhs(n, 0, UINT8_MAX)
+#define DO_UQXTN_S(n) do_sat_bhs(n, 0, UINT16_MAX)
+#define DO_UQXTN_D(n) do_sat_bhs(n, 0, UINT32_MAX)
+
+DO_XTNB(sve2_uqxtnb_h, uint16_t, DO_UQXTN_H)
+DO_XTNB(sve2_uqxtnb_s, uint32_t, DO_UQXTN_S)
+DO_XTNB(sve2_uqxtnb_d, uint64_t, DO_UQXTN_D)
+
+DO_XTNT(sve2_uqxtnt_h, uint16_t, uint8_t, H1, DO_UQXTN_H)
+DO_XTNT(sve2_uqxtnt_s, uint32_t, uint16_t, H1_2, DO_UQXTN_S)
+DO_XTNT(sve2_uqxtnt_d, uint64_t, uint32_t, H1_4, DO_UQXTN_D)
+
+DO_XTNB(sve2_sqxtunb_h, int16_t, DO_UQXTN_H)
+DO_XTNB(sve2_sqxtunb_s, int32_t, DO_UQXTN_S)
+DO_XTNB(sve2_sqxtunb_d, int64_t, DO_UQXTN_D)
+
+DO_XTNT(sve2_sqxtunt_h, int16_t, int8_t, H1, DO_UQXTN_H)
+DO_XTNT(sve2_sqxtunt_s, int32_t, int16_t, H1_2, DO_UQXTN_S)
+DO_XTNT(sve2_sqxtunt_d, int64_t, int32_t, H1_4, DO_UQXTN_D)
+
+#undef DO_XTNB
+#undef DO_XTNT
+
void HELPER(sve2_adcl_s)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
{
intptr_t i, opr_sz = simd_oprsz(desc);
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index e21f078af7..6b29d2580b 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6203,22 +6203,22 @@ static void gen_ushll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm)
static bool do_sve2_shll_tb(DisasContext *s, arg_rri_esz *a,
bool sel, bool uns)
{
- static const TCGOpcode sshll_list[] = {
+ static const TCGOpcode sshll_list[] = {
INDEX_op_shli_vec, INDEX_op_sari_vec, 0
};
- static const TCGOpcode ushll_list[] = {
+ static const TCGOpcode ushll_list[] = {
INDEX_op_shli_vec, INDEX_op_shri_vec, 0
};
static const GVecGen2i ops[2][3] = {
- { { .fniv = gen_sshll_vec,
+ { { .fniv = gen_sshll_vec,
.opt_opc = sshll_list,
.fno = gen_helper_sve2_sshll_h,
.vece = MO_16 },
- { .fniv = gen_sshll_vec,
+ { .fniv = gen_sshll_vec,
.opt_opc = sshll_list,
.fno = gen_helper_sve2_sshll_s,
.vece = MO_32 },
- { .fniv = gen_sshll_vec,
+ { .fniv = gen_sshll_vec,
.opt_opc = sshll_list,
.fno = gen_helper_sve2_sshll_d,
.vece = MO_64 } },
@@ -6469,3 +6469,241 @@ static bool trans_UABA(DisasContext *s, arg_rrr_esz *a)
{
return do_sve2_fn_zzz(s, a, gen_gvec_uaba);
}
+
+static bool do_sve2_narrow_extract(DisasContext *s, arg_rri_esz *a,
+ const GVecGen2 ops[3])
+{
+ if (a->esz < 0 || a->esz > MO_32 || a->imm != 0 ||
+ !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ tcg_gen_gvec_2(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vsz, vsz, &ops[a->esz]);
+ }
+ return true;
+}
+
+static const TCGOpcode sqxtn_list[] = {
+ INDEX_op_shli_vec, INDEX_op_smin_vec, INDEX_op_smax_vec, 0
+};
+
+static void gen_sqxtnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
+{
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ int halfbits = 4 << vece;
+ int64_t mask = (1ull << halfbits) - 1;
+ int64_t min = -1ull << (halfbits - 1);
+ int64_t max = -min - 1;
+
+ tcg_gen_dupi_vec(vece, t, min);
+ tcg_gen_smax_vec(vece, d, n, t);
+ tcg_gen_dupi_vec(vece, t, max);
+ tcg_gen_smin_vec(vece, d, d, t);
+ tcg_gen_dupi_vec(vece, t, mask);
+ tcg_gen_and_vec(vece, d, d, t);
+ tcg_temp_free_vec(t);
+}
+
+static bool trans_SQXTNB(DisasContext *s, arg_rri_esz *a)
+{
+ static const GVecGen2 ops[3] = {
+ { .fniv = gen_sqxtnb_vec,
+ .opt_opc = sqxtn_list,
+ .fno = gen_helper_sve2_sqxtnb_h,
+ .vece = MO_16 },
+ { .fniv = gen_sqxtnb_vec,
+ .opt_opc = sqxtn_list,
+ .fno = gen_helper_sve2_sqxtnb_s,
+ .vece = MO_32 },
+ { .fniv = gen_sqxtnb_vec,
+ .opt_opc = sqxtn_list,
+ .fno = gen_helper_sve2_sqxtnb_d,
+ .vece = MO_64 },
+ };
+ return do_sve2_narrow_extract(s, a, ops);
+}
+
+static void gen_sqxtnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
+{
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ int halfbits = 4 << vece;
+ int64_t mask = (1ull << halfbits) - 1;
+ int64_t min = -1ull << (halfbits - 1);
+ int64_t max = -min - 1;
+
+ tcg_gen_dupi_vec(vece, t, min);
+ tcg_gen_smax_vec(vece, n, n, t);
+ tcg_gen_dupi_vec(vece, t, max);
+ tcg_gen_smin_vec(vece, n, n, t);
+ tcg_gen_shli_vec(vece, n, n, halfbits);
+ tcg_gen_dupi_vec(vece, t, mask);
+ tcg_gen_bitsel_vec(vece, d, t, d, n);
+ tcg_temp_free_vec(t);
+}
+
+static bool trans_SQXTNT(DisasContext *s, arg_rri_esz *a)
+{
+ static const GVecGen2 ops[3] = {
+ { .fniv = gen_sqxtnt_vec,
+ .opt_opc = sqxtn_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_sqxtnt_h,
+ .vece = MO_16 },
+ { .fniv = gen_sqxtnt_vec,
+ .opt_opc = sqxtn_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_sqxtnt_s,
+ .vece = MO_32 },
+ { .fniv = gen_sqxtnt_vec,
+ .opt_opc = sqxtn_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_sqxtnt_d,
+ .vece = MO_64 },
+ };
+ return do_sve2_narrow_extract(s, a, ops);
+}
+
+static const TCGOpcode uqxtn_list[] = {
+ INDEX_op_shli_vec, INDEX_op_umin_vec, 0
+};
+
+static void gen_uqxtnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
+{
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ int halfbits = 4 << vece;
+ int64_t max = (1ull << halfbits) - 1;
+
+ tcg_gen_dupi_vec(vece, t, max);
+ tcg_gen_umin_vec(vece, d, n, t);
+ tcg_temp_free_vec(t);
+}
+
+static bool trans_UQXTNB(DisasContext *s, arg_rri_esz *a)
+{
+ static const GVecGen2 ops[3] = {
+ { .fniv = gen_uqxtnb_vec,
+ .opt_opc = uqxtn_list,
+ .fno = gen_helper_sve2_uqxtnb_h,
+ .vece = MO_16 },
+ { .fniv = gen_uqxtnb_vec,
+ .opt_opc = uqxtn_list,
+ .fno = gen_helper_sve2_uqxtnb_s,
+ .vece = MO_32 },
+ { .fniv = gen_uqxtnb_vec,
+ .opt_opc = uqxtn_list,
+ .fno = gen_helper_sve2_uqxtnb_d,
+ .vece = MO_64 },
+ };
+ return do_sve2_narrow_extract(s, a, ops);
+}
+
+static void gen_uqxtnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
+{
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ int halfbits = 4 << vece;
+ int64_t max = (1ull << halfbits) - 1;
+
+ tcg_gen_dupi_vec(vece, t, max);
+ tcg_gen_umin_vec(vece, n, n, t);
+ tcg_gen_shli_vec(vece, n, n, halfbits);
+ tcg_gen_bitsel_vec(vece, d, t, d, n);
+ tcg_temp_free_vec(t);
+}
+
+static bool trans_UQXTNT(DisasContext *s, arg_rri_esz *a)
+{
+ static const GVecGen2 ops[3] = {
+ { .fniv = gen_uqxtnt_vec,
+ .opt_opc = uqxtn_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_uqxtnt_h,
+ .vece = MO_16 },
+ { .fniv = gen_uqxtnt_vec,
+ .opt_opc = uqxtn_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_uqxtnt_s,
+ .vece = MO_32 },
+ { .fniv = gen_uqxtnt_vec,
+ .opt_opc = uqxtn_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_uqxtnt_d,
+ .vece = MO_64 },
+ };
+ return do_sve2_narrow_extract(s, a, ops);
+}
+
+static const TCGOpcode sqxtun_list[] = {
+ INDEX_op_shli_vec, INDEX_op_umin_vec, INDEX_op_smax_vec, 0
+};
+
+static void gen_sqxtunb_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
+{
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ int halfbits = 4 << vece;
+ int64_t max = (1ull << halfbits) - 1;
+
+ tcg_gen_dupi_vec(vece, t, 0);
+ tcg_gen_smax_vec(vece, d, n, t);
+ tcg_gen_dupi_vec(vece, t, max);
+ tcg_gen_umin_vec(vece, d, d, t);
+ tcg_temp_free_vec(t);
+}
+
+static bool trans_SQXTUNB(DisasContext *s, arg_rri_esz *a)
+{
+ static const GVecGen2 ops[3] = {
+ { .fniv = gen_sqxtunb_vec,
+ .opt_opc = sqxtun_list,
+ .fno = gen_helper_sve2_sqxtunb_h,
+ .vece = MO_16 },
+ { .fniv = gen_sqxtunb_vec,
+ .opt_opc = sqxtun_list,
+ .fno = gen_helper_sve2_sqxtunb_s,
+ .vece = MO_32 },
+ { .fniv = gen_sqxtunb_vec,
+ .opt_opc = sqxtun_list,
+ .fno = gen_helper_sve2_sqxtunb_d,
+ .vece = MO_64 },
+ };
+ return do_sve2_narrow_extract(s, a, ops);
+}
+
+static void gen_sqxtunt_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
+{
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ int halfbits = 4 << vece;
+ int64_t max = (1ull << halfbits) - 1;
+
+ tcg_gen_dupi_vec(vece, t, 0);
+ tcg_gen_smax_vec(vece, n, n, t);
+ tcg_gen_dupi_vec(vece, t, max);
+ tcg_gen_umin_vec(vece, n, n, t);
+ tcg_gen_shli_vec(vece, n, n, halfbits);
+ tcg_gen_bitsel_vec(vece, d, t, d, n);
+ tcg_temp_free_vec(t);
+}
+
+static bool trans_SQXTUNT(DisasContext *s, arg_rri_esz *a)
+{
+ static const GVecGen2 ops[3] = {
+ { .fniv = gen_sqxtunt_vec,
+ .opt_opc = sqxtun_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_sqxtunt_h,
+ .vece = MO_16 },
+ { .fniv = gen_sqxtunt_vec,
+ .opt_opc = sqxtun_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_sqxtunt_s,
+ .vece = MO_32 },
+ { .fniv = gen_sqxtunt_vec,
+ .opt_opc = sqxtun_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_sqxtunt_d,
+ .vece = MO_64 },
+ };
+ return do_sve2_narrow_extract(s, a, ops);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 28/81] target/arm: Implement SVE2 floating-point pairwise
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (26 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 27/81] target/arm: Implement SVE2 saturating extract narrow Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:36 ` [PATCH v3 29/81] target/arm: Implement SVE2 SHRN, RSHRN Richard Henderson
` (54 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Signed-off-by: Stephen Long <steplong@quicinc.com>
Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Load all inputs before writing any output (laurent desnogues)
---
target/arm/helper-sve.h | 35 +++++++++++++++++++++++++++++
target/arm/sve.decode | 8 +++++++
target/arm/sve_helper.c | 46 ++++++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 25 +++++++++++++++++++++
4 files changed, 114 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 8abc06f358..12cded4210 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2443,3 +2443,38 @@ DEF_HELPER_FLAGS_3(sve2_uqxtnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve2_fmaxnmp_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fmaxnmp_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fmaxnmp_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve2_fminnmp_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fminnmp_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fminnmp_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve2_fmaxp_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fmaxp_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fmaxp_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 733f9a3db4..54657d996a 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1284,3 +1284,11 @@ UQXTNB 01000101 .. 1 ..... 010 010 ..... ..... @rd_rn_tszimm_shl
UQXTNT 01000101 .. 1 ..... 010 011 ..... ..... @rd_rn_tszimm_shl
SQXTUNB 01000101 .. 1 ..... 010 100 ..... ..... @rd_rn_tszimm_shl
SQXTUNT 01000101 .. 1 ..... 010 101 ..... ..... @rd_rn_tszimm_shl
+
+## SVE2 floating-point pairwise operations
+
+FADDP 01100100 .. 010 00 0 100 ... ..... ..... @rdn_pg_rm
+FMAXNMP 01100100 .. 010 10 0 100 ... ..... ..... @rdn_pg_rm
+FMINNMP 01100100 .. 010 10 1 100 ... ..... ..... @rdn_pg_rm
+FMAXP 01100100 .. 010 11 0 100 ... ..... ..... @rdn_pg_rm
+FMINP 01100100 .. 010 11 1 100 ... ..... ..... @rdn_pg_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 07c3e8f02a..dd8e122765 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -890,6 +890,52 @@ DO_ZPZZ_PAIR_D(sve2_sminp_zpzz_d, int64_t, DO_MIN)
#undef DO_ZPZZ_PAIR
#undef DO_ZPZZ_PAIR_D
+#define DO_ZPZZ_PAIR_FP(NAME, TYPE, H, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, \
+ void *status, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ for (i = 0; i < opr_sz; ) { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ TYPE n0 = *(TYPE *)(vn + H(i)); \
+ TYPE m0 = *(TYPE *)(vm + H(i)); \
+ TYPE n1 = *(TYPE *)(vn + H(i + sizeof(TYPE))); \
+ TYPE m1 = *(TYPE *)(vm + H(i + sizeof(TYPE))); \
+ if (pg & 1) { \
+ *(TYPE *)(vd + H(i)) = OP(n0, n1, status); \
+ } \
+ i += sizeof(TYPE), pg >>= sizeof(TYPE); \
+ if (pg & 1) { \
+ *(TYPE *)(vd + H(i)) = OP(m0, m1, status); \
+ } \
+ i += sizeof(TYPE), pg >>= sizeof(TYPE); \
+ } while (i & 15); \
+ } \
+}
+
+DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_h, float16, H1_2, float16_add)
+DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_s, float32, H1_4, float32_add)
+DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_d, float64, , float64_add)
+
+DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_h, float16, H1_2, float16_maxnum)
+DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_s, float32, H1_4, float32_maxnum)
+DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_d, float64, , float64_maxnum)
+
+DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_h, float16, H1_2, float16_minnum)
+DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_s, float32, H1_4, float32_minnum)
+DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_d, float64, , float64_minnum)
+
+DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_h, float16, H1_2, float16_max)
+DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_s, float32, H1_4, float32_max)
+DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_d, float64, , float64_max)
+
+DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_h, float16, H1_2, float16_min)
+DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_s, float32, H1_4, float32_min)
+DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_d, float64, , float64_min)
+
+#undef DO_ZPZZ_PAIR_FP
+
/* Three-operand expander, controlled by a predicate, in which the
* third operand is "wide". That is, for D = N op M, the same 64-bit
* value of M is used with all of the narrower values of N.
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 6b29d2580b..c7f9c7b561 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6707,3 +6707,28 @@ static bool trans_SQXTUNT(DisasContext *s, arg_rri_esz *a)
};
return do_sve2_narrow_extract(s, a, ops);
}
+
+static bool do_sve2_zpzz_fp(DisasContext *s, arg_rprr_esz *a,
+ gen_helper_gvec_4_ptr *fn)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_zpzz_fp(s, a, fn);
+}
+
+#define DO_SVE2_ZPZZ_FP(NAME, name) \
+static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
+{ \
+ static gen_helper_gvec_4_ptr * const fns[4] = { \
+ NULL, gen_helper_sve2_##name##_zpzz_h, \
+ gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d \
+ }; \
+ return do_sve2_zpzz_fp(s, a, fns[a->esz]); \
+}
+
+DO_SVE2_ZPZZ_FP(FADDP, faddp)
+DO_SVE2_ZPZZ_FP(FMAXNMP, fmaxnmp)
+DO_SVE2_ZPZZ_FP(FMINNMP, fminnmp)
+DO_SVE2_ZPZZ_FP(FMAXP, fmaxp)
+DO_SVE2_ZPZZ_FP(FMINP, fminp)
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 29/81] target/arm: Implement SVE2 SHRN, RSHRN
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (27 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 28/81] target/arm: Implement SVE2 floating-point pairwise Richard Henderson
@ 2020-09-18 18:36 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 30/81] target/arm: Implement SVE2 SQSHRUN, SQRSHRUN Richard Henderson
` (53 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:36 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Fix typo in gen_shrnb_vec (laurent desnogues)
v3: Replace DO_RSHR with an inline function
---
target/arm/helper-sve.h | 16 ++++
target/arm/sve.decode | 8 ++
target/arm/sve_helper.c | 54 ++++++++++++-
target/arm/translate-sve.c | 160 +++++++++++++++++++++++++++++++++++++
4 files changed, 236 insertions(+), 2 deletions(-)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 12cded4210..f4bf3f9a40 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2444,6 +2444,22 @@ DEF_HELPER_FLAGS_3(sve2_sqxtunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqxtunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_shrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_shrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_shrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_shrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_shrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_shrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_rshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_rshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_rshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_rshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_rshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_rshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 54657d996a..7cc4b6cc43 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1285,6 +1285,14 @@ UQXTNT 01000101 .. 1 ..... 010 011 ..... ..... @rd_rn_tszimm_shl
SQXTUNB 01000101 .. 1 ..... 010 100 ..... ..... @rd_rn_tszimm_shl
SQXTUNT 01000101 .. 1 ..... 010 101 ..... ..... @rd_rn_tszimm_shl
+## SVE2 bitwise shift right narrow
+
+# Bit 23 == 0 is handled by esz > 0 in the translator.
+SHRNB 01000101 .. 1 ..... 00 0100 ..... ..... @rd_rn_tszimm_shr
+SHRNT 01000101 .. 1 ..... 00 0101 ..... ..... @rd_rn_tszimm_shr
+RSHRNB 01000101 .. 1 ..... 00 0110 ..... ..... @rd_rn_tszimm_shr
+RSHRNT 01000101 .. 1 ..... 00 0111 ..... ..... @rd_rn_tszimm_shr
+
## SVE2 floating-point pairwise operations
FADDP 01100100 .. 010 00 0 100 ... ..... ..... @rdn_pg_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index dd8e122765..27907ed1a5 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1862,6 +1862,17 @@ void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc) \
when N is negative, add 2**M-1. */
#define DO_ASRD(N, M) ((N + (N < 0 ? ((__typeof(N))1 << M) - 1 : 0)) >> M)
+static inline uint64_t do_urshr(uint64_t x, unsigned sh)
+{
+ if (likely(sh < 64)) {
+ return (x >> sh) + ((x >> (sh - 1)) & 1);
+ } else if (sh == 64) {
+ return x >> 63;
+ } else {
+ return 0;
+ }
+}
+
DO_ZPZI(sve_asr_zpzi_b, int8_t, H1, DO_SHR)
DO_ZPZI(sve_asr_zpzi_h, int16_t, H1_2, DO_SHR)
DO_ZPZI(sve_asr_zpzi_s, int32_t, H1_4, DO_SHR)
@@ -1882,12 +1893,51 @@ DO_ZPZI(sve_asrd_h, int16_t, H1_2, DO_ASRD)
DO_ZPZI(sve_asrd_s, int32_t, H1_4, DO_ASRD)
DO_ZPZI_D(sve_asrd_d, int64_t, DO_ASRD)
-#undef DO_SHR
-#undef DO_SHL
#undef DO_ASRD
#undef DO_ZPZI
#undef DO_ZPZI_D
+#define DO_SHRNB(NAME, TYPEW, TYPEN, OP) \
+void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ int shift = simd_data(desc); \
+ for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
+ TYPEW nn = *(TYPEW *)(vn + i); \
+ *(TYPEW *)(vd + i) = (TYPEN)OP(nn, shift); \
+ } \
+}
+
+#define DO_SHRNT(NAME, TYPEW, TYPEN, HW, HN, OP) \
+void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ int shift = simd_data(desc); \
+ for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
+ TYPEW nn = *(TYPEW *)(vn + HW(i)); \
+ *(TYPEN *)(vd + HN(i + sizeof(TYPEN))) = OP(nn, shift); \
+ } \
+}
+
+DO_SHRNB(sve2_shrnb_h, uint16_t, uint8_t, DO_SHR)
+DO_SHRNB(sve2_shrnb_s, uint32_t, uint16_t, DO_SHR)
+DO_SHRNB(sve2_shrnb_d, uint64_t, uint32_t, DO_SHR)
+
+DO_SHRNT(sve2_shrnt_h, uint16_t, uint8_t, H1_2, H1, DO_SHR)
+DO_SHRNT(sve2_shrnt_s, uint32_t, uint16_t, H1_4, H1_2, DO_SHR)
+DO_SHRNT(sve2_shrnt_d, uint64_t, uint32_t, , H1_4, DO_SHR)
+
+DO_SHRNB(sve2_rshrnb_h, uint16_t, uint8_t, do_urshr)
+DO_SHRNB(sve2_rshrnb_s, uint32_t, uint16_t, do_urshr)
+DO_SHRNB(sve2_rshrnb_d, uint64_t, uint32_t, do_urshr)
+
+DO_SHRNT(sve2_rshrnt_h, uint16_t, uint8_t, H1_2, H1, do_urshr)
+DO_SHRNT(sve2_rshrnt_s, uint32_t, uint16_t, H1_4, H1_2, do_urshr)
+DO_SHRNT(sve2_rshrnt_d, uint64_t, uint32_t, , H1_4, do_urshr)
+
+#undef DO_SHRNB
+#undef DO_SHRNT
+
/* Fully general four-operand expander, controlled by a predicate.
*/
#define DO_ZPZZZ(NAME, TYPE, H, OP) \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index c7f9c7b561..ad31066296 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6708,6 +6708,166 @@ static bool trans_SQXTUNT(DisasContext *s, arg_rri_esz *a)
return do_sve2_narrow_extract(s, a, ops);
}
+static bool do_sve2_shr_narrow(DisasContext *s, arg_rri_esz *a,
+ const GVecGen2i ops[3])
+{
+ if (a->esz < 0 || a->esz > MO_32 || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ assert(a->imm > 0 && a->imm <= (8 << a->esz));
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vsz, vsz, a->imm, &ops[a->esz]);
+ }
+ return true;
+}
+
+static void gen_shrnb_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int shr)
+{
+ int halfbits = 4 << vece;
+ uint64_t mask = dup_const(vece, MAKE_64BIT_MASK(0, halfbits));
+
+ tcg_gen_shri_i64(d, n, shr);
+ tcg_gen_andi_i64(d, d, mask);
+}
+
+static void gen_shrnb16_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
+{
+ gen_shrnb_i64(MO_16, d, n, shr);
+}
+
+static void gen_shrnb32_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
+{
+ gen_shrnb_i64(MO_32, d, n, shr);
+}
+
+static void gen_shrnb64_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
+{
+ gen_shrnb_i64(MO_64, d, n, shr);
+}
+
+static void gen_shrnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr)
+{
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ int halfbits = 4 << vece;
+ uint64_t mask = MAKE_64BIT_MASK(0, halfbits);
+
+ tcg_gen_shri_vec(vece, n, n, shr);
+ tcg_gen_dupi_vec(vece, t, mask);
+ tcg_gen_and_vec(vece, d, n, t);
+ tcg_temp_free_vec(t);
+}
+
+static bool trans_SHRNB(DisasContext *s, arg_rri_esz *a)
+{
+ static const TCGOpcode vec_list[] = { INDEX_op_shri_vec, 0 };
+ static const GVecGen2i ops[3] = {
+ { .fni8 = gen_shrnb16_i64,
+ .fniv = gen_shrnb_vec,
+ .opt_opc = vec_list,
+ .fno = gen_helper_sve2_shrnb_h,
+ .vece = MO_16 },
+ { .fni8 = gen_shrnb32_i64,
+ .fniv = gen_shrnb_vec,
+ .opt_opc = vec_list,
+ .fno = gen_helper_sve2_shrnb_s,
+ .vece = MO_32 },
+ { .fni8 = gen_shrnb64_i64,
+ .fniv = gen_shrnb_vec,
+ .opt_opc = vec_list,
+ .fno = gen_helper_sve2_shrnb_d,
+ .vece = MO_64 },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
+static void gen_shrnt_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int shr)
+{
+ int halfbits = 4 << vece;
+ uint64_t mask = dup_const(vece, MAKE_64BIT_MASK(0, halfbits));
+
+ tcg_gen_shli_i64(n, n, halfbits - shr);
+ tcg_gen_andi_i64(n, n, ~mask);
+ tcg_gen_andi_i64(d, d, mask);
+ tcg_gen_or_i64(d, d, n);
+}
+
+static void gen_shrnt16_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
+{
+ gen_shrnt_i64(MO_16, d, n, shr);
+}
+
+static void gen_shrnt32_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
+{
+ gen_shrnt_i64(MO_32, d, n, shr);
+}
+
+static void gen_shrnt64_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
+{
+ tcg_gen_shri_i64(n, n, shr);
+ tcg_gen_deposit_i64(d, d, n, 32, 32);
+}
+
+static void gen_shrnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr)
+{
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ int halfbits = 4 << vece;
+ uint64_t mask = MAKE_64BIT_MASK(0, halfbits);
+
+ tcg_gen_shli_vec(vece, n, n, halfbits - shr);
+ tcg_gen_dupi_vec(vece, t, mask);
+ tcg_gen_bitsel_vec(vece, d, t, d, n);
+ tcg_temp_free_vec(t);
+}
+
+static bool trans_SHRNT(DisasContext *s, arg_rri_esz *a)
+{
+ static const TCGOpcode vec_list[] = { INDEX_op_shli_vec, 0 };
+ static const GVecGen2i ops[3] = {
+ { .fni8 = gen_shrnt16_i64,
+ .fniv = gen_shrnt_vec,
+ .opt_opc = vec_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_shrnt_h,
+ .vece = MO_16 },
+ { .fni8 = gen_shrnt32_i64,
+ .fniv = gen_shrnt_vec,
+ .opt_opc = vec_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_shrnt_s,
+ .vece = MO_32 },
+ { .fni8 = gen_shrnt64_i64,
+ .fniv = gen_shrnt_vec,
+ .opt_opc = vec_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_shrnt_d,
+ .vece = MO_64 },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
+static bool trans_RSHRNB(DisasContext *s, arg_rri_esz *a)
+{
+ static const GVecGen2i ops[3] = {
+ { .fno = gen_helper_sve2_rshrnb_h },
+ { .fno = gen_helper_sve2_rshrnb_s },
+ { .fno = gen_helper_sve2_rshrnb_d },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
+static bool trans_RSHRNT(DisasContext *s, arg_rri_esz *a)
+{
+ static const GVecGen2i ops[3] = {
+ { .fno = gen_helper_sve2_rshrnt_h },
+ { .fno = gen_helper_sve2_rshrnt_s },
+ { .fno = gen_helper_sve2_rshrnt_d },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
static bool do_sve2_zpzz_fp(DisasContext *s, arg_rprr_esz *a,
gen_helper_gvec_4_ptr *fn)
{
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 30/81] target/arm: Implement SVE2 SQSHRUN, SQRSHRUN
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (28 preceding siblings ...)
2020-09-18 18:36 ` [PATCH v3 29/81] target/arm: Implement SVE2 SHRN, RSHRN Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 31/81] target/arm: Implement SVE2 UQSHRN, UQRSHRN Richard Henderson
` (52 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 16 +++++++
target/arm/sve.decode | 4 ++
target/arm/sve_helper.c | 35 ++++++++++++++
target/arm/translate-sve.c | 98 ++++++++++++++++++++++++++++++++++++++
4 files changed, 153 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index f4bf3f9a40..8ca99b5c3c 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2460,6 +2460,22 @@ DEF_HELPER_FLAGS_3(sve2_rshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_rshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_rshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrunb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrunb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrunb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_sqshrunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_sqrshrunb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrunb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrunb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_sqrshrunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 7cc4b6cc43..cade628cfd 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1288,6 +1288,10 @@ SQXTUNT 01000101 .. 1 ..... 010 101 ..... ..... @rd_rn_tszimm_shl
## SVE2 bitwise shift right narrow
# Bit 23 == 0 is handled by esz > 0 in the translator.
+SQSHRUNB 01000101 .. 1 ..... 00 0000 ..... ..... @rd_rn_tszimm_shr
+SQSHRUNT 01000101 .. 1 ..... 00 0001 ..... ..... @rd_rn_tszimm_shr
+SQRSHRUNB 01000101 .. 1 ..... 00 0010 ..... ..... @rd_rn_tszimm_shr
+SQRSHRUNT 01000101 .. 1 ..... 00 0011 ..... ..... @rd_rn_tszimm_shr
SHRNB 01000101 .. 1 ..... 00 0100 ..... ..... @rd_rn_tszimm_shr
SHRNT 01000101 .. 1 ..... 00 0101 ..... ..... @rd_rn_tszimm_shr
RSHRNB 01000101 .. 1 ..... 00 0110 ..... ..... @rd_rn_tszimm_shr
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 27907ed1a5..df324f0498 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1873,6 +1873,16 @@ static inline uint64_t do_urshr(uint64_t x, unsigned sh)
}
}
+static inline int64_t do_srshr(int64_t x, unsigned sh)
+{
+ if (likely(sh < 64)) {
+ return (x >> sh) + ((x >> (sh - 1)) & 1);
+ } else {
+ /* Rounding the sign bit always produces 0. */
+ return 0;
+ }
+}
+
DO_ZPZI(sve_asr_zpzi_b, int8_t, H1, DO_SHR)
DO_ZPZI(sve_asr_zpzi_h, int16_t, H1_2, DO_SHR)
DO_ZPZI(sve_asr_zpzi_s, int32_t, H1_4, DO_SHR)
@@ -1935,6 +1945,31 @@ DO_SHRNT(sve2_rshrnt_h, uint16_t, uint8_t, H1_2, H1, do_urshr)
DO_SHRNT(sve2_rshrnt_s, uint32_t, uint16_t, H1_4, H1_2, do_urshr)
DO_SHRNT(sve2_rshrnt_d, uint64_t, uint32_t, , H1_4, do_urshr)
+#define DO_SQSHRUN_H(x, sh) do_sat_bhs((int64_t)(x) >> sh, 0, UINT8_MAX)
+#define DO_SQSHRUN_S(x, sh) do_sat_bhs((int64_t)(x) >> sh, 0, UINT16_MAX)
+#define DO_SQSHRUN_D(x, sh) \
+ do_sat_bhs((int64_t)(x) >> (sh < 64 ? sh : 63), 0, UINT32_MAX)
+
+DO_SHRNB(sve2_sqshrunb_h, int16_t, uint8_t, DO_SQSHRUN_H)
+DO_SHRNB(sve2_sqshrunb_s, int32_t, uint16_t, DO_SQSHRUN_S)
+DO_SHRNB(sve2_sqshrunb_d, int64_t, uint32_t, DO_SQSHRUN_D)
+
+DO_SHRNT(sve2_sqshrunt_h, int16_t, uint8_t, H1_2, H1, DO_SQSHRUN_H)
+DO_SHRNT(sve2_sqshrunt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQSHRUN_S)
+DO_SHRNT(sve2_sqshrunt_d, int64_t, uint32_t, , H1_4, DO_SQSHRUN_D)
+
+#define DO_SQRSHRUN_H(x, sh) do_sat_bhs(do_srshr(x, sh), 0, UINT8_MAX)
+#define DO_SQRSHRUN_S(x, sh) do_sat_bhs(do_srshr(x, sh), 0, UINT16_MAX)
+#define DO_SQRSHRUN_D(x, sh) do_sat_bhs(do_srshr(x, sh), 0, UINT32_MAX)
+
+DO_SHRNB(sve2_sqrshrunb_h, int16_t, uint8_t, DO_SQRSHRUN_H)
+DO_SHRNB(sve2_sqrshrunb_s, int32_t, uint16_t, DO_SQRSHRUN_S)
+DO_SHRNB(sve2_sqrshrunb_d, int64_t, uint32_t, DO_SQRSHRUN_D)
+
+DO_SHRNT(sve2_sqrshrunt_h, int16_t, uint8_t, H1_2, H1, DO_SQRSHRUN_H)
+DO_SHRNT(sve2_sqrshrunt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQRSHRUN_S)
+DO_SHRNT(sve2_sqrshrunt_d, int64_t, uint32_t, , H1_4, DO_SQRSHRUN_D)
+
#undef DO_SHRNB
#undef DO_SHRNT
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index ad31066296..3751544e95 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6868,6 +6868,104 @@ static bool trans_RSHRNT(DisasContext *s, arg_rri_esz *a)
return do_sve2_shr_narrow(s, a, ops);
}
+static void gen_sqshrunb_vec(unsigned vece, TCGv_vec d,
+ TCGv_vec n, int64_t shr)
+{
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ int halfbits = 4 << vece;
+
+ tcg_gen_sari_vec(vece, n, n, shr);
+ tcg_gen_dupi_vec(vece, t, 0);
+ tcg_gen_smax_vec(vece, n, n, t);
+ tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
+ tcg_gen_umin_vec(vece, d, n, t);
+ tcg_temp_free_vec(t);
+}
+
+static bool trans_SQSHRUNB(DisasContext *s, arg_rri_esz *a)
+{
+ static const TCGOpcode vec_list[] = {
+ INDEX_op_sari_vec, INDEX_op_smax_vec, INDEX_op_umin_vec, 0
+ };
+ static const GVecGen2i ops[3] = {
+ { .fniv = gen_sqshrunb_vec,
+ .opt_opc = vec_list,
+ .fno = gen_helper_sve2_sqshrunb_h,
+ .vece = MO_16 },
+ { .fniv = gen_sqshrunb_vec,
+ .opt_opc = vec_list,
+ .fno = gen_helper_sve2_sqshrunb_s,
+ .vece = MO_32 },
+ { .fniv = gen_sqshrunb_vec,
+ .opt_opc = vec_list,
+ .fno = gen_helper_sve2_sqshrunb_d,
+ .vece = MO_64 },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
+static void gen_sqshrunt_vec(unsigned vece, TCGv_vec d,
+ TCGv_vec n, int64_t shr)
+{
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ int halfbits = 4 << vece;
+
+ tcg_gen_sari_vec(vece, n, n, shr);
+ tcg_gen_dupi_vec(vece, t, 0);
+ tcg_gen_smax_vec(vece, n, n, t);
+ tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
+ tcg_gen_umin_vec(vece, n, n, t);
+ tcg_gen_shli_vec(vece, n, n, halfbits);
+ tcg_gen_bitsel_vec(vece, d, t, d, n);
+ tcg_temp_free_vec(t);
+}
+
+static bool trans_SQSHRUNT(DisasContext *s, arg_rri_esz *a)
+{
+ static const TCGOpcode vec_list[] = {
+ INDEX_op_shli_vec, INDEX_op_sari_vec,
+ INDEX_op_smax_vec, INDEX_op_umin_vec, 0
+ };
+ static const GVecGen2i ops[3] = {
+ { .fniv = gen_sqshrunt_vec,
+ .opt_opc = vec_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_sqshrunt_h,
+ .vece = MO_16 },
+ { .fniv = gen_sqshrunt_vec,
+ .opt_opc = vec_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_sqshrunt_s,
+ .vece = MO_32 },
+ { .fniv = gen_sqshrunt_vec,
+ .opt_opc = vec_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_sqshrunt_d,
+ .vece = MO_64 },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
+static bool trans_SQRSHRUNB(DisasContext *s, arg_rri_esz *a)
+{
+ static const GVecGen2i ops[3] = {
+ { .fno = gen_helper_sve2_sqrshrunb_h },
+ { .fno = gen_helper_sve2_sqrshrunb_s },
+ { .fno = gen_helper_sve2_sqrshrunb_d },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
+static bool trans_SQRSHRUNT(DisasContext *s, arg_rri_esz *a)
+{
+ static const GVecGen2i ops[3] = {
+ { .fno = gen_helper_sve2_sqrshrunt_h },
+ { .fno = gen_helper_sve2_sqrshrunt_s },
+ { .fno = gen_helper_sve2_sqrshrunt_d },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
static bool do_sve2_zpzz_fp(DisasContext *s, arg_rprr_esz *a,
gen_helper_gvec_4_ptr *fn)
{
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 31/81] target/arm: Implement SVE2 UQSHRN, UQRSHRN
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (29 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 30/81] target/arm: Implement SVE2 SQSHRUN, SQRSHRUN Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 32/81] target/arm: Implement SVE2 SQSHRN, SQRSHRN Richard Henderson
` (51 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 16 +++++++
target/arm/sve.decode | 4 ++
target/arm/sve_helper.c | 24 ++++++++++
target/arm/translate-sve.c | 93 ++++++++++++++++++++++++++++++++++++++
4 files changed, 137 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 8ca99b5c3c..3a13064980 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2476,6 +2476,22 @@ DEF_HELPER_FLAGS_3(sve2_sqrshrunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_uqshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_uqrshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqrshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqrshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_uqrshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqrshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqrshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index cade628cfd..69915398e7 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1296,6 +1296,10 @@ SHRNB 01000101 .. 1 ..... 00 0100 ..... ..... @rd_rn_tszimm_shr
SHRNT 01000101 .. 1 ..... 00 0101 ..... ..... @rd_rn_tszimm_shr
RSHRNB 01000101 .. 1 ..... 00 0110 ..... ..... @rd_rn_tszimm_shr
RSHRNT 01000101 .. 1 ..... 00 0111 ..... ..... @rd_rn_tszimm_shr
+UQSHRNB 01000101 .. 1 ..... 00 1100 ..... ..... @rd_rn_tszimm_shr
+UQSHRNT 01000101 .. 1 ..... 00 1101 ..... ..... @rd_rn_tszimm_shr
+UQRSHRNB 01000101 .. 1 ..... 00 1110 ..... ..... @rd_rn_tszimm_shr
+UQRSHRNT 01000101 .. 1 ..... 00 1111 ..... ..... @rd_rn_tszimm_shr
## SVE2 floating-point pairwise operations
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index df324f0498..c478a1f68d 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1970,6 +1970,30 @@ DO_SHRNT(sve2_sqrshrunt_h, int16_t, uint8_t, H1_2, H1, DO_SQRSHRUN_H)
DO_SHRNT(sve2_sqrshrunt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQRSHRUN_S)
DO_SHRNT(sve2_sqrshrunt_d, int64_t, uint32_t, , H1_4, DO_SQRSHRUN_D)
+#define DO_UQSHRN_H(x, sh) MIN(x >> sh, UINT8_MAX)
+#define DO_UQSHRN_S(x, sh) MIN(x >> sh, UINT16_MAX)
+#define DO_UQSHRN_D(x, sh) MIN(x >> sh, UINT32_MAX)
+
+DO_SHRNB(sve2_uqshrnb_h, uint16_t, uint8_t, DO_UQSHRN_H)
+DO_SHRNB(sve2_uqshrnb_s, uint32_t, uint16_t, DO_UQSHRN_S)
+DO_SHRNB(sve2_uqshrnb_d, uint64_t, uint32_t, DO_UQSHRN_D)
+
+DO_SHRNT(sve2_uqshrnt_h, uint16_t, uint8_t, H1_2, H1, DO_UQSHRN_H)
+DO_SHRNT(sve2_uqshrnt_s, uint32_t, uint16_t, H1_4, H1_2, DO_UQSHRN_S)
+DO_SHRNT(sve2_uqshrnt_d, uint64_t, uint32_t, , H1_4, DO_UQSHRN_D)
+
+#define DO_UQRSHRN_H(x, sh) MIN(do_urshr(x, sh), UINT8_MAX)
+#define DO_UQRSHRN_S(x, sh) MIN(do_urshr(x, sh), UINT16_MAX)
+#define DO_UQRSHRN_D(x, sh) MIN(do_urshr(x, sh), UINT32_MAX)
+
+DO_SHRNB(sve2_uqrshrnb_h, uint16_t, uint8_t, DO_UQRSHRN_H)
+DO_SHRNB(sve2_uqrshrnb_s, uint32_t, uint16_t, DO_UQRSHRN_S)
+DO_SHRNB(sve2_uqrshrnb_d, uint64_t, uint32_t, DO_UQRSHRN_D)
+
+DO_SHRNT(sve2_uqrshrnt_h, uint16_t, uint8_t, H1_2, H1, DO_UQRSHRN_H)
+DO_SHRNT(sve2_uqrshrnt_s, uint32_t, uint16_t, H1_4, H1_2, DO_UQRSHRN_S)
+DO_SHRNT(sve2_uqrshrnt_d, uint64_t, uint32_t, , H1_4, DO_UQRSHRN_D)
+
#undef DO_SHRNB
#undef DO_SHRNT
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 3751544e95..fe7c916344 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6966,6 +6966,99 @@ static bool trans_SQRSHRUNT(DisasContext *s, arg_rri_esz *a)
return do_sve2_shr_narrow(s, a, ops);
}
+static void gen_uqshrnb_vec(unsigned vece, TCGv_vec d,
+ TCGv_vec n, int64_t shr)
+{
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ int halfbits = 4 << vece;
+
+ tcg_gen_shri_vec(vece, n, n, shr);
+ tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
+ tcg_gen_umin_vec(vece, d, n, t);
+ tcg_temp_free_vec(t);
+}
+
+static bool trans_UQSHRNB(DisasContext *s, arg_rri_esz *a)
+{
+ static const TCGOpcode vec_list[] = {
+ INDEX_op_shri_vec, INDEX_op_umin_vec, 0
+ };
+ static const GVecGen2i ops[3] = {
+ { .fniv = gen_uqshrnb_vec,
+ .opt_opc = vec_list,
+ .fno = gen_helper_sve2_uqshrnb_h,
+ .vece = MO_16 },
+ { .fniv = gen_uqshrnb_vec,
+ .opt_opc = vec_list,
+ .fno = gen_helper_sve2_uqshrnb_s,
+ .vece = MO_32 },
+ { .fniv = gen_uqshrnb_vec,
+ .opt_opc = vec_list,
+ .fno = gen_helper_sve2_uqshrnb_d,
+ .vece = MO_64 },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
+static void gen_uqshrnt_vec(unsigned vece, TCGv_vec d,
+ TCGv_vec n, int64_t shr)
+{
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ int halfbits = 4 << vece;
+
+ tcg_gen_shri_vec(vece, n, n, shr);
+ tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
+ tcg_gen_umin_vec(vece, n, n, t);
+ tcg_gen_shli_vec(vece, n, n, halfbits);
+ tcg_gen_bitsel_vec(vece, d, t, d, n);
+ tcg_temp_free_vec(t);
+}
+
+static bool trans_UQSHRNT(DisasContext *s, arg_rri_esz *a)
+{
+ static const TCGOpcode vec_list[] = {
+ INDEX_op_shli_vec, INDEX_op_shri_vec, INDEX_op_umin_vec, 0
+ };
+ static const GVecGen2i ops[3] = {
+ { .fniv = gen_uqshrnt_vec,
+ .opt_opc = vec_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_uqshrnt_h,
+ .vece = MO_16 },
+ { .fniv = gen_uqshrnt_vec,
+ .opt_opc = vec_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_uqshrnt_s,
+ .vece = MO_32 },
+ { .fniv = gen_uqshrnt_vec,
+ .opt_opc = vec_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_uqshrnt_d,
+ .vece = MO_64 },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
+static bool trans_UQRSHRNB(DisasContext *s, arg_rri_esz *a)
+{
+ static const GVecGen2i ops[3] = {
+ { .fno = gen_helper_sve2_uqrshrnb_h },
+ { .fno = gen_helper_sve2_uqrshrnb_s },
+ { .fno = gen_helper_sve2_uqrshrnb_d },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
+static bool trans_UQRSHRNT(DisasContext *s, arg_rri_esz *a)
+{
+ static const GVecGen2i ops[3] = {
+ { .fno = gen_helper_sve2_uqrshrnt_h },
+ { .fno = gen_helper_sve2_uqrshrnt_s },
+ { .fno = gen_helper_sve2_uqrshrnt_d },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
static bool do_sve2_zpzz_fp(DisasContext *s, arg_rprr_esz *a,
gen_helper_gvec_4_ptr *fn)
{
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 32/81] target/arm: Implement SVE2 SQSHRN, SQRSHRN
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (30 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 31/81] target/arm: Implement SVE2 UQSHRN, UQRSHRN Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 33/81] target/arm: Implement SVE2 WHILEGT, WHILEGE, WHILEHI, WHILEHS Richard Henderson
` (50 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
This completes the section "SVE2 bitwise shift right narrow".
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 16 ++++++
target/arm/sve.decode | 4 ++
target/arm/sve_helper.c | 24 +++++++++
target/arm/translate-sve.c | 105 +++++++++++++++++++++++++++++++++++++
4 files changed, 149 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 3a13064980..04b3ae3219 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2476,6 +2476,22 @@ DEF_HELPER_FLAGS_3(sve2_sqrshrunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_sqrshrunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_sqshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_sqrshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve2_sqrshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_3(sve2_uqshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 69915398e7..5f76a95139 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1296,6 +1296,10 @@ SHRNB 01000101 .. 1 ..... 00 0100 ..... ..... @rd_rn_tszimm_shr
SHRNT 01000101 .. 1 ..... 00 0101 ..... ..... @rd_rn_tszimm_shr
RSHRNB 01000101 .. 1 ..... 00 0110 ..... ..... @rd_rn_tszimm_shr
RSHRNT 01000101 .. 1 ..... 00 0111 ..... ..... @rd_rn_tszimm_shr
+SQSHRNB 01000101 .. 1 ..... 00 1000 ..... ..... @rd_rn_tszimm_shr
+SQSHRNT 01000101 .. 1 ..... 00 1001 ..... ..... @rd_rn_tszimm_shr
+SQRSHRNB 01000101 .. 1 ..... 00 1010 ..... ..... @rd_rn_tszimm_shr
+SQRSHRNT 01000101 .. 1 ..... 00 1011 ..... ..... @rd_rn_tszimm_shr
UQSHRNB 01000101 .. 1 ..... 00 1100 ..... ..... @rd_rn_tszimm_shr
UQSHRNT 01000101 .. 1 ..... 00 1101 ..... ..... @rd_rn_tszimm_shr
UQRSHRNB 01000101 .. 1 ..... 00 1110 ..... ..... @rd_rn_tszimm_shr
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index c478a1f68d..2a3dc76d87 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1970,6 +1970,30 @@ DO_SHRNT(sve2_sqrshrunt_h, int16_t, uint8_t, H1_2, H1, DO_SQRSHRUN_H)
DO_SHRNT(sve2_sqrshrunt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQRSHRUN_S)
DO_SHRNT(sve2_sqrshrunt_d, int64_t, uint32_t, , H1_4, DO_SQRSHRUN_D)
+#define DO_SQSHRN_H(x, sh) do_sat_bhs(x >> sh, INT8_MIN, INT8_MAX)
+#define DO_SQSHRN_S(x, sh) do_sat_bhs(x >> sh, INT16_MIN, INT16_MAX)
+#define DO_SQSHRN_D(x, sh) do_sat_bhs(x >> sh, INT32_MIN, INT32_MAX)
+
+DO_SHRNB(sve2_sqshrnb_h, int16_t, uint8_t, DO_SQSHRN_H)
+DO_SHRNB(sve2_sqshrnb_s, int32_t, uint16_t, DO_SQSHRN_S)
+DO_SHRNB(sve2_sqshrnb_d, int64_t, uint32_t, DO_SQSHRN_D)
+
+DO_SHRNT(sve2_sqshrnt_h, int16_t, uint8_t, H1_2, H1, DO_SQSHRN_H)
+DO_SHRNT(sve2_sqshrnt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQSHRN_S)
+DO_SHRNT(sve2_sqshrnt_d, int64_t, uint32_t, , H1_4, DO_SQSHRN_D)
+
+#define DO_SQRSHRN_H(x, sh) do_sat_bhs(do_srshr(x, sh), INT8_MIN, INT8_MAX)
+#define DO_SQRSHRN_S(x, sh) do_sat_bhs(do_srshr(x, sh), INT16_MIN, INT16_MAX)
+#define DO_SQRSHRN_D(x, sh) do_sat_bhs(do_srshr(x, sh), INT32_MIN, INT32_MAX)
+
+DO_SHRNB(sve2_sqrshrnb_h, int16_t, uint8_t, DO_SQRSHRN_H)
+DO_SHRNB(sve2_sqrshrnb_s, int32_t, uint16_t, DO_SQRSHRN_S)
+DO_SHRNB(sve2_sqrshrnb_d, int64_t, uint32_t, DO_SQRSHRN_D)
+
+DO_SHRNT(sve2_sqrshrnt_h, int16_t, uint8_t, H1_2, H1, DO_SQRSHRN_H)
+DO_SHRNT(sve2_sqrshrnt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQRSHRN_S)
+DO_SHRNT(sve2_sqrshrnt_d, int64_t, uint32_t, , H1_4, DO_SQRSHRN_D)
+
#define DO_UQSHRN_H(x, sh) MIN(x >> sh, UINT8_MAX)
#define DO_UQSHRN_S(x, sh) MIN(x >> sh, UINT16_MAX)
#define DO_UQSHRN_D(x, sh) MIN(x >> sh, UINT32_MAX)
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index fe7c916344..5091c0bc67 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6966,6 +6966,111 @@ static bool trans_SQRSHRUNT(DisasContext *s, arg_rri_esz *a)
return do_sve2_shr_narrow(s, a, ops);
}
+static void gen_sqshrnb_vec(unsigned vece, TCGv_vec d,
+ TCGv_vec n, int64_t shr)
+{
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ int halfbits = 4 << vece;
+ int64_t max = MAKE_64BIT_MASK(0, halfbits - 1);
+ int64_t min = -max - 1;
+
+ tcg_gen_sari_vec(vece, n, n, shr);
+ tcg_gen_dupi_vec(vece, t, min);
+ tcg_gen_smax_vec(vece, n, n, t);
+ tcg_gen_dupi_vec(vece, t, max);
+ tcg_gen_smin_vec(vece, n, n, t);
+ tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
+ tcg_gen_and_vec(vece, d, n, t);
+ tcg_temp_free_vec(t);
+}
+
+static bool trans_SQSHRNB(DisasContext *s, arg_rri_esz *a)
+{
+ static const TCGOpcode vec_list[] = {
+ INDEX_op_sari_vec, INDEX_op_smax_vec, INDEX_op_smin_vec, 0
+ };
+ static const GVecGen2i ops[3] = {
+ { .fniv = gen_sqshrnb_vec,
+ .opt_opc = vec_list,
+ .fno = gen_helper_sve2_sqshrnb_h,
+ .vece = MO_16 },
+ { .fniv = gen_sqshrnb_vec,
+ .opt_opc = vec_list,
+ .fno = gen_helper_sve2_sqshrnb_s,
+ .vece = MO_32 },
+ { .fniv = gen_sqshrnb_vec,
+ .opt_opc = vec_list,
+ .fno = gen_helper_sve2_sqshrnb_d,
+ .vece = MO_64 },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
+static void gen_sqshrnt_vec(unsigned vece, TCGv_vec d,
+ TCGv_vec n, int64_t shr)
+{
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ int halfbits = 4 << vece;
+ int64_t max = MAKE_64BIT_MASK(0, halfbits - 1);
+ int64_t min = -max - 1;
+
+ tcg_gen_sari_vec(vece, n, n, shr);
+ tcg_gen_dupi_vec(vece, t, min);
+ tcg_gen_smax_vec(vece, n, n, t);
+ tcg_gen_dupi_vec(vece, t, max);
+ tcg_gen_smin_vec(vece, n, n, t);
+ tcg_gen_shli_vec(vece, n, n, halfbits);
+ tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
+ tcg_gen_bitsel_vec(vece, d, t, d, n);
+ tcg_temp_free_vec(t);
+}
+
+static bool trans_SQSHRNT(DisasContext *s, arg_rri_esz *a)
+{
+ static const TCGOpcode vec_list[] = {
+ INDEX_op_shli_vec, INDEX_op_sari_vec,
+ INDEX_op_smax_vec, INDEX_op_smin_vec, 0
+ };
+ static const GVecGen2i ops[3] = {
+ { .fniv = gen_sqshrnt_vec,
+ .opt_opc = vec_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_sqshrnt_h,
+ .vece = MO_16 },
+ { .fniv = gen_sqshrnt_vec,
+ .opt_opc = vec_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_sqshrnt_s,
+ .vece = MO_32 },
+ { .fniv = gen_sqshrnt_vec,
+ .opt_opc = vec_list,
+ .load_dest = true,
+ .fno = gen_helper_sve2_sqshrnt_d,
+ .vece = MO_64 },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
+static bool trans_SQRSHRNB(DisasContext *s, arg_rri_esz *a)
+{
+ static const GVecGen2i ops[3] = {
+ { .fno = gen_helper_sve2_sqrshrnb_h },
+ { .fno = gen_helper_sve2_sqrshrnb_s },
+ { .fno = gen_helper_sve2_sqrshrnb_d },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
+static bool trans_SQRSHRNT(DisasContext *s, arg_rri_esz *a)
+{
+ static const GVecGen2i ops[3] = {
+ { .fno = gen_helper_sve2_sqrshrnt_h },
+ { .fno = gen_helper_sve2_sqrshrnt_s },
+ { .fno = gen_helper_sve2_sqrshrnt_d },
+ };
+ return do_sve2_shr_narrow(s, a, ops);
+}
+
static void gen_uqshrnb_vec(unsigned vece, TCGv_vec d,
TCGv_vec n, int64_t shr)
{
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 33/81] target/arm: Implement SVE2 WHILEGT, WHILEGE, WHILEHI, WHILEHS
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (31 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 32/81] target/arm: Implement SVE2 SQSHRN, SQRSHRN Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 34/81] target/arm: Implement SVE2 WHILERW, WHILEWR Richard Henderson
` (49 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Rename the existing sve_while (less-than) helper to sve_whilel
to make room for a new sve_whileg helper for greater-than.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Use a new helper function to implement this.
---
target/arm/helper-sve.h | 3 +-
target/arm/sve.decode | 2 +-
target/arm/sve_helper.c | 38 +++++++++++++++++++++++++-
target/arm/translate-sve.c | 56 ++++++++++++++++++++++++++++----------
4 files changed, 82 insertions(+), 17 deletions(-)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 04b3ae3219..6b4ae8f98b 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -913,7 +913,8 @@ DEF_HELPER_FLAGS_4(sve_brkns, TCG_CALL_NO_RWG, i32, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve_cntp, TCG_CALL_NO_RWG, i64, ptr, ptr, i32)
-DEF_HELPER_FLAGS_3(sve_while, TCG_CALL_NO_RWG, i32, ptr, i32, i32)
+DEF_HELPER_FLAGS_3(sve_whilel, TCG_CALL_NO_RWG, i32, ptr, i32, i32)
+DEF_HELPER_FLAGS_3(sve_whileg, TCG_CALL_NO_RWG, i32, ptr, i32, i32)
DEF_HELPER_FLAGS_4(sve_subri_b, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32)
DEF_HELPER_FLAGS_4(sve_subri_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 5f76a95139..b7038f9f57 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -700,7 +700,7 @@ SINCDECP_z 00100101 .. 1010 d:1 u:1 10000 00 .... ..... @incdec2_pred
CTERM 00100101 1 sf:1 1 rm:5 001000 rn:5 ne:1 0000
# SVE integer compare scalar count and limit
-WHILE 00100101 esz:2 1 rm:5 000 sf:1 u:1 1 rn:5 eq:1 rd:4
+WHILE 00100101 esz:2 1 rm:5 000 sf:1 u:1 lt:1 rn:5 eq:1 rd:4
### SVE Integer Wide Immediate - Unpredicated Group
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 2a3dc76d87..0bb9d73cbf 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -3746,7 +3746,7 @@ uint64_t HELPER(sve_cntp)(void *vn, void *vg, uint32_t pred_desc)
return sum;
}
-uint32_t HELPER(sve_while)(void *vd, uint32_t count, uint32_t pred_desc)
+uint32_t HELPER(sve_whilel)(void *vd, uint32_t count, uint32_t pred_desc)
{
uintptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
intptr_t esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
@@ -3772,6 +3772,42 @@ uint32_t HELPER(sve_while)(void *vd, uint32_t count, uint32_t pred_desc)
return predtest_ones(d, oprsz, esz_mask);
}
+uint32_t HELPER(sve_whileg)(void *vd, uint32_t count, uint32_t pred_desc)
+{
+ uintptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+ intptr_t esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
+ uint64_t esz_mask = pred_esz_masks[esz];
+ ARMPredicateReg *d = vd;
+ intptr_t i, invcount, oprbits;
+ uint64_t bits;
+
+ if (count == 0) {
+ return do_zero(d, oprsz);
+ }
+
+ oprbits = oprsz * 8;
+ tcg_debug_assert(count <= oprbits);
+
+ bits = esz_mask;
+ if (oprbits & 63) {
+ bits &= MAKE_64BIT_MASK(0, oprbits & 63);
+ }
+
+ invcount = oprbits - count;
+ for (i = (oprsz - 1) / 8; i > invcount / 64; --i) {
+ d->p[i] = bits;
+ bits = esz_mask;
+ }
+
+ d->p[i] = bits & MAKE_64BIT_MASK(invcount & 63, 64);
+
+ while (--i >= 0) {
+ d->p[i] = 0;
+ }
+
+ return predtest_ones(d, oprsz, esz_mask);
+}
+
/* Recursive reduction on a function;
* C.f. the ARM ARM function ReducePredicated.
*
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 5091c0bc67..f1bc4c63e6 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -3121,7 +3121,14 @@ static bool trans_WHILE(DisasContext *s, arg_WHILE *a)
TCGv_ptr ptr;
unsigned desc, vsz = vec_full_reg_size(s);
TCGCond cond;
+ uint64_t maxval;
+ /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */
+ bool eq = a->eq == a->lt;
+ /* The greater-than conditions are all SVE2. */
+ if (!a->lt && !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
if (!sve_access_check(s)) {
return true;
}
@@ -3144,22 +3151,42 @@ static bool trans_WHILE(DisasContext *s, arg_WHILE *a)
*/
t0 = tcg_temp_new_i64();
t1 = tcg_temp_new_i64();
- tcg_gen_sub_i64(t0, op1, op0);
+
+ if (a->lt) {
+ tcg_gen_sub_i64(t0, op1, op0);
+ if (a->u) {
+ maxval = a->sf ? UINT64_MAX : UINT32_MAX;
+ cond = eq ? TCG_COND_LEU : TCG_COND_LTU;
+ } else {
+ maxval = a->sf ? INT64_MAX : INT32_MAX;
+ cond = eq ? TCG_COND_LE : TCG_COND_LT;
+ }
+ } else {
+ tcg_gen_sub_i64(t0, op0, op1);
+ if (a->u) {
+ maxval = 0;
+ cond = eq ? TCG_COND_GEU : TCG_COND_GTU;
+ } else {
+ maxval = a->sf ? INT64_MIN : INT32_MIN;
+ cond = eq ? TCG_COND_GE : TCG_COND_GT;
+ }
+ }
tmax = tcg_const_i64(vsz >> a->esz);
- if (a->eq) {
+ if (eq) {
/* Equality means one more iteration. */
tcg_gen_addi_i64(t0, t0, 1);
- /* If op1 is max (un)signed integer (and the only time the addition
- * above could overflow), then we produce an all-true predicate by
- * setting the count to the vector length. This is because the
- * pseudocode is described as an increment + compare loop, and the
- * max integer would always compare true.
+ /*
+ * For the less-than while, if op1 is maxval (and the only time
+ * the addition above could overflow), then we produce an all-true
+ * predicate by setting the count to the vector length. This is
+ * because the pseudocode is described as an increment + compare
+ * loop, and the maximum integer would always compare true.
+ * Similarly, the greater-than while has the same issue with the
+ * minimum integer due to the decrement + compare loop.
*/
- tcg_gen_movi_i64(t1, (a->sf
- ? (a->u ? UINT64_MAX : INT64_MAX)
- : (a->u ? UINT32_MAX : INT32_MAX)));
+ tcg_gen_movi_i64(t1, maxval);
tcg_gen_movcond_i64(TCG_COND_EQ, t0, op1, t1, tmax, t0);
}
@@ -3168,9 +3195,6 @@ static bool trans_WHILE(DisasContext *s, arg_WHILE *a)
tcg_temp_free_i64(tmax);
/* Set the count to zero if the condition is false. */
- cond = (a->u
- ? (a->eq ? TCG_COND_LEU : TCG_COND_LTU)
- : (a->eq ? TCG_COND_LE : TCG_COND_LT));
tcg_gen_movi_i64(t1, 0);
tcg_gen_movcond_i64(cond, t0, op0, op1, t0, t1);
tcg_temp_free_i64(t1);
@@ -3190,7 +3214,11 @@ static bool trans_WHILE(DisasContext *s, arg_WHILE *a)
ptr = tcg_temp_new_ptr();
tcg_gen_addi_ptr(ptr, cpu_env, pred_full_reg_offset(s, a->rd));
- gen_helper_sve_while(t2, ptr, t2, t3);
+ if (a->lt) {
+ gen_helper_sve_whilel(t2, ptr, t2, t3);
+ } else {
+ gen_helper_sve_whileg(t2, ptr, t2, t3);
+ }
do_pred_flags(t2);
tcg_temp_free_ptr(ptr);
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 34/81] target/arm: Implement SVE2 WHILERW, WHILEWR
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (32 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 33/81] target/arm: Implement SVE2 WHILEGT, WHILEGE, WHILEHI, WHILEHS Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-10-13 2:33 ` LIU Zhiwei
2020-09-18 18:37 ` [PATCH v3 35/81] target/arm: Implement SVE2 bitwise ternary operations Richard Henderson
` (48 subsequent siblings)
82 siblings, 1 reply; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Fix decodetree typo
---
target/arm/sve.decode | 3 ++
target/arm/translate-sve.c | 62 ++++++++++++++++++++++++++++++++++++++
2 files changed, 65 insertions(+)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index b7038f9f57..19d503e2f4 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -702,6 +702,9 @@ CTERM 00100101 1 sf:1 1 rm:5 001000 rn:5 ne:1 0000
# SVE integer compare scalar count and limit
WHILE 00100101 esz:2 1 rm:5 000 sf:1 u:1 lt:1 rn:5 eq:1 rd:4
+# SVE2 pointer conflict compare
+WHILE_ptr 00100101 esz:2 1 rm:5 001 100 rn:5 rw:1 rd:4
+
### SVE Integer Wide Immediate - Unpredicated Group
# SVE broadcast floating-point immediate (unpredicated)
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index f1bc4c63e6..d3241ce167 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -3227,6 +3227,68 @@ static bool trans_WHILE(DisasContext *s, arg_WHILE *a)
return true;
}
+static bool trans_WHILE_ptr(DisasContext *s, arg_WHILE_ptr *a)
+{
+ TCGv_i64 op0, op1, diff, t1, tmax;
+ TCGv_i32 t2, t3;
+ TCGv_ptr ptr;
+ unsigned desc, vsz = vec_full_reg_size(s);
+
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (!sve_access_check(s)) {
+ return true;
+ }
+
+ op0 = read_cpu_reg(s, a->rn, 1);
+ op1 = read_cpu_reg(s, a->rm, 1);
+
+ tmax = tcg_const_i64(vsz);
+ diff = tcg_temp_new_i64();
+
+ if (a->rw) {
+ /* WHILERW */
+ /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
+ t1 = tcg_temp_new_i64();
+ tcg_gen_sub_i64(diff, op0, op1);
+ tcg_gen_sub_i64(t1, op1, op0);
+ tcg_gen_movcond_i64(TCG_COND_LTU, diff, op0, op1, diff, t1);
+ tcg_temp_free_i64(t1);
+ /* If op1 == op0, diff == 0, and the condition is always true. */
+ tcg_gen_movcond_i64(TCG_COND_EQ, diff, op0, op1, tmax, diff);
+ } else {
+ /* WHILEWR */
+ tcg_gen_sub_i64(diff, op1, op0);
+ /* If op0 >= op1, diff <= 0, the condition is always true. */
+ tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, tmax, diff);
+ }
+
+ /* Bound to the maximum. */
+ tcg_gen_umin_i64(diff, diff, tmax);
+ tcg_temp_free_i64(tmax);
+
+ /* Since we're bounded, pass as a 32-bit type. */
+ t2 = tcg_temp_new_i32();
+ tcg_gen_extrl_i64_i32(t2, diff);
+ tcg_temp_free_i64(diff);
+
+ desc = (vsz / 8) - 2;
+ desc = deposit32(desc, SIMD_DATA_SHIFT, 2, a->esz);
+ t3 = tcg_const_i32(desc);
+
+ ptr = tcg_temp_new_ptr();
+ tcg_gen_addi_ptr(ptr, cpu_env, pred_full_reg_offset(s, a->rd));
+
+ gen_helper_sve_whilel(t2, ptr, t2, t3);
+ do_pred_flags(t2);
+
+ tcg_temp_free_ptr(ptr);
+ tcg_temp_free_i32(t2);
+ tcg_temp_free_i32(t3);
+ return true;
+}
+
/*
*** SVE Integer Wide Immediate - Unpredicated Group
*/
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* Re: [PATCH v3 34/81] target/arm: Implement SVE2 WHILERW, WHILEWR
2020-09-18 18:37 ` [PATCH v3 34/81] target/arm: Implement SVE2 WHILERW, WHILEWR Richard Henderson
@ 2020-10-13 2:33 ` LIU Zhiwei
2020-10-19 21:58 ` Richard Henderson
0 siblings, 1 reply; 94+ messages in thread
From: LIU Zhiwei @ 2020-10-13 2:33 UTC (permalink / raw)
To: Richard Henderson, qemu-devel; +Cc: peter.maydell, alex.bennee
[-- Attachment #1: Type: text/plain, Size: 3424 bytes --]
On 2020/9/19 2:37, Richard Henderson wrote:
> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
> ---
> v2: Fix decodetree typo
> ---
> target/arm/sve.decode | 3 ++
> target/arm/translate-sve.c | 62 ++++++++++++++++++++++++++++++++++++++
> 2 files changed, 65 insertions(+)
>
> diff --git a/target/arm/sve.decode b/target/arm/sve.decode
> index b7038f9f57..19d503e2f4 100644
> --- a/target/arm/sve.decode
> +++ b/target/arm/sve.decode
> @@ -702,6 +702,9 @@ CTERM 00100101 1 sf:1 1 rm:5 001000 rn:5 ne:1 0000
> # SVE integer compare scalar count and limit
> WHILE 00100101 esz:2 1 rm:5 000 sf:1 u:1 lt:1 rn:5 eq:1 rd:4
>
> +# SVE2 pointer conflict compare
> +WHILE_ptr 00100101 esz:2 1 rm:5 001 100 rn:5 rw:1 rd:4
> +
> ### SVE Integer Wide Immediate - Unpredicated Group
>
> # SVE broadcast floating-point immediate (unpredicated)
> diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
> index f1bc4c63e6..d3241ce167 100644
> --- a/target/arm/translate-sve.c
> +++ b/target/arm/translate-sve.c
> @@ -3227,6 +3227,68 @@ static bool trans_WHILE(DisasContext *s, arg_WHILE *a)
> return true;
> }
>
> +static bool trans_WHILE_ptr(DisasContext *s, arg_WHILE_ptr *a)
> +{
> + TCGv_i64 op0, op1, diff, t1, tmax;
> + TCGv_i32 t2, t3;
> + TCGv_ptr ptr;
> + unsigned desc, vsz = vec_full_reg_size(s);
> +
> + if (!dc_isar_feature(aa64_sve2, s)) {
> + return false;
> + }
> + if (!sve_access_check(s)) {
> + return true;
> + }
> +
> + op0 = read_cpu_reg(s, a->rn, 1);
> + op1 = read_cpu_reg(s, a->rm, 1);
> +
> + tmax = tcg_const_i64(vsz);
> + diff = tcg_temp_new_i64();
> +
> + if (a->rw) {
> + /* WHILERW */
> + /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
> + t1 = tcg_temp_new_i64();
> + tcg_gen_sub_i64(diff, op0, op1);
> + tcg_gen_sub_i64(t1, op1, op0);
> + tcg_gen_movcond_i64(TCG_COND_LTU, diff, op0, op1, diff, t1);
It should be:
tcg_gen_movcond_i64(TCG_COND_GTU, diff, op0, op1, diff, t1);
> + tcg_temp_free_i64(t1);
> + /* If op1 == op0, diff == 0, and the condition is always true. */
> + tcg_gen_movcond_i64(TCG_COND_EQ, diff, op0, op1, tmax, diff);
> + } else {
> + /* WHILEWR */
> + tcg_gen_sub_i64(diff, op1, op0);
> + /* If op0 >= op1, diff <= 0, the condition is always true. */
> + tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, tmax, diff);
> + }
> +
> + /* Bound to the maximum. */
> + tcg_gen_umin_i64(diff, diff, tmax);
> + tcg_temp_free_i64(tmax);
> +
> + /* Since we're bounded, pass as a 32-bit type. */
> + t2 = tcg_temp_new_i32();
> + tcg_gen_extrl_i64_i32(t2, diff);
We should align count down to (1 << esz),
tcg_gen_andi_i32(t2，~MAKE_64BIT_MASK(0, esz));
Best Regards,
Zhiwei
> + tcg_temp_free_i64(diff);
> +
> + desc = (vsz / 8) - 2;
> + desc = deposit32(desc, SIMD_DATA_SHIFT, 2, a->esz);
> + t3 = tcg_const_i32(desc);
> +
> + ptr = tcg_temp_new_ptr();
> + tcg_gen_addi_ptr(ptr, cpu_env, pred_full_reg_offset(s, a->rd));
> +
> + gen_helper_sve_whilel(t2, ptr, t2, t3);
> + do_pred_flags(t2);
> +
> + tcg_temp_free_ptr(ptr);
> + tcg_temp_free_i32(t2);
> + tcg_temp_free_i32(t3);
> + return true;
> +}
> +
> /*
> *** SVE Integer Wide Immediate - Unpredicated Group
> */
[-- Attachment #2: Type: text/html, Size: 4265 bytes --]
^ permalink raw reply [flat|nested] 94+ messages in thread* Re: [PATCH v3 34/81] target/arm: Implement SVE2 WHILERW, WHILEWR
2020-10-13 2:33 ` LIU Zhiwei
@ 2020-10-19 21:58 ` Richard Henderson
0 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-10-19 21:58 UTC (permalink / raw)
To: LIU Zhiwei, qemu-devel; +Cc: peter.maydell, alex.bennee
On 10/12/20 7:33 PM, LIU Zhiwei wrote:
>> + if (a->rw) {
>> + /* WHILERW */
>> + /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
>> + t1 = tcg_temp_new_i64();
>> + tcg_gen_sub_i64(diff, op0, op1);
>> + tcg_gen_sub_i64(t1, op1, op0);
>> + tcg_gen_movcond_i64(TCG_COND_LTU, diff, op0, op1, diff, t1);
> It should be:
>
> tcg_gen_movcond_i64(TCG_COND_GTU, diff, op0, op1, diff, t1);
Yep.
>
>> + tcg_temp_free_i64(t1);
>> + /* If op1 == op0, diff == 0, and the condition is always true. */
>> + tcg_gen_movcond_i64(TCG_COND_EQ, diff, op0, op1, tmax, diff);
>> + } else {
>> + /* WHILEWR */
>> + tcg_gen_sub_i64(diff, op1, op0);
>> + /* If op0 >= op1, diff <= 0, the condition is always true. */
>> + tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, tmax, diff);
>> + }
>> +
>> + /* Bound to the maximum. */
>> + tcg_gen_umin_i64(diff, diff, tmax);
>> + tcg_temp_free_i64(tmax);
>> +
>> + /* Since we're bounded, pass as a 32-bit type. */
>> + t2 = tcg_temp_new_i32();
>> + tcg_gen_extrl_i64_i32(t2, diff);
> We should align count down to (1 << esz),
>
> tcg_gen_andi_i32(t2，~MAKE_64BIT_MASK(0, esz));
Yep, this corresponds to the "DIV (esize DIV 8)" portion of the psuedo code.
But it needs to go earlier, before we compare diff against 0 in the two movcond
above.
Will fix. Thanks,
r~
^ permalink raw reply [flat|nested] 94+ messages in thread
* [PATCH v3 35/81] target/arm: Implement SVE2 bitwise ternary operations
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (33 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 34/81] target/arm: Implement SVE2 WHILERW, WHILEWR Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 36/81] target/arm: Implement SVE2 MATCH, NMATCH Richard Henderson
` (47 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 6 ++
target/arm/sve.decode | 12 +++
target/arm/sve_helper.c | 50 +++++++++
target/arm/translate-sve.c | 213 +++++++++++++++++++++++++++++++++++++
4 files changed, 281 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 6b4ae8f98b..923911ca21 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2543,3 +2543,9 @@ DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_eor3, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_bcax, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_bsl1n, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_bsl2n, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_nbsl, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 19d503e2f4..a50afd40c2 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -124,6 +124,10 @@
@rda_rn_rm ........ esz:2 . rm:5 ... ... rn:5 rd:5 \
&rrrr_esz ra=%reg_movprfx
+# Four operand with unused vector element size
+@rdn_ra_rm_e0 ........ ... rm:5 ... ... ra:5 rd:5 \
+ &rrrr_esz esz=0 rn=%reg_movprfx
+
# Three operand with "memory" size, aka immediate left shift
@rd_rn_msz_rm ........ ... rm:5 .... imm:2 rn:5 rd:5 &rrri
@@ -379,6 +383,14 @@ ORR_zzz 00000100 01 1 ..... 001 100 ..... ..... @rd_rn_rm_e0
EOR_zzz 00000100 10 1 ..... 001 100 ..... ..... @rd_rn_rm_e0
BIC_zzz 00000100 11 1 ..... 001 100 ..... ..... @rd_rn_rm_e0
+# SVE2 bitwise ternary operations
+EOR3 00000100 00 1 ..... 001 110 ..... ..... @rdn_ra_rm_e0
+BSL 00000100 00 1 ..... 001 111 ..... ..... @rdn_ra_rm_e0
+BCAX 00000100 01 1 ..... 001 110 ..... ..... @rdn_ra_rm_e0
+BSL1N 00000100 01 1 ..... 001 111 ..... ..... @rdn_ra_rm_e0
+BSL2N 00000100 10 1 ..... 001 111 ..... ..... @rdn_ra_rm_e0
+NBSL 00000100 11 1 ..... 001 111 ..... ..... @rdn_ra_rm_e0
+
### SVE Index Generation Group
# SVE index generation (immediate start, immediate increment)
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 0bb9d73cbf..aa7ff94812 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -6825,3 +6825,53 @@ DO_ST1_ZPZ_D(dd_be, zd, MO_64)
#undef DO_ST1_ZPZ_S
#undef DO_ST1_ZPZ_D
+
+void HELPER(sve2_eor3)(void *vd, void *vn, void *vm, void *vk, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+ uint64_t *d = vd, *n = vn, *m = vm, *k = vk;
+
+ for (i = 0; i < opr_sz; ++i) {
+ d[i] = n[i] ^ m[i] ^ k[i];
+ }
+}
+
+void HELPER(sve2_bcax)(void *vd, void *vn, void *vm, void *vk, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+ uint64_t *d = vd, *n = vn, *m = vm, *k = vk;
+
+ for (i = 0; i < opr_sz; ++i) {
+ d[i] = n[i] ^ (m[i] & ~k[i]);
+ }
+}
+
+void HELPER(sve2_bsl1n)(void *vd, void *vn, void *vm, void *vk, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+ uint64_t *d = vd, *n = vn, *m = vm, *k = vk;
+
+ for (i = 0; i < opr_sz; ++i) {
+ d[i] = (~n[i] & k[i]) | (m[i] & ~k[i]);
+ }
+}
+
+void HELPER(sve2_bsl2n)(void *vd, void *vn, void *vm, void *vk, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+ uint64_t *d = vd, *n = vn, *m = vm, *k = vk;
+
+ for (i = 0; i < opr_sz; ++i) {
+ d[i] = (n[i] & k[i]) | (~m[i] & ~k[i]);
+ }
+}
+
+void HELPER(sve2_nbsl)(void *vd, void *vn, void *vm, void *vk, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+ uint64_t *d = vd, *n = vn, *m = vm, *k = vk;
+
+ for (i = 0; i < opr_sz; ++i) {
+ d[i] = ~((n[i] & k[i]) | (m[i] & ~k[i]));
+ }
+}
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index d3241ce167..540e926c18 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -217,6 +217,17 @@ static void gen_gvec_fn_zzz(DisasContext *s, GVecGen3Fn *gvec_fn,
vec_full_reg_offset(s, rm), vsz, vsz);
}
+/* Invoke a vector expander on four Zregs. */
+static void gen_gvec_fn_zzzz(DisasContext *s, GVecGen4Fn *gvec_fn,
+ int esz, int rd, int rn, int rm, int ra)
+{
+ unsigned vsz = vec_full_reg_size(s);
+ gvec_fn(esz, vec_full_reg_offset(s, rd),
+ vec_full_reg_offset(s, rn),
+ vec_full_reg_offset(s, rm),
+ vec_full_reg_offset(s, ra), vsz, vsz);
+}
+
/* Invoke a vector move on two Zregs. */
static bool do_mov_z(DisasContext *s, int rd, int rn)
{
@@ -329,6 +340,208 @@ static bool trans_BIC_zzz(DisasContext *s, arg_rrr_esz *a)
return do_zzz_fn(s, a, tcg_gen_gvec_andc);
}
+static bool do_sve2_zzzz_fn(DisasContext *s, arg_rrrr_esz *a, GVecGen4Fn *fn)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_fn_zzzz(s, fn, a->esz, a->rd, a->rn, a->rm, a->ra);
+ }
+ return true;
+}
+
+static void gen_eor3_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
+{
+ tcg_gen_xor_i64(d, n, m);
+ tcg_gen_xor_i64(d, d, k);
+}
+
+static void gen_eor3_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
+ TCGv_vec m, TCGv_vec k)
+{
+ tcg_gen_xor_vec(vece, d, n, m);
+ tcg_gen_xor_vec(vece, d, d, k);
+}
+
+static void gen_eor3(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
+ uint32_t a, uint32_t oprsz, uint32_t maxsz)
+{
+ static const GVecGen4 op = {
+ .fni8 = gen_eor3_i64,
+ .fniv = gen_eor3_vec,
+ .fno = gen_helper_sve2_eor3,
+ .vece = MO_64,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ };
+ tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
+}
+
+static bool trans_EOR3(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_sve2_zzzz_fn(s, a, gen_eor3);
+}
+
+static void gen_bcax_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
+{
+ tcg_gen_andc_i64(d, m, k);
+ tcg_gen_xor_i64(d, d, n);
+}
+
+static void gen_bcax_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
+ TCGv_vec m, TCGv_vec k)
+{
+ tcg_gen_andc_vec(vece, d, m, k);
+ tcg_gen_xor_vec(vece, d, d, n);
+}
+
+static void gen_bcax(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
+ uint32_t a, uint32_t oprsz, uint32_t maxsz)
+{
+ static const GVecGen4 op = {
+ .fni8 = gen_bcax_i64,
+ .fniv = gen_bcax_vec,
+ .fno = gen_helper_sve2_bcax,
+ .vece = MO_64,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ };
+ tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
+}
+
+static bool trans_BCAX(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_sve2_zzzz_fn(s, a, gen_bcax);
+}
+
+static void gen_bsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
+ uint32_t a, uint32_t oprsz, uint32_t maxsz)
+{
+ /* BSL differs from the generic bitsel in argument ordering. */
+ tcg_gen_gvec_bitsel(vece, d, a, n, m, oprsz, maxsz);
+}
+
+static bool trans_BSL(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_sve2_zzzz_fn(s, a, gen_bsl);
+}
+
+static void gen_bsl1n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
+{
+ tcg_gen_andc_i64(n, k, n);
+ tcg_gen_andc_i64(m, m, k);
+ tcg_gen_or_i64(d, n, m);
+}
+
+static void gen_bsl1n_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
+ TCGv_vec m, TCGv_vec k)
+{
+ if (TCG_TARGET_HAS_bitsel_vec) {
+ tcg_gen_not_vec(vece, n, n);
+ tcg_gen_bitsel_vec(vece, d, k, n, m);
+ } else {
+ tcg_gen_andc_vec(vece, n, k, n);
+ tcg_gen_andc_vec(vece, m, m, k);
+ tcg_gen_or_vec(vece, d, n, m);
+ }
+}
+
+static void gen_bsl1n(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
+ uint32_t a, uint32_t oprsz, uint32_t maxsz)
+{
+ static const GVecGen4 op = {
+ .fni8 = gen_bsl1n_i64,
+ .fniv = gen_bsl1n_vec,
+ .fno = gen_helper_sve2_bsl1n,
+ .vece = MO_64,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ };
+ tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
+}
+
+static bool trans_BSL1N(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_sve2_zzzz_fn(s, a, gen_bsl1n);
+}
+
+static void gen_bsl2n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
+{
+ /*
+ * Z[dn] = (n & k) | (~m & ~k)
+ * = | ~(m | k)
+ */
+ tcg_gen_and_i64(n, n, k);
+ if (TCG_TARGET_HAS_orc_i64) {
+ tcg_gen_or_i64(m, m, k);
+ tcg_gen_orc_i64(d, n, m);
+ } else {
+ tcg_gen_nor_i64(m, m, k);
+ tcg_gen_or_i64(d, n, m);
+ }
+}
+
+static void gen_bsl2n_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
+ TCGv_vec m, TCGv_vec k)
+{
+ if (TCG_TARGET_HAS_bitsel_vec) {
+ tcg_gen_not_vec(vece, m, m);
+ tcg_gen_bitsel_vec(vece, d, k, n, m);
+ } else {
+ tcg_gen_and_vec(vece, n, n, k);
+ tcg_gen_or_vec(vece, m, m, k);
+ tcg_gen_orc_vec(vece, d, n, m);
+ }
+}
+
+static void gen_bsl2n(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
+ uint32_t a, uint32_t oprsz, uint32_t maxsz)
+{
+ static const GVecGen4 op = {
+ .fni8 = gen_bsl2n_i64,
+ .fniv = gen_bsl2n_vec,
+ .fno = gen_helper_sve2_bsl2n,
+ .vece = MO_64,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ };
+ tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
+}
+
+static bool trans_BSL2N(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_sve2_zzzz_fn(s, a, gen_bsl2n);
+}
+
+static void gen_nbsl_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
+{
+ tcg_gen_and_i64(n, n, k);
+ tcg_gen_andc_i64(m, m, k);
+ tcg_gen_nor_i64(d, n, m);
+}
+
+static void gen_nbsl_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
+ TCGv_vec m, TCGv_vec k)
+{
+ tcg_gen_bitsel_vec(vece, d, k, n, m);
+ tcg_gen_not_vec(vece, d, d);
+}
+
+static void gen_nbsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
+ uint32_t a, uint32_t oprsz, uint32_t maxsz)
+{
+ static const GVecGen4 op = {
+ .fni8 = gen_nbsl_i64,
+ .fniv = gen_nbsl_vec,
+ .fno = gen_helper_sve2_nbsl,
+ .vece = MO_64,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ };
+ tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
+}
+
+static bool trans_NBSL(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_sve2_zzzz_fn(s, a, gen_nbsl);
+}
+
/*
*** SVE Integer Arithmetic - Unpredicated Group
*/
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 36/81] target/arm: Implement SVE2 MATCH, NMATCH
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (34 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 35/81] target/arm: Implement SVE2 bitwise ternary operations Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 37/81] target/arm: Implement SVE2 saturating multiply-add long Richard Henderson
` (46 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200415145915.2859-1-steplong@quicinc.com>
[rth: Expanded comment for do_match2]
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Apply esz_mask to input pg to fix output flags.
---
target/arm/helper-sve.h | 10 ++++++
target/arm/sve.decode | 5 +++
target/arm/sve_helper.c | 64 ++++++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 22 +++++++++++++
4 files changed, 101 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 923911ca21..7f53287f0d 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2509,6 +2509,16 @@ DEF_HELPER_FLAGS_3(sve2_uqrshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqrshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqrshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_match_ppzz_b, TCG_CALL_NO_RWG,
+ i32, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_match_ppzz_h, TCG_CALL_NO_RWG,
+ i32, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_b, TCG_CALL_NO_RWG,
+ i32, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_h, TCG_CALL_NO_RWG,
+ i32, ptr, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index a50afd40c2..2207693d28 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1320,6 +1320,11 @@ UQSHRNT 01000101 .. 1 ..... 00 1101 ..... ..... @rd_rn_tszimm_shr
UQRSHRNB 01000101 .. 1 ..... 00 1110 ..... ..... @rd_rn_tszimm_shr
UQRSHRNT 01000101 .. 1 ..... 00 1111 ..... ..... @rd_rn_tszimm_shr
+### SVE2 Character Match
+
+MATCH 01000101 .. 1 ..... 100 ... ..... 0 .... @pd_pg_rn_rm
+NMATCH 01000101 .. 1 ..... 100 ... ..... 1 .... @pd_pg_rn_rm
+
## SVE2 floating-point pairwise operations
FADDP 01100100 .. 010 00 0 100 ... ..... ..... @rdn_pg_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index aa7ff94812..139dea423f 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -6875,3 +6875,67 @@ void HELPER(sve2_nbsl)(void *vd, void *vn, void *vm, void *vk, uint32_t desc)
d[i] = ~((n[i] & k[i]) | (m[i] & ~k[i]));
}
}
+
+/*
+ * Returns true if m0 or m1 contains the low uint8_t/uint16_t in n.
+ * See hasless(v,1) from
+ * https://graphics.stanford.edu/~seander/bithacks.html#ZeroInWord
+ */
+static inline bool do_match2(uint64_t n, uint64_t m0, uint64_t m1, int esz)
+{
+ int bits = 8 << esz;
+ uint64_t ones = dup_const(esz, 1);
+ uint64_t signs = ones << (bits - 1);
+ uint64_t cmp0, cmp1;
+
+ cmp1 = dup_const(esz, n);
+ cmp0 = cmp1 ^ m0;
+ cmp1 = cmp1 ^ m1;
+ cmp0 = (cmp0 - ones) & ~cmp0;
+ cmp1 = (cmp1 - ones) & ~cmp1;
+ return (cmp0 | cmp1) & signs;
+}
+
+static inline uint32_t do_match(void *vd, void *vn, void *vm, void *vg,
+ uint32_t desc, int esz, bool nmatch)
+{
+ uint16_t esz_mask = pred_esz_masks[esz];
+ intptr_t opr_sz = simd_oprsz(desc);
+ uint32_t flags = PREDTEST_INIT;
+ intptr_t i, j, k;
+
+ for (i = 0; i < opr_sz; i += 16) {
+ uint64_t m0 = *(uint64_t *)(vm + i);
+ uint64_t m1 = *(uint64_t *)(vm + i + 8);
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)) & esz_mask;
+ uint16_t out = 0;
+
+ for (j = 0; j < 16; j += 8) {
+ uint64_t n = *(uint64_t *)(vn + i + j);
+
+ for (k = 0; k < 8; k += 1 << esz) {
+ if (pg & (1 << (j + k))) {
+ bool o = do_match2(n >> (k * 8), m0, m1, esz);
+ out |= (o ^ nmatch) << (j + k);
+ }
+ }
+ }
+ *(uint16_t *)(vd + H1_2(i >> 3)) = out;
+ flags = iter_predtest_fwd(out, pg, flags);
+ }
+ return flags;
+}
+
+#define DO_PPZZ_MATCH(NAME, ESZ, INV) \
+uint32_t HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
+{ \
+ return do_match(vd, vn, vm, vg, desc, ESZ, INV); \
+}
+
+DO_PPZZ_MATCH(sve2_match_ppzz_b, MO_8, false)
+DO_PPZZ_MATCH(sve2_match_ppzz_h, MO_16, false)
+
+DO_PPZZ_MATCH(sve2_nmatch_ppzz_b, MO_8, true)
+DO_PPZZ_MATCH(sve2_nmatch_ppzz_h, MO_16, true)
+
+#undef DO_PPZZ_MATCH
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 540e926c18..ffb7f5c8a0 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -7467,6 +7467,28 @@ static bool trans_UQRSHRNT(DisasContext *s, arg_rri_esz *a)
return do_sve2_shr_narrow(s, a, ops);
}
+static bool do_sve2_ppzz_flags(DisasContext *s, arg_rprr_esz *a,
+ gen_helper_gvec_flags_4 *fn)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_ppzz_flags(s, a, fn);
+}
+
+#define DO_SVE2_PPZZ_MATCH(NAME, name) \
+static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
+{ \
+ static gen_helper_gvec_flags_4 * const fns[4] = { \
+ gen_helper_sve2_##name##_ppzz_b, gen_helper_sve2_##name##_ppzz_h, \
+ NULL, NULL \
+ }; \
+ return do_sve2_ppzz_flags(s, a, fns[a->esz]); \
+}
+
+DO_SVE2_PPZZ_MATCH(MATCH, match)
+DO_SVE2_PPZZ_MATCH(NMATCH, nmatch)
+
static bool do_sve2_zpzz_fp(DisasContext *s, arg_rprr_esz *a,
gen_helper_gvec_4_ptr *fn)
{
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 37/81] target/arm: Implement SVE2 saturating multiply-add long
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (35 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 36/81] target/arm: Implement SVE2 MATCH, NMATCH Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 38/81] target/arm: Implement SVE2 saturating multiply-add high Richard Henderson
` (45 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 14 ++++++++++
target/arm/sve.decode | 14 ++++++++++
target/arm/sve_helper.c | 30 +++++++++++++++++++++
target/arm/translate-sve.c | 54 ++++++++++++++++++++++++++++++++++++++
4 files changed, 112 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 7f53287f0d..ff5f744b9a 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2559,3 +2559,17 @@ DEF_HELPER_FLAGS_5(sve2_bcax, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_bsl1n, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_bsl2n, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_nbsl, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_sqdmlal_zzzw_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlal_zzzw_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlal_zzzw_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 2207693d28..d0d24978bb 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1332,3 +1332,17 @@ FMAXNMP 01100100 .. 010 10 0 100 ... ..... ..... @rdn_pg_rm
FMINNMP 01100100 .. 010 10 1 100 ... ..... ..... @rdn_pg_rm
FMAXP 01100100 .. 010 11 0 100 ... ..... ..... @rdn_pg_rm
FMINP 01100100 .. 010 11 1 100 ... ..... ..... @rdn_pg_rm
+
+#### SVE Integer Multiply-Add (unpredicated)
+
+## SVE2 saturating multiply-add long
+
+SQDMLALB_zzzw 01000100 .. 0 ..... 0110 00 ..... ..... @rda_rn_rm
+SQDMLALT_zzzw 01000100 .. 0 ..... 0110 01 ..... ..... @rda_rn_rm
+SQDMLSLB_zzzw 01000100 .. 0 ..... 0110 10 ..... ..... @rda_rn_rm
+SQDMLSLT_zzzw 01000100 .. 0 ..... 0110 11 ..... ..... @rda_rn_rm
+
+## SVE2 saturating multiply-add interleaved long
+
+SQDMLALBT 01000100 .. 0 ..... 00001 0 ..... ..... @rda_rn_rm
+SQDMLSLBT 01000100 .. 0 ..... 00001 1 ..... ..... @rda_rn_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 139dea423f..9822675892 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1400,6 +1400,36 @@ void HELPER(sve2_adcl_d)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
}
}
+#define DO_SQDMLAL(NAME, TYPEW, TYPEN, HW, HN, DMUL_OP, SUM_OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ int sel1 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \
+ int sel2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(TYPEN); \
+ for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
+ TYPEW nn = *(TYPEN *)(vn + HN(i + sel1)); \
+ TYPEW mm = *(TYPEN *)(vm + HN(i + sel2)); \
+ TYPEW aa = *(TYPEW *)(va + HW(i)); \
+ *(TYPEW *)(vd + HW(i)) = SUM_OP(aa, DMUL_OP(nn, mm)); \
+ } \
+}
+
+DO_SQDMLAL(sve2_sqdmlal_zzzw_h, int16_t, int8_t, H1_2, H1,
+ do_sqdmull_h, DO_SQADD_H)
+DO_SQDMLAL(sve2_sqdmlal_zzzw_s, int32_t, int16_t, H1_4, H1_2,
+ do_sqdmull_s, DO_SQADD_S)
+DO_SQDMLAL(sve2_sqdmlal_zzzw_d, int64_t, int32_t, , H1_4,
+ do_sqdmull_d, do_sqadd_d)
+
+DO_SQDMLAL(sve2_sqdmlsl_zzzw_h, int16_t, int8_t, H1_2, H1,
+ do_sqdmull_h, DO_SQSUB_H)
+DO_SQDMLAL(sve2_sqdmlsl_zzzw_s, int32_t, int16_t, H1_4, H1_2,
+ do_sqdmull_s, DO_SQSUB_S)
+DO_SQDMLAL(sve2_sqdmlsl_zzzw_d, int64_t, int32_t, , H1_4,
+ do_sqdmull_d, do_sqsub_d)
+
+#undef DO_SQDMLAL
+
#define DO_BITPERM(NAME, TYPE, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
{ \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index ffb7f5c8a0..5da4793d49 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -7513,3 +7513,57 @@ DO_SVE2_ZPZZ_FP(FMAXNMP, fmaxnmp)
DO_SVE2_ZPZZ_FP(FMINNMP, fminnmp)
DO_SVE2_ZPZZ_FP(FMAXP, fmaxp)
DO_SVE2_ZPZZ_FP(FMINP, fminp)
+
+/*
+ * SVE Integer Multiply-Add (unpredicated)
+ */
+
+static bool do_sqdmlal_zzzw(DisasContext *s, arg_rrrr_esz *a,
+ bool sel1, bool sel2)
+{
+ static gen_helper_gvec_4 * const fns[] = {
+ NULL, gen_helper_sve2_sqdmlal_zzzw_h,
+ gen_helper_sve2_sqdmlal_zzzw_s, gen_helper_sve2_sqdmlal_zzzw_d,
+ };
+ return do_sve2_zzzz_ool(s, a, fns[a->esz], (sel2 << 1) | sel1);
+}
+
+static bool do_sqdmlsl_zzzw(DisasContext *s, arg_rrrr_esz *a,
+ bool sel1, bool sel2)
+{
+ static gen_helper_gvec_4 * const fns[] = {
+ NULL, gen_helper_sve2_sqdmlsl_zzzw_h,
+ gen_helper_sve2_sqdmlsl_zzzw_s, gen_helper_sve2_sqdmlsl_zzzw_d,
+ };
+ return do_sve2_zzzz_ool(s, a, fns[a->esz], (sel2 << 1) | sel1);
+}
+
+static bool trans_SQDMLALB_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_sqdmlal_zzzw(s, a, false, false);
+}
+
+static bool trans_SQDMLALT_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_sqdmlal_zzzw(s, a, true, true);
+}
+
+static bool trans_SQDMLALBT(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_sqdmlal_zzzw(s, a, false, true);
+}
+
+static bool trans_SQDMLSLB_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_sqdmlsl_zzzw(s, a, false, false);
+}
+
+static bool trans_SQDMLSLT_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_sqdmlsl_zzzw(s, a, true, true);
+}
+
+static bool trans_SQDMLSLBT(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_sqdmlsl_zzzw(s, a, false, true);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 38/81] target/arm: Implement SVE2 saturating multiply-add high
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (36 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 37/81] target/arm: Implement SVE2 saturating multiply-add long Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 39/81] target/arm: Implement SVE2 integer multiply-add long Richard Henderson
` (44 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
SVE2 has two additional sizes of the operation and unlike NEON,
there is no saturation flag. Create new entry points for SVE2
that do not set QC.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper.h | 17 ++++
target/arm/sve.decode | 5 ++
target/arm/translate-sve.c | 18 +++++
target/arm/vec_helper.c | 161 +++++++++++++++++++++++++++++++++++--
4 files changed, 195 insertions(+), 6 deletions(-)
diff --git a/target/arm/helper.h b/target/arm/helper.h
index 94658169b2..c8eb165989 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -578,6 +578,23 @@ DEF_HELPER_FLAGS_5(gvec_qrdmlah_s32, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(gvec_qrdmlsh_s32, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlah_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlah_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlah_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlah_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_4(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index d0d24978bb..177b3cc803 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1346,3 +1346,8 @@ SQDMLSLT_zzzw 01000100 .. 0 ..... 0110 11 ..... ..... @rda_rn_rm
SQDMLALBT 01000100 .. 0 ..... 00001 0 ..... ..... @rda_rn_rm
SQDMLSLBT 01000100 .. 0 ..... 00001 1 ..... ..... @rda_rn_rm
+
+## SVE2 saturating multiply-add high
+
+SQRDMLAH_zzzz 01000100 .. 0 ..... 01110 0 ..... ..... @rda_rn_rm
+SQRDMLSH_zzzz 01000100 .. 0 ..... 01110 1 ..... ..... @rda_rn_rm
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 5da4793d49..1f75489868 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -7567,3 +7567,21 @@ static bool trans_SQDMLSLBT(DisasContext *s, arg_rrrr_esz *a)
{
return do_sqdmlsl_zzzw(s, a, false, true);
}
+
+static bool trans_SQRDMLAH_zzzz(DisasContext *s, arg_rrrr_esz *a)
+{
+ static gen_helper_gvec_4 * const fns[] = {
+ gen_helper_sve2_sqrdmlah_b, gen_helper_sve2_sqrdmlah_h,
+ gen_helper_sve2_sqrdmlah_s, gen_helper_sve2_sqrdmlah_d,
+ };
+ return do_sve2_zzzz_ool(s, a, fns[a->esz], 0);
+}
+
+static bool trans_SQRDMLSH_zzzz(DisasContext *s, arg_rrrr_esz *a)
+{
+ static gen_helper_gvec_4 * const fns[] = {
+ gen_helper_sve2_sqrdmlsh_b, gen_helper_sve2_sqrdmlsh_h,
+ gen_helper_sve2_sqrdmlsh_s, gen_helper_sve2_sqrdmlsh_d,
+ };
+ return do_sve2_zzzz_ool(s, a, fns[a->esz], 0);
+}
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index dcaf3926e1..b40d9688c5 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -22,6 +22,7 @@
#include "exec/helper-proto.h"
#include "tcg/tcg-gvec-desc.h"
#include "fpu/softfloat.h"
+#include "qemu/int128.h"
#include "vec_internal.h"
/* Note that vector data is stored in host-endian 64-bit chunks,
@@ -36,15 +37,55 @@
#define H4(x) (x)
#endif
+/* Signed saturating rounding doubling multiply-accumulate high half, 8-bit */
+static int8_t do_sqrdmlah_b(int8_t src1, int8_t src2, int8_t src3,
+ bool neg, bool round)
+{
+ /*
+ * Simplify:
+ * = ((a3 << 8) + ((e1 * e2) << 1) + (round << 7)) >> 8
+ * = ((a3 << 7) + (e1 * e2) + (round << 6)) >> 7
+ */
+ int32_t ret = (int32_t)src1 * src2;
+ if (neg) {
+ ret = -ret;
+ }
+ ret += ((int32_t)src3 << 7) + (round << 6);
+ ret >>= 7;
+
+ if (ret != (int8_t)ret) {
+ ret = (ret < 0 ? INT8_MIN : INT8_MAX);
+ }
+ return ret;
+}
+
+void HELPER(sve2_sqrdmlah_b)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int8_t *d = vd, *n = vn, *m = vm, *a = va;
+
+ for (i = 0; i < opr_sz; ++i) {
+ d[i] = do_sqrdmlah_b(n[i], m[i], a[i], false, true);
+ }
+}
+
+void HELPER(sve2_sqrdmlsh_b)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int8_t *d = vd, *n = vn, *m = vm, *a = va;
+
+ for (i = 0; i < opr_sz; ++i) {
+ d[i] = do_sqrdmlah_b(n[i], m[i], a[i], true, true);
+ }
+}
+
/* Signed saturating rounding doubling multiply-accumulate high half, 16-bit */
static int16_t do_sqrdmlah_h(int16_t src1, int16_t src2, int16_t src3,
bool neg, bool round, uint32_t *sat)
{
- /*
- * Simplify:
- * = ((a3 << 16) + ((e1 * e2) << 1) + (1 << 15)) >> 16
- * = ((a3 << 15) + (e1 * e2) + (1 << 14)) >> 15
- */
+ /* Simplify similarly to do_sqrdmlah_b above. */
int32_t ret = (int32_t)src1 * src2;
if (neg) {
ret = -ret;
@@ -133,11 +174,35 @@ void HELPER(neon_sqrdmulh_h)(void *vd, void *vn, void *vm,
clear_tail(d, opr_sz, simd_maxsz(desc));
}
+void HELPER(sve2_sqrdmlah_h)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int16_t *d = vd, *n = vn, *m = vm, *a = va;
+ uint32_t discard;
+
+ for (i = 0; i < opr_sz / 2; ++i) {
+ d[i] = do_sqrdmlah_h(n[i], m[i], a[i], false, true, &discard);
+ }
+}
+
+void HELPER(sve2_sqrdmlsh_h)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int16_t *d = vd, *n = vn, *m = vm, *a = va;
+ uint32_t discard;
+
+ for (i = 0; i < opr_sz / 2; ++i) {
+ d[i] = do_sqrdmlah_h(n[i], m[i], a[i], true, true, &discard);
+ }
+}
+
/* Signed saturating rounding doubling multiply-accumulate high half, 32-bit */
static int32_t do_sqrdmlah_s(int32_t src1, int32_t src2, int32_t src3,
bool neg, bool round, uint32_t *sat)
{
- /* Simplify similarly to int_qrdmlah_s16 above. */
+ /* Simplify similarly to do_sqrdmlah_b above. */
int64_t ret = (int64_t)src1 * src2;
if (neg) {
ret = -ret;
@@ -220,6 +285,90 @@ void HELPER(neon_sqrdmulh_s)(void *vd, void *vn, void *vm,
clear_tail(d, opr_sz, simd_maxsz(desc));
}
+void HELPER(sve2_sqrdmlah_s)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int32_t *d = vd, *n = vn, *m = vm, *a = va;
+ uint32_t discard;
+
+ for (i = 0; i < opr_sz / 4; ++i) {
+ d[i] = do_sqrdmlah_s(n[i], m[i], a[i], false, true, &discard);
+ }
+}
+
+void HELPER(sve2_sqrdmlsh_s)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int32_t *d = vd, *n = vn, *m = vm, *a = va;
+ uint32_t discard;
+
+ for (i = 0; i < opr_sz / 4; ++i) {
+ d[i] = do_sqrdmlah_s(n[i], m[i], a[i], true, true, &discard);
+ }
+}
+
+/* Signed saturating rounding doubling multiply-accumulate high half, 64-bit */
+static int64_t do_sat128_d(Int128 r)
+{
+ int64_t ls = int128_getlo(r);
+ int64_t hs = int128_gethi(r);
+
+ if (unlikely(hs != (ls >> 63))) {
+ return hs < 0 ? INT64_MIN : INT64_MAX;
+ }
+ return ls;
+}
+
+static int64_t do_sqrdmlah_d(int64_t n, int64_t m, int64_t a,
+ bool neg, bool round)
+{
+ uint64_t l, h;
+ Int128 r, t;
+
+ /* As in do_sqrdmlah_b, but with 128-bit arithmetic. */
+ muls64(&l, &h, m, n);
+ r = int128_make128(l, h);
+ if (neg) {
+ r = int128_neg(r);
+ }
+ if (a) {
+ t = int128_exts64(a);
+ t = int128_lshift(t, 63);
+ r = int128_add(r, t);
+ }
+ if (round) {
+ t = int128_exts64(1ll << 62);
+ r = int128_add(r, t);
+ }
+ r = int128_rshift(r, 63);
+
+ return do_sat128_d(r);
+}
+
+void HELPER(sve2_sqrdmlah_d)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int64_t *d = vd, *n = vn, *m = vm, *a = va;
+
+ for (i = 0; i < opr_sz / 8; ++i) {
+ d[i] = do_sqrdmlah_d(n[i], m[i], a[i], false, true);
+ }
+}
+
+void HELPER(sve2_sqrdmlsh_d)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int64_t *d = vd, *n = vn, *m = vm, *a = va;
+
+ for (i = 0; i < opr_sz / 8; ++i) {
+ d[i] = do_sqrdmlah_d(n[i], m[i], a[i], true, true);
+ }
+}
+
/* Integer 8 and 16-bit dot-product.
*
* Note that for the loops herein, host endianness does not matter
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 39/81] target/arm: Implement SVE2 integer multiply-add long
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (37 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 38/81] target/arm: Implement SVE2 saturating multiply-add high Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 40/81] target/arm: Implement SVE2 complex integer multiply-add Richard Henderson
` (43 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 28 ++++++++++++++
target/arm/sve.decode | 11 ++++++
target/arm/sve_helper.c | 18 +++++++++
target/arm/translate-sve.c | 76 ++++++++++++++++++++++++++++++++++++++
4 files changed, 133 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index ff5f744b9a..3054215e9d 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2573,3 +2573,31 @@ DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_smlal_zzzw_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlal_zzzw_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlal_zzzw_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_umlal_zzzw_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlal_zzzw_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlal_zzzw_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_smlsl_zzzw_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlsl_zzzw_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlsl_zzzw_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 177b3cc803..19c5013ddd 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1351,3 +1351,14 @@ SQDMLSLBT 01000100 .. 0 ..... 00001 1 ..... ..... @rda_rn_rm
SQRDMLAH_zzzz 01000100 .. 0 ..... 01110 0 ..... ..... @rda_rn_rm
SQRDMLSH_zzzz 01000100 .. 0 ..... 01110 1 ..... ..... @rda_rn_rm
+
+## SVE2 integer multiply-add long
+
+SMLALB_zzzw 01000100 .. 0 ..... 010 000 ..... ..... @rda_rn_rm
+SMLALT_zzzw 01000100 .. 0 ..... 010 001 ..... ..... @rda_rn_rm
+UMLALB_zzzw 01000100 .. 0 ..... 010 010 ..... ..... @rda_rn_rm
+UMLALT_zzzw 01000100 .. 0 ..... 010 011 ..... ..... @rda_rn_rm
+SMLSLB_zzzw 01000100 .. 0 ..... 010 100 ..... ..... @rda_rn_rm
+SMLSLT_zzzw 01000100 .. 0 ..... 010 101 ..... ..... @rda_rn_rm
+UMLSLB_zzzw 01000100 .. 0 ..... 010 110 ..... ..... @rda_rn_rm
+UMLSLT_zzzw 01000100 .. 0 ..... 010 111 ..... ..... @rda_rn_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 9822675892..76e8db5dd3 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1308,6 +1308,24 @@ DO_ZZZW_ACC(sve2_uabal_h, uint16_t, uint8_t, H1_2, H1, DO_ABD)
DO_ZZZW_ACC(sve2_uabal_s, uint32_t, uint16_t, H1_4, H1_2, DO_ABD)
DO_ZZZW_ACC(sve2_uabal_d, uint64_t, uint32_t, , H1_4, DO_ABD)
+DO_ZZZW_ACC(sve2_smlal_zzzw_h, int16_t, int8_t, H1_2, H1, DO_MUL)
+DO_ZZZW_ACC(sve2_smlal_zzzw_s, int32_t, int16_t, H1_4, H1_2, DO_MUL)
+DO_ZZZW_ACC(sve2_smlal_zzzw_d, int64_t, int32_t, , H1_4, DO_MUL)
+
+DO_ZZZW_ACC(sve2_umlal_zzzw_h, uint16_t, uint8_t, H1_2, H1, DO_MUL)
+DO_ZZZW_ACC(sve2_umlal_zzzw_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL)
+DO_ZZZW_ACC(sve2_umlal_zzzw_d, uint64_t, uint32_t, , H1_4, DO_MUL)
+
+#define DO_NMUL(N, M) -(N * M)
+
+DO_ZZZW_ACC(sve2_smlsl_zzzw_h, int16_t, int8_t, H1_2, H1, DO_NMUL)
+DO_ZZZW_ACC(sve2_smlsl_zzzw_s, int32_t, int16_t, H1_4, H1_2, DO_NMUL)
+DO_ZZZW_ACC(sve2_smlsl_zzzw_d, int64_t, int32_t, , H1_4, DO_NMUL)
+
+DO_ZZZW_ACC(sve2_umlsl_zzzw_h, uint16_t, uint8_t, H1_2, H1, DO_NMUL)
+DO_ZZZW_ACC(sve2_umlsl_zzzw_s, uint32_t, uint16_t, H1_4, H1_2, DO_NMUL)
+DO_ZZZW_ACC(sve2_umlsl_zzzw_d, uint64_t, uint32_t, , H1_4, DO_NMUL)
+
#undef DO_ZZZW_ACC
#define DO_XTNB(NAME, TYPE, OP) \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 1f75489868..9fdf3eb9ef 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -7585,3 +7585,79 @@ static bool trans_SQRDMLSH_zzzz(DisasContext *s, arg_rrrr_esz *a)
};
return do_sve2_zzzz_ool(s, a, fns[a->esz], 0);
}
+
+static bool do_smlal_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel)
+{
+ static gen_helper_gvec_4 * const fns[] = {
+ NULL, gen_helper_sve2_smlal_zzzw_h,
+ gen_helper_sve2_smlal_zzzw_s, gen_helper_sve2_smlal_zzzw_d,
+ };
+ return do_sve2_zzzz_ool(s, a, fns[a->esz], sel);
+}
+
+static bool trans_SMLALB_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_smlal_zzzw(s, a, false);
+}
+
+static bool trans_SMLALT_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_smlal_zzzw(s, a, true);
+}
+
+static bool do_umlal_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel)
+{
+ static gen_helper_gvec_4 * const fns[] = {
+ NULL, gen_helper_sve2_umlal_zzzw_h,
+ gen_helper_sve2_umlal_zzzw_s, gen_helper_sve2_umlal_zzzw_d,
+ };
+ return do_sve2_zzzz_ool(s, a, fns[a->esz], sel);
+}
+
+static bool trans_UMLALB_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_umlal_zzzw(s, a, false);
+}
+
+static bool trans_UMLALT_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_umlal_zzzw(s, a, true);
+}
+
+static bool do_smlsl_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel)
+{
+ static gen_helper_gvec_4 * const fns[] = {
+ NULL, gen_helper_sve2_smlsl_zzzw_h,
+ gen_helper_sve2_smlsl_zzzw_s, gen_helper_sve2_smlsl_zzzw_d,
+ };
+ return do_sve2_zzzz_ool(s, a, fns[a->esz], sel);
+}
+
+static bool trans_SMLSLB_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_smlsl_zzzw(s, a, false);
+}
+
+static bool trans_SMLSLT_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_smlsl_zzzw(s, a, true);
+}
+
+static bool do_umlsl_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel)
+{
+ static gen_helper_gvec_4 * const fns[] = {
+ NULL, gen_helper_sve2_umlsl_zzzw_h,
+ gen_helper_sve2_umlsl_zzzw_s, gen_helper_sve2_umlsl_zzzw_d,
+ };
+ return do_sve2_zzzz_ool(s, a, fns[a->esz], sel);
+}
+
+static bool trans_UMLSLB_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_umlsl_zzzw(s, a, false);
+}
+
+static bool trans_UMLSLT_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_umlsl_zzzw(s, a, true);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 40/81] target/arm: Implement SVE2 complex integer multiply-add
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (38 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 39/81] target/arm: Implement SVE2 integer multiply-add long Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 41/81] target/arm: Implement SVE2 ADDHNB, ADDHNT Richard Henderson
` (42 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Fix do_sqrdmlah_d (laurent desnogues)
---
target/arm/helper-sve.h | 18 ++++++++++++++++
target/arm/vec_internal.h | 5 +++++
target/arm/sve.decode | 5 +++++
target/arm/sve_helper.c | 42 ++++++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 32 +++++++++++++++++++++++++++++
target/arm/vec_helper.c | 15 +++++++-------
6 files changed, 109 insertions(+), 8 deletions(-)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 3054215e9d..4cd43d3ecc 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2601,3 +2601,21 @@ DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/vec_internal.h b/target/arm/vec_internal.h
index 372fe76523..38ce31b4ca 100644
--- a/target/arm/vec_internal.h
+++ b/target/arm/vec_internal.h
@@ -168,4 +168,9 @@ static inline int64_t do_suqrshl_d(int64_t src, int64_t shift,
return do_uqrshl_d(src, shift, round, sat);
}
+int8_t do_sqrdmlah_b(int8_t, int8_t, int8_t, bool, bool);
+int16_t do_sqrdmlah_h(int16_t, int16_t, int16_t, bool, bool, uint32_t *);
+int32_t do_sqrdmlah_s(int32_t, int32_t, int32_t, bool, bool, uint32_t *);
+int64_t do_sqrdmlah_d(int64_t, int64_t, int64_t, bool, bool);
+
#endif /* TARGET_ARM_VEC_INTERNALS_H */
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 19c5013ddd..a03d6107da 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1362,3 +1362,8 @@ SMLSLB_zzzw 01000100 .. 0 ..... 010 100 ..... ..... @rda_rn_rm
SMLSLT_zzzw 01000100 .. 0 ..... 010 101 ..... ..... @rda_rn_rm
UMLSLB_zzzw 01000100 .. 0 ..... 010 110 ..... ..... @rda_rn_rm
UMLSLT_zzzw 01000100 .. 0 ..... 010 111 ..... ..... @rda_rn_rm
+
+## SVE2 complex integer multiply-add
+
+CMLA_zzzz 01000100 esz:2 0 rm:5 0010 rot:2 rn:5 rd:5 ra=%reg_movprfx
+SQRDCMLAH_zzzz 01000100 esz:2 0 rm:5 0011 rot:2 rn:5 rd:5 ra=%reg_movprfx
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 76e8db5dd3..10e3bd8415 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1448,6 +1448,48 @@ DO_SQDMLAL(sve2_sqdmlsl_zzzw_d, int64_t, int32_t, , H1_4,
#undef DO_SQDMLAL
+#define DO_CMLA(NAME, TYPE, H, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(TYPE); \
+ int rot = simd_data(desc); \
+ int sel_a = rot & 1, sel_b = sel_a ^ 1; \
+ bool sub_r = rot == 1 || rot == 2; \
+ bool sub_i = rot >= 2; \
+ TYPE *d = vd, *n = vn, *m = vm, *a = va; \
+ for (i = 0; i < opr_sz; i += 2) { \
+ TYPE elt1_a = n[H(i + sel_a)]; \
+ TYPE elt2_a = m[H(i + sel_a)]; \
+ TYPE elt2_b = m[H(i + sel_b)]; \
+ d[H(i)] = OP(elt1_a, elt2_a, a[H(i)], sub_r); \
+ d[H(i + 1)] = OP(elt1_a, elt2_b, a[H(i + 1)], sub_i); \
+ } \
+}
+
+#define do_cmla(N, M, A, S) (A + (N * M) * (S ? -1 : 1))
+
+DO_CMLA(sve2_cmla_zzzz_b, uint8_t, H1, do_cmla)
+DO_CMLA(sve2_cmla_zzzz_h, uint16_t, H2, do_cmla)
+DO_CMLA(sve2_cmla_zzzz_s, uint32_t, H4, do_cmla)
+DO_CMLA(sve2_cmla_zzzz_d, uint64_t, , do_cmla)
+
+#define DO_SQRDMLAH_B(N, M, A, S) \
+ do_sqrdmlah_b(N, M, A, S, true)
+#define DO_SQRDMLAH_H(N, M, A, S) \
+ ({ uint32_t discard; do_sqrdmlah_h(N, M, A, S, true, &discard); })
+#define DO_SQRDMLAH_S(N, M, A, S) \
+ ({ uint32_t discard; do_sqrdmlah_s(N, M, A, S, true, &discard); })
+#define DO_SQRDMLAH_D(N, M, A, S) \
+ do_sqrdmlah_d(N, M, A, S, true)
+
+DO_CMLA(sve2_sqrdcmlah_zzzz_b, int8_t, H1, DO_SQRDMLAH_B)
+DO_CMLA(sve2_sqrdcmlah_zzzz_h, int16_t, H2, DO_SQRDMLAH_H)
+DO_CMLA(sve2_sqrdcmlah_zzzz_s, int32_t, H4, DO_SQRDMLAH_S)
+DO_CMLA(sve2_sqrdcmlah_zzzz_d, int64_t, , DO_SQRDMLAH_D)
+
+#undef do_cmla
+#undef DO_CMLA
+
#define DO_BITPERM(NAME, TYPE, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
{ \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 9fdf3eb9ef..5f4d879193 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -7661,3 +7661,35 @@ static bool trans_UMLSLT_zzzw(DisasContext *s, arg_rrrr_esz *a)
{
return do_umlsl_zzzw(s, a, true);
}
+
+static bool trans_CMLA_zzzz(DisasContext *s, arg_CMLA_zzzz *a)
+{
+ static gen_helper_gvec_4 * const fns[] = {
+ gen_helper_sve2_cmla_zzzz_b, gen_helper_sve2_cmla_zzzz_h,
+ gen_helper_sve2_cmla_zzzz_s, gen_helper_sve2_cmla_zzzz_d,
+ };
+
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_ool_zzzz(s, fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot);
+ }
+ return true;
+}
+
+static bool trans_SQRDCMLAH_zzzz(DisasContext *s, arg_SQRDCMLAH_zzzz *a)
+{
+ static gen_helper_gvec_4 * const fns[] = {
+ gen_helper_sve2_sqrdcmlah_zzzz_b, gen_helper_sve2_sqrdcmlah_zzzz_h,
+ gen_helper_sve2_sqrdcmlah_zzzz_s, gen_helper_sve2_sqrdcmlah_zzzz_d,
+ };
+
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_ool_zzzz(s, fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot);
+ }
+ return true;
+}
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index b40d9688c5..ed33e1ee36 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -38,8 +38,8 @@
#endif
/* Signed saturating rounding doubling multiply-accumulate high half, 8-bit */
-static int8_t do_sqrdmlah_b(int8_t src1, int8_t src2, int8_t src3,
- bool neg, bool round)
+int8_t do_sqrdmlah_b(int8_t src1, int8_t src2, int8_t src3,
+ bool neg, bool round)
{
/*
* Simplify:
@@ -82,8 +82,8 @@ void HELPER(sve2_sqrdmlsh_b)(void *vd, void *vn, void *vm,
}
/* Signed saturating rounding doubling multiply-accumulate high half, 16-bit */
-static int16_t do_sqrdmlah_h(int16_t src1, int16_t src2, int16_t src3,
- bool neg, bool round, uint32_t *sat)
+int16_t do_sqrdmlah_h(int16_t src1, int16_t src2, int16_t src3,
+ bool neg, bool round, uint32_t *sat)
{
/* Simplify similarly to do_sqrdmlah_b above. */
int32_t ret = (int32_t)src1 * src2;
@@ -199,8 +199,8 @@ void HELPER(sve2_sqrdmlsh_h)(void *vd, void *vn, void *vm,
}
/* Signed saturating rounding doubling multiply-accumulate high half, 32-bit */
-static int32_t do_sqrdmlah_s(int32_t src1, int32_t src2, int32_t src3,
- bool neg, bool round, uint32_t *sat)
+int32_t do_sqrdmlah_s(int32_t src1, int32_t src2, int32_t src3,
+ bool neg, bool round, uint32_t *sat)
{
/* Simplify similarly to do_sqrdmlah_b above. */
int64_t ret = (int64_t)src1 * src2;
@@ -321,8 +321,7 @@ static int64_t do_sat128_d(Int128 r)
return ls;
}
-static int64_t do_sqrdmlah_d(int64_t n, int64_t m, int64_t a,
- bool neg, bool round)
+int64_t do_sqrdmlah_d(int64_t n, int64_t m, int64_t a, bool neg, bool round)
{
uint64_t l, h;
Int128 r, t;
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 41/81] target/arm: Implement SVE2 ADDHNB, ADDHNT
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (39 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 40/81] target/arm: Implement SVE2 complex integer multiply-add Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 42/81] target/arm: Implement SVE2 RADDHNB, RADDHNT Richard Henderson
` (41 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200417162231.10374-2-steplong@quicinc.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 8 ++++++++
target/arm/sve.decode | 5 +++++
target/arm/sve_helper.c | 36 ++++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 13 +++++++++++++
4 files changed, 62 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 4cd43d3ecc..5cf5473487 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2509,6 +2509,14 @@ DEF_HELPER_FLAGS_3(sve2_uqrshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqrshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve2_uqrshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_addhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_addhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_addhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_addhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_addhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_addhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_5(sve2_match_ppzz_b, TCG_CALL_NO_RWG,
i32, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_match_ppzz_h, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index a03d6107da..af9e87e88d 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1320,6 +1320,11 @@ UQSHRNT 01000101 .. 1 ..... 00 1101 ..... ..... @rd_rn_tszimm_shr
UQRSHRNB 01000101 .. 1 ..... 00 1110 ..... ..... @rd_rn_tszimm_shr
UQRSHRNT 01000101 .. 1 ..... 00 1111 ..... ..... @rd_rn_tszimm_shr
+## SVE2 integer add/subtract narrow high part
+
+ADDHNB 01000101 .. 1 ..... 011 000 ..... ..... @rd_rn_rm
+ADDHNT 01000101 .. 1 ..... 011 001 ..... ..... @rd_rn_rm
+
### SVE2 Character Match
MATCH 01000101 .. 1 ..... 100 ... ..... 0 .... @pd_pg_rn_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 10e3bd8415..6b1c0ad266 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -2111,6 +2111,42 @@ DO_SHRNT(sve2_uqrshrnt_d, uint64_t, uint32_t, , H1_4, DO_UQRSHRN_D)
#undef DO_SHRNB
#undef DO_SHRNT
+#define DO_BINOPNB(NAME, TYPEW, TYPEN, SHIFT, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
+ TYPEW nn = *(TYPEW *)(vn + i); \
+ TYPEW mm = *(TYPEW *)(vm + i); \
+ *(TYPEW *)(vd + i) = (TYPEN)OP(nn, mm, SHIFT); \
+ } \
+}
+
+#define DO_BINOPNT(NAME, TYPEW, TYPEN, SHIFT, HW, HN, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
+{ \
+ intptr_t i, opr_sz = simd_oprsz(desc); \
+ for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
+ TYPEW nn = *(TYPEW *)(vn + HW(i)); \
+ TYPEW mm = *(TYPEW *)(vm + HW(i)); \
+ *(TYPEN *)(vd + HN(i + sizeof(TYPEN))) = OP(nn, mm, SHIFT); \
+ } \
+}
+
+#define DO_ADDHN(N, M, SH) ((N + M) >> SH)
+
+DO_BINOPNB(sve2_addhnb_h, uint16_t, uint8_t, 8, DO_ADDHN)
+DO_BINOPNB(sve2_addhnb_s, uint32_t, uint16_t, 16, DO_ADDHN)
+DO_BINOPNB(sve2_addhnb_d, uint64_t, uint32_t, 32, DO_ADDHN)
+
+DO_BINOPNT(sve2_addhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_ADDHN)
+DO_BINOPNT(sve2_addhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_ADDHN)
+DO_BINOPNT(sve2_addhnt_d, uint64_t, uint32_t, 32, , H1_4, DO_ADDHN)
+
+#undef DO_ADDHN
+
+#undef DO_BINOPNB
+
/* Fully general four-operand expander, controlled by a predicate.
*/
#define DO_ZPZZZ(NAME, TYPE, H, OP) \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 5f4d879193..aa21aa49d0 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -7467,6 +7467,19 @@ static bool trans_UQRSHRNT(DisasContext *s, arg_rri_esz *a)
return do_sve2_shr_narrow(s, a, ops);
}
+#define DO_SVE2_ZZZ_NARROW(NAME, name) \
+static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
+{ \
+ static gen_helper_gvec_3 * const fns[4] = { \
+ NULL, gen_helper_sve2_##name##_h, \
+ gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
+ }; \
+ return do_sve2_zzz_ool(s, a, fns[a->esz]); \
+}
+
+DO_SVE2_ZZZ_NARROW(ADDHNB, addhnb)
+DO_SVE2_ZZZ_NARROW(ADDHNT, addhnt)
+
static bool do_sve2_ppzz_flags(DisasContext *s, arg_rprr_esz *a,
gen_helper_gvec_flags_4 *fn)
{
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 42/81] target/arm: Implement SVE2 RADDHNB, RADDHNT
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (40 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 41/81] target/arm: Implement SVE2 ADDHNB, ADDHNT Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 43/81] target/arm: Implement SVE2 SUBHNB, SUBHNT Richard Henderson
` (40 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200417162231.10374-3-steplong@quicinc.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Fix round bit type (laurent desnogues)
---
target/arm/helper-sve.h | 8 ++++++++
target/arm/sve.decode | 2 ++
target/arm/sve_helper.c | 10 ++++++++++
target/arm/translate-sve.c | 2 ++
4 files changed, 22 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 5cf5473487..4d395d3965 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2517,6 +2517,14 @@ DEF_HELPER_FLAGS_4(sve2_addhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_addhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_addhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_raddhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_raddhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_raddhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_raddhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_raddhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_raddhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_5(sve2_match_ppzz_b, TCG_CALL_NO_RWG,
i32, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_match_ppzz_h, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index af9e87e88d..a33825066c 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1324,6 +1324,8 @@ UQRSHRNT 01000101 .. 1 ..... 00 1111 ..... ..... @rd_rn_tszimm_shr
ADDHNB 01000101 .. 1 ..... 011 000 ..... ..... @rd_rn_rm
ADDHNT 01000101 .. 1 ..... 011 001 ..... ..... @rd_rn_rm
+RADDHNB 01000101 .. 1 ..... 011 010 ..... ..... @rd_rn_rm
+RADDHNT 01000101 .. 1 ..... 011 011 ..... ..... @rd_rn_rm
### SVE2 Character Match
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 6b1c0ad266..b778fad142 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -2134,6 +2134,7 @@ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
}
#define DO_ADDHN(N, M, SH) ((N + M) >> SH)
+#define DO_RADDHN(N, M, SH) ((N + M + ((__typeof(N))1 << (SH - 1))) >> SH)
DO_BINOPNB(sve2_addhnb_h, uint16_t, uint8_t, 8, DO_ADDHN)
DO_BINOPNB(sve2_addhnb_s, uint32_t, uint16_t, 16, DO_ADDHN)
@@ -2143,6 +2144,15 @@ DO_BINOPNT(sve2_addhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_ADDHN)
DO_BINOPNT(sve2_addhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_ADDHN)
DO_BINOPNT(sve2_addhnt_d, uint64_t, uint32_t, 32, , H1_4, DO_ADDHN)
+DO_BINOPNB(sve2_raddhnb_h, uint16_t, uint8_t, 8, DO_RADDHN)
+DO_BINOPNB(sve2_raddhnb_s, uint32_t, uint16_t, 16, DO_RADDHN)
+DO_BINOPNB(sve2_raddhnb_d, uint64_t, uint32_t, 32, DO_RADDHN)
+
+DO_BINOPNT(sve2_raddhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_RADDHN)
+DO_BINOPNT(sve2_raddhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_RADDHN)
+DO_BINOPNT(sve2_raddhnt_d, uint64_t, uint32_t, 32, , H1_4, DO_RADDHN)
+
+#undef DO_RADDHN
#undef DO_ADDHN
#undef DO_BINOPNB
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index aa21aa49d0..e384bed108 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -7479,6 +7479,8 @@ static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
DO_SVE2_ZZZ_NARROW(ADDHNB, addhnb)
DO_SVE2_ZZZ_NARROW(ADDHNT, addhnt)
+DO_SVE2_ZZZ_NARROW(RADDHNB, raddhnb)
+DO_SVE2_ZZZ_NARROW(RADDHNT, raddhnt)
static bool do_sve2_ppzz_flags(DisasContext *s, arg_rprr_esz *a,
gen_helper_gvec_flags_4 *fn)
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 43/81] target/arm: Implement SVE2 SUBHNB, SUBHNT
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (41 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 42/81] target/arm: Implement SVE2 RADDHNB, RADDHNT Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 44/81] target/arm: Implement SVE2 RSUBHNB, RSUBHNT Richard Henderson
` (39 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200417162231.10374-4-steplong@quicinc.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 8 ++++++++
target/arm/sve.decode | 2 ++
target/arm/sve_helper.c | 10 ++++++++++
target/arm/translate-sve.c | 3 +++
4 files changed, 23 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 4d395d3965..4c57dde9a9 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2525,6 +2525,14 @@ DEF_HELPER_FLAGS_4(sve2_raddhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_raddhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_raddhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_subhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_subhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_subhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_subhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_subhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_subhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_5(sve2_match_ppzz_b, TCG_CALL_NO_RWG,
i32, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_match_ppzz_h, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index a33825066c..8ad2698bcf 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1326,6 +1326,8 @@ ADDHNB 01000101 .. 1 ..... 011 000 ..... ..... @rd_rn_rm
ADDHNT 01000101 .. 1 ..... 011 001 ..... ..... @rd_rn_rm
RADDHNB 01000101 .. 1 ..... 011 010 ..... ..... @rd_rn_rm
RADDHNT 01000101 .. 1 ..... 011 011 ..... ..... @rd_rn_rm
+SUBHNB 01000101 .. 1 ..... 011 100 ..... ..... @rd_rn_rm
+SUBHNT 01000101 .. 1 ..... 011 101 ..... ..... @rd_rn_rm
### SVE2 Character Match
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index b778fad142..35681cf0c6 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -2135,6 +2135,7 @@ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
#define DO_ADDHN(N, M, SH) ((N + M) >> SH)
#define DO_RADDHN(N, M, SH) ((N + M + ((__typeof(N))1 << (SH - 1))) >> SH)
+#define DO_SUBHN(N, M, SH) ((N - M) >> SH)
DO_BINOPNB(sve2_addhnb_h, uint16_t, uint8_t, 8, DO_ADDHN)
DO_BINOPNB(sve2_addhnb_s, uint32_t, uint16_t, 16, DO_ADDHN)
@@ -2152,6 +2153,15 @@ DO_BINOPNT(sve2_raddhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_RADDHN)
DO_BINOPNT(sve2_raddhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_RADDHN)
DO_BINOPNT(sve2_raddhnt_d, uint64_t, uint32_t, 32, , H1_4, DO_RADDHN)
+DO_BINOPNB(sve2_subhnb_h, uint16_t, uint8_t, 8, DO_SUBHN)
+DO_BINOPNB(sve2_subhnb_s, uint32_t, uint16_t, 16, DO_SUBHN)
+DO_BINOPNB(sve2_subhnb_d, uint64_t, uint32_t, 32, DO_SUBHN)
+
+DO_BINOPNT(sve2_subhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_SUBHN)
+DO_BINOPNT(sve2_subhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_SUBHN)
+DO_BINOPNT(sve2_subhnt_d, uint64_t, uint32_t, 32, , H1_4, DO_SUBHN)
+
+#undef DO_SUBHN
#undef DO_RADDHN
#undef DO_ADDHN
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index e384bed108..1c730a835f 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -7482,6 +7482,9 @@ DO_SVE2_ZZZ_NARROW(ADDHNT, addhnt)
DO_SVE2_ZZZ_NARROW(RADDHNB, raddhnb)
DO_SVE2_ZZZ_NARROW(RADDHNT, raddhnt)
+DO_SVE2_ZZZ_NARROW(SUBHNB, subhnb)
+DO_SVE2_ZZZ_NARROW(SUBHNT, subhnt)
+
static bool do_sve2_ppzz_flags(DisasContext *s, arg_rprr_esz *a,
gen_helper_gvec_flags_4 *fn)
{
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 44/81] target/arm: Implement SVE2 RSUBHNB, RSUBHNT
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (42 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 43/81] target/arm: Implement SVE2 SUBHNB, SUBHNT Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 45/81] target/arm: Implement SVE2 HISTCNT, HISTSEG Richard Henderson
` (38 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
This completes the section 'SVE2 integer add/subtract narrow high part'
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200417162231.10374-5-steplong@quicinc.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Fix round bit type (laurent desnogues)
---
target/arm/helper-sve.h | 8 ++++++++
target/arm/sve.decode | 2 ++
target/arm/sve_helper.c | 10 ++++++++++
target/arm/translate-sve.c | 2 ++
4 files changed, 22 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 4c57dde9a9..9e8641e1c0 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2533,6 +2533,14 @@ DEF_HELPER_FLAGS_4(sve2_subhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_subhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_subhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_rsubhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_rsubhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_rsubhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_rsubhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_rsubhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_rsubhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_5(sve2_match_ppzz_b, TCG_CALL_NO_RWG,
i32, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_match_ppzz_h, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 8ad2698bcf..3121eabbf8 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1328,6 +1328,8 @@ RADDHNB 01000101 .. 1 ..... 011 010 ..... ..... @rd_rn_rm
RADDHNT 01000101 .. 1 ..... 011 011 ..... ..... @rd_rn_rm
SUBHNB 01000101 .. 1 ..... 011 100 ..... ..... @rd_rn_rm
SUBHNT 01000101 .. 1 ..... 011 101 ..... ..... @rd_rn_rm
+RSUBHNB 01000101 .. 1 ..... 011 110 ..... ..... @rd_rn_rm
+RSUBHNT 01000101 .. 1 ..... 011 111 ..... ..... @rd_rn_rm
### SVE2 Character Match
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 35681cf0c6..19fbf94189 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -2136,6 +2136,7 @@ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
#define DO_ADDHN(N, M, SH) ((N + M) >> SH)
#define DO_RADDHN(N, M, SH) ((N + M + ((__typeof(N))1 << (SH - 1))) >> SH)
#define DO_SUBHN(N, M, SH) ((N - M) >> SH)
+#define DO_RSUBHN(N, M, SH) ((N - M + ((__typeof(N))1 << (SH - 1))) >> SH)
DO_BINOPNB(sve2_addhnb_h, uint16_t, uint8_t, 8, DO_ADDHN)
DO_BINOPNB(sve2_addhnb_s, uint32_t, uint16_t, 16, DO_ADDHN)
@@ -2161,6 +2162,15 @@ DO_BINOPNT(sve2_subhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_SUBHN)
DO_BINOPNT(sve2_subhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_SUBHN)
DO_BINOPNT(sve2_subhnt_d, uint64_t, uint32_t, 32, , H1_4, DO_SUBHN)
+DO_BINOPNB(sve2_rsubhnb_h, uint16_t, uint8_t, 8, DO_RSUBHN)
+DO_BINOPNB(sve2_rsubhnb_s, uint32_t, uint16_t, 16, DO_RSUBHN)
+DO_BINOPNB(sve2_rsubhnb_d, uint64_t, uint32_t, 32, DO_RSUBHN)
+
+DO_BINOPNT(sve2_rsubhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_RSUBHN)
+DO_BINOPNT(sve2_rsubhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_RSUBHN)
+DO_BINOPNT(sve2_rsubhnt_d, uint64_t, uint32_t, 32, , H1_4, DO_RSUBHN)
+
+#undef DO_RSUBHN
#undef DO_SUBHN
#undef DO_RADDHN
#undef DO_ADDHN
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 1c730a835f..e947a0ff25 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -7484,6 +7484,8 @@ DO_SVE2_ZZZ_NARROW(RADDHNT, raddhnt)
DO_SVE2_ZZZ_NARROW(SUBHNB, subhnb)
DO_SVE2_ZZZ_NARROW(SUBHNT, subhnt)
+DO_SVE2_ZZZ_NARROW(RSUBHNB, rsubhnb)
+DO_SVE2_ZZZ_NARROW(RSUBHNT, rsubhnt)
static bool do_sve2_ppzz_flags(DisasContext *s, arg_rprr_esz *a,
gen_helper_gvec_flags_4 *fn)
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 45/81] target/arm: Implement SVE2 HISTCNT, HISTSEG
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (43 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 44/81] target/arm: Implement SVE2 RSUBHNB, RSUBHNT Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-10-09 6:13 ` LIU Zhiwei
2020-09-18 18:37 ` [PATCH v3 46/81] target/arm: Implement SVE2 XAR Richard Henderson
` (37 subsequent siblings)
82 siblings, 1 reply; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200416173109.8856-1-steplong@quicinc.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Fix overlap between output and input vectors.
---
target/arm/helper-sve.h | 7 +++
target/arm/sve.decode | 6 ++
target/arm/sve_helper.c | 124 +++++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 19 ++++++
4 files changed, 156 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 9e8641e1c0..34bbb767ef 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2551,6 +2551,13 @@ DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_b, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_h, TCG_CALL_NO_RWG,
i32, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_histcnt_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_histcnt_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_histseg, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 3121eabbf8..0edb72d4fb 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -146,6 +146,7 @@
&rprrr_esz rn=%reg_movprfx
@rdn_pg_rm_ra ........ esz:2 . ra:5 ... pg:3 rm:5 rd:5 \
&rprrr_esz rn=%reg_movprfx
+@rd_pg_rn_rm ........ esz:2 . rm:5 ... pg:3 rn:5 rd:5 &rprr_esz
# One register operand, with governing predicate, vector element size
@rd_pg_rn ........ esz:2 ... ... ... pg:3 rn:5 rd:5 &rpr_esz
@@ -1336,6 +1337,11 @@ RSUBHNT 01000101 .. 1 ..... 011 111 ..... ..... @rd_rn_rm
MATCH 01000101 .. 1 ..... 100 ... ..... 0 .... @pd_pg_rn_rm
NMATCH 01000101 .. 1 ..... 100 ... ..... 1 .... @pd_pg_rn_rm
+### SVE2 Histogram Computation
+
+HISTCNT 01000101 .. 1 ..... 110 ... ..... ..... @rd_pg_rn_rm
+HISTSEG 01000101 .. 1 ..... 101 000 ..... ..... @rd_rn_rm
+
## SVE2 floating-point pairwise operations
FADDP 01100100 .. 010 00 0 100 ... ..... ..... @rdn_pg_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 19fbf94189..fa4848bc5c 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -7095,3 +7095,127 @@ DO_PPZZ_MATCH(sve2_nmatch_ppzz_b, MO_8, true)
DO_PPZZ_MATCH(sve2_nmatch_ppzz_h, MO_16, true)
#undef DO_PPZZ_MATCH
+
+void HELPER(sve2_histcnt_s)(void *vd, void *vn, void *vm, void *vg,
+ uint32_t desc)
+{
+ ARMVectorReg scratch;
+ intptr_t i, j;
+ intptr_t opr_sz = simd_oprsz(desc);
+ uint32_t *d = vd, *n = vn, *m = vm;
+ uint8_t *pg = vg;
+
+ if (d == n) {
+ n = memcpy(&scratch, n, opr_sz);
+ if (d == m) {
+ m = n;
+ }
+ } else if (d == m) {
+ m = memcpy(&scratch, m, opr_sz);
+ }
+
+ for (i = 0; i < opr_sz; i += 4) {
+ uint64_t count = 0;
+ uint8_t pred;
+
+ pred = pg[H1(i >> 3)] >> (i & 7);
+ if (pred & 1) {
+ uint32_t nn = n[H4(i >> 2)];
+
+ for (j = 0; j <= i; j += 4) {
+ pred = pg[H1(j >> 3)] >> (j & 7);
+ if ((pred & 1) && nn == m[H4(j >> 2)]) {
+ ++count;
+ }
+ }
+ }
+ d[H4(i >> 2)] = count;
+ }
+}
+
+void HELPER(sve2_histcnt_d)(void *vd, void *vn, void *vm, void *vg,
+ uint32_t desc)
+{
+ ARMVectorReg scratch;
+ intptr_t i, j;
+ intptr_t opr_sz = simd_oprsz(desc);
+ uint64_t *d = vd, *n = vn, *m = vm;
+ uint8_t *pg = vg;
+
+ if (d == n) {
+ n = memcpy(&scratch, n, opr_sz);
+ if (d == m) {
+ m = n;
+ }
+ } else if (d == m) {
+ m = memcpy(&scratch, m, opr_sz);
+ }
+
+ for (i = 0; i < opr_sz / 8; ++i) {
+ uint64_t count = 0;
+ if (pg[H1(i)] & 1) {
+ uint64_t nn = n[i];
+ for (j = 0; j <= i; ++j) {
+ if ((pg[H1(j)] & 1) && nn == m[j]) {
+ ++count;
+ }
+ }
+ }
+ d[i] = count;
+ }
+}
+
+/*
+ * Returns the number of bytes in m0 and m1 that match n.
+ * See comment for do_match2().
+ * */
+static inline uint64_t do_histseg_cnt(uint8_t n, uint64_t m0, uint64_t m1)
+{
+ int esz = MO_8;
+ int bits = 8 << esz;
+ uint64_t ones = dup_const(esz, 1);
+ uint64_t signs = ones << (bits - 1);
+ uint64_t cmp0, cmp1;
+
+ cmp1 = dup_const(esz, n);
+ cmp0 = cmp1 ^ m0;
+ cmp1 = cmp1 ^ m1;
+ cmp0 = (cmp0 - ones) & ~cmp0 & signs;
+ cmp1 = (cmp1 - ones) & ~cmp1 & signs;
+
+ /*
+ * Combine the two compares in a way that the bits do
+ * not overlap, and so preserves the count of set bits.
+ * If the host has an efficient instruction for ctpop,
+ * then ctpop(x) + ctpop(y) has the same number of
+ * operations as ctpop(x | (y >> 1)). If the host does
+ * not have an efficient ctpop, then we only want to
+ * use it once.
+ */
+ return ctpop64(cmp0 | (cmp1 >> 1));
+}
+
+void HELPER(sve2_histseg)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, j;
+ intptr_t opr_sz = simd_oprsz(desc);
+
+ for (i = 0; i < opr_sz; i += 16) {
+ uint64_t n0 = *(uint64_t *)(vn + i);
+ uint64_t m0 = *(uint64_t *)(vm + i);
+ uint64_t n1 = *(uint64_t *)(vn + i + 8);
+ uint64_t m1 = *(uint64_t *)(vm + i + 8);
+ uint64_t out0 = 0;
+ uint64_t out1 = 0;
+
+ for (j = 0; j < 64; j += 8) {
+ uint64_t cnt0 = do_histseg_cnt(n0 >> j, m0, m1);
+ uint64_t cnt1 = do_histseg_cnt(n1 >> j, m0, m1);
+ out0 |= cnt0 << j;
+ out1 |= cnt1 << j;
+ }
+
+ *(uint64_t *)(vd + i) = out0;
+ *(uint64_t *)(vd + i + 8) = out1;
+ }
+}
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index e947a0ff25..4e792c9b9a 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -7509,6 +7509,25 @@ static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
DO_SVE2_PPZZ_MATCH(MATCH, match)
DO_SVE2_PPZZ_MATCH(NMATCH, nmatch)
+static bool trans_HISTCNT(DisasContext *s, arg_rprr_esz *a)
+{
+ static gen_helper_gvec_4 * const fns[2] = {
+ gen_helper_sve2_histcnt_s, gen_helper_sve2_histcnt_d
+ };
+ if (a->esz < 2) {
+ return false;
+ }
+ return do_sve2_zpzz_ool(s, a, fns[a->esz - 2]);
+}
+
+static bool trans_HISTSEG(DisasContext *s, arg_rrr_esz *a)
+{
+ if (a->esz != 0) {
+ return false;
+ }
+ return do_sve2_zzz_ool(s, a, gen_helper_sve2_histseg);
+}
+
static bool do_sve2_zpzz_fp(DisasContext *s, arg_rprr_esz *a,
gen_helper_gvec_4_ptr *fn)
{
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* Re: [PATCH v3 45/81] target/arm: Implement SVE2 HISTCNT, HISTSEG
2020-09-18 18:37 ` [PATCH v3 45/81] target/arm: Implement SVE2 HISTCNT, HISTSEG Richard Henderson
@ 2020-10-09 6:13 ` LIU Zhiwei
2020-10-09 12:35 ` Richard Henderson
0 siblings, 1 reply; 94+ messages in thread
From: LIU Zhiwei @ 2020-10-09 6:13 UTC (permalink / raw)
To: qemu-devel, Richard Henderson, Stephen Long
On 2020/9/19 2:37, Richard Henderson wrote:
> From: Stephen Long <steplong@quicinc.com>
>
> Signed-off-by: Stephen Long <steplong@quicinc.com>
> Message-Id: <20200416173109.8856-1-steplong@quicinc.com>
> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
> ---
> v2: Fix overlap between output and input vectors.
> ---
> target/arm/helper-sve.h | 7 +++
> target/arm/sve.decode | 6 ++
> target/arm/sve_helper.c | 124 +++++++++++++++++++++++++++++++++++++
> target/arm/translate-sve.c | 19 ++++++
> 4 files changed, 156 insertions(+)
>
> diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
> index 9e8641e1c0..34bbb767ef 100644
> --- a/target/arm/helper-sve.h
> +++ b/target/arm/helper-sve.h
> @@ -2551,6 +2551,13 @@ DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_b, TCG_CALL_NO_RWG,
> DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_h, TCG_CALL_NO_RWG,
> i32, ptr, ptr, ptr, ptr, i32)
>
> +DEF_HELPER_FLAGS_5(sve2_histcnt_s, TCG_CALL_NO_RWG,
> + void, ptr, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(sve2_histcnt_d, TCG_CALL_NO_RWG,
> + void, ptr, ptr, ptr, ptr, i32)
> +
> +DEF_HELPER_FLAGS_4(sve2_histseg, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> +
> DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG,
> void, ptr, ptr, ptr, ptr, ptr, i32)
> DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG,
> diff --git a/target/arm/sve.decode b/target/arm/sve.decode
> index 3121eabbf8..0edb72d4fb 100644
> --- a/target/arm/sve.decode
> +++ b/target/arm/sve.decode
> @@ -146,6 +146,7 @@
> &rprrr_esz rn=%reg_movprfx
> @rdn_pg_rm_ra ........ esz:2 . ra:5 ... pg:3 rm:5 rd:5 \
> &rprrr_esz rn=%reg_movprfx
> +@rd_pg_rn_rm ........ esz:2 . rm:5 ... pg:3 rn:5 rd:5 &rprr_esz
>
> # One register operand, with governing predicate, vector element size
> @rd_pg_rn ........ esz:2 ... ... ... pg:3 rn:5 rd:5 &rpr_esz
> @@ -1336,6 +1337,11 @@ RSUBHNT 01000101 .. 1 ..... 011 111 ..... ..... @rd_rn_rm
> MATCH 01000101 .. 1 ..... 100 ... ..... 0 .... @pd_pg_rn_rm
> NMATCH 01000101 .. 1 ..... 100 ... ..... 1 .... @pd_pg_rn_rm
>
> +### SVE2 Histogram Computation
> +
> +HISTCNT 01000101 .. 1 ..... 110 ... ..... ..... @rd_pg_rn_rm
> +HISTSEG 01000101 .. 1 ..... 101 000 ..... ..... @rd_rn_rm
> +
> ## SVE2 floating-point pairwise operations
>
> FADDP 01100100 .. 010 00 0 100 ... ..... ..... @rdn_pg_rm
> diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
> index 19fbf94189..fa4848bc5c 100644
> --- a/target/arm/sve_helper.c
> +++ b/target/arm/sve_helper.c
> @@ -7095,3 +7095,127 @@ DO_PPZZ_MATCH(sve2_nmatch_ppzz_b, MO_8, true)
> DO_PPZZ_MATCH(sve2_nmatch_ppzz_h, MO_16, true)
>
> #undef DO_PPZZ_MATCH
> +
> +void HELPER(sve2_histcnt_s)(void *vd, void *vn, void *vm, void *vg,
> + uint32_t desc)
> +{
> + ARMVectorReg scratch;
> + intptr_t i, j;
> + intptr_t opr_sz = simd_oprsz(desc);
> + uint32_t *d = vd, *n = vn, *m = vm;
> + uint8_t *pg = vg;
> +
> + if (d == n) {
> + n = memcpy(&scratch, n, opr_sz);
> + if (d == m) {
> + m = n;
> + }
> + } else if (d == m) {
> + m = memcpy(&scratch, m, opr_sz);
> + }
> +
> + for (i = 0; i < opr_sz; i += 4) {
> + uint64_t count = 0;
> + uint8_t pred;
> +
> + pred = pg[H1(i >> 3)] >> (i & 7);
> + if (pred & 1) {
> + uint32_t nn = n[H4(i >> 2)];
> +
> + for (j = 0; j <= i; j += 4) {
> + pred = pg[H1(j >> 3)] >> (j & 7);
> + if ((pred & 1) && nn == m[H4(j >> 2)]) {
> + ++count;
> + }
> + }
> + }
> + d[H4(i >> 2)] = count;
> + }
> +}
> +
> +void HELPER(sve2_histcnt_d)(void *vd, void *vn, void *vm, void *vg,
> + uint32_t desc)
> +{
> + ARMVectorReg scratch;
> + intptr_t i, j;
> + intptr_t opr_sz = simd_oprsz(desc);
> + uint64_t *d = vd, *n = vn, *m = vm;
> + uint8_t *pg = vg;
> +
> + if (d == n) {
> + n = memcpy(&scratch, n, opr_sz);
> + if (d == m) {
> + m = n;
> + }
> + } else if (d == m) {
> + m = memcpy(&scratch, m, opr_sz);
> + }
> +
> + for (i = 0; i < opr_sz / 8; ++i) {
> + uint64_t count = 0;
> + if (pg[H1(i)] & 1) {
> + uint64_t nn = n[i];
> + for (j = 0; j <= i; ++j) {
> + if ((pg[H1(j)] & 1) && nn == m[j]) {
> + ++count;
> + }
> + }
> + }
> + d[i] = count;
> + }
> +}
> +
> +/*
> + * Returns the number of bytes in m0 and m1 that match n.
> + * See comment for do_match2().
> + * */
> +static inline uint64_t do_histseg_cnt(uint8_t n, uint64_t m0, uint64_t m1)
> +{
> + int esz = MO_8;
> + int bits = 8 << esz;
> + uint64_t ones = dup_const(esz, 1);
> + uint64_t signs = ones << (bits - 1);
> + uint64_t cmp0, cmp1;
> +
> + cmp1 = dup_const(esz, n);
> + cmp0 = cmp1 ^ m0;
> + cmp1 = cmp1 ^ m1;
> + cmp0 = (cmp0 - ones) & ~cmp0 & signs;
> + cmp1 = (cmp1 - ones) & ~cmp1 & signs;
> +
Hi Richard,
Although we can detect zero byte with this method, we can't use it to
count the zero bytes.
For example,
IF
cmp1 = 0x0100010001000100 , ones = 0x101010101010101, signs =
0x8080808080808080,
THEN
cmp1 = (cmp1 - ones) & ~cmp1 & signs = 0x8080808080808080
So
cmp1 will have 6 zeros by this method. In fact, cmp1 only have 4
zeros instead of 6 zeros.
I don't find a "bit twiddling" way, if you find it, please let me know.
Best Regards,
Zhiwei
> + /*
> + * Combine the two compares in a way that the bits do
> + * not overlap, and so preserves the count of set bits.
> + * If the host has an efficient instruction for ctpop,
> + * then ctpop(x) + ctpop(y) has the same number of
> + * operations as ctpop(x | (y >> 1)). If the host does
> + * not have an efficient ctpop, then we only want to
> + * use it once.
> + */
> + return ctpop64(cmp0 | (cmp1 >> 1));
> +}
> +
> +void HELPER(sve2_histseg)(void *vd, void *vn, void *vm, uint32_t desc)
> +{
> + intptr_t i, j;
> + intptr_t opr_sz = simd_oprsz(desc);
> +
> + for (i = 0; i < opr_sz; i += 16) {
> + uint64_t n0 = *(uint64_t *)(vn + i);
> + uint64_t m0 = *(uint64_t *)(vm + i);
> + uint64_t n1 = *(uint64_t *)(vn + i + 8);
> + uint64_t m1 = *(uint64_t *)(vm + i + 8);
> + uint64_t out0 = 0;
> + uint64_t out1 = 0;
> +
> + for (j = 0; j < 64; j += 8) {
> + uint64_t cnt0 = do_histseg_cnt(n0 >> j, m0, m1);
> + uint64_t cnt1 = do_histseg_cnt(n1 >> j, m0, m1);
> + out0 |= cnt0 << j;
> + out1 |= cnt1 << j;
> + }
> +
> + *(uint64_t *)(vd + i) = out0;
> + *(uint64_t *)(vd + i + 8) = out1;
> + }
> +}
> diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
> index e947a0ff25..4e792c9b9a 100644
> --- a/target/arm/translate-sve.c
> +++ b/target/arm/translate-sve.c
> @@ -7509,6 +7509,25 @@ static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
> DO_SVE2_PPZZ_MATCH(MATCH, match)
> DO_SVE2_PPZZ_MATCH(NMATCH, nmatch)
>
> +static bool trans_HISTCNT(DisasContext *s, arg_rprr_esz *a)
> +{
> + static gen_helper_gvec_4 * const fns[2] = {
> + gen_helper_sve2_histcnt_s, gen_helper_sve2_histcnt_d
> + };
> + if (a->esz < 2) {
> + return false;
> + }
> + return do_sve2_zpzz_ool(s, a, fns[a->esz - 2]);
> +}
> +
> +static bool trans_HISTSEG(DisasContext *s, arg_rrr_esz *a)
> +{
> + if (a->esz != 0) {
> + return false;
> + }
> + return do_sve2_zzz_ool(s, a, gen_helper_sve2_histseg);
> +}
> +
> static bool do_sve2_zpzz_fp(DisasContext *s, arg_rprr_esz *a,
> gen_helper_gvec_4_ptr *fn)
> {
^ permalink raw reply [flat|nested] 94+ messages in thread* Re: [PATCH v3 45/81] target/arm: Implement SVE2 HISTCNT, HISTSEG
2020-10-09 6:13 ` LIU Zhiwei
@ 2020-10-09 12:35 ` Richard Henderson
0 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-10-09 12:35 UTC (permalink / raw)
To: LIU Zhiwei, qemu-devel, Stephen Long
On 10/9/20 1:13 AM, LIU Zhiwei wrote:
>
>
> On 2020/9/19 2:37, Richard Henderson wrote:
>> From: Stephen Long <steplong@quicinc.com>
>>
>> Signed-off-by: Stephen Long <steplong@quicinc.com>
>> Message-Id: <20200416173109.8856-1-steplong@quicinc.com>
>> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
>> ---
>> v2: Fix overlap between output and input vectors.
>> ---
>> target/arm/helper-sve.h | 7 +++
>> target/arm/sve.decode | 6 ++
>> target/arm/sve_helper.c | 124 +++++++++++++++++++++++++++++++++++++
>> target/arm/translate-sve.c | 19 ++++++
>> 4 files changed, 156 insertions(+)
>>
>> diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
>> index 9e8641e1c0..34bbb767ef 100644
>> --- a/target/arm/helper-sve.h
>> +++ b/target/arm/helper-sve.h
>> @@ -2551,6 +2551,13 @@ DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_b, TCG_CALL_NO_RWG,
>> DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_h, TCG_CALL_NO_RWG,
>> i32, ptr, ptr, ptr, ptr, i32)
>> +DEF_HELPER_FLAGS_5(sve2_histcnt_s, TCG_CALL_NO_RWG,
>> + void, ptr, ptr, ptr, ptr, i32)
>> +DEF_HELPER_FLAGS_5(sve2_histcnt_d, TCG_CALL_NO_RWG,
>> + void, ptr, ptr, ptr, ptr, i32)
>> +
>> +DEF_HELPER_FLAGS_4(sve2_histseg, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
>> +
>> DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG,
>> void, ptr, ptr, ptr, ptr, ptr, i32)
>> DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG,
>> diff --git a/target/arm/sve.decode b/target/arm/sve.decode
>> index 3121eabbf8..0edb72d4fb 100644
>> --- a/target/arm/sve.decode
>> +++ b/target/arm/sve.decode
>> @@ -146,6 +146,7 @@
>> &rprrr_esz rn=%reg_movprfx
>> @rdn_pg_rm_ra ........ esz:2 . ra:5 ... pg:3 rm:5 rd:5 \
>> &rprrr_esz rn=%reg_movprfx
>> +@rd_pg_rn_rm ........ esz:2 . rm:5 ... pg:3 rn:5 rd:5 &rprr_esz
>> # One register operand, with governing predicate, vector element size
>> @rd_pg_rn ........ esz:2 ... ... ... pg:3 rn:5 rd:5 &rpr_esz
>> @@ -1336,6 +1337,11 @@ RSUBHNT 01000101 .. 1 ..... 011 111 .....
>> ..... @rd_rn_rm
>> MATCH 01000101 .. 1 ..... 100 ... ..... 0 .... @pd_pg_rn_rm
>> NMATCH 01000101 .. 1 ..... 100 ... ..... 1 .... @pd_pg_rn_rm
>> +### SVE2 Histogram Computation
>> +
>> +HISTCNT 01000101 .. 1 ..... 110 ... ..... ..... @rd_pg_rn_rm
>> +HISTSEG 01000101 .. 1 ..... 101 000 ..... ..... @rd_rn_rm
>> +
>> ## SVE2 floating-point pairwise operations
>> FADDP 01100100 .. 010 00 0 100 ... ..... ..... @rdn_pg_rm
>> diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
>> index 19fbf94189..fa4848bc5c 100644
>> --- a/target/arm/sve_helper.c
>> +++ b/target/arm/sve_helper.c
>> @@ -7095,3 +7095,127 @@ DO_PPZZ_MATCH(sve2_nmatch_ppzz_b, MO_8, true)
>> DO_PPZZ_MATCH(sve2_nmatch_ppzz_h, MO_16, true)
>> #undef DO_PPZZ_MATCH
>> +
>> +void HELPER(sve2_histcnt_s)(void *vd, void *vn, void *vm, void *vg,
>> + uint32_t desc)
>> +{
>> + ARMVectorReg scratch;
>> + intptr_t i, j;
>> + intptr_t opr_sz = simd_oprsz(desc);
>> + uint32_t *d = vd, *n = vn, *m = vm;
>> + uint8_t *pg = vg;
>> +
>> + if (d == n) {
>> + n = memcpy(&scratch, n, opr_sz);
>> + if (d == m) {
>> + m = n;
>> + }
>> + } else if (d == m) {
>> + m = memcpy(&scratch, m, opr_sz);
>> + }
>> +
>> + for (i = 0; i < opr_sz; i += 4) {
>> + uint64_t count = 0;
>> + uint8_t pred;
>> +
>> + pred = pg[H1(i >> 3)] >> (i & 7);
>> + if (pred & 1) {
>> + uint32_t nn = n[H4(i >> 2)];
>> +
>> + for (j = 0; j <= i; j += 4) {
>> + pred = pg[H1(j >> 3)] >> (j & 7);
>> + if ((pred & 1) && nn == m[H4(j >> 2)]) {
>> + ++count;
>> + }
>> + }
>> + }
>> + d[H4(i >> 2)] = count;
>> + }
>> +}
>> +
>> +void HELPER(sve2_histcnt_d)(void *vd, void *vn, void *vm, void *vg,
>> + uint32_t desc)
>> +{
>> + ARMVectorReg scratch;
>> + intptr_t i, j;
>> + intptr_t opr_sz = simd_oprsz(desc);
>> + uint64_t *d = vd, *n = vn, *m = vm;
>> + uint8_t *pg = vg;
>> +
>> + if (d == n) {
>> + n = memcpy(&scratch, n, opr_sz);
>> + if (d == m) {
>> + m = n;
>> + }
>> + } else if (d == m) {
>> + m = memcpy(&scratch, m, opr_sz);
>> + }
>> +
>> + for (i = 0; i < opr_sz / 8; ++i) {
>> + uint64_t count = 0;
>> + if (pg[H1(i)] & 1) {
>> + uint64_t nn = n[i];
>> + for (j = 0; j <= i; ++j) {
>> + if ((pg[H1(j)] & 1) && nn == m[j]) {
>> + ++count;
>> + }
>> + }
>> + }
>> + d[i] = count;
>> + }
>> +}
>> +
>> +/*
>> + * Returns the number of bytes in m0 and m1 that match n.
>> + * See comment for do_match2().
>> + * */
>> +static inline uint64_t do_histseg_cnt(uint8_t n, uint64_t m0, uint64_t m1)
>> +{
>> + int esz = MO_8;
>> + int bits = 8 << esz;
>> + uint64_t ones = dup_const(esz, 1);
>> + uint64_t signs = ones << (bits - 1);
>> + uint64_t cmp0, cmp1;
>> +
>> + cmp1 = dup_const(esz, n);
>> + cmp0 = cmp1 ^ m0;
>> + cmp1 = cmp1 ^ m1;
>> + cmp0 = (cmp0 - ones) & ~cmp0 & signs;
>> + cmp1 = (cmp1 - ones) & ~cmp1 & signs;
>> +
> Hi Richard,
>
> Although we can detect zero byte with this method, we can't use it to count the
> zero bytes.
>
> For example,
> IF
> cmp1 = 0x0100010001000100 , ones = 0x101010101010101, signs =
> 0x8080808080808080,
> THEN
> cmp1 = (cmp1 - ones) & ~cmp1 & signs = 0x8080808080808080
> So
> cmp1 will have 6 zeros by this method. In fact, cmp1 only have 4 zeros
> instead of 6 zeros.
>
> I don't find a "bit twiddling" way, if you find it, please let me know.
Thanks for noticing the error. We already have a bit twiddling example in qemu
for this in target/alpha:
uint64_t helper_cmpbe0(uint64_t a)
{
uint64_t m = 0x7f7f7f7f7f7f7f7fULL;
uint64_t c = ~(((a & m) + m) | a | m);
...
which produces the exact results that we need here.
r~
^ permalink raw reply [flat|nested] 94+ messages in thread
* [PATCH v3 46/81] target/arm: Implement SVE2 XAR
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (44 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 45/81] target/arm: Implement SVE2 HISTCNT, HISTSEG Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 47/81] target/arm: Implement SVE2 scatter store insns Richard Henderson
` (36 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
In addition, use the same vector generator interface for AdvSIMD.
This fixes a bug in which the AdvSIMD insn failed to clear the
high bits of the SVE register.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 4 ++
target/arm/helper.h | 2 +
target/arm/translate-a64.h | 3 ++
target/arm/sve.decode | 4 ++
target/arm/sve_helper.c | 39 ++++++++++++++
target/arm/translate-a64.c | 25 ++-------
target/arm/translate-sve.c | 104 +++++++++++++++++++++++++++++++++++++
target/arm/vec_helper.c | 12 +++++
8 files changed, 172 insertions(+), 21 deletions(-)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 34bbb767ef..d472fc009d 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2558,6 +2558,10 @@ DEF_HELPER_FLAGS_5(sve2_histcnt_d, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_4(sve2_histseg, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_xar_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_xar_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_xar_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG,
diff --git a/target/arm/helper.h b/target/arm/helper.h
index c8eb165989..8294055cab 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -940,6 +940,8 @@ DEF_HELPER_FLAGS_5(neon_sqrdmulh_h, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(neon_sqrdmulh_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_xar_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
#ifdef TARGET_AARCH64
#include "helper-a64.h"
#include "helper-sve.h"
diff --git a/target/arm/translate-a64.h b/target/arm/translate-a64.h
index 2e0d16da25..83729d0280 100644
--- a/target/arm/translate-a64.h
+++ b/target/arm/translate-a64.h
@@ -122,5 +122,8 @@ bool disas_sve(DisasContext *, uint32_t);
void gen_gvec_rax1(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz);
+void gen_gvec_xar(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
+ uint32_t rm_ofs, int64_t shift,
+ uint32_t opr_sz, uint32_t max_sz);
#endif /* TARGET_ARM_TRANSLATE_A64_H */
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 0edb72d4fb..a375ce31f1 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -65,6 +65,7 @@
&rr_dbm rd rn dbm
&rrri rd rn rm imm
&rri_esz rd rn imm esz
+&rrri_esz rd rn rm imm esz
&rrr_esz rd rn rm esz
&rpr_esz rd pg rn esz
&rpr_s rd pg rn s
@@ -384,6 +385,9 @@ ORR_zzz 00000100 01 1 ..... 001 100 ..... ..... @rd_rn_rm_e0
EOR_zzz 00000100 10 1 ..... 001 100 ..... ..... @rd_rn_rm_e0
BIC_zzz 00000100 11 1 ..... 001 100 ..... ..... @rd_rn_rm_e0
+XAR 00000100 .. 1 ..... 001 101 rm:5 rd:5 &rrri_esz \
+ rn=%reg_movprfx esz=%tszimm16_esz imm=%tszimm16_shr
+
# SVE2 bitwise ternary operations
EOR3 00000100 00 1 ..... 001 110 ..... ..... @rdn_ra_rm_e0
BSL 00000100 00 1 ..... 001 111 ..... ..... @rdn_ra_rm_e0
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index fa4848bc5c..e0bd0fbe78 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -7219,3 +7219,42 @@ void HELPER(sve2_histseg)(void *vd, void *vn, void *vm, uint32_t desc)
*(uint64_t *)(vd + i + 8) = out1;
}
}
+
+void HELPER(sve2_xar_b)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+ int shr = simd_data(desc);
+ int shl = 8 - shr;
+ uint64_t mask = dup_const(MO_8, 0xff >> shr);
+ uint64_t *d = vd, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz; ++i) {
+ uint64_t t = n[i] ^ m[i];
+ d[i] = ((t >> shr) & mask) | ((t << shl) & ~mask);
+ }
+}
+
+void HELPER(sve2_xar_h)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+ int shr = simd_data(desc);
+ int shl = 16 - shr;
+ uint64_t mask = dup_const(MO_16, 0xffff >> shr);
+ uint64_t *d = vd, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz; ++i) {
+ uint64_t t = n[i] ^ m[i];
+ d[i] = ((t >> shr) & mask) | ((t << shl) & ~mask);
+ }
+}
+
+void HELPER(sve2_xar_s)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc) / 4;
+ int shr = simd_data(desc);
+ uint32_t *d = vd, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz; ++i) {
+ d[i] = ror32(n[i] ^ m[i], shr);
+ }
+}
diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c
index 7188808341..76e54c1a4e 100644
--- a/target/arm/translate-a64.c
+++ b/target/arm/translate-a64.c
@@ -14368,8 +14368,6 @@ static void disas_crypto_xar(DisasContext *s, uint32_t insn)
int imm6 = extract32(insn, 10, 6);
int rn = extract32(insn, 5, 5);
int rd = extract32(insn, 0, 5);
- TCGv_i64 tcg_op1, tcg_op2, tcg_res[2];
- int pass;
if (!dc_isar_feature(aa64_sha3, s)) {
unallocated_encoding(s);
@@ -14380,25 +14378,10 @@ static void disas_crypto_xar(DisasContext *s, uint32_t insn)
return;
}
- tcg_op1 = tcg_temp_new_i64();
- tcg_op2 = tcg_temp_new_i64();
- tcg_res[0] = tcg_temp_new_i64();
- tcg_res[1] = tcg_temp_new_i64();
-
- for (pass = 0; pass < 2; pass++) {
- read_vec_element(s, tcg_op1, rn, pass, MO_64);
- read_vec_element(s, tcg_op2, rm, pass, MO_64);
-
- tcg_gen_xor_i64(tcg_res[pass], tcg_op1, tcg_op2);
- tcg_gen_rotri_i64(tcg_res[pass], tcg_res[pass], imm6);
- }
- write_vec_element(s, tcg_res[0], rd, 0, MO_64);
- write_vec_element(s, tcg_res[1], rd, 1, MO_64);
-
- tcg_temp_free_i64(tcg_op1);
- tcg_temp_free_i64(tcg_op2);
- tcg_temp_free_i64(tcg_res[0]);
- tcg_temp_free_i64(tcg_res[1]);
+ gen_gvec_xar(MO_64, vec_full_reg_offset(s, rd),
+ vec_full_reg_offset(s, rn),
+ vec_full_reg_offset(s, rm), imm6, 16,
+ vec_full_reg_size(s));
}
/* Crypto three-reg imm2
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 4e792c9b9a..0d0e0c3b46 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -340,6 +340,110 @@ static bool trans_BIC_zzz(DisasContext *s, arg_rrr_esz *a)
return do_zzz_fn(s, a, tcg_gen_gvec_andc);
}
+static void gen_xar8_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh)
+{
+ TCGv_i64 t = tcg_temp_new_i64();
+ uint64_t mask = dup_const(MO_8, 0xff >> sh);
+
+ tcg_gen_xor_i64(t, n, m);
+ tcg_gen_shri_i64(d, t, sh);
+ tcg_gen_shli_i64(t, t, 8 - sh);
+ tcg_gen_andi_i64(d, d, mask);
+ tcg_gen_andi_i64(t, t, ~mask);
+ tcg_gen_or_i64(d, d, t);
+ tcg_temp_free_i64(t);
+}
+
+static void gen_xar16_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh)
+{
+ TCGv_i64 t = tcg_temp_new_i64();
+ uint64_t mask = dup_const(MO_16, 0xffff >> sh);
+
+ tcg_gen_xor_i64(t, n, m);
+ tcg_gen_shri_i64(d, t, sh);
+ tcg_gen_shli_i64(t, t, 16 - sh);
+ tcg_gen_andi_i64(d, d, mask);
+ tcg_gen_andi_i64(t, t, ~mask);
+ tcg_gen_or_i64(d, d, t);
+ tcg_temp_free_i64(t);
+}
+
+static void gen_xar_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, int32_t sh)
+{
+ tcg_gen_xor_i32(d, n, m);
+ tcg_gen_rotri_i32(d, d, sh);
+}
+
+static void gen_xar_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh)
+{
+ tcg_gen_xor_i64(d, n, m);
+ tcg_gen_rotri_i64(d, d, sh);
+}
+
+static void gen_xar_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
+ TCGv_vec m, int64_t sh)
+{
+ tcg_gen_xor_vec(vece, d, n, m);
+ tcg_gen_rotri_vec(vece, d, d, sh);
+}
+
+void gen_gvec_xar(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
+ uint32_t rm_ofs, int64_t shift,
+ uint32_t opr_sz, uint32_t max_sz)
+{
+ static const TCGOpcode vecop[] = { INDEX_op_rotli_vec, 0 };
+ static const GVecGen3i ops[4] = {
+ { .fni8 = gen_xar8_i64,
+ .fniv = gen_xar_vec,
+ .fno = gen_helper_sve2_xar_b,
+ .opt_opc = vecop,
+ .vece = MO_8 },
+ { .fni8 = gen_xar16_i64,
+ .fniv = gen_xar_vec,
+ .fno = gen_helper_sve2_xar_h,
+ .opt_opc = vecop,
+ .vece = MO_16 },
+ { .fni4 = gen_xar_i32,
+ .fniv = gen_xar_vec,
+ .fno = gen_helper_sve2_xar_s,
+ .opt_opc = vecop,
+ .vece = MO_32 },
+ { .fni8 = gen_xar_i64,
+ .fniv = gen_xar_vec,
+ .fno = gen_helper_gvec_xar_d,
+ .opt_opc = vecop,
+ .vece = MO_64 }
+ };
+ int esize = 8 << vece;
+
+ /* The SVE2 range is 1 .. esize; the AdvSIMD range is 0 .. esize-1. */
+ tcg_debug_assert(shift >= 0);
+ tcg_debug_assert(shift <= esize);
+ shift &= esize - 1;
+
+ if (shift == 0) {
+ /* xar with no rotate devolves to xor. */
+ tcg_gen_gvec_xor(vece, rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz);
+ } else {
+ tcg_gen_gvec_3i(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz,
+ shift, &ops[vece]);
+ }
+}
+
+static bool trans_XAR(DisasContext *s, arg_rrri_esz *a)
+{
+ if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ gen_gvec_xar(a->esz, vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm), a->imm, vsz, vsz);
+ }
+ return true;
+}
+
static bool do_sve2_zzzz_fn(DisasContext *s, arg_rrrr_esz *a, GVecGen4Fn *fn)
{
if (!dc_isar_feature(aa64_sve2, s)) {
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index ed33e1ee36..32a4403256 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -2205,3 +2205,15 @@ void HELPER(gvec_umulh_d)(void *vd, void *vn, void *vm, uint32_t desc)
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
+
+void HELPER(gvec_xar_d)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+ int shr = simd_data(desc);
+ uint64_t *d = vd, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz; ++i) {
+ d[i] = ror64(n[i] ^ m[i], shr);
+ }
+ clear_tail(d, opr_sz * 8, simd_maxsz(desc));
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 47/81] target/arm: Implement SVE2 scatter store insns
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (45 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 46/81] target/arm: Implement SVE2 XAR Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 48/81] target/arm: Implement SVE2 gather load insns Richard Henderson
` (35 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Add decoding logic for SVE2 64-bit/32-bit scatter non-temporal
store insns.
64-bit
* STNT1B (vector plus scalar)
* STNT1H (vector plus scalar)
* STNT1W (vector plus scalar)
* STNT1D (vector plus scalar)
32-bit
* STNT1B (vector plus scalar)
* STNT1H (vector plus scalar)
* STNT1W (vector plus scalar)
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200422141553.8037-1-steplong@quicinc.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve.decode | 10 ++++++++++
target/arm/translate-sve.c | 8 ++++++++
2 files changed, 18 insertions(+)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index a375ce31f1..dc784dcabe 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1388,3 +1388,13 @@ UMLSLT_zzzw 01000100 .. 0 ..... 010 111 ..... ..... @rda_rn_rm
CMLA_zzzz 01000100 esz:2 0 rm:5 0010 rot:2 rn:5 rd:5 ra=%reg_movprfx
SQRDCMLAH_zzzz 01000100 esz:2 0 rm:5 0011 rot:2 rn:5 rd:5 ra=%reg_movprfx
+
+### SVE2 Memory Store Group
+
+# SVE2 64-bit scatter non-temporal store (vector plus scalar)
+STNT1_zprz 1110010 .. 00 ..... 001 ... ..... ..... \
+ @rprr_scatter_store xs=2 esz=3 scale=0
+
+# SVE2 32-bit scatter non-temporal store (vector plus scalar)
+STNT1_zprz 1110010 .. 10 ..... 001 ... ..... ..... \
+ @rprr_scatter_store xs=0 esz=2 scale=0
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 0d0e0c3b46..af8feff707 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6172,6 +6172,14 @@ static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a)
return true;
}
+static bool trans_STNT1_zprz(DisasContext *s, arg_ST1_zprz *a)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return trans_ST1_zprz(s, a);
+}
+
/*
* Prefetches
*/
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 48/81] target/arm: Implement SVE2 gather load insns
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (46 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 47/81] target/arm: Implement SVE2 scatter store insns Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 49/81] target/arm: Implement SVE2 FMMLA Richard Henderson
` (34 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Add decoding logic for SVE2 64-bit/32-bit gather non-temporal
load insns.
64-bit
* LDNT1SB
* LDNT1B (vector plus scalar)
* LDNT1SH
* LDNT1H (vector plus scalar)
* LDNT1SW
* LDNT1W (vector plus scalar)
* LDNT1D (vector plus scalar)
32-bit
* LDNT1SB
* LDNT1B (vector plus scalar)
* LDNT1SH
* LDNT1H (vector plus scalar)
* LDNT1W (vector plus scalar)
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200422152343.12493-1-steplong@quicinc.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve.decode | 11 +++++++++++
target/arm/translate-sve.c | 8 ++++++++
2 files changed, 19 insertions(+)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index dc784dcabe..1b5bd2d193 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1389,6 +1389,17 @@ UMLSLT_zzzw 01000100 .. 0 ..... 010 111 ..... ..... @rda_rn_rm
CMLA_zzzz 01000100 esz:2 0 rm:5 0010 rot:2 rn:5 rd:5 ra=%reg_movprfx
SQRDCMLAH_zzzz 01000100 esz:2 0 rm:5 0011 rot:2 rn:5 rd:5 ra=%reg_movprfx
+### SVE2 Memory Gather Load Group
+
+# SVE2 64-bit gather non-temporal load
+# (scalar plus unpacked 32-bit unscaled offsets)
+LDNT1_zprz 1100010 msz:2 00 rm:5 1 u:1 0 pg:3 rn:5 rd:5 \
+ &rprr_gather_load xs=0 esz=3 scale=0 ff=0
+
+# SVE2 32-bit gather non-temporal load (scalar plus 32-bit unscaled offsets)
+LDNT1_zprz 1000010 msz:2 00 rm:5 10 u:1 pg:3 rn:5 rd:5 \
+ &rprr_gather_load xs=0 esz=2 scale=0 ff=0
+
### SVE2 Memory Store Group
# SVE2 64-bit scatter non-temporal store (vector plus scalar)
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index af8feff707..0cb10ac3e2 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6020,6 +6020,14 @@ static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a)
return true;
}
+static bool trans_LDNT1_zprz(DisasContext *s, arg_LD1_zprz *a)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return trans_LD1_zprz(s, a);
+}
+
/* Indexed by [mte][be][xs][msz]. */
static gen_helper_gvec_mem_scatter * const scatter_store_fn32[2][2][2][3] = {
{ /* MTE Inactive */
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 49/81] target/arm: Implement SVE2 FMMLA
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (47 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 48/81] target/arm: Implement SVE2 gather load insns Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 50/81] target/arm: Implement SVE2 SPLICE, EXT Richard Henderson
` (33 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200422165503.13511-1-steplong@quicinc.com>
[rth: Fix indexing in helpers, expand macro to straight functions.]
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/cpu.h | 10 ++++++
target/arm/helper-sve.h | 3 ++
target/arm/sve.decode | 4 +++
target/arm/sve_helper.c | 74 ++++++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 34 ++++++++++++++++++
5 files changed, 125 insertions(+)
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index 5ed2f2c65b..aaefbf0167 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -3917,6 +3917,16 @@ static inline bool isar_feature_aa64_sve2_bitperm(const ARMISARegisters *id)
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, BITPERM) != 0;
}
+static inline bool isar_feature_aa64_sve2_f32mm(const ARMISARegisters *id)
+{
+ return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, F32MM) != 0;
+}
+
+static inline bool isar_feature_aa64_sve2_f64mm(const ARMISARegisters *id)
+{
+ return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, F64MM) != 0;
+}
+
/*
* Feature tests for "does this exist in either 32-bit or 64-bit?"
*/
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index d472fc009d..2c6d3431ec 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2662,3 +2662,6 @@ DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(fmmla_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(fmmla_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 1b5bd2d193..11e724d3a2 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1389,6 +1389,10 @@ UMLSLT_zzzw 01000100 .. 0 ..... 010 111 ..... ..... @rda_rn_rm
CMLA_zzzz 01000100 esz:2 0 rm:5 0010 rot:2 rn:5 rd:5 ra=%reg_movprfx
SQRDCMLAH_zzzz 01000100 esz:2 0 rm:5 0011 rot:2 rn:5 rd:5 ra=%reg_movprfx
+### SVE2 floating point matrix multiply accumulate
+
+FMMLA 01100100 .. 1 ..... 111001 ..... ..... @rda_rn_rm
+
### SVE2 Memory Gather Load Group
# SVE2 64-bit gather non-temporal load
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index e0bd0fbe78..aaf336118b 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -7258,3 +7258,77 @@ void HELPER(sve2_xar_s)(void *vd, void *vn, void *vm, uint32_t desc)
d[i] = ror32(n[i] ^ m[i], shr);
}
}
+
+void HELPER(fmmla_s)(void *vd, void *vn, void *vm, void *va,
+ void *status, uint32_t desc)
+{
+ intptr_t s, opr_sz = simd_oprsz(desc) / (sizeof(float32) * 4);
+
+ for (s = 0; s < opr_sz; ++s) {
+ float32 *n = vn + s * sizeof(float32) * 4;
+ float32 *m = vm + s * sizeof(float32) * 4;
+ float32 *a = va + s * sizeof(float32) * 4;
+ float32 *d = vd + s * sizeof(float32) * 4;
+ float32 n00 = n[H4(0)], n01 = n[H4(1)];
+ float32 n10 = n[H4(2)], n11 = n[H4(3)];
+ float32 m00 = m[H4(0)], m01 = m[H4(1)];
+ float32 m10 = m[H4(2)], m11 = m[H4(3)];
+ float32 p0, p1;
+
+ /* i = 0, j = 0 */
+ p0 = float32_mul(n00, m00, status);
+ p1 = float32_mul(n01, m01, status);
+ d[H4(0)] = float32_add(a[H4(0)], float32_add(p0, p1, status), status);
+
+ /* i = 0, j = 1 */
+ p0 = float32_mul(n00, m10, status);
+ p1 = float32_mul(n01, m11, status);
+ d[H4(1)] = float32_add(a[H4(1)], float32_add(p0, p1, status), status);
+
+ /* i = 1, j = 0 */
+ p0 = float32_mul(n10, m00, status);
+ p1 = float32_mul(n11, m01, status);
+ d[H4(2)] = float32_add(a[H4(2)], float32_add(p0, p1, status), status);
+
+ /* i = 1, j = 1 */
+ p0 = float32_mul(n10, m10, status);
+ p1 = float32_mul(n11, m11, status);
+ d[H4(3)] = float32_add(a[H4(3)], float32_add(p0, p1, status), status);
+ }
+}
+
+void HELPER(fmmla_d)(void *vd, void *vn, void *vm, void *va,
+ void *status, uint32_t desc)
+{
+ intptr_t s, opr_sz = simd_oprsz(desc) / (sizeof(float64) * 4);
+
+ for (s = 0; s < opr_sz; ++s) {
+ float64 *n = vn + s * sizeof(float64) * 4;
+ float64 *m = vm + s * sizeof(float64) * 4;
+ float64 *a = va + s * sizeof(float64) * 4;
+ float64 *d = vd + s * sizeof(float64) * 4;
+ float64 n00 = n[0], n01 = n[1], n10 = n[2], n11 = n[3];
+ float64 m00 = m[0], m01 = m[1], m10 = m[2], m11 = m[3];
+ float64 p0, p1;
+
+ /* i = 0, j = 0 */
+ p0 = float64_mul(n00, m00, status);
+ p1 = float64_mul(n01, m01, status);
+ d[0] = float64_add(a[0], float64_add(p0, p1, status), status);
+
+ /* i = 0, j = 1 */
+ p0 = float64_mul(n00, m10, status);
+ p1 = float64_mul(n01, m11, status);
+ d[1] = float64_add(a[1], float64_add(p0, p1, status), status);
+
+ /* i = 1, j = 0 */
+ p0 = float64_mul(n10, m00, status);
+ p1 = float64_mul(n11, m01, status);
+ d[2] = float64_add(a[2], float64_add(p0, p1, status), status);
+
+ /* i = 1, j = 1 */
+ p0 = float64_mul(n10, m10, status);
+ p1 = float64_mul(n11, m11, status);
+ d[3] = float64_add(a[3], float64_add(p0, p1, status), status);
+ }
+}
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 0cb10ac3e2..63acb25921 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -7677,6 +7677,40 @@ DO_SVE2_ZPZZ_FP(FMINP, fminp)
* SVE Integer Multiply-Add (unpredicated)
*/
+static bool trans_FMMLA(DisasContext *s, arg_rrrr_esz *a)
+{
+ gen_helper_gvec_4_ptr *fn;
+
+ switch (a->esz) {
+ case MO_32:
+ if (!dc_isar_feature(aa64_sve2_f32mm, s)) {
+ return false;
+ }
+ fn = gen_helper_fmmla_s;
+ break;
+ case MO_64:
+ if (!dc_isar_feature(aa64_sve2_f64mm, s)) {
+ return false;
+ }
+ fn = gen_helper_fmmla_d;
+ break;
+ default:
+ return false;
+ }
+
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
+ tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ vec_full_reg_offset(s, a->ra),
+ status, vsz, vsz, 0, fn);
+ tcg_temp_free_ptr(status);
+ }
+ return true;
+}
+
static bool do_sqdmlal_zzzw(DisasContext *s, arg_rrrr_esz *a,
bool sel1, bool sel2)
{
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 50/81] target/arm: Implement SVE2 SPLICE, EXT
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (48 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 49/81] target/arm: Implement SVE2 FMMLA Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 51/81] target/arm: Pass separate addend to {U, S}DOT helpers Richard Henderson
` (32 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200423180347.9403-1-steplong@quicinc.com>
[rth: Rename the trans_* functions to *_sve2.]
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve.decode | 11 +++++++++--
target/arm/translate-sve.c | 35 ++++++++++++++++++++++++++++++-----
2 files changed, 39 insertions(+), 7 deletions(-)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 11e724d3a2..0688dae450 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -494,10 +494,14 @@ CPY_z_i 00000101 .. 01 .... 00 . ........ ..... @rdn_pg4 imm=%sh8_i8s
### SVE Permute - Extract Group
-# SVE extract vector (immediate offset)
+# SVE extract vector (destructive)
EXT 00000101 001 ..... 000 ... rm:5 rd:5 \
&rrri rn=%reg_movprfx imm=%imm8_16_10
+# SVE2 extract vector (constructive)
+EXT_sve2 00000101 011 ..... 000 ... rn:5 rd:5 \
+ &rri imm=%imm8_16_10
+
### SVE Permute - Unpredicated Group
# SVE broadcast general register
@@ -588,9 +592,12 @@ REVH 00000101 .. 1001 01 100 ... ..... ..... @rd_pg_rn
REVW 00000101 .. 1001 10 100 ... ..... ..... @rd_pg_rn
RBIT 00000101 .. 1001 11 100 ... ..... ..... @rd_pg_rn
-# SVE vector splice (predicated)
+# SVE vector splice (predicated, destructive)
SPLICE 00000101 .. 101 100 100 ... ..... ..... @rdn_pg_rm
+# SVE2 vector splice (predicated, constructive)
+SPLICE_sve2 00000101 .. 101 101 100 ... ..... ..... @rd_pg_rn
+
### SVE Select Vectors Group
# SVE select vector elements (predicated)
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 63acb25921..5e8291e44b 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -2266,18 +2266,18 @@ static bool trans_CPY_z_i(DisasContext *s, arg_CPY_z_i *a)
*** SVE Permute Extract Group
*/
-static bool trans_EXT(DisasContext *s, arg_EXT *a)
+static bool do_EXT(DisasContext *s, int rd, int rn, int rm, int imm)
{
if (!sve_access_check(s)) {
return true;
}
unsigned vsz = vec_full_reg_size(s);
- unsigned n_ofs = a->imm >= vsz ? 0 : a->imm;
+ unsigned n_ofs = imm >= vsz ? 0 : imm;
unsigned n_siz = vsz - n_ofs;
- unsigned d = vec_full_reg_offset(s, a->rd);
- unsigned n = vec_full_reg_offset(s, a->rn);
- unsigned m = vec_full_reg_offset(s, a->rm);
+ unsigned d = vec_full_reg_offset(s, rd);
+ unsigned n = vec_full_reg_offset(s, rn);
+ unsigned m = vec_full_reg_offset(s, rm);
/* Use host vector move insns if we have appropriate sizes
* and no unfortunate overlap.
@@ -2296,6 +2296,19 @@ static bool trans_EXT(DisasContext *s, arg_EXT *a)
return true;
}
+static bool trans_EXT(DisasContext *s, arg_EXT *a)
+{
+ return do_EXT(s, a->rd, a->rn, a->rm, a->imm);
+}
+
+static bool trans_EXT_sve2(DisasContext *s, arg_rri *a)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_EXT(s, a->rd, a->rn, (a->rn + 1) % 32, a->imm);
+}
+
/*
*** SVE Permute - Unpredicated Group
*/
@@ -3023,6 +3036,18 @@ static bool trans_SPLICE(DisasContext *s, arg_rprr_esz *a)
return true;
}
+static bool trans_SPLICE_sve2(DisasContext *s, arg_rpr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_ool_zzzp(s, gen_helper_sve_splice,
+ a->rd, a->rn, (a->rn + 1) % 32, a->pg, a->esz);
+ }
+ return true;
+}
+
/*
*** SVE Integer Compare - Vectors Group
*/
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 51/81] target/arm: Pass separate addend to {U, S}DOT helpers
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (49 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 50/81] target/arm: Implement SVE2 SPLICE, EXT Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-23 10:01 ` LIU Zhiwei
2020-09-23 11:48 ` LIU Zhiwei
2020-09-18 18:37 ` [PATCH v3 52/81] target/arm: Pass separate addend to FCMLA helpers Richard Henderson
` (31 subsequent siblings)
82 siblings, 2 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
For SVE, we potentially have a 4th argument coming from the
movprfx instruction. Currently we do not optimize movprfx,
so the problem is not visible.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper.h | 20 +++---
target/arm/sve.decode | 7 +-
target/arm/translate-a64.c | 15 ++++-
target/arm/translate-sve.c | 13 ++--
target/arm/vec_helper.c | 112 ++++++++++++++++++--------------
target/arm/translate-neon.c.inc | 10 +--
6 files changed, 105 insertions(+), 72 deletions(-)
diff --git a/target/arm/helper.h b/target/arm/helper.h
index 8294055cab..97222bd256 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -595,15 +595,19 @@ DEF_HELPER_FLAGS_5(sve2_sqrdmlah_d, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_4(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_4(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_4(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_4(gvec_udot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_udot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_4(gvec_sdot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_4(gvec_udot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_4(gvec_sdot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_4(gvec_udot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_sdot_idx_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_udot_idx_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_sdot_idx_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_udot_idx_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_fcaddh, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 0688dae450..5815ba9b1c 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -756,12 +756,13 @@ UMIN_zzi 00100101 .. 101 011 110 ........ ..... @rdn_i8u
MUL_zzi 00100101 .. 110 000 110 ........ ..... @rdn_i8s
# SVE integer dot product (unpredicated)
-DOT_zzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 ra=%reg_movprfx
+DOT_zzzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 \
+ ra=%reg_movprfx
# SVE integer dot product (indexed)
-DOT_zzx 01000100 101 index:2 rm:3 00000 u:1 rn:5 rd:5 \
+DOT_zzxw 01000100 101 index:2 rm:3 00000 u:1 rn:5 rd:5 \
sz=0 ra=%reg_movprfx
-DOT_zzx 01000100 111 index:1 rm:4 00000 u:1 rn:5 rd:5 \
+DOT_zzxw 01000100 111 index:1 rm:4 00000 u:1 rn:5 rd:5 \
sz=1 ra=%reg_movprfx
# SVE floating-point complex add (predicated)
diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c
index 76e54c1a4e..1a9251db67 100644
--- a/target/arm/translate-a64.c
+++ b/target/arm/translate-a64.c
@@ -692,6 +692,17 @@ static void gen_gvec_op3_qc(DisasContext *s, bool is_q, int rd, int rn,
tcg_temp_free_ptr(qc_ptr);
}
+/* Expand a 4-operand operation using an out-of-line helper. */
+static void gen_gvec_op4_ool(DisasContext *s, bool is_q, int rd, int rn,
+ int rm, int ra, int data, gen_helper_gvec_4 *fn)
+{
+ tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
+ vec_full_reg_offset(s, rn),
+ vec_full_reg_offset(s, rm),
+ vec_full_reg_offset(s, ra),
+ is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
+}
+
/* Set ZF and NF based on a 64 bit result. This is alas fiddlier
* than the 32 bit equivalent.
*/
@@ -12202,7 +12213,7 @@ static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn)
return;
case 0x2: /* SDOT / UDOT */
- gen_gvec_op3_ool(s, is_q, rd, rn, rm, 0,
+ gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0,
u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b);
return;
@@ -13461,7 +13472,7 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn)
switch (16 * u + opcode) {
case 0x0e: /* SDOT */
case 0x1e: /* UDOT */
- gen_gvec_op3_ool(s, is_q, rd, rn, rm, index,
+ gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
u ? gen_helper_gvec_udot_idx_b
: gen_helper_gvec_sdot_idx_b);
return;
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 5e8291e44b..66303dac54 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -3804,28 +3804,29 @@ DO_ZZI(UMIN, umin)
#undef DO_ZZI
-static bool trans_DOT_zzz(DisasContext *s, arg_DOT_zzz *a)
+static bool trans_DOT_zzzz(DisasContext *s, arg_DOT_zzzz *a)
{
- static gen_helper_gvec_3 * const fns[2][2] = {
+ static gen_helper_gvec_4 * const fns[2][2] = {
{ gen_helper_gvec_sdot_b, gen_helper_gvec_sdot_h },
{ gen_helper_gvec_udot_b, gen_helper_gvec_udot_h }
};
if (sve_access_check(s)) {
- gen_gvec_ool_zzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, 0);
+ gen_gvec_ool_zzzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, a->ra, 0);
}
return true;
}
-static bool trans_DOT_zzx(DisasContext *s, arg_DOT_zzx *a)
+static bool trans_DOT_zzxw(DisasContext *s, arg_DOT_zzxw *a)
{
- static gen_helper_gvec_3 * const fns[2][2] = {
+ static gen_helper_gvec_4 * const fns[2][2] = {
{ gen_helper_gvec_sdot_idx_b, gen_helper_gvec_sdot_idx_h },
{ gen_helper_gvec_udot_idx_b, gen_helper_gvec_udot_idx_h }
};
if (sve_access_check(s)) {
- gen_gvec_ool_zzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, a->index);
+ gen_gvec_ool_zzzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm,
+ a->ra, a->index);
}
return true;
}
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index 32a4403256..d7ef31915b 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -375,71 +375,76 @@ void HELPER(sve2_sqrdmlsh_d)(void *vd, void *vn, void *vm,
* All elements are treated equally, no matter where they are.
*/
-void HELPER(gvec_sdot_b)(void *vd, void *vn, void *vm, uint32_t desc)
+void HELPER(gvec_sdot_b)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
{
intptr_t i, opr_sz = simd_oprsz(desc);
- uint32_t *d = vd;
+ int32_t *d = vd, *a = va;
int8_t *n = vn, *m = vm;
for (i = 0; i < opr_sz / 4; ++i) {
- d[i] += n[i * 4 + 0] * m[i * 4 + 0]
- + n[i * 4 + 1] * m[i * 4 + 1]
- + n[i * 4 + 2] * m[i * 4 + 2]
- + n[i * 4 + 3] * m[i * 4 + 3];
+ d[i] = (a[i] +
+ n[i * 4 + 0] * m[i * 4 + 0] +
+ n[i * 4 + 1] * m[i * 4 + 1] +
+ n[i * 4 + 2] * m[i * 4 + 2] +
+ n[i * 4 + 3] * m[i * 4 + 3]);
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
-void HELPER(gvec_udot_b)(void *vd, void *vn, void *vm, uint32_t desc)
+void HELPER(gvec_udot_b)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
{
intptr_t i, opr_sz = simd_oprsz(desc);
- uint32_t *d = vd;
+ uint32_t *d = vd, *a = va;
uint8_t *n = vn, *m = vm;
for (i = 0; i < opr_sz / 4; ++i) {
- d[i] += n[i * 4 + 0] * m[i * 4 + 0]
- + n[i * 4 + 1] * m[i * 4 + 1]
- + n[i * 4 + 2] * m[i * 4 + 2]
- + n[i * 4 + 3] * m[i * 4 + 3];
+ d[i] = (a[i] +
+ n[i * 4 + 0] * m[i * 4 + 0] +
+ n[i * 4 + 1] * m[i * 4 + 1] +
+ n[i * 4 + 2] * m[i * 4 + 2] +
+ n[i * 4 + 3] * m[i * 4 + 3]);
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
-void HELPER(gvec_sdot_h)(void *vd, void *vn, void *vm, uint32_t desc)
+void HELPER(gvec_sdot_h)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
{
intptr_t i, opr_sz = simd_oprsz(desc);
- uint64_t *d = vd;
+ int64_t *d = vd, *a = va;
int16_t *n = vn, *m = vm;
for (i = 0; i < opr_sz / 8; ++i) {
- d[i] += (int64_t)n[i * 4 + 0] * m[i * 4 + 0]
- + (int64_t)n[i * 4 + 1] * m[i * 4 + 1]
- + (int64_t)n[i * 4 + 2] * m[i * 4 + 2]
- + (int64_t)n[i * 4 + 3] * m[i * 4 + 3];
+ d[i] = (a[i] +
+ (int64_t)n[i * 4 + 0] * m[i * 4 + 0] +
+ (int64_t)n[i * 4 + 1] * m[i * 4 + 1] +
+ (int64_t)n[i * 4 + 2] * m[i * 4 + 2] +
+ (int64_t)n[i * 4 + 3] * m[i * 4 + 3]);
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
-void HELPER(gvec_udot_h)(void *vd, void *vn, void *vm, uint32_t desc)
+void HELPER(gvec_udot_h)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
{
intptr_t i, opr_sz = simd_oprsz(desc);
- uint64_t *d = vd;
+ uint64_t *d = vd, *a = va;
uint16_t *n = vn, *m = vm;
for (i = 0; i < opr_sz / 8; ++i) {
- d[i] += (uint64_t)n[i * 4 + 0] * m[i * 4 + 0]
- + (uint64_t)n[i * 4 + 1] * m[i * 4 + 1]
- + (uint64_t)n[i * 4 + 2] * m[i * 4 + 2]
- + (uint64_t)n[i * 4 + 3] * m[i * 4 + 3];
+ d[i] = (a[i] +
+ (uint64_t)n[i * 4 + 0] * m[i * 4 + 0] +
+ (uint64_t)n[i * 4 + 1] * m[i * 4 + 1] +
+ (uint64_t)n[i * 4 + 2] * m[i * 4 + 2] +
+ (uint64_t)n[i * 4 + 3] * m[i * 4 + 3]);
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
-void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
+void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
{
intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4;
intptr_t index = simd_data(desc);
- uint32_t *d = vd;
+ int32_t *d = vd, *a = va;
int8_t *n = vn;
int8_t *m_indexed = (int8_t *)vm + index * 4;
@@ -455,10 +460,11 @@ void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
int8_t m3 = m_indexed[i * 4 + 3];
do {
- d[i] += n[i * 4 + 0] * m0
- + n[i * 4 + 1] * m1
- + n[i * 4 + 2] * m2
- + n[i * 4 + 3] * m3;
+ d[i] = (a[i] +
+ n[i * 4 + 0] * m0 +
+ n[i * 4 + 1] * m1 +
+ n[i * 4 + 2] * m2 +
+ n[i * 4 + 3] * m3);
} while (++i < segend);
segend = i + 4;
} while (i < opr_sz_4);
@@ -466,11 +472,12 @@ void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
clear_tail(d, opr_sz, simd_maxsz(desc));
}
-void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
+void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
{
intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4;
intptr_t index = simd_data(desc);
- uint32_t *d = vd;
+ uint32_t *d = vd, *a = va;
uint8_t *n = vn;
uint8_t *m_indexed = (uint8_t *)vm + index * 4;
@@ -486,10 +493,11 @@ void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
uint8_t m3 = m_indexed[i * 4 + 3];
do {
- d[i] += n[i * 4 + 0] * m0
- + n[i * 4 + 1] * m1
- + n[i * 4 + 2] * m2
- + n[i * 4 + 3] * m3;
+ d[i] = (a[i] +
+ n[i * 4 + 0] * m0 +
+ n[i * 4 + 1] * m1 +
+ n[i * 4 + 2] * m2 +
+ n[i * 4 + 3] * m3);
} while (++i < segend);
segend = i + 4;
} while (i < opr_sz_4);
@@ -497,11 +505,12 @@ void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
clear_tail(d, opr_sz, simd_maxsz(desc));
}
-void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
+void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
{
intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8;
intptr_t index = simd_data(desc);
- uint64_t *d = vd;
+ int64_t *d = vd, *a = va;
int16_t *n = vn;
int16_t *m_indexed = (int16_t *)vm + index * 4;
@@ -509,14 +518,17 @@ void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
* Process the entire segment all at once, writing back the results
* only after we've consumed all of the inputs.
*/
- for (i = 0; i < opr_sz_8 ; i += 2) {
- uint64_t d0, d1;
+ for (i = 0; i < opr_sz_8; i += 2) {
+ int64_t d0, d1;
- d0 = n[i * 4 + 0] * (int64_t)m_indexed[i * 4 + 0];
+ d0 = a[i + 0];
+ d0 += n[i * 4 + 0] * (int64_t)m_indexed[i * 4 + 0];
d0 += n[i * 4 + 1] * (int64_t)m_indexed[i * 4 + 1];
d0 += n[i * 4 + 2] * (int64_t)m_indexed[i * 4 + 2];
d0 += n[i * 4 + 3] * (int64_t)m_indexed[i * 4 + 3];
- d1 = n[i * 4 + 4] * (int64_t)m_indexed[i * 4 + 0];
+
+ d1 = a[i + 1];
+ d1 += n[i * 4 + 4] * (int64_t)m_indexed[i * 4 + 0];
d1 += n[i * 4 + 5] * (int64_t)m_indexed[i * 4 + 1];
d1 += n[i * 4 + 6] * (int64_t)m_indexed[i * 4 + 2];
d1 += n[i * 4 + 7] * (int64_t)m_indexed[i * 4 + 3];
@@ -524,15 +536,15 @@ void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
d[i + 0] += d0;
d[i + 1] += d1;
}
-
clear_tail(d, opr_sz, simd_maxsz(desc));
}
-void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
+void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
{
intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8;
intptr_t index = simd_data(desc);
- uint64_t *d = vd;
+ uint64_t *d = vd, *a = va;
uint16_t *n = vn;
uint16_t *m_indexed = (uint16_t *)vm + index * 4;
@@ -540,14 +552,17 @@ void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
* Process the entire segment all at once, writing back the results
* only after we've consumed all of the inputs.
*/
- for (i = 0; i < opr_sz_8 ; i += 2) {
+ for (i = 0; i < opr_sz_8; i += 2) {
uint64_t d0, d1;
- d0 = n[i * 4 + 0] * (uint64_t)m_indexed[i * 4 + 0];
+ d0 = a[i + 0];
+ d0 += n[i * 4 + 0] * (uint64_t)m_indexed[i * 4 + 0];
d0 += n[i * 4 + 1] * (uint64_t)m_indexed[i * 4 + 1];
d0 += n[i * 4 + 2] * (uint64_t)m_indexed[i * 4 + 2];
d0 += n[i * 4 + 3] * (uint64_t)m_indexed[i * 4 + 3];
- d1 = n[i * 4 + 4] * (uint64_t)m_indexed[i * 4 + 0];
+
+ d1 = a[i + 1];
+ d1 += n[i * 4 + 4] * (uint64_t)m_indexed[i * 4 + 0];
d1 += n[i * 4 + 5] * (uint64_t)m_indexed[i * 4 + 1];
d1 += n[i * 4 + 6] * (uint64_t)m_indexed[i * 4 + 2];
d1 += n[i * 4 + 7] * (uint64_t)m_indexed[i * 4 + 3];
@@ -555,7 +570,6 @@ void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
d[i + 0] += d0;
d[i + 1] += d1;
}
-
clear_tail(d, opr_sz, simd_maxsz(desc));
}
diff --git a/target/arm/translate-neon.c.inc b/target/arm/translate-neon.c.inc
index 4d1a292981..7efe3d9556 100644
--- a/target/arm/translate-neon.c.inc
+++ b/target/arm/translate-neon.c.inc
@@ -240,7 +240,7 @@ static bool trans_VCADD(DisasContext *s, arg_VCADD *a)
static bool trans_VDOT(DisasContext *s, arg_VDOT *a)
{
int opr_sz;
- gen_helper_gvec_3 *fn_gvec;
+ gen_helper_gvec_4 *fn_gvec;
if (!dc_isar_feature(aa32_dp, s)) {
return false;
@@ -262,9 +262,10 @@ static bool trans_VDOT(DisasContext *s, arg_VDOT *a)
opr_sz = (1 + a->q) * 8;
fn_gvec = a->u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b;
- tcg_gen_gvec_3_ool(vfp_reg_offset(1, a->vd),
+ tcg_gen_gvec_4_ool(vfp_reg_offset(1, a->vd),
vfp_reg_offset(1, a->vn),
vfp_reg_offset(1, a->vm),
+ vfp_reg_offset(1, a->vd),
opr_sz, opr_sz, 0, fn_gvec);
return true;
}
@@ -342,7 +343,7 @@ static bool trans_VCMLA_scalar(DisasContext *s, arg_VCMLA_scalar *a)
static bool trans_VDOT_scalar(DisasContext *s, arg_VDOT_scalar *a)
{
- gen_helper_gvec_3 *fn_gvec;
+ gen_helper_gvec_4 *fn_gvec;
int opr_sz;
TCGv_ptr fpst;
@@ -367,9 +368,10 @@ static bool trans_VDOT_scalar(DisasContext *s, arg_VDOT_scalar *a)
fn_gvec = a->u ? gen_helper_gvec_udot_idx_b : gen_helper_gvec_sdot_idx_b;
opr_sz = (1 + a->q) * 8;
fpst = fpstatus_ptr(FPST_STD);
- tcg_gen_gvec_3_ool(vfp_reg_offset(1, a->vd),
+ tcg_gen_gvec_4_ool(vfp_reg_offset(1, a->vd),
vfp_reg_offset(1, a->vn),
vfp_reg_offset(1, a->rm),
+ vfp_reg_offset(1, a->vd),
opr_sz, opr_sz, a->index, fn_gvec);
tcg_temp_free_ptr(fpst);
return true;
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* Re: [PATCH v3 51/81] target/arm: Pass separate addend to {U, S}DOT helpers
2020-09-18 18:37 ` [PATCH v3 51/81] target/arm: Pass separate addend to {U, S}DOT helpers Richard Henderson
@ 2020-09-23 10:01 ` LIU Zhiwei
2020-09-23 14:46 ` Richard Henderson
2020-09-23 11:48 ` LIU Zhiwei
1 sibling, 1 reply; 94+ messages in thread
From: LIU Zhiwei @ 2020-09-23 10:01 UTC (permalink / raw)
To: Richard Henderson, qemu-devel; +Cc: peter.maydell, alex.bennee
On 2020/9/19 2:37, Richard Henderson wrote:
> For SVE, we potentially have a 4th argument coming from the
> movprfx instruction. Currently we do not optimize movprfx,
> so the problem is not visible.
Hi Richard,
I am a little confused. If it is not immediately preceded by a MOVPRFX
instruction, the addend will
still be used.
Is it right?
Zhiwei
> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
> ---
> target/arm/helper.h | 20 +++---
> target/arm/sve.decode | 7 +-
> target/arm/translate-a64.c | 15 ++++-
> target/arm/translate-sve.c | 13 ++--
> target/arm/vec_helper.c | 112 ++++++++++++++++++--------------
> target/arm/translate-neon.c.inc | 10 +--
> 6 files changed, 105 insertions(+), 72 deletions(-)
>
> diff --git a/target/arm/helper.h b/target/arm/helper.h
> index 8294055cab..97222bd256 100644
> --- a/target/arm/helper.h
> +++ b/target/arm/helper.h
> @@ -595,15 +595,19 @@ DEF_HELPER_FLAGS_5(sve2_sqrdmlah_d, TCG_CALL_NO_RWG,
> DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_d, TCG_CALL_NO_RWG,
> void, ptr, ptr, ptr, ptr, i32)
>
> -DEF_HELPER_FLAGS_4(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> -DEF_HELPER_FLAGS_4(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> -DEF_HELPER_FLAGS_4(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> -DEF_HELPER_FLAGS_4(gvec_udot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_udot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
>
> -DEF_HELPER_FLAGS_4(gvec_sdot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> -DEF_HELPER_FLAGS_4(gvec_udot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> -DEF_HELPER_FLAGS_4(gvec_sdot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> -DEF_HELPER_FLAGS_4(gvec_udot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_sdot_idx_b, TCG_CALL_NO_RWG,
> + void, ptr, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_udot_idx_b, TCG_CALL_NO_RWG,
> + void, ptr, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_sdot_idx_h, TCG_CALL_NO_RWG,
> + void, ptr, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_udot_idx_h, TCG_CALL_NO_RWG,
> + void, ptr, ptr, ptr, ptr, i32)
>
> DEF_HELPER_FLAGS_5(gvec_fcaddh, TCG_CALL_NO_RWG,
> void, ptr, ptr, ptr, ptr, i32)
> diff --git a/target/arm/sve.decode b/target/arm/sve.decode
> index 0688dae450..5815ba9b1c 100644
> --- a/target/arm/sve.decode
> +++ b/target/arm/sve.decode
> @@ -756,12 +756,13 @@ UMIN_zzi 00100101 .. 101 011 110 ........ ..... @rdn_i8u
> MUL_zzi 00100101 .. 110 000 110 ........ ..... @rdn_i8s
>
> # SVE integer dot product (unpredicated)
> -DOT_zzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 ra=%reg_movprfx
> +DOT_zzzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 \
> + ra=%reg_movprfx
>
> # SVE integer dot product (indexed)
> -DOT_zzx 01000100 101 index:2 rm:3 00000 u:1 rn:5 rd:5 \
> +DOT_zzxw 01000100 101 index:2 rm:3 00000 u:1 rn:5 rd:5 \
> sz=0 ra=%reg_movprfx
> -DOT_zzx 01000100 111 index:1 rm:4 00000 u:1 rn:5 rd:5 \
> +DOT_zzxw 01000100 111 index:1 rm:4 00000 u:1 rn:5 rd:5 \
> sz=1 ra=%reg_movprfx
>
> # SVE floating-point complex add (predicated)
> diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c
> index 76e54c1a4e..1a9251db67 100644
> --- a/target/arm/translate-a64.c
> +++ b/target/arm/translate-a64.c
> @@ -692,6 +692,17 @@ static void gen_gvec_op3_qc(DisasContext *s, bool is_q, int rd, int rn,
> tcg_temp_free_ptr(qc_ptr);
> }
>
> +/* Expand a 4-operand operation using an out-of-line helper. */
> +static void gen_gvec_op4_ool(DisasContext *s, bool is_q, int rd, int rn,
> + int rm, int ra, int data, gen_helper_gvec_4 *fn)
> +{
> + tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
> + vec_full_reg_offset(s, rn),
> + vec_full_reg_offset(s, rm),
> + vec_full_reg_offset(s, ra),
> + is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
> +}
> +
> /* Set ZF and NF based on a 64 bit result. This is alas fiddlier
> * than the 32 bit equivalent.
> */
> @@ -12202,7 +12213,7 @@ static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn)
> return;
>
> case 0x2: /* SDOT / UDOT */
> - gen_gvec_op3_ool(s, is_q, rd, rn, rm, 0,
> + gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0,
> u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b);
> return;
>
> @@ -13461,7 +13472,7 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn)
> switch (16 * u + opcode) {
> case 0x0e: /* SDOT */
> case 0x1e: /* UDOT */
> - gen_gvec_op3_ool(s, is_q, rd, rn, rm, index,
> + gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
> u ? gen_helper_gvec_udot_idx_b
> : gen_helper_gvec_sdot_idx_b);
> return;
> diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
> index 5e8291e44b..66303dac54 100644
> --- a/target/arm/translate-sve.c
> +++ b/target/arm/translate-sve.c
> @@ -3804,28 +3804,29 @@ DO_ZZI(UMIN, umin)
>
> #undef DO_ZZI
>
> -static bool trans_DOT_zzz(DisasContext *s, arg_DOT_zzz *a)
> +static bool trans_DOT_zzzz(DisasContext *s, arg_DOT_zzzz *a)
> {
> - static gen_helper_gvec_3 * const fns[2][2] = {
> + static gen_helper_gvec_4 * const fns[2][2] = {
> { gen_helper_gvec_sdot_b, gen_helper_gvec_sdot_h },
> { gen_helper_gvec_udot_b, gen_helper_gvec_udot_h }
> };
>
> if (sve_access_check(s)) {
> - gen_gvec_ool_zzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, 0);
> + gen_gvec_ool_zzzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, a->ra, 0);
> }
> return true;
> }
>
> -static bool trans_DOT_zzx(DisasContext *s, arg_DOT_zzx *a)
> +static bool trans_DOT_zzxw(DisasContext *s, arg_DOT_zzxw *a)
> {
> - static gen_helper_gvec_3 * const fns[2][2] = {
> + static gen_helper_gvec_4 * const fns[2][2] = {
> { gen_helper_gvec_sdot_idx_b, gen_helper_gvec_sdot_idx_h },
> { gen_helper_gvec_udot_idx_b, gen_helper_gvec_udot_idx_h }
> };
>
> if (sve_access_check(s)) {
> - gen_gvec_ool_zzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, a->index);
> + gen_gvec_ool_zzzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm,
> + a->ra, a->index);
> }
> return true;
> }
> diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
> index 32a4403256..d7ef31915b 100644
> --- a/target/arm/vec_helper.c
> +++ b/target/arm/vec_helper.c
> @@ -375,71 +375,76 @@ void HELPER(sve2_sqrdmlsh_d)(void *vd, void *vn, void *vm,
> * All elements are treated equally, no matter where they are.
> */
>
> -void HELPER(gvec_sdot_b)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_sdot_b)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
> {
> intptr_t i, opr_sz = simd_oprsz(desc);
> - uint32_t *d = vd;
> + int32_t *d = vd, *a = va;
> int8_t *n = vn, *m = vm;
>
> for (i = 0; i < opr_sz / 4; ++i) {
> - d[i] += n[i * 4 + 0] * m[i * 4 + 0]
> - + n[i * 4 + 1] * m[i * 4 + 1]
> - + n[i * 4 + 2] * m[i * 4 + 2]
> - + n[i * 4 + 3] * m[i * 4 + 3];
> + d[i] = (a[i] +
> + n[i * 4 + 0] * m[i * 4 + 0] +
> + n[i * 4 + 1] * m[i * 4 + 1] +
> + n[i * 4 + 2] * m[i * 4 + 2] +
> + n[i * 4 + 3] * m[i * 4 + 3]);
> }
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> -void HELPER(gvec_udot_b)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_udot_b)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
> {
> intptr_t i, opr_sz = simd_oprsz(desc);
> - uint32_t *d = vd;
> + uint32_t *d = vd, *a = va;
> uint8_t *n = vn, *m = vm;
>
> for (i = 0; i < opr_sz / 4; ++i) {
> - d[i] += n[i * 4 + 0] * m[i * 4 + 0]
> - + n[i * 4 + 1] * m[i * 4 + 1]
> - + n[i * 4 + 2] * m[i * 4 + 2]
> - + n[i * 4 + 3] * m[i * 4 + 3];
> + d[i] = (a[i] +
> + n[i * 4 + 0] * m[i * 4 + 0] +
> + n[i * 4 + 1] * m[i * 4 + 1] +
> + n[i * 4 + 2] * m[i * 4 + 2] +
> + n[i * 4 + 3] * m[i * 4 + 3]);
> }
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> -void HELPER(gvec_sdot_h)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_sdot_h)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
> {
> intptr_t i, opr_sz = simd_oprsz(desc);
> - uint64_t *d = vd;
> + int64_t *d = vd, *a = va;
> int16_t *n = vn, *m = vm;
>
> for (i = 0; i < opr_sz / 8; ++i) {
> - d[i] += (int64_t)n[i * 4 + 0] * m[i * 4 + 0]
> - + (int64_t)n[i * 4 + 1] * m[i * 4 + 1]
> - + (int64_t)n[i * 4 + 2] * m[i * 4 + 2]
> - + (int64_t)n[i * 4 + 3] * m[i * 4 + 3];
> + d[i] = (a[i] +
> + (int64_t)n[i * 4 + 0] * m[i * 4 + 0] +
> + (int64_t)n[i * 4 + 1] * m[i * 4 + 1] +
> + (int64_t)n[i * 4 + 2] * m[i * 4 + 2] +
> + (int64_t)n[i * 4 + 3] * m[i * 4 + 3]);
> }
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> -void HELPER(gvec_udot_h)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_udot_h)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
> {
> intptr_t i, opr_sz = simd_oprsz(desc);
> - uint64_t *d = vd;
> + uint64_t *d = vd, *a = va;
> uint16_t *n = vn, *m = vm;
>
> for (i = 0; i < opr_sz / 8; ++i) {
> - d[i] += (uint64_t)n[i * 4 + 0] * m[i * 4 + 0]
> - + (uint64_t)n[i * 4 + 1] * m[i * 4 + 1]
> - + (uint64_t)n[i * 4 + 2] * m[i * 4 + 2]
> - + (uint64_t)n[i * 4 + 3] * m[i * 4 + 3];
> + d[i] = (a[i] +
> + (uint64_t)n[i * 4 + 0] * m[i * 4 + 0] +
> + (uint64_t)n[i * 4 + 1] * m[i * 4 + 1] +
> + (uint64_t)n[i * 4 + 2] * m[i * 4 + 2] +
> + (uint64_t)n[i * 4 + 3] * m[i * 4 + 3]);
> }
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> -void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm,
> + void *va, uint32_t desc)
> {
> intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4;
> intptr_t index = simd_data(desc);
> - uint32_t *d = vd;
> + int32_t *d = vd, *a = va;
> int8_t *n = vn;
> int8_t *m_indexed = (int8_t *)vm + index * 4;
>
> @@ -455,10 +460,11 @@ void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
> int8_t m3 = m_indexed[i * 4 + 3];
>
> do {
> - d[i] += n[i * 4 + 0] * m0
> - + n[i * 4 + 1] * m1
> - + n[i * 4 + 2] * m2
> - + n[i * 4 + 3] * m3;
> + d[i] = (a[i] +
> + n[i * 4 + 0] * m0 +
> + n[i * 4 + 1] * m1 +
> + n[i * 4 + 2] * m2 +
> + n[i * 4 + 3] * m3);
> } while (++i < segend);
> segend = i + 4;
> } while (i < opr_sz_4);
> @@ -466,11 +472,12 @@ void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> -void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm,
> + void *va, uint32_t desc)
> {
> intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4;
> intptr_t index = simd_data(desc);
> - uint32_t *d = vd;
> + uint32_t *d = vd, *a = va;
> uint8_t *n = vn;
> uint8_t *m_indexed = (uint8_t *)vm + index * 4;
>
> @@ -486,10 +493,11 @@ void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
> uint8_t m3 = m_indexed[i * 4 + 3];
>
> do {
> - d[i] += n[i * 4 + 0] * m0
> - + n[i * 4 + 1] * m1
> - + n[i * 4 + 2] * m2
> - + n[i * 4 + 3] * m3;
> + d[i] = (a[i] +
> + n[i * 4 + 0] * m0 +
> + n[i * 4 + 1] * m1 +
> + n[i * 4 + 2] * m2 +
> + n[i * 4 + 3] * m3);
> } while (++i < segend);
> segend = i + 4;
> } while (i < opr_sz_4);
> @@ -497,11 +505,12 @@ void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> -void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm,
> + void *va, uint32_t desc)
> {
> intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8;
> intptr_t index = simd_data(desc);
> - uint64_t *d = vd;
> + int64_t *d = vd, *a = va;
> int16_t *n = vn;
> int16_t *m_indexed = (int16_t *)vm + index * 4;
>
> @@ -509,14 +518,17 @@ void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
> * Process the entire segment all at once, writing back the results
> * only after we've consumed all of the inputs.
> */
> - for (i = 0; i < opr_sz_8 ; i += 2) {
> - uint64_t d0, d1;
> + for (i = 0; i < opr_sz_8; i += 2) {
> + int64_t d0, d1;
>
> - d0 = n[i * 4 + 0] * (int64_t)m_indexed[i * 4 + 0];
> + d0 = a[i + 0];
> + d0 += n[i * 4 + 0] * (int64_t)m_indexed[i * 4 + 0];
> d0 += n[i * 4 + 1] * (int64_t)m_indexed[i * 4 + 1];
> d0 += n[i * 4 + 2] * (int64_t)m_indexed[i * 4 + 2];
> d0 += n[i * 4 + 3] * (int64_t)m_indexed[i * 4 + 3];
> - d1 = n[i * 4 + 4] * (int64_t)m_indexed[i * 4 + 0];
> +
> + d1 = a[i + 1];
> + d1 += n[i * 4 + 4] * (int64_t)m_indexed[i * 4 + 0];
> d1 += n[i * 4 + 5] * (int64_t)m_indexed[i * 4 + 1];
> d1 += n[i * 4 + 6] * (int64_t)m_indexed[i * 4 + 2];
> d1 += n[i * 4 + 7] * (int64_t)m_indexed[i * 4 + 3];
> @@ -524,15 +536,15 @@ void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
> d[i + 0] += d0;
> d[i + 1] += d1;
> }
> -
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> -void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm,
> + void *va, uint32_t desc)
> {
> intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8;
> intptr_t index = simd_data(desc);
> - uint64_t *d = vd;
> + uint64_t *d = vd, *a = va;
> uint16_t *n = vn;
> uint16_t *m_indexed = (uint16_t *)vm + index * 4;
>
> @@ -540,14 +552,17 @@ void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
> * Process the entire segment all at once, writing back the results
> * only after we've consumed all of the inputs.
> */
> - for (i = 0; i < opr_sz_8 ; i += 2) {
> + for (i = 0; i < opr_sz_8; i += 2) {
> uint64_t d0, d1;
>
> - d0 = n[i * 4 + 0] * (uint64_t)m_indexed[i * 4 + 0];
> + d0 = a[i + 0];
> + d0 += n[i * 4 + 0] * (uint64_t)m_indexed[i * 4 + 0];
> d0 += n[i * 4 + 1] * (uint64_t)m_indexed[i * 4 + 1];
> d0 += n[i * 4 + 2] * (uint64_t)m_indexed[i * 4 + 2];
> d0 += n[i * 4 + 3] * (uint64_t)m_indexed[i * 4 + 3];
> - d1 = n[i * 4 + 4] * (uint64_t)m_indexed[i * 4 + 0];
> +
> + d1 = a[i + 1];
> + d1 += n[i * 4 + 4] * (uint64_t)m_indexed[i * 4 + 0];
> d1 += n[i * 4 + 5] * (uint64_t)m_indexed[i * 4 + 1];
> d1 += n[i * 4 + 6] * (uint64_t)m_indexed[i * 4 + 2];
> d1 += n[i * 4 + 7] * (uint64_t)m_indexed[i * 4 + 3];
> @@ -555,7 +570,6 @@ void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
> d[i + 0] += d0;
> d[i + 1] += d1;
> }
> -
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> diff --git a/target/arm/translate-neon.c.inc b/target/arm/translate-neon.c.inc
> index 4d1a292981..7efe3d9556 100644
> --- a/target/arm/translate-neon.c.inc
> +++ b/target/arm/translate-neon.c.inc
> @@ -240,7 +240,7 @@ static bool trans_VCADD(DisasContext *s, arg_VCADD *a)
> static bool trans_VDOT(DisasContext *s, arg_VDOT *a)
> {
> int opr_sz;
> - gen_helper_gvec_3 *fn_gvec;
> + gen_helper_gvec_4 *fn_gvec;
>
> if (!dc_isar_feature(aa32_dp, s)) {
> return false;
> @@ -262,9 +262,10 @@ static bool trans_VDOT(DisasContext *s, arg_VDOT *a)
>
> opr_sz = (1 + a->q) * 8;
> fn_gvec = a->u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b;
> - tcg_gen_gvec_3_ool(vfp_reg_offset(1, a->vd),
> + tcg_gen_gvec_4_ool(vfp_reg_offset(1, a->vd),
> vfp_reg_offset(1, a->vn),
> vfp_reg_offset(1, a->vm),
> + vfp_reg_offset(1, a->vd),
> opr_sz, opr_sz, 0, fn_gvec);
> return true;
> }
> @@ -342,7 +343,7 @@ static bool trans_VCMLA_scalar(DisasContext *s, arg_VCMLA_scalar *a)
>
> static bool trans_VDOT_scalar(DisasContext *s, arg_VDOT_scalar *a)
> {
> - gen_helper_gvec_3 *fn_gvec;
> + gen_helper_gvec_4 *fn_gvec;
> int opr_sz;
> TCGv_ptr fpst;
>
> @@ -367,9 +368,10 @@ static bool trans_VDOT_scalar(DisasContext *s, arg_VDOT_scalar *a)
> fn_gvec = a->u ? gen_helper_gvec_udot_idx_b : gen_helper_gvec_sdot_idx_b;
> opr_sz = (1 + a->q) * 8;
> fpst = fpstatus_ptr(FPST_STD);
> - tcg_gen_gvec_3_ool(vfp_reg_offset(1, a->vd),
> + tcg_gen_gvec_4_ool(vfp_reg_offset(1, a->vd),
> vfp_reg_offset(1, a->vn),
> vfp_reg_offset(1, a->rm),
> + vfp_reg_offset(1, a->vd),
> opr_sz, opr_sz, a->index, fn_gvec);
> tcg_temp_free_ptr(fpst);
> return true;
^ permalink raw reply [flat|nested] 94+ messages in thread* Re: [PATCH v3 51/81] target/arm: Pass separate addend to {U, S}DOT helpers
2020-09-23 10:01 ` LIU Zhiwei
@ 2020-09-23 14:46 ` Richard Henderson
2020-09-24 1:29 ` LIU Zhiwei
0 siblings, 1 reply; 94+ messages in thread
From: Richard Henderson @ 2020-09-23 14:46 UTC (permalink / raw)
To: LIU Zhiwei, qemu-devel; +Cc: peter.maydell, alex.bennee
On 9/23/20 3:01 AM, LIU Zhiwei wrote:
>
>
> On 2020/9/19 2:37, Richard Henderson wrote:
>> For SVE, we potentially have a 4th argument coming from the
>> movprfx instruction. Currently we do not optimize movprfx,
>> so the problem is not visible.
> Hi Richard,
>
> I am a little confused. If it is not immediately preceded by a MOVPRFX
> instruction, the addend will
> still be used.
>
> Is it right?
If movprfx is not used, then the addend register will be the same as the
destination register.
r~
^ permalink raw reply [flat|nested] 94+ messages in thread* Re: [PATCH v3 51/81] target/arm: Pass separate addend to {U, S}DOT helpers
2020-09-23 14:46 ` Richard Henderson
@ 2020-09-24 1:29 ` LIU Zhiwei
0 siblings, 0 replies; 94+ messages in thread
From: LIU Zhiwei @ 2020-09-24 1:29 UTC (permalink / raw)
To: Richard Henderson, qemu-devel; +Cc: peter.maydell, alex.bennee
[-- Attachment #1: Type: text/plain, Size: 903 bytes --]
On 2020/9/23 22:46, Richard Henderson wrote:
> On 9/23/20 3:01 AM, LIU Zhiwei wrote:
>>
>> On 2020/9/19 2:37, Richard Henderson wrote:
>>> For SVE, we potentially have a 4th argument coming from the
>>> movprfx instruction. Currently we do not optimize movprfx,
>>> so the problem is not visible.
>> Hi Richard,
>>
>> I am a little confused. If it is not immediately preceded by a MOVPRFX
>> instruction, the addend will
>> still be used.
>>
>> Is it right?
> If movprfx is not used, then the addend register will be the same as the
> destination register.
Get it.
Could you see again the definition of HELPER(gvec_udot_idx_h) or the
HELPER(gvec_sdot_idx_h)?
I think it is wrong there, it code sequence is like this:
d0 = a[i+0]
//dot calculation
d[i+0] += d0
Because when addend is the destination register, it has no reason to add
destination register twice.
Best Regards,
Zhiwei
>
> r~
[-- Attachment #2: Type: text/html, Size: 1780 bytes --]
^ permalink raw reply [flat|nested] 94+ messages in thread
* Re: [PATCH v3 51/81] target/arm: Pass separate addend to {U, S}DOT helpers
2020-09-18 18:37 ` [PATCH v3 51/81] target/arm: Pass separate addend to {U, S}DOT helpers Richard Henderson
2020-09-23 10:01 ` LIU Zhiwei
@ 2020-09-23 11:48 ` LIU Zhiwei
2020-10-09 12:42 ` Richard Henderson
1 sibling, 1 reply; 94+ messages in thread
From: LIU Zhiwei @ 2020-09-23 11:48 UTC (permalink / raw)
To: Richard Henderson, qemu-devel; +Cc: peter.maydell, alex.bennee
On 2020/9/19 2:37, Richard Henderson wrote:
> For SVE, we potentially have a 4th argument coming from the
> movprfx instruction. Currently we do not optimize movprfx,
> so the problem is not visible.
>
> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
> ---
> target/arm/helper.h | 20 +++---
> target/arm/sve.decode | 7 +-
> target/arm/translate-a64.c | 15 ++++-
> target/arm/translate-sve.c | 13 ++--
> target/arm/vec_helper.c | 112 ++++++++++++++++++--------------
> target/arm/translate-neon.c.inc | 10 +--
> 6 files changed, 105 insertions(+), 72 deletions(-)
>
> diff --git a/target/arm/helper.h b/target/arm/helper.h
> index 8294055cab..97222bd256 100644
> --- a/target/arm/helper.h
> +++ b/target/arm/helper.h
> @@ -595,15 +595,19 @@ DEF_HELPER_FLAGS_5(sve2_sqrdmlah_d, TCG_CALL_NO_RWG,
> DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_d, TCG_CALL_NO_RWG,
> void, ptr, ptr, ptr, ptr, i32)
>
> -DEF_HELPER_FLAGS_4(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> -DEF_HELPER_FLAGS_4(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> -DEF_HELPER_FLAGS_4(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> -DEF_HELPER_FLAGS_4(gvec_udot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_udot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
>
> -DEF_HELPER_FLAGS_4(gvec_sdot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> -DEF_HELPER_FLAGS_4(gvec_udot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> -DEF_HELPER_FLAGS_4(gvec_sdot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> -DEF_HELPER_FLAGS_4(gvec_udot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_sdot_idx_b, TCG_CALL_NO_RWG,
> + void, ptr, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_udot_idx_b, TCG_CALL_NO_RWG,
> + void, ptr, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_sdot_idx_h, TCG_CALL_NO_RWG,
> + void, ptr, ptr, ptr, ptr, i32)
> +DEF_HELPER_FLAGS_5(gvec_udot_idx_h, TCG_CALL_NO_RWG,
> + void, ptr, ptr, ptr, ptr, i32)
>
> DEF_HELPER_FLAGS_5(gvec_fcaddh, TCG_CALL_NO_RWG,
> void, ptr, ptr, ptr, ptr, i32)
> diff --git a/target/arm/sve.decode b/target/arm/sve.decode
> index 0688dae450..5815ba9b1c 100644
> --- a/target/arm/sve.decode
> +++ b/target/arm/sve.decode
> @@ -756,12 +756,13 @@ UMIN_zzi 00100101 .. 101 011 110 ........ ..... @rdn_i8u
> MUL_zzi 00100101 .. 110 000 110 ........ ..... @rdn_i8s
>
> # SVE integer dot product (unpredicated)
> -DOT_zzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 ra=%reg_movprfx
> +DOT_zzzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 \
> + ra=%reg_movprfx
>
> # SVE integer dot product (indexed)
> -DOT_zzx 01000100 101 index:2 rm:3 00000 u:1 rn:5 rd:5 \
> +DOT_zzxw 01000100 101 index:2 rm:3 00000 u:1 rn:5 rd:5 \
> sz=0 ra=%reg_movprfx
> -DOT_zzx 01000100 111 index:1 rm:4 00000 u:1 rn:5 rd:5 \
> +DOT_zzxw 01000100 111 index:1 rm:4 00000 u:1 rn:5 rd:5 \
> sz=1 ra=%reg_movprfx
>
> # SVE floating-point complex add (predicated)
> diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c
> index 76e54c1a4e..1a9251db67 100644
> --- a/target/arm/translate-a64.c
> +++ b/target/arm/translate-a64.c
> @@ -692,6 +692,17 @@ static void gen_gvec_op3_qc(DisasContext *s, bool is_q, int rd, int rn,
> tcg_temp_free_ptr(qc_ptr);
> }
>
> +/* Expand a 4-operand operation using an out-of-line helper. */
> +static void gen_gvec_op4_ool(DisasContext *s, bool is_q, int rd, int rn,
> + int rm, int ra, int data, gen_helper_gvec_4 *fn)
> +{
> + tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
> + vec_full_reg_offset(s, rn),
> + vec_full_reg_offset(s, rm),
> + vec_full_reg_offset(s, ra),
> + is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
> +}
> +
> /* Set ZF and NF based on a 64 bit result. This is alas fiddlier
> * than the 32 bit equivalent.
> */
> @@ -12202,7 +12213,7 @@ static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn)
> return;
>
> case 0x2: /* SDOT / UDOT */
> - gen_gvec_op3_ool(s, is_q, rd, rn, rm, 0,
> + gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0,
> u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b);
> return;
>
> @@ -13461,7 +13472,7 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn)
> switch (16 * u + opcode) {
> case 0x0e: /* SDOT */
> case 0x1e: /* UDOT */
> - gen_gvec_op3_ool(s, is_q, rd, rn, rm, index,
> + gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
> u ? gen_helper_gvec_udot_idx_b
> : gen_helper_gvec_sdot_idx_b);
> return;
> diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
> index 5e8291e44b..66303dac54 100644
> --- a/target/arm/translate-sve.c
> +++ b/target/arm/translate-sve.c
> @@ -3804,28 +3804,29 @@ DO_ZZI(UMIN, umin)
>
> #undef DO_ZZI
>
> -static bool trans_DOT_zzz(DisasContext *s, arg_DOT_zzz *a)
> +static bool trans_DOT_zzzz(DisasContext *s, arg_DOT_zzzz *a)
> {
> - static gen_helper_gvec_3 * const fns[2][2] = {
> + static gen_helper_gvec_4 * const fns[2][2] = {
> { gen_helper_gvec_sdot_b, gen_helper_gvec_sdot_h },
> { gen_helper_gvec_udot_b, gen_helper_gvec_udot_h }
> };
>
> if (sve_access_check(s)) {
> - gen_gvec_ool_zzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, 0);
> + gen_gvec_ool_zzzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, a->ra, 0);
> }
> return true;
> }
>
> -static bool trans_DOT_zzx(DisasContext *s, arg_DOT_zzx *a)
> +static bool trans_DOT_zzxw(DisasContext *s, arg_DOT_zzxw *a)
> {
> - static gen_helper_gvec_3 * const fns[2][2] = {
> + static gen_helper_gvec_4 * const fns[2][2] = {
> { gen_helper_gvec_sdot_idx_b, gen_helper_gvec_sdot_idx_h },
> { gen_helper_gvec_udot_idx_b, gen_helper_gvec_udot_idx_h }
> };
>
> if (sve_access_check(s)) {
> - gen_gvec_ool_zzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, a->index);
> + gen_gvec_ool_zzzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm,
> + a->ra, a->index);
> }
> return true;
> }
> diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
> index 32a4403256..d7ef31915b 100644
> --- a/target/arm/vec_helper.c
> +++ b/target/arm/vec_helper.c
> @@ -375,71 +375,76 @@ void HELPER(sve2_sqrdmlsh_d)(void *vd, void *vn, void *vm,
> * All elements are treated equally, no matter where they are.
> */
>
> -void HELPER(gvec_sdot_b)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_sdot_b)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
> {
> intptr_t i, opr_sz = simd_oprsz(desc);
> - uint32_t *d = vd;
> + int32_t *d = vd, *a = va;
> int8_t *n = vn, *m = vm;
>
> for (i = 0; i < opr_sz / 4; ++i) {
> - d[i] += n[i * 4 + 0] * m[i * 4 + 0]
> - + n[i * 4 + 1] * m[i * 4 + 1]
> - + n[i * 4 + 2] * m[i * 4 + 2]
> - + n[i * 4 + 3] * m[i * 4 + 3];
> + d[i] = (a[i] +
> + n[i * 4 + 0] * m[i * 4 + 0] +
> + n[i * 4 + 1] * m[i * 4 + 1] +
> + n[i * 4 + 2] * m[i * 4 + 2] +
> + n[i * 4 + 3] * m[i * 4 + 3]);
> }
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> -void HELPER(gvec_udot_b)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_udot_b)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
> {
> intptr_t i, opr_sz = simd_oprsz(desc);
> - uint32_t *d = vd;
> + uint32_t *d = vd, *a = va;
> uint8_t *n = vn, *m = vm;
>
> for (i = 0; i < opr_sz / 4; ++i) {
> - d[i] += n[i * 4 + 0] * m[i * 4 + 0]
> - + n[i * 4 + 1] * m[i * 4 + 1]
> - + n[i * 4 + 2] * m[i * 4 + 2]
> - + n[i * 4 + 3] * m[i * 4 + 3];
> + d[i] = (a[i] +
> + n[i * 4 + 0] * m[i * 4 + 0] +
> + n[i * 4 + 1] * m[i * 4 + 1] +
> + n[i * 4 + 2] * m[i * 4 + 2] +
> + n[i * 4 + 3] * m[i * 4 + 3]);
> }
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> -void HELPER(gvec_sdot_h)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_sdot_h)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
> {
> intptr_t i, opr_sz = simd_oprsz(desc);
> - uint64_t *d = vd;
> + int64_t *d = vd, *a = va;
> int16_t *n = vn, *m = vm;
>
> for (i = 0; i < opr_sz / 8; ++i) {
> - d[i] += (int64_t)n[i * 4 + 0] * m[i * 4 + 0]
> - + (int64_t)n[i * 4 + 1] * m[i * 4 + 1]
> - + (int64_t)n[i * 4 + 2] * m[i * 4 + 2]
> - + (int64_t)n[i * 4 + 3] * m[i * 4 + 3];
> + d[i] = (a[i] +
> + (int64_t)n[i * 4 + 0] * m[i * 4 + 0] +
> + (int64_t)n[i * 4 + 1] * m[i * 4 + 1] +
> + (int64_t)n[i * 4 + 2] * m[i * 4 + 2] +
> + (int64_t)n[i * 4 + 3] * m[i * 4 + 3]);
> }
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> -void HELPER(gvec_udot_h)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_udot_h)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
> {
> intptr_t i, opr_sz = simd_oprsz(desc);
> - uint64_t *d = vd;
> + uint64_t *d = vd, *a = va;
> uint16_t *n = vn, *m = vm;
>
> for (i = 0; i < opr_sz / 8; ++i) {
> - d[i] += (uint64_t)n[i * 4 + 0] * m[i * 4 + 0]
> - + (uint64_t)n[i * 4 + 1] * m[i * 4 + 1]
> - + (uint64_t)n[i * 4 + 2] * m[i * 4 + 2]
> - + (uint64_t)n[i * 4 + 3] * m[i * 4 + 3];
> + d[i] = (a[i] +
> + (uint64_t)n[i * 4 + 0] * m[i * 4 + 0] +
> + (uint64_t)n[i * 4 + 1] * m[i * 4 + 1] +
> + (uint64_t)n[i * 4 + 2] * m[i * 4 + 2] +
> + (uint64_t)n[i * 4 + 3] * m[i * 4 + 3]);
> }
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> -void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm,
> + void *va, uint32_t desc)
> {
> intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4;
> intptr_t index = simd_data(desc);
> - uint32_t *d = vd;
> + int32_t *d = vd, *a = va;
> int8_t *n = vn;
> int8_t *m_indexed = (int8_t *)vm + index * 4;
>
> @@ -455,10 +460,11 @@ void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
> int8_t m3 = m_indexed[i * 4 + 3];
>
> do {
> - d[i] += n[i * 4 + 0] * m0
> - + n[i * 4 + 1] * m1
> - + n[i * 4 + 2] * m2
> - + n[i * 4 + 3] * m3;
> + d[i] = (a[i] +
> + n[i * 4 + 0] * m0 +
> + n[i * 4 + 1] * m1 +
> + n[i * 4 + 2] * m2 +
> + n[i * 4 + 3] * m3);
> } while (++i < segend);
> segend = i + 4;
> } while (i < opr_sz_4);
> @@ -466,11 +472,12 @@ void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> -void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm,
> + void *va, uint32_t desc)
> {
> intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4;
> intptr_t index = simd_data(desc);
> - uint32_t *d = vd;
> + uint32_t *d = vd, *a = va;
> uint8_t *n = vn;
> uint8_t *m_indexed = (uint8_t *)vm + index * 4;
>
> @@ -486,10 +493,11 @@ void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
> uint8_t m3 = m_indexed[i * 4 + 3];
>
> do {
> - d[i] += n[i * 4 + 0] * m0
> - + n[i * 4 + 1] * m1
> - + n[i * 4 + 2] * m2
> - + n[i * 4 + 3] * m3;
> + d[i] = (a[i] +
> + n[i * 4 + 0] * m0 +
> + n[i * 4 + 1] * m1 +
> + n[i * 4 + 2] * m2 +
> + n[i * 4 + 3] * m3);
> } while (++i < segend);
> segend = i + 4;
> } while (i < opr_sz_4);
> @@ -497,11 +505,12 @@ void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> -void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm,
> + void *va, uint32_t desc)
> {
> intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8;
> intptr_t index = simd_data(desc);
> - uint64_t *d = vd;
> + int64_t *d = vd, *a = va;
> int16_t *n = vn;
> int16_t *m_indexed = (int16_t *)vm + index * 4;
>
> @@ -509,14 +518,17 @@ void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
> * Process the entire segment all at once, writing back the results
> * only after we've consumed all of the inputs.
> */
> - for (i = 0; i < opr_sz_8 ; i += 2) {
> - uint64_t d0, d1;
> + for (i = 0; i < opr_sz_8; i += 2) {
> + int64_t d0, d1;
>
> - d0 = n[i * 4 + 0] * (int64_t)m_indexed[i * 4 + 0];
> + d0 = a[i + 0];
> + d0 += n[i * 4 + 0] * (int64_t)m_indexed[i * 4 + 0];
> d0 += n[i * 4 + 1] * (int64_t)m_indexed[i * 4 + 1];
> d0 += n[i * 4 + 2] * (int64_t)m_indexed[i * 4 + 2];
> d0 += n[i * 4 + 3] * (int64_t)m_indexed[i * 4 + 3];
> - d1 = n[i * 4 + 4] * (int64_t)m_indexed[i * 4 + 0];
> +
> + d1 = a[i + 1];
> + d1 += n[i * 4 + 4] * (int64_t)m_indexed[i * 4 + 0];
> d1 += n[i * 4 + 5] * (int64_t)m_indexed[i * 4 + 1];
> d1 += n[i * 4 + 6] * (int64_t)m_indexed[i * 4 + 2];
> d1 += n[i * 4 + 7] * (int64_t)m_indexed[i * 4 + 3];
> @@ -524,15 +536,15 @@ void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
> d[i + 0] += d0;
> d[i + 1] += d1;
> }
> -
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> -void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
> +void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm,
> + void *va, uint32_t desc)
> {
> intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8;
> intptr_t index = simd_data(desc);
> - uint64_t *d = vd;
> + uint64_t *d = vd, *a = va;
> uint16_t *n = vn;
> uint16_t *m_indexed = (uint16_t *)vm + index * 4;
>
> @@ -540,14 +552,17 @@ void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
> * Process the entire segment all at once, writing back the results
> * only after we've consumed all of the inputs.
> */
> - for (i = 0; i < opr_sz_8 ; i += 2) {
> + for (i = 0; i < opr_sz_8; i += 2) {
> uint64_t d0, d1;
>
> - d0 = n[i * 4 + 0] * (uint64_t)m_indexed[i * 4 + 0];
> + d0 = a[i + 0];
Add once.
> + d0 += n[i * 4 + 0] * (uint64_t)m_indexed[i * 4 + 0];
> d0 += n[i * 4 + 1] * (uint64_t)m_indexed[i * 4 + 1];
> d0 += n[i * 4 + 2] * (uint64_t)m_indexed[i * 4 + 2];
> d0 += n[i * 4 + 3] * (uint64_t)m_indexed[i * 4 + 3];
> - d1 = n[i * 4 + 4] * (uint64_t)m_indexed[i * 4 + 0];
> +
> + d1 = a[i + 1];
> + d1 += n[i * 4 + 4] * (uint64_t)m_indexed[i * 4 + 0];
> d1 += n[i * 4 + 5] * (uint64_t)m_indexed[i * 4 + 1];
> d1 += n[i * 4 + 6] * (uint64_t)m_indexed[i * 4 + 2];
> d1 += n[i * 4 + 7] * (uint64_t)m_indexed[i * 4 + 3];
> @@ -555,7 +570,6 @@ void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
> d[i + 0] += d0;
Add twice.
I think it is wrong here. Do you thinks so?
Zhiwei
> d[i + 1] += d1;
> }
> -
> clear_tail(d, opr_sz, simd_maxsz(desc));
> }
>
> diff --git a/target/arm/translate-neon.c.inc b/target/arm/translate-neon.c.inc
> index 4d1a292981..7efe3d9556 100644
> --- a/target/arm/translate-neon.c.inc
> +++ b/target/arm/translate-neon.c.inc
> @@ -240,7 +240,7 @@ static bool trans_VCADD(DisasContext *s, arg_VCADD *a)
> static bool trans_VDOT(DisasContext *s, arg_VDOT *a)
> {
> int opr_sz;
> - gen_helper_gvec_3 *fn_gvec;
> + gen_helper_gvec_4 *fn_gvec;
>
> if (!dc_isar_feature(aa32_dp, s)) {
> return false;
> @@ -262,9 +262,10 @@ static bool trans_VDOT(DisasContext *s, arg_VDOT *a)
>
> opr_sz = (1 + a->q) * 8;
> fn_gvec = a->u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b;
> - tcg_gen_gvec_3_ool(vfp_reg_offset(1, a->vd),
> + tcg_gen_gvec_4_ool(vfp_reg_offset(1, a->vd),
> vfp_reg_offset(1, a->vn),
> vfp_reg_offset(1, a->vm),
> + vfp_reg_offset(1, a->vd),
> opr_sz, opr_sz, 0, fn_gvec);
> return true;
> }
> @@ -342,7 +343,7 @@ static bool trans_VCMLA_scalar(DisasContext *s, arg_VCMLA_scalar *a)
>
> static bool trans_VDOT_scalar(DisasContext *s, arg_VDOT_scalar *a)
> {
> - gen_helper_gvec_3 *fn_gvec;
> + gen_helper_gvec_4 *fn_gvec;
> int opr_sz;
> TCGv_ptr fpst;
>
> @@ -367,9 +368,10 @@ static bool trans_VDOT_scalar(DisasContext *s, arg_VDOT_scalar *a)
> fn_gvec = a->u ? gen_helper_gvec_udot_idx_b : gen_helper_gvec_sdot_idx_b;
> opr_sz = (1 + a->q) * 8;
> fpst = fpstatus_ptr(FPST_STD);
> - tcg_gen_gvec_3_ool(vfp_reg_offset(1, a->vd),
> + tcg_gen_gvec_4_ool(vfp_reg_offset(1, a->vd),
> vfp_reg_offset(1, a->vn),
> vfp_reg_offset(1, a->rm),
> + vfp_reg_offset(1, a->vd),
> opr_sz, opr_sz, a->index, fn_gvec);
> tcg_temp_free_ptr(fpst);
> return true;
^ permalink raw reply [flat|nested] 94+ messages in thread* Re: [PATCH v3 51/81] target/arm: Pass separate addend to {U, S}DOT helpers
2020-09-23 11:48 ` LIU Zhiwei
@ 2020-10-09 12:42 ` Richard Henderson
0 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-10-09 12:42 UTC (permalink / raw)
To: LIU Zhiwei, qemu-devel; +Cc: peter.maydell, alex.bennee
On 9/23/20 6:48 AM, LIU Zhiwei wrote:
>> + for (i = 0; i < opr_sz_8; i += 2) {
>> uint64_t d0, d1;
>> - d0 = n[i * 4 + 0] * (uint64_t)m_indexed[i * 4 + 0];
>> + d0 = a[i + 0];
> Add once.
>> + d0 += n[i * 4 + 0] * (uint64_t)m_indexed[i * 4 + 0];
>> d0 += n[i * 4 + 1] * (uint64_t)m_indexed[i * 4 + 1];
>> d0 += n[i * 4 + 2] * (uint64_t)m_indexed[i * 4 + 2];
>> d0 += n[i * 4 + 3] * (uint64_t)m_indexed[i * 4 + 3];
>> - d1 = n[i * 4 + 4] * (uint64_t)m_indexed[i * 4 + 0];
>> +
>> + d1 = a[i + 1];
>> + d1 += n[i * 4 + 4] * (uint64_t)m_indexed[i * 4 + 0];
>> d1 += n[i * 4 + 5] * (uint64_t)m_indexed[i * 4 + 1];
>> d1 += n[i * 4 + 6] * (uint64_t)m_indexed[i * 4 + 2];
>> d1 += n[i * 4 + 7] * (uint64_t)m_indexed[i * 4 + 3];
>> @@ -555,7 +570,6 @@ void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void
>> *vm, uint32_t desc)
>> d[i + 0] += d0;
> Add twice.
>
> I think it is wrong here. Do you thinks so?
Yep. Thanks for noticing.
r~
^ permalink raw reply [flat|nested] 94+ messages in thread
* [PATCH v3 52/81] target/arm: Pass separate addend to FCMLA helpers
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (50 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 51/81] target/arm: Pass separate addend to {U, S}DOT helpers Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 53/81] target/arm: Split out formats for 2 vectors + 1 index Richard Henderson
` (30 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
For SVE, we potentially have a 4th argument coming from the
movprfx instruction. Currently we do not optimize movprfx,
so the problem is not visible.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper.h | 20 ++++++-------
target/arm/translate-a64.c | 27 ++++++++++++++----
target/arm/translate-sve.c | 5 ++--
target/arm/vec_helper.c | 50 +++++++++++++--------------------
target/arm/translate-neon.c.inc | 10 ++++---
5 files changed, 61 insertions(+), 51 deletions(-)
diff --git a/target/arm/helper.h b/target/arm/helper.h
index 97222bd256..8b3dc9b4e3 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -616,16 +616,16 @@ DEF_HELPER_FLAGS_5(gvec_fcadds, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(gvec_fcaddd, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_5(gvec_fcmlah, TCG_CALL_NO_RWG,
- void, ptr, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_5(gvec_fcmlah_idx, TCG_CALL_NO_RWG,
- void, ptr, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_5(gvec_fcmlas, TCG_CALL_NO_RWG,
- void, ptr, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_5(gvec_fcmlas_idx, TCG_CALL_NO_RWG,
- void, ptr, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_5(gvec_fcmlad, TCG_CALL_NO_RWG,
- void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(gvec_fcmlah, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(gvec_fcmlah_idx, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(gvec_fcmlas, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(gvec_fcmlas_idx, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(gvec_fcmlad, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(neon_paddh, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(neon_pmaxh, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c
index 1a9251db67..0a463058f0 100644
--- a/target/arm/translate-a64.c
+++ b/target/arm/translate-a64.c
@@ -703,6 +703,22 @@ static void gen_gvec_op4_ool(DisasContext *s, bool is_q, int rd, int rn,
is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
}
+/* Expand a 4-operand + fpstatus pointer + simd data value operation using
+ * an out-of-line helper.
+ */
+static void gen_gvec_op4_fpst(DisasContext *s, bool is_q, int rd, int rn,
+ int rm, int ra, bool is_fp16, int data,
+ gen_helper_gvec_4_ptr *fn)
+{
+ TCGv_ptr fpst = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR);
+ tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
+ vec_full_reg_offset(s, rn),
+ vec_full_reg_offset(s, rm),
+ vec_full_reg_offset(s, ra), fpst,
+ is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
+ tcg_temp_free_ptr(fpst);
+}
+
/* Set ZF and NF based on a 64 bit result. This is alas fiddlier
* than the 32 bit equivalent.
*/
@@ -12224,15 +12240,15 @@ static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn)
rot = extract32(opcode, 0, 2);
switch (size) {
case 1:
- gen_gvec_op3_fpst(s, is_q, rd, rn, rm, true, rot,
+ gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, true, rot,
gen_helper_gvec_fcmlah);
break;
case 2:
- gen_gvec_op3_fpst(s, is_q, rd, rn, rm, false, rot,
+ gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, false, rot,
gen_helper_gvec_fcmlas);
break;
case 3:
- gen_gvec_op3_fpst(s, is_q, rd, rn, rm, false, rot,
+ gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, false, rot,
gen_helper_gvec_fcmlad);
break;
default:
@@ -13483,9 +13499,10 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn)
{
int rot = extract32(insn, 13, 2);
int data = (index << 2) | rot;
- tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
+ tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
vec_full_reg_offset(s, rn),
- vec_full_reg_offset(s, rm), fpst,
+ vec_full_reg_offset(s, rm),
+ vec_full_reg_offset(s, rd), fpst,
is_q ? 16 : 8, vec_full_reg_size(s), data,
size == MO_64
? gen_helper_gvec_fcmlas_idx
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 66303dac54..10200f0207 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -4387,7 +4387,7 @@ static bool trans_FCMLA_zpzzz(DisasContext *s, arg_FCMLA_zpzzz *a)
static bool trans_FCMLA_zzxz(DisasContext *s, arg_FCMLA_zzxz *a)
{
- static gen_helper_gvec_3_ptr * const fns[2] = {
+ static gen_helper_gvec_4_ptr * const fns[2] = {
gen_helper_gvec_fcmlah_idx,
gen_helper_gvec_fcmlas_idx,
};
@@ -4397,9 +4397,10 @@ static bool trans_FCMLA_zzxz(DisasContext *s, arg_FCMLA_zzxz *a)
if (sve_access_check(s)) {
unsigned vsz = vec_full_reg_size(s);
TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
- tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
+ tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, a->rd),
vec_full_reg_offset(s, a->rn),
vec_full_reg_offset(s, a->rm),
+ vec_full_reg_offset(s, a->ra),
status, vsz, vsz,
a->index * 4 + a->rot,
fns[a->esz - 1]);
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index d7ef31915b..82e12b51c1 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -657,13 +657,11 @@ void HELPER(gvec_fcaddd)(void *vd, void *vn, void *vm,
clear_tail(d, opr_sz, simd_maxsz(desc));
}
-void HELPER(gvec_fcmlah)(void *vd, void *vn, void *vm,
+void HELPER(gvec_fcmlah)(void *vd, void *vn, void *vm, void *va,
void *vfpst, uint32_t desc)
{
uintptr_t opr_sz = simd_oprsz(desc);
- float16 *d = vd;
- float16 *n = vn;
- float16 *m = vm;
+ float16 *d = vd, *n = vn, *m = vm, *a = va;
float_status *fpst = vfpst;
intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
@@ -680,19 +678,17 @@ void HELPER(gvec_fcmlah)(void *vd, void *vn, void *vm,
float16 e4 = e2;
float16 e3 = m[H2(i + 1 - flip)] ^ neg_imag;
- d[H2(i)] = float16_muladd(e2, e1, d[H2(i)], 0, fpst);
- d[H2(i + 1)] = float16_muladd(e4, e3, d[H2(i + 1)], 0, fpst);
+ d[H2(i)] = float16_muladd(e2, e1, a[H2(i)], 0, fpst);
+ d[H2(i + 1)] = float16_muladd(e4, e3, a[H2(i + 1)], 0, fpst);
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
-void HELPER(gvec_fcmlah_idx)(void *vd, void *vn, void *vm,
+void HELPER(gvec_fcmlah_idx)(void *vd, void *vn, void *vm, void *va,
void *vfpst, uint32_t desc)
{
uintptr_t opr_sz = simd_oprsz(desc);
- float16 *d = vd;
- float16 *n = vn;
- float16 *m = vm;
+ float16 *d = vd, *n = vn, *m = vm, *a = va;
float_status *fpst = vfpst;
intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
@@ -716,20 +712,18 @@ void HELPER(gvec_fcmlah_idx)(void *vd, void *vn, void *vm,
float16 e2 = n[H2(j + flip)];
float16 e4 = e2;
- d[H2(j)] = float16_muladd(e2, e1, d[H2(j)], 0, fpst);
- d[H2(j + 1)] = float16_muladd(e4, e3, d[H2(j + 1)], 0, fpst);
+ d[H2(j)] = float16_muladd(e2, e1, a[H2(j)], 0, fpst);
+ d[H2(j + 1)] = float16_muladd(e4, e3, a[H2(j + 1)], 0, fpst);
}
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
-void HELPER(gvec_fcmlas)(void *vd, void *vn, void *vm,
+void HELPER(gvec_fcmlas)(void *vd, void *vn, void *vm, void *va,
void *vfpst, uint32_t desc)
{
uintptr_t opr_sz = simd_oprsz(desc);
- float32 *d = vd;
- float32 *n = vn;
- float32 *m = vm;
+ float32 *d = vd, *n = vn, *m = vm, *a = va;
float_status *fpst = vfpst;
intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
@@ -746,19 +740,17 @@ void HELPER(gvec_fcmlas)(void *vd, void *vn, void *vm,
float32 e4 = e2;
float32 e3 = m[H4(i + 1 - flip)] ^ neg_imag;
- d[H4(i)] = float32_muladd(e2, e1, d[H4(i)], 0, fpst);
- d[H4(i + 1)] = float32_muladd(e4, e3, d[H4(i + 1)], 0, fpst);
+ d[H4(i)] = float32_muladd(e2, e1, a[H4(i)], 0, fpst);
+ d[H4(i + 1)] = float32_muladd(e4, e3, a[H4(i + 1)], 0, fpst);
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
-void HELPER(gvec_fcmlas_idx)(void *vd, void *vn, void *vm,
+void HELPER(gvec_fcmlas_idx)(void *vd, void *vn, void *vm, void *va,
void *vfpst, uint32_t desc)
{
uintptr_t opr_sz = simd_oprsz(desc);
- float32 *d = vd;
- float32 *n = vn;
- float32 *m = vm;
+ float32 *d = vd, *n = vn, *m = vm, *a = va;
float_status *fpst = vfpst;
intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
@@ -782,20 +774,18 @@ void HELPER(gvec_fcmlas_idx)(void *vd, void *vn, void *vm,
float32 e2 = n[H4(j + flip)];
float32 e4 = e2;
- d[H4(j)] = float32_muladd(e2, e1, d[H4(j)], 0, fpst);
- d[H4(j + 1)] = float32_muladd(e4, e3, d[H4(j + 1)], 0, fpst);
+ d[H4(j)] = float32_muladd(e2, e1, a[H4(j)], 0, fpst);
+ d[H4(j + 1)] = float32_muladd(e4, e3, a[H4(j + 1)], 0, fpst);
}
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
-void HELPER(gvec_fcmlad)(void *vd, void *vn, void *vm,
+void HELPER(gvec_fcmlad)(void *vd, void *vn, void *vm, void *va,
void *vfpst, uint32_t desc)
{
uintptr_t opr_sz = simd_oprsz(desc);
- float64 *d = vd;
- float64 *n = vn;
- float64 *m = vm;
+ float64 *d = vd, *n = vn, *m = vm, *a = va;
float_status *fpst = vfpst;
intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
uint64_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
@@ -812,8 +802,8 @@ void HELPER(gvec_fcmlad)(void *vd, void *vn, void *vm,
float64 e4 = e2;
float64 e3 = m[i + 1 - flip] ^ neg_imag;
- d[i] = float64_muladd(e2, e1, d[i], 0, fpst);
- d[i + 1] = float64_muladd(e4, e3, d[i + 1], 0, fpst);
+ d[i] = float64_muladd(e2, e1, a[i], 0, fpst);
+ d[i + 1] = float64_muladd(e4, e3, a[i + 1], 0, fpst);
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
diff --git a/target/arm/translate-neon.c.inc b/target/arm/translate-neon.c.inc
index 7efe3d9556..499ebeb325 100644
--- a/target/arm/translate-neon.c.inc
+++ b/target/arm/translate-neon.c.inc
@@ -165,7 +165,7 @@ static bool trans_VCMLA(DisasContext *s, arg_VCMLA *a)
{
int opr_sz;
TCGv_ptr fpst;
- gen_helper_gvec_3_ptr *fn_gvec_ptr;
+ gen_helper_gvec_4_ptr *fn_gvec_ptr;
if (!dc_isar_feature(aa32_vcma, s)
|| (a->size == MO_16 && !dc_isar_feature(aa32_fp16_arith, s))) {
@@ -190,9 +190,10 @@ static bool trans_VCMLA(DisasContext *s, arg_VCMLA *a)
fpst = fpstatus_ptr(a->size == MO_16 ? FPST_STD_F16 : FPST_STD);
fn_gvec_ptr = (a->size == MO_16) ?
gen_helper_gvec_fcmlah : gen_helper_gvec_fcmlas;
- tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd),
+ tcg_gen_gvec_4_ptr(vfp_reg_offset(1, a->vd),
vfp_reg_offset(1, a->vn),
vfp_reg_offset(1, a->vm),
+ vfp_reg_offset(1, a->vd),
fpst, opr_sz, opr_sz, a->rot,
fn_gvec_ptr);
tcg_temp_free_ptr(fpst);
@@ -303,7 +304,7 @@ static bool trans_VFML(DisasContext *s, arg_VFML *a)
static bool trans_VCMLA_scalar(DisasContext *s, arg_VCMLA_scalar *a)
{
- gen_helper_gvec_3_ptr *fn_gvec_ptr;
+ gen_helper_gvec_4_ptr *fn_gvec_ptr;
int opr_sz;
TCGv_ptr fpst;
@@ -332,9 +333,10 @@ static bool trans_VCMLA_scalar(DisasContext *s, arg_VCMLA_scalar *a)
gen_helper_gvec_fcmlah_idx : gen_helper_gvec_fcmlas_idx;
opr_sz = (1 + a->q) * 8;
fpst = fpstatus_ptr(a->size == MO_16 ? FPST_STD_F16 : FPST_STD);
- tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd),
+ tcg_gen_gvec_4_ptr(vfp_reg_offset(1, a->vd),
vfp_reg_offset(1, a->vn),
vfp_reg_offset(1, a->vm),
+ vfp_reg_offset(1, a->vd),
fpst, opr_sz, opr_sz,
(a->index << 2) | a->rot, fn_gvec_ptr);
tcg_temp_free_ptr(fpst);
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 53/81] target/arm: Split out formats for 2 vectors + 1 index
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (51 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 52/81] target/arm: Pass separate addend to FCMLA helpers Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 54/81] target/arm: Split out formats for 3 " Richard Henderson
` (29 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Currently only used by FMUL, but will shortly be used more.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve.decode | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 5815ba9b1c..a121e55f07 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -67,6 +67,7 @@
&rri_esz rd rn imm esz
&rrri_esz rd rn rm imm esz
&rrr_esz rd rn rm esz
+&rrx_esz rd rn rm index esz
&rpr_esz rd pg rn esz
&rpr_s rd pg rn s
&rprr_s rd pg rn rm s
@@ -245,6 +246,14 @@
@rpri_scatter_store ....... msz:2 .. imm:5 ... pg:3 rn:5 rd:5 \
&rpri_scatter_store
+# Two registers and a scalar by index
+@rrx_h ........ 0. . .. rm:3 ...... rn:5 rd:5 \
+ &rrx_esz index=%index3_22_19 esz=1
+@rrx_s ........ 10 . index:2 rm:3 ...... rn:5 rd:5 \
+ &rrx_esz esz=2
+@rrx_d ........ 11 . index:1 rm:4 ...... rn:5 rd:5 \
+ &rrx_esz esz=3
+
###########################################################################
# Instruction patterns. Grouped according to the SVE encodingindex.xhtml.
@@ -792,10 +801,9 @@ FMLA_zzxz 01100100 111 index:1 rm:4 00000 sub:1 rn:5 rd:5 \
### SVE FP Multiply Indexed Group
# SVE floating-point multiply (indexed)
-FMUL_zzx 01100100 0.1 .. rm:3 001000 rn:5 rd:5 \
- index=%index3_22_19 esz=1
-FMUL_zzx 01100100 101 index:2 rm:3 001000 rn:5 rd:5 esz=2
-FMUL_zzx 01100100 111 index:1 rm:4 001000 rn:5 rd:5 esz=3
+FMUL_zzx 01100100 .. 1 ..... 001000 ..... ..... @rrx_h
+FMUL_zzx 01100100 .. 1 ..... 001000 ..... ..... @rrx_s
+FMUL_zzx 01100100 .. 1 ..... 001000 ..... ..... @rrx_d
### SVE FP Fast Reduction Group
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 54/81] target/arm: Split out formats for 3 vectors + 1 index
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (52 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 53/81] target/arm: Split out formats for 2 vectors + 1 index Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 55/81] target/arm: Implement SVE2 integer multiply (indexed) Richard Henderson
` (28 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Used by FMLA and DOT, but will shortly be used more.
Split FMLA from FMLS to avoid an extra sub field;
similarly for SDOT from UDOT.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve.decode | 29 +++++++++++++++++++----------
target/arm/translate-sve.c | 38 ++++++++++++++++++++++++++++----------
2 files changed, 47 insertions(+), 20 deletions(-)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index a121e55f07..ea6ec5f198 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -73,6 +73,7 @@
&rprr_s rd pg rn rm s
&rprr_esz rd pg rn rm esz
&rrrr_esz rd ra rn rm esz
+&rrxr_esz rd rn rm ra index esz
&rprrr_esz rd pg rn rm ra esz
&rpri_esz rd pg rn imm esz
&ptrue rd esz pat s
@@ -254,6 +255,14 @@
@rrx_d ........ 11 . index:1 rm:4 ...... rn:5 rd:5 \
&rrx_esz esz=3
+# Three registers and a scalar by index
+@rrxr_h ........ 0. . .. rm:3 ...... rn:5 rd:5 \
+ &rrxr_esz ra=%reg_movprfx index=%index3_22_19 esz=1
+@rrxr_s ........ 10 . index:2 rm:3 ...... rn:5 rd:5 \
+ &rrxr_esz ra=%reg_movprfx esz=2
+@rrxr_d ........ 11 . index:1 rm:4 ...... rn:5 rd:5 \
+ &rrxr_esz ra=%reg_movprfx esz=3
+
###########################################################################
# Instruction patterns. Grouped according to the SVE encodingindex.xhtml.
@@ -769,10 +778,10 @@ DOT_zzzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 \
ra=%reg_movprfx
# SVE integer dot product (indexed)
-DOT_zzxw 01000100 101 index:2 rm:3 00000 u:1 rn:5 rd:5 \
- sz=0 ra=%reg_movprfx
-DOT_zzxw 01000100 111 index:1 rm:4 00000 u:1 rn:5 rd:5 \
- sz=1 ra=%reg_movprfx
+SDOT_zzxw_s 01000100 .. 1 ..... 000000 ..... ..... @rrxr_s
+SDOT_zzxw_d 01000100 .. 1 ..... 000000 ..... ..... @rrxr_d
+UDOT_zzxw_s 01000100 .. 1 ..... 000001 ..... ..... @rrxr_s
+UDOT_zzxw_d 01000100 .. 1 ..... 000001 ..... ..... @rrxr_d
# SVE floating-point complex add (predicated)
FCADD 01100100 esz:2 00000 rot:1 100 pg:3 rm:5 rd:5 \
@@ -791,12 +800,12 @@ FCMLA_zzxz 01100100 11 1 index:1 rm:4 0001 rot:2 rn:5 rd:5 \
### SVE FP Multiply-Add Indexed Group
# SVE floating-point multiply-add (indexed)
-FMLA_zzxz 01100100 0.1 .. rm:3 00000 sub:1 rn:5 rd:5 \
- ra=%reg_movprfx index=%index3_22_19 esz=1
-FMLA_zzxz 01100100 101 index:2 rm:3 00000 sub:1 rn:5 rd:5 \
- ra=%reg_movprfx esz=2
-FMLA_zzxz 01100100 111 index:1 rm:4 00000 sub:1 rn:5 rd:5 \
- ra=%reg_movprfx esz=3
+FMLA_zzxz 01100100 .. 1 ..... 000000 ..... ..... @rrxr_h
+FMLA_zzxz 01100100 .. 1 ..... 000000 ..... ..... @rrxr_s
+FMLA_zzxz 01100100 .. 1 ..... 000000 ..... ..... @rrxr_d
+FMLS_zzxz 01100100 .. 1 ..... 000001 ..... ..... @rrxr_h
+FMLS_zzxz 01100100 .. 1 ..... 000001 ..... ..... @rrxr_s
+FMLS_zzxz 01100100 .. 1 ..... 000001 ..... ..... @rrxr_d
### SVE FP Multiply Indexed Group
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 10200f0207..7c50298302 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -3817,26 +3817,34 @@ static bool trans_DOT_zzzz(DisasContext *s, arg_DOT_zzzz *a)
return true;
}
-static bool trans_DOT_zzxw(DisasContext *s, arg_DOT_zzxw *a)
+static bool do_zzxz_ool(DisasContext *s, arg_rrxr_esz *a,
+ gen_helper_gvec_4 *fn)
{
- static gen_helper_gvec_4 * const fns[2][2] = {
- { gen_helper_gvec_sdot_idx_b, gen_helper_gvec_sdot_idx_h },
- { gen_helper_gvec_udot_idx_b, gen_helper_gvec_udot_idx_h }
- };
-
+ if (fn == NULL) {
+ return false;
+ }
if (sve_access_check(s)) {
- gen_gvec_ool_zzzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm,
- a->ra, a->index);
+ gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index);
}
return true;
}
+#define DO_RRXR(NAME, FUNC) \
+ static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
+ { return do_zzxz_ool(s, a, FUNC); }
+
+DO_RRXR(trans_SDOT_zzxw_s, gen_helper_gvec_sdot_idx_b)
+DO_RRXR(trans_SDOT_zzxw_d, gen_helper_gvec_sdot_idx_h)
+DO_RRXR(trans_UDOT_zzxw_s, gen_helper_gvec_udot_idx_b)
+DO_RRXR(trans_UDOT_zzxw_d, gen_helper_gvec_udot_idx_h)
+
+#undef DO_RRXR
/*
*** SVE Floating Point Multiply-Add Indexed Group
*/
-static bool trans_FMLA_zzxz(DisasContext *s, arg_FMLA_zzxz *a)
+static bool do_FMLA_zzxz(DisasContext *s, arg_rrxr_esz *a, bool sub)
{
static gen_helper_gvec_4_ptr * const fns[3] = {
gen_helper_gvec_fmla_idx_h,
@@ -3851,13 +3859,23 @@ static bool trans_FMLA_zzxz(DisasContext *s, arg_FMLA_zzxz *a)
vec_full_reg_offset(s, a->rn),
vec_full_reg_offset(s, a->rm),
vec_full_reg_offset(s, a->ra),
- status, vsz, vsz, (a->index << 1) | a->sub,
+ status, vsz, vsz, (a->index << 1) | sub,
fns[a->esz - 1]);
tcg_temp_free_ptr(status);
}
return true;
}
+static bool trans_FMLA_zzxz(DisasContext *s, arg_FMLA_zzxz *a)
+{
+ return do_FMLA_zzxz(s, a, false);
+}
+
+static bool trans_FMLS_zzxz(DisasContext *s, arg_FMLA_zzxz *a)
+{
+ return do_FMLA_zzxz(s, a, true);
+}
+
/*
*** SVE Floating Point Multiply Indexed Group
*/
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 55/81] target/arm: Implement SVE2 integer multiply (indexed)
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (53 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 54/81] target/arm: Split out formats for 3 " Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 56/81] target/arm: Implement SVE2 integer multiply-add (indexed) Richard Henderson
` (27 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve.decode | 7 +++++++
target/arm/translate-sve.c | 30 ++++++++++++++++++++++++++++++
2 files changed, 37 insertions(+)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index ea6ec5f198..fa0a572da6 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -777,12 +777,19 @@ MUL_zzi 00100101 .. 110 000 110 ........ ..... @rdn_i8s
DOT_zzzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 \
ra=%reg_movprfx
+#### SVE Multiply - Indexed
+
# SVE integer dot product (indexed)
SDOT_zzxw_s 01000100 .. 1 ..... 000000 ..... ..... @rrxr_s
SDOT_zzxw_d 01000100 .. 1 ..... 000000 ..... ..... @rrxr_d
UDOT_zzxw_s 01000100 .. 1 ..... 000001 ..... ..... @rrxr_s
UDOT_zzxw_d 01000100 .. 1 ..... 000001 ..... ..... @rrxr_d
+# SVE2 integer multiply (indexed)
+MUL_zzx_h 01000100 .. 1 ..... 111110 ..... ..... @rrx_h
+MUL_zzx_s 01000100 .. 1 ..... 111110 ..... ..... @rrx_s
+MUL_zzx_d 01000100 .. 1 ..... 111110 ..... ..... @rrx_d
+
# SVE floating-point complex add (predicated)
FCADD 01100100 esz:2 00000 rot:1 100 pg:3 rm:5 rd:5 \
rn=%reg_movprfx
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 7c50298302..90f0b1ff9b 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -3817,6 +3817,10 @@ static bool trans_DOT_zzzz(DisasContext *s, arg_DOT_zzzz *a)
return true;
}
+/*
+ * SVE Multiply - Indexed
+ */
+
static bool do_zzxz_ool(DisasContext *s, arg_rrxr_esz *a,
gen_helper_gvec_4 *fn)
{
@@ -3840,6 +3844,32 @@ DO_RRXR(trans_UDOT_zzxw_d, gen_helper_gvec_udot_idx_h)
#undef DO_RRXR
+static bool do_sve2_zzx_ool(DisasContext *s, arg_rrx_esz *a,
+ gen_helper_gvec_3 *fn)
+{
+ if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ vsz, vsz, a->index, fn);
+ }
+ return true;
+}
+
+#define DO_SVE2_RRX(NAME, FUNC) \
+ static bool NAME(DisasContext *s, arg_rrx_esz *a) \
+ { return do_sve2_zzx_ool(s, a, FUNC); }
+
+DO_SVE2_RRX(trans_MUL_zzx_h, gen_helper_gvec_mul_idx_h)
+DO_SVE2_RRX(trans_MUL_zzx_s, gen_helper_gvec_mul_idx_s)
+DO_SVE2_RRX(trans_MUL_zzx_d, gen_helper_gvec_mul_idx_d)
+
+#undef DO_SVE2_RRX
+
/*
*** SVE Floating Point Multiply-Add Indexed Group
*/
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 56/81] target/arm: Implement SVE2 integer multiply-add (indexed)
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (54 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 55/81] target/arm: Implement SVE2 integer multiply (indexed) Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 57/81] target/arm: Implement SVE2 saturating multiply-add high (indexed) Richard Henderson
` (26 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve.decode | 8 ++++++++
target/arm/translate-sve.c | 23 +++++++++++++++++++++++
2 files changed, 31 insertions(+)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index fa0a572da6..467a93052f 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -785,6 +785,14 @@ SDOT_zzxw_d 01000100 .. 1 ..... 000000 ..... ..... @rrxr_d
UDOT_zzxw_s 01000100 .. 1 ..... 000001 ..... ..... @rrxr_s
UDOT_zzxw_d 01000100 .. 1 ..... 000001 ..... ..... @rrxr_d
+# SVE2 integer multiply-add (indexed)
+MLA_zzxz_h 01000100 .. 1 ..... 000010 ..... ..... @rrxr_h
+MLA_zzxz_s 01000100 .. 1 ..... 000010 ..... ..... @rrxr_s
+MLA_zzxz_d 01000100 .. 1 ..... 000010 ..... ..... @rrxr_d
+MLS_zzxz_h 01000100 .. 1 ..... 000011 ..... ..... @rrxr_h
+MLS_zzxz_s 01000100 .. 1 ..... 000011 ..... ..... @rrxr_s
+MLS_zzxz_d 01000100 .. 1 ..... 000011 ..... ..... @rrxr_d
+
# SVE2 integer multiply (indexed)
MUL_zzx_h 01000100 .. 1 ..... 111110 ..... ..... @rrx_h
MUL_zzx_s 01000100 .. 1 ..... 111110 ..... ..... @rrx_s
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 90f0b1ff9b..a6235d78d5 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -3870,6 +3870,29 @@ DO_SVE2_RRX(trans_MUL_zzx_d, gen_helper_gvec_mul_idx_d)
#undef DO_SVE2_RRX
+static bool do_sve2_zzxz_ool(DisasContext *s, arg_rrxr_esz *a,
+ gen_helper_gvec_4 *fn)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_zzxz_ool(s, a, fn);
+}
+
+#define DO_SVE2_RRXR(NAME, FUNC) \
+ static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
+ { return do_sve2_zzxz_ool(s, a, FUNC); }
+
+DO_SVE2_RRXR(trans_MLA_zzxz_h, gen_helper_gvec_mla_idx_h)
+DO_SVE2_RRXR(trans_MLA_zzxz_s, gen_helper_gvec_mla_idx_s)
+DO_SVE2_RRXR(trans_MLA_zzxz_d, gen_helper_gvec_mla_idx_d)
+
+DO_SVE2_RRXR(trans_MLS_zzxz_h, gen_helper_gvec_mls_idx_h)
+DO_SVE2_RRXR(trans_MLS_zzxz_s, gen_helper_gvec_mls_idx_s)
+DO_SVE2_RRXR(trans_MLS_zzxz_d, gen_helper_gvec_mls_idx_d)
+
+#undef DO_SVE2_RRXR
+
/*
*** SVE Floating Point Multiply-Add Indexed Group
*/
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 57/81] target/arm: Implement SVE2 saturating multiply-add high (indexed)
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (55 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 56/81] target/arm: Implement SVE2 integer multiply-add (indexed) Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 58/81] target/arm: Implement SVE2 saturating multiply-add (indexed) Richard Henderson
` (25 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 14 +++++++++++++
target/arm/sve.decode | 8 ++++++++
target/arm/sve_helper.c | 40 ++++++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 8 ++++++++
4 files changed, 70 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 2c6d3431ec..4ff7298f67 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2665,3 +2665,17 @@ DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_d, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_6(fmmla_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(fmmla_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_sqrdmlah_idx_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlah_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlah_idx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 467a93052f..5fc76b7fc3 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -793,6 +793,14 @@ MLS_zzxz_h 01000100 .. 1 ..... 000011 ..... ..... @rrxr_h
MLS_zzxz_s 01000100 .. 1 ..... 000011 ..... ..... @rrxr_s
MLS_zzxz_d 01000100 .. 1 ..... 000011 ..... ..... @rrxr_d
+# SVE2 saturating multiply-add high (indexed)
+SQRDMLAH_zzxz_h 01000100 .. 1 ..... 000100 ..... ..... @rrxr_h
+SQRDMLAH_zzxz_s 01000100 .. 1 ..... 000100 ..... ..... @rrxr_s
+SQRDMLAH_zzxz_d 01000100 .. 1 ..... 000100 ..... ..... @rrxr_d
+SQRDMLSH_zzxz_h 01000100 .. 1 ..... 000101 ..... ..... @rrxr_h
+SQRDMLSH_zzxz_s 01000100 .. 1 ..... 000101 ..... ..... @rrxr_s
+SQRDMLSH_zzxz_d 01000100 .. 1 ..... 000101 ..... ..... @rrxr_d
+
# SVE2 integer multiply (indexed)
MUL_zzx_h 01000100 .. 1 ..... 111110 ..... ..... @rrx_h
MUL_zzx_s 01000100 .. 1 ..... 111110 ..... ..... @rrx_s
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index aaf336118b..1facec8ce0 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1487,9 +1487,49 @@ DO_CMLA(sve2_sqrdcmlah_zzzz_h, int16_t, H2, DO_SQRDMLAH_H)
DO_CMLA(sve2_sqrdcmlah_zzzz_s, int32_t, H4, DO_SQRDMLAH_S)
DO_CMLA(sve2_sqrdcmlah_zzzz_d, int64_t, , DO_SQRDMLAH_D)
+#undef DO_SQRDMLAH_B
+#undef DO_SQRDMLAH_H
+#undef DO_SQRDMLAH_S
+#undef DO_SQRDMLAH_D
#undef do_cmla
#undef DO_CMLA
+#define DO_ZZXZ(NAME, TYPE, H, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
+{ \
+ intptr_t oprsz = simd_oprsz(desc), segment = 16 / sizeof(TYPE); \
+ intptr_t i, j, idx = simd_data(desc); \
+ TYPE *d = vd, *a = va, *n = vn, *m = (TYPE *)vm + H(idx); \
+ for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \
+ TYPE mm = m[i]; \
+ for (j = 0; j < segment; j++) { \
+ d[i + j] = OP(n[i + j], mm, a[i + j]); \
+ } \
+ } \
+}
+
+#define DO_SQRDMLAH_H(N, M, A) \
+ ({ uint32_t discard; do_sqrdmlah_h(N, M, A, false, true, &discard); })
+#define DO_SQRDMLAH_S(N, M, A) \
+ ({ uint32_t discard; do_sqrdmlah_s(N, M, A, false, true, &discard); })
+#define DO_SQRDMLAH_D(N, M, A) do_sqrdmlah_d(N, M, A, false, true)
+
+DO_ZZXZ(sve2_sqrdmlah_idx_h, int16_t, H2, DO_SQRDMLAH_H)
+DO_ZZXZ(sve2_sqrdmlah_idx_s, int32_t, H4, DO_SQRDMLAH_S)
+DO_ZZXZ(sve2_sqrdmlah_idx_d, int64_t, , DO_SQRDMLAH_D)
+
+#define DO_SQRDMLSH_H(N, M, A) \
+ ({ uint32_t discard; do_sqrdmlah_h(N, M, A, true, true, &discard); })
+#define DO_SQRDMLSH_S(N, M, A) \
+ ({ uint32_t discard; do_sqrdmlah_s(N, M, A, true, true, &discard); })
+#define DO_SQRDMLSH_D(N, M, A) do_sqrdmlah_d(N, M, A, true, true)
+
+DO_ZZXZ(sve2_sqrdmlsh_idx_h, int16_t, H2, DO_SQRDMLSH_H)
+DO_ZZXZ(sve2_sqrdmlsh_idx_s, int32_t, H4, DO_SQRDMLSH_S)
+DO_ZZXZ(sve2_sqrdmlsh_idx_d, int64_t, , DO_SQRDMLSH_D)
+
+#undef DO_ZZXZ
+
#define DO_BITPERM(NAME, TYPE, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
{ \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index a6235d78d5..1dd03ae21e 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -3891,6 +3891,14 @@ DO_SVE2_RRXR(trans_MLS_zzxz_h, gen_helper_gvec_mls_idx_h)
DO_SVE2_RRXR(trans_MLS_zzxz_s, gen_helper_gvec_mls_idx_s)
DO_SVE2_RRXR(trans_MLS_zzxz_d, gen_helper_gvec_mls_idx_d)
+DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_h, gen_helper_sve2_sqrdmlah_idx_h)
+DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_s, gen_helper_sve2_sqrdmlah_idx_s)
+DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_d, gen_helper_sve2_sqrdmlah_idx_d)
+
+DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_h, gen_helper_sve2_sqrdmlsh_idx_h)
+DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_s, gen_helper_sve2_sqrdmlsh_idx_s)
+DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_d, gen_helper_sve2_sqrdmlsh_idx_d)
+
#undef DO_SVE2_RRXR
/*
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 58/81] target/arm: Implement SVE2 saturating multiply-add (indexed)
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (56 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 57/81] target/arm: Implement SVE2 saturating multiply-add high (indexed) Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 59/81] target/arm: Implement SVE2 integer multiply long (indexed) Richard Henderson
` (24 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 9 +++++++++
target/arm/sve.decode | 18 ++++++++++++++++++
target/arm/sve_helper.c | 30 ++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 32 ++++++++++++++++++++++++--------
4 files changed, 81 insertions(+), 8 deletions(-)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 4ff7298f67..d6e94a8b9c 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2679,3 +2679,12 @@ DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_sqdmlal_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlal_idx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlsl_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlsl_idx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 5fc76b7fc3..36cdd9dab4 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -30,6 +30,8 @@
%size_23 23:2
%dtype_23_13 23:2 13:2
%index3_22_19 22:1 19:2
+%index3_19_11 19:2 11:1
+%index2_20_11 20:1 11:1
# A combination of tsz:imm3 -- extract esize.
%tszimm_esz 22:2 5:5 !function=tszimm_esz
@@ -263,6 +265,12 @@
@rrxr_d ........ 11 . index:1 rm:4 ...... rn:5 rd:5 \
&rrxr_esz ra=%reg_movprfx esz=3
+# Three registers and a scalar by index, wide
+@rrxw_s ........ 10 ... rm:3 ...... rn:5 rd:5 \
+ &rrxr_esz ra=%reg_movprfx index=%index3_19_11 esz=2
+@rrxw_d ........ 11 .. rm:4 ...... rn:5 rd:5 \
+ &rrxr_esz ra=%reg_movprfx index=%index2_20_11 esz=3
+
###########################################################################
# Instruction patterns. Grouped according to the SVE encodingindex.xhtml.
@@ -801,6 +809,16 @@ SQRDMLSH_zzxz_h 01000100 .. 1 ..... 000101 ..... ..... @rrxr_h
SQRDMLSH_zzxz_s 01000100 .. 1 ..... 000101 ..... ..... @rrxr_s
SQRDMLSH_zzxz_d 01000100 .. 1 ..... 000101 ..... ..... @rrxr_d
+# SVE2 saturating multiply-add (indexed)
+SQDMLALB_zzxw_s 01000100 .. 1 ..... 0010.0 ..... ..... @rrxw_s
+SQDMLALB_zzxw_d 01000100 .. 1 ..... 0010.0 ..... ..... @rrxw_d
+SQDMLALT_zzxw_s 01000100 .. 1 ..... 0010.1 ..... ..... @rrxw_s
+SQDMLALT_zzxw_d 01000100 .. 1 ..... 0010.1 ..... ..... @rrxw_d
+SQDMLSLB_zzxw_s 01000100 .. 1 ..... 0011.0 ..... ..... @rrxw_s
+SQDMLSLB_zzxw_d 01000100 .. 1 ..... 0011.0 ..... ..... @rrxw_d
+SQDMLSLT_zzxw_s 01000100 .. 1 ..... 0011.1 ..... ..... @rrxw_s
+SQDMLSLT_zzxw_d 01000100 .. 1 ..... 0011.1 ..... ..... @rrxw_d
+
# SVE2 integer multiply (indexed)
MUL_zzx_h 01000100 .. 1 ..... 111110 ..... ..... @rrx_h
MUL_zzx_s 01000100 .. 1 ..... 111110 ..... ..... @rrx_s
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 1facec8ce0..125da036b8 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1530,6 +1530,36 @@ DO_ZZXZ(sve2_sqrdmlsh_idx_d, int64_t, , DO_SQRDMLSH_D)
#undef DO_ZZXZ
+#define DO_ZZXW(NAME, TYPEW, TYPEN, HW, HN, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
+{ \
+ intptr_t i, j, oprsz = simd_oprsz(desc); \
+ intptr_t sel = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \
+ intptr_t idx = extract32(desc, SIMD_DATA_SHIFT + 1, 3) * sizeof(TYPEN); \
+ for (i = 0; i < oprsz; i += 16) { \
+ TYPEW mm = *(TYPEN *)(vm + i + idx); \
+ for (j = 0; j < 16; j += sizeof(TYPEW)) { \
+ TYPEW nn = *(TYPEN *)(vn + HN(i + j + sel)); \
+ TYPEW aa = *(TYPEW *)(va + HW(i + j)); \
+ *(TYPEW *)(vd + HW(i + j)) = OP(nn, mm, aa); \
+ } \
+ } \
+}
+
+#define DO_SQDMLAL_S(N, M, A) DO_SQADD_S(A, do_sqdmull_s(N, M))
+#define DO_SQDMLAL_D(N, M, A) do_sqadd_d(A, do_sqdmull_d(N, M))
+
+DO_ZZXW(sve2_sqdmlal_idx_s, int32_t, int16_t, H1_4, H1_2, DO_SQDMLAL_S)
+DO_ZZXW(sve2_sqdmlal_idx_d, int64_t, int32_t, , H1_4, DO_SQDMLAL_D)
+
+#define DO_SQDMLSL_S(N, M, A) DO_SQSUB_S(A, do_sqdmull_s(N, M))
+#define DO_SQDMLSL_D(N, M, A) do_sqsub_d(A, do_sqdmull_d(N, M))
+
+DO_ZZXW(sve2_sqdmlsl_idx_s, int32_t, int16_t, H1_4, H1_2, DO_SQDMLSL_S)
+DO_ZZXW(sve2_sqdmlsl_idx_d, int64_t, int32_t, , H1_4, DO_SQDMLSL_D)
+
+#undef DO_ZZXW
+
#define DO_BITPERM(NAME, TYPE, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
{ \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 1dd03ae21e..5b694a9243 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -3821,21 +3821,21 @@ static bool trans_DOT_zzzz(DisasContext *s, arg_DOT_zzzz *a)
* SVE Multiply - Indexed
*/
-static bool do_zzxz_ool(DisasContext *s, arg_rrxr_esz *a,
- gen_helper_gvec_4 *fn)
+static bool do_zzxz_data(DisasContext *s, arg_rrxr_esz *a,
+ gen_helper_gvec_4 *fn, int data)
{
if (fn == NULL) {
return false;
}
if (sve_access_check(s)) {
- gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index);
+ gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data);
}
return true;
}
#define DO_RRXR(NAME, FUNC) \
static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
- { return do_zzxz_ool(s, a, FUNC); }
+ { return do_zzxz_data(s, a, FUNC, a->index); }
DO_RRXR(trans_SDOT_zzxw_s, gen_helper_gvec_sdot_idx_b)
DO_RRXR(trans_SDOT_zzxw_d, gen_helper_gvec_sdot_idx_h)
@@ -3870,18 +3870,18 @@ DO_SVE2_RRX(trans_MUL_zzx_d, gen_helper_gvec_mul_idx_d)
#undef DO_SVE2_RRX
-static bool do_sve2_zzxz_ool(DisasContext *s, arg_rrxr_esz *a,
- gen_helper_gvec_4 *fn)
+static bool do_sve2_zzxz_data(DisasContext *s, arg_rrxr_esz *a,
+ gen_helper_gvec_4 *fn, int data)
{
if (!dc_isar_feature(aa64_sve2, s)) {
return false;
}
- return do_zzxz_ool(s, a, fn);
+ return do_zzxz_data(s, a, fn, data);
}
#define DO_SVE2_RRXR(NAME, FUNC) \
static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
- { return do_sve2_zzxz_ool(s, a, FUNC); }
+ { return do_sve2_zzxz_data(s, a, FUNC, a->index); }
DO_SVE2_RRXR(trans_MLA_zzxz_h, gen_helper_gvec_mla_idx_h)
DO_SVE2_RRXR(trans_MLA_zzxz_s, gen_helper_gvec_mla_idx_s)
@@ -3901,6 +3901,22 @@ DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_d, gen_helper_sve2_sqrdmlsh_idx_d)
#undef DO_SVE2_RRXR
+#define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
+ static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
+ { return do_sve2_zzxz_data(s, a, FUNC, (a->index << 1) | TOP); }
+
+DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, false)
+DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, false)
+DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, true)
+DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, true)
+
+DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, false)
+DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, false)
+DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, true)
+DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, true)
+
+#undef DO_SVE2_RRXR_TB
+
/*
*** SVE Floating Point Multiply-Add Indexed Group
*/
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 59/81] target/arm: Implement SVE2 integer multiply long (indexed)
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (57 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 58/81] target/arm: Implement SVE2 saturating multiply-add (indexed) Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 60/81] target/arm: Implement SVE2 saturating multiply (indexed) Richard Henderson
` (23 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 5 +++++
target/arm/sve.decode | 16 ++++++++++++++++
target/arm/sve_helper.c | 23 +++++++++++++++++++++++
target/arm/translate-sve.c | 24 ++++++++++++++++++++----
4 files changed, 64 insertions(+), 4 deletions(-)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index d6e94a8b9c..d11598a75a 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2688,3 +2688,8 @@ DEF_HELPER_FLAGS_5(sve2_sqdmlsl_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqdmlsl_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_smull_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_smull_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_umull_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_umull_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 36cdd9dab4..f0a4d86428 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -257,6 +257,12 @@
@rrx_d ........ 11 . index:1 rm:4 ...... rn:5 rd:5 \
&rrx_esz esz=3
+# Two registers and a scalar by index, wide
+@rrxl_s ........ 10 ... rm:3 ...... rn:5 rd:5 \
+ &rrx_esz index=%index3_19_11 esz=2
+@rrxl_d ........ 11 .. rm:4 ...... rn:5 rd:5 \
+ &rrx_esz index=%index2_20_11 esz=3
+
# Three registers and a scalar by index
@rrxr_h ........ 0. . .. rm:3 ...... rn:5 rd:5 \
&rrxr_esz ra=%reg_movprfx index=%index3_22_19 esz=1
@@ -819,6 +825,16 @@ SQDMLSLB_zzxw_d 01000100 .. 1 ..... 0011.0 ..... ..... @rrxw_d
SQDMLSLT_zzxw_s 01000100 .. 1 ..... 0011.1 ..... ..... @rrxw_s
SQDMLSLT_zzxw_d 01000100 .. 1 ..... 0011.1 ..... ..... @rrxw_d
+# SVE2 integer multiply long (indexed)
+SMULLB_zzx_s 01000100 .. 1 ..... 1100.0 ..... ..... @rrxl_s
+SMULLB_zzx_d 01000100 .. 1 ..... 1100.0 ..... ..... @rrxl_d
+SMULLT_zzx_s 01000100 .. 1 ..... 1100.1 ..... ..... @rrxl_s
+SMULLT_zzx_d 01000100 .. 1 ..... 1100.1 ..... ..... @rrxl_d
+UMULLB_zzx_s 01000100 .. 1 ..... 1101.0 ..... ..... @rrxl_s
+UMULLB_zzx_d 01000100 .. 1 ..... 1101.0 ..... ..... @rrxl_d
+UMULLT_zzx_s 01000100 .. 1 ..... 1101.1 ..... ..... @rrxl_s
+UMULLT_zzx_d 01000100 .. 1 ..... 1101.1 ..... ..... @rrxl_d
+
# SVE2 integer multiply (indexed)
MUL_zzx_h 01000100 .. 1 ..... 111110 ..... ..... @rrx_h
MUL_zzx_s 01000100 .. 1 ..... 111110 ..... ..... @rrx_s
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 125da036b8..80449cc569 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1560,6 +1560,29 @@ DO_ZZXW(sve2_sqdmlsl_idx_d, int64_t, int32_t, , H1_4, DO_SQDMLSL_D)
#undef DO_ZZXW
+#define DO_ZZX(NAME, TYPEW, TYPEN, HW, HN, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
+{ \
+ intptr_t i, j, oprsz = simd_oprsz(desc); \
+ intptr_t sel = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \
+ intptr_t idx = extract32(desc, SIMD_DATA_SHIFT + 1, 3) * sizeof(TYPEN); \
+ for (i = 0; i < oprsz; i += 16) { \
+ TYPEW mm = *(TYPEN *)(vm + i + idx); \
+ for (j = 0; j < 16; j += sizeof(TYPEW)) { \
+ TYPEW nn = *(TYPEN *)(vn + HN(i + j + sel)); \
+ *(TYPEW *)(vd + HW(i + j)) = OP(nn, mm); \
+ } \
+ } \
+}
+
+DO_ZZX(sve2_smull_idx_s, int32_t, int16_t, H1_4, H1_2, DO_MUL)
+DO_ZZX(sve2_smull_idx_d, int64_t, int32_t, , H1_4, DO_MUL)
+
+DO_ZZX(sve2_umull_idx_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL)
+DO_ZZX(sve2_umull_idx_d, uint64_t, uint32_t, , H1_4, DO_MUL)
+
+#undef DO_ZZX
+
#define DO_BITPERM(NAME, TYPE, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
{ \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 5b694a9243..39e5160e25 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -3844,8 +3844,8 @@ DO_RRXR(trans_UDOT_zzxw_d, gen_helper_gvec_udot_idx_h)
#undef DO_RRXR
-static bool do_sve2_zzx_ool(DisasContext *s, arg_rrx_esz *a,
- gen_helper_gvec_3 *fn)
+static bool do_sve2_zzx_data(DisasContext *s, arg_rrx_esz *a,
+ gen_helper_gvec_3 *fn, int data)
{
if (fn == NULL || !dc_isar_feature(aa64_sve2, s)) {
return false;
@@ -3855,14 +3855,14 @@ static bool do_sve2_zzx_ool(DisasContext *s, arg_rrx_esz *a,
tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd),
vec_full_reg_offset(s, a->rn),
vec_full_reg_offset(s, a->rm),
- vsz, vsz, a->index, fn);
+ vsz, vsz, data, fn);
}
return true;
}
#define DO_SVE2_RRX(NAME, FUNC) \
static bool NAME(DisasContext *s, arg_rrx_esz *a) \
- { return do_sve2_zzx_ool(s, a, FUNC); }
+ { return do_sve2_zzx_data(s, a, FUNC, a->index); }
DO_SVE2_RRX(trans_MUL_zzx_h, gen_helper_gvec_mul_idx_h)
DO_SVE2_RRX(trans_MUL_zzx_s, gen_helper_gvec_mul_idx_s)
@@ -3870,6 +3870,22 @@ DO_SVE2_RRX(trans_MUL_zzx_d, gen_helper_gvec_mul_idx_d)
#undef DO_SVE2_RRX
+#define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \
+ static bool NAME(DisasContext *s, arg_rrx_esz *a) \
+ { return do_sve2_zzx_data(s, a, FUNC, (a->index << 1) | TOP); }
+
+DO_SVE2_RRX_TB(trans_SMULLB_zzx_s, gen_helper_sve2_smull_idx_s, false)
+DO_SVE2_RRX_TB(trans_SMULLB_zzx_d, gen_helper_sve2_smull_idx_d, false)
+DO_SVE2_RRX_TB(trans_SMULLT_zzx_s, gen_helper_sve2_smull_idx_s, true)
+DO_SVE2_RRX_TB(trans_SMULLT_zzx_d, gen_helper_sve2_smull_idx_d, true)
+
+DO_SVE2_RRX_TB(trans_UMULLB_zzx_s, gen_helper_sve2_umull_idx_s, false)
+DO_SVE2_RRX_TB(trans_UMULLB_zzx_d, gen_helper_sve2_umull_idx_d, false)
+DO_SVE2_RRX_TB(trans_UMULLT_zzx_s, gen_helper_sve2_umull_idx_s, true)
+DO_SVE2_RRX_TB(trans_UMULLT_zzx_d, gen_helper_sve2_umull_idx_d, true)
+
+#undef DO_SVE2_RRX_TB
+
static bool do_sve2_zzxz_data(DisasContext *s, arg_rrxr_esz *a,
gen_helper_gvec_4 *fn, int data)
{
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 60/81] target/arm: Implement SVE2 saturating multiply (indexed)
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (58 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 59/81] target/arm: Implement SVE2 integer multiply long (indexed) Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 61/81] target/arm: Implement SVE2 signed saturating doubling multiply high Richard Henderson
` (22 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 5 +++++
target/arm/sve.decode | 6 ++++++
target/arm/sve_helper.c | 3 +++
target/arm/translate-sve.c | 5 +++++
4 files changed, 19 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index d11598a75a..ff85e1327b 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2693,3 +2693,8 @@ DEF_HELPER_FLAGS_4(sve2_smull_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_smull_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_umull_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_umull_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_sqdmull_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmull_idx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index f0a4d86428..400940a18d 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -835,6 +835,12 @@ UMULLB_zzx_d 01000100 .. 1 ..... 1101.0 ..... ..... @rrxl_d
UMULLT_zzx_s 01000100 .. 1 ..... 1101.1 ..... ..... @rrxl_s
UMULLT_zzx_d 01000100 .. 1 ..... 1101.1 ..... ..... @rrxl_d
+# SVE2 saturating multiply (indexed)
+SQDMULLB_zzx_s 01000100 .. 1 ..... 1110.0 ..... ..... @rrxl_s
+SQDMULLB_zzx_d 01000100 .. 1 ..... 1110.0 ..... ..... @rrxl_d
+SQDMULLT_zzx_s 01000100 .. 1 ..... 1110.1 ..... ..... @rrxl_s
+SQDMULLT_zzx_d 01000100 .. 1 ..... 1110.1 ..... ..... @rrxl_d
+
# SVE2 integer multiply (indexed)
MUL_zzx_h 01000100 .. 1 ..... 111110 ..... ..... @rrx_h
MUL_zzx_s 01000100 .. 1 ..... 111110 ..... ..... @rrx_s
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 80449cc569..926fe98eb5 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1581,6 +1581,9 @@ DO_ZZX(sve2_smull_idx_d, int64_t, int32_t, , H1_4, DO_MUL)
DO_ZZX(sve2_umull_idx_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL)
DO_ZZX(sve2_umull_idx_d, uint64_t, uint32_t, , H1_4, DO_MUL)
+DO_ZZX(sve2_sqdmull_idx_s, int32_t, int16_t, H1_4, H1_2, do_sqdmull_s)
+DO_ZZX(sve2_sqdmull_idx_d, int64_t, int32_t, , H1_4, do_sqdmull_d)
+
#undef DO_ZZX
#define DO_BITPERM(NAME, TYPE, OP) \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 39e5160e25..c30aed211f 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -3884,6 +3884,11 @@ DO_SVE2_RRX_TB(trans_UMULLB_zzx_d, gen_helper_sve2_umull_idx_d, false)
DO_SVE2_RRX_TB(trans_UMULLT_zzx_s, gen_helper_sve2_umull_idx_s, true)
DO_SVE2_RRX_TB(trans_UMULLT_zzx_d, gen_helper_sve2_umull_idx_d, true)
+DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_s, gen_helper_sve2_sqdmull_idx_s, false)
+DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_d, gen_helper_sve2_sqdmull_idx_d, false)
+DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_s, gen_helper_sve2_sqdmull_idx_s, true)
+DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_d, gen_helper_sve2_sqdmull_idx_d, true)
+
#undef DO_SVE2_RRX_TB
static bool do_sve2_zzxz_data(DisasContext *s, arg_rrxr_esz *a,
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 61/81] target/arm: Implement SVE2 signed saturating doubling multiply high
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (59 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 60/81] target/arm: Implement SVE2 saturating multiply (indexed) Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 62/81] target/arm: Implement SVE2 saturating multiply high (indexed) Richard Henderson
` (21 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper.h | 10 +++++
target/arm/sve.decode | 4 ++
target/arm/translate-sve.c | 18 ++++++++
target/arm/vec_helper.c | 84 ++++++++++++++++++++++++++++++++++++++
4 files changed, 116 insertions(+)
diff --git a/target/arm/helper.h b/target/arm/helper.h
index 8b3dc9b4e3..fa7b2b7dd7 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -944,6 +944,16 @@ DEF_HELPER_FLAGS_5(neon_sqrdmulh_h, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(neon_sqrdmulh_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmulh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_sqrdmulh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqrdmulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqrdmulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqrdmulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_4(gvec_xar_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
#ifdef TARGET_AARCH64
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 400940a18d..6879870cc1 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1214,6 +1214,10 @@ SMULH_zzz 00000100 .. 1 ..... 0110 10 ..... ..... @rd_rn_rm
UMULH_zzz 00000100 .. 1 ..... 0110 11 ..... ..... @rd_rn_rm
PMUL_zzz 00000100 00 1 ..... 0110 01 ..... ..... @rd_rn_rm_e0
+# SVE2 signed saturating doubling multiply high (unpredicated)
+SQDMULH_zzz 00000100 .. 1 ..... 0111 00 ..... ..... @rd_rn_rm
+SQRDMULH_zzz 00000100 .. 1 ..... 0111 01 ..... ..... @rd_rn_rm
+
### SVE2 Integer - Predicated
SADALP_zpzz 01000100 .. 000 100 101 ... ..... ..... @rdm_pg_rn
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index c30aed211f..298c328ce0 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -6434,6 +6434,24 @@ static bool trans_PMUL_zzz(DisasContext *s, arg_rrr_esz *a)
return do_sve2_zzz_ool(s, a, gen_helper_gvec_pmul_b);
}
+static bool trans_SQDMULH_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve2_sqdmulh_b, gen_helper_sve2_sqdmulh_h,
+ gen_helper_sve2_sqdmulh_s, gen_helper_sve2_sqdmulh_d,
+ };
+ return do_sve2_zzz_ool(s, a, fns[a->esz]);
+}
+
+static bool trans_SQRDMULH_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve2_sqrdmulh_b, gen_helper_sve2_sqrdmulh_h,
+ gen_helper_sve2_sqrdmulh_s, gen_helper_sve2_sqrdmulh_d,
+ };
+ return do_sve2_zzz_ool(s, a, fns[a->esz]);
+}
+
/*
* SVE2 Integer - Predicated
*/
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index 82e12b51c1..f3a19821cd 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -81,6 +81,26 @@ void HELPER(sve2_sqrdmlsh_b)(void *vd, void *vn, void *vm,
}
}
+void HELPER(sve2_sqdmulh_b)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int8_t *d = vd, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz; ++i) {
+ d[i] = do_sqrdmlah_b(n[i], m[i], 0, false, false);
+ }
+}
+
+void HELPER(sve2_sqrdmulh_b)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int8_t *d = vd, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz; ++i) {
+ d[i] = do_sqrdmlah_b(n[i], m[i], 0, false, true);
+ }
+}
+
/* Signed saturating rounding doubling multiply-accumulate high half, 16-bit */
int16_t do_sqrdmlah_h(int16_t src1, int16_t src2, int16_t src3,
bool neg, bool round, uint32_t *sat)
@@ -198,6 +218,28 @@ void HELPER(sve2_sqrdmlsh_h)(void *vd, void *vn, void *vm,
}
}
+void HELPER(sve2_sqdmulh_h)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int16_t *d = vd, *n = vn, *m = vm;
+ uint32_t discard;
+
+ for (i = 0; i < opr_sz / 2; ++i) {
+ d[i] = do_sqrdmlah_h(n[i], m[i], 0, false, false, &discard);
+ }
+}
+
+void HELPER(sve2_sqrdmulh_h)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int16_t *d = vd, *n = vn, *m = vm;
+ uint32_t discard;
+
+ for (i = 0; i < opr_sz / 2; ++i) {
+ d[i] = do_sqrdmlah_h(n[i], m[i], 0, false, true, &discard);
+ }
+}
+
/* Signed saturating rounding doubling multiply-accumulate high half, 32-bit */
int32_t do_sqrdmlah_s(int32_t src1, int32_t src2, int32_t src3,
bool neg, bool round, uint32_t *sat)
@@ -309,6 +351,28 @@ void HELPER(sve2_sqrdmlsh_s)(void *vd, void *vn, void *vm,
}
}
+void HELPER(sve2_sqdmulh_s)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int32_t *d = vd, *n = vn, *m = vm;
+ uint32_t discard;
+
+ for (i = 0; i < opr_sz / 4; ++i) {
+ d[i] = do_sqrdmlah_s(n[i], m[i], 0, false, false, &discard);
+ }
+}
+
+void HELPER(sve2_sqrdmulh_s)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int32_t *d = vd, *n = vn, *m = vm;
+ uint32_t discard;
+
+ for (i = 0; i < opr_sz / 4; ++i) {
+ d[i] = do_sqrdmlah_s(n[i], m[i], 0, false, true, &discard);
+ }
+}
+
/* Signed saturating rounding doubling multiply-accumulate high half, 64-bit */
static int64_t do_sat128_d(Int128 r)
{
@@ -368,6 +432,26 @@ void HELPER(sve2_sqrdmlsh_d)(void *vd, void *vn, void *vm,
}
}
+void HELPER(sve2_sqdmulh_d)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int64_t *d = vd, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz / 8; ++i) {
+ d[i] = do_sqrdmlah_d(n[i], m[i], 0, false, false);
+ }
+}
+
+void HELPER(sve2_sqrdmulh_d)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int64_t *d = vd, *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz / 8; ++i) {
+ d[i] = do_sqrdmlah_d(n[i], m[i], 0, false, true);
+ }
+}
+
/* Integer 8 and 16-bit dot-product.
*
* Note that for the loops herein, host endianness does not matter
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 62/81] target/arm: Implement SVE2 saturating multiply high (indexed)
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (60 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 61/81] target/arm: Implement SVE2 signed saturating doubling multiply high Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 63/81] target/arm: Implement SVE2 multiply-add long (indexed) Richard Henderson
` (20 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper.h | 14 ++++++
target/arm/sve.decode | 8 ++++
target/arm/translate-sve.c | 8 ++++
target/arm/vec_helper.c | 88 ++++++++++++++++++++++++++++++++++++++
4 files changed, 118 insertions(+)
diff --git a/target/arm/helper.h b/target/arm/helper.h
index fa7b2b7dd7..828fcc0fc9 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -954,6 +954,20 @@ DEF_HELPER_FLAGS_4(sve2_sqrdmulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqrdmulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqrdmulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmulh_idx_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmulh_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmulh_idx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_sqrdmulh_idx_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqrdmulh_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqrdmulh_idx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_4(gvec_xar_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
#ifdef TARGET_AARCH64
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 6879870cc1..80d76982e8 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -841,6 +841,14 @@ SQDMULLB_zzx_d 01000100 .. 1 ..... 1110.0 ..... ..... @rrxl_d
SQDMULLT_zzx_s 01000100 .. 1 ..... 1110.1 ..... ..... @rrxl_s
SQDMULLT_zzx_d 01000100 .. 1 ..... 1110.1 ..... ..... @rrxl_d
+# SVE2 saturating multiply high (indexed)
+SQDMULH_zzx_h 01000100 .. 1 ..... 111100 ..... ..... @rrx_h
+SQDMULH_zzx_s 01000100 .. 1 ..... 111100 ..... ..... @rrx_s
+SQDMULH_zzx_d 01000100 .. 1 ..... 111100 ..... ..... @rrx_d
+SQRDMULH_zzx_h 01000100 .. 1 ..... 111101 ..... ..... @rrx_h
+SQRDMULH_zzx_s 01000100 .. 1 ..... 111101 ..... ..... @rrx_s
+SQRDMULH_zzx_d 01000100 .. 1 ..... 111101 ..... ..... @rrx_d
+
# SVE2 integer multiply (indexed)
MUL_zzx_h 01000100 .. 1 ..... 111110 ..... ..... @rrx_h
MUL_zzx_s 01000100 .. 1 ..... 111110 ..... ..... @rrx_s
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 298c328ce0..bdee644c78 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -3868,6 +3868,14 @@ DO_SVE2_RRX(trans_MUL_zzx_h, gen_helper_gvec_mul_idx_h)
DO_SVE2_RRX(trans_MUL_zzx_s, gen_helper_gvec_mul_idx_s)
DO_SVE2_RRX(trans_MUL_zzx_d, gen_helper_gvec_mul_idx_d)
+DO_SVE2_RRX(trans_SQDMULH_zzx_h, gen_helper_sve2_sqdmulh_idx_h)
+DO_SVE2_RRX(trans_SQDMULH_zzx_s, gen_helper_sve2_sqdmulh_idx_s)
+DO_SVE2_RRX(trans_SQDMULH_zzx_d, gen_helper_sve2_sqdmulh_idx_d)
+
+DO_SVE2_RRX(trans_SQRDMULH_zzx_h, gen_helper_sve2_sqrdmulh_idx_h)
+DO_SVE2_RRX(trans_SQRDMULH_zzx_s, gen_helper_sve2_sqrdmulh_idx_s)
+DO_SVE2_RRX(trans_SQRDMULH_zzx_d, gen_helper_sve2_sqrdmulh_idx_d)
+
#undef DO_SVE2_RRX
#define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index f3a19821cd..339e722383 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -240,6 +240,36 @@ void HELPER(sve2_sqrdmulh_h)(void *vd, void *vn, void *vm, uint32_t desc)
}
}
+void HELPER(sve2_sqdmulh_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, j, opr_sz = simd_oprsz(desc);
+ int idx = simd_data(desc);
+ int16_t *d = vd, *n = vn, *m = (int16_t *)vm + H2(idx);
+ uint32_t discard;
+
+ for (i = 0; i < opr_sz / 2; i += 16 / 2) {
+ int16_t mm = m[i];
+ for (j = 0; j < 16 / 2; ++j) {
+ d[i + j] = do_sqrdmlah_h(n[i + j], mm, 0, false, false, &discard);
+ }
+ }
+}
+
+void HELPER(sve2_sqrdmulh_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, j, opr_sz = simd_oprsz(desc);
+ int idx = simd_data(desc);
+ int16_t *d = vd, *n = vn, *m = (int16_t *)vm + H2(idx);
+ uint32_t discard;
+
+ for (i = 0; i < opr_sz / 2; i += 16 / 2) {
+ int16_t mm = m[i];
+ for (j = 0; j < 16 / 2; ++j) {
+ d[i + j] = do_sqrdmlah_h(n[i + j], mm, 0, false, true, &discard);
+ }
+ }
+}
+
/* Signed saturating rounding doubling multiply-accumulate high half, 32-bit */
int32_t do_sqrdmlah_s(int32_t src1, int32_t src2, int32_t src3,
bool neg, bool round, uint32_t *sat)
@@ -373,6 +403,36 @@ void HELPER(sve2_sqrdmulh_s)(void *vd, void *vn, void *vm, uint32_t desc)
}
}
+void HELPER(sve2_sqdmulh_idx_s)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, j, opr_sz = simd_oprsz(desc);
+ int idx = simd_data(desc);
+ int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx);
+ uint32_t discard;
+
+ for (i = 0; i < opr_sz / 4; i += 16 / 4) {
+ int32_t mm = m[i];
+ for (j = 0; j < 16 / 4; ++j) {
+ d[i + j] = do_sqrdmlah_s(n[i + j], mm, 0, false, false, &discard);
+ }
+ }
+}
+
+void HELPER(sve2_sqrdmulh_idx_s)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, j, opr_sz = simd_oprsz(desc);
+ int idx = simd_data(desc);
+ int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx);
+ uint32_t discard;
+
+ for (i = 0; i < opr_sz / 4; i += 16 / 4) {
+ int32_t mm = m[i];
+ for (j = 0; j < 16 / 4; ++j) {
+ d[i + j] = do_sqrdmlah_s(n[i + j], mm, 0, false, true, &discard);
+ }
+ }
+}
+
/* Signed saturating rounding doubling multiply-accumulate high half, 64-bit */
static int64_t do_sat128_d(Int128 r)
{
@@ -452,6 +512,34 @@ void HELPER(sve2_sqrdmulh_d)(void *vd, void *vn, void *vm, uint32_t desc)
}
}
+void HELPER(sve2_sqdmulh_idx_d)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, j, opr_sz = simd_oprsz(desc);
+ int idx = simd_data(desc);
+ int64_t *d = vd, *n = vn, *m = (int64_t *)vm + idx;
+
+ for (i = 0; i < opr_sz / 8; i += 16 / 8) {
+ int64_t mm = m[i];
+ for (j = 0; j < 16 / 8; ++j) {
+ d[i + j] = do_sqrdmlah_d(n[i + j], mm, 0, false, false);
+ }
+ }
+}
+
+void HELPER(sve2_sqrdmulh_idx_d)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, j, opr_sz = simd_oprsz(desc);
+ int idx = simd_data(desc);
+ int64_t *d = vd, *n = vn, *m = (int64_t *)vm + idx;
+
+ for (i = 0; i < opr_sz / 8; i += 16 / 8) {
+ int64_t mm = m[i];
+ for (j = 0; j < 16 / 8; ++j) {
+ d[i + j] = do_sqrdmlah_d(n[i + j], mm, 0, false, true);
+ }
+ }
+}
+
/* Integer 8 and 16-bit dot-product.
*
* Note that for the loops herein, host endianness does not matter
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 63/81] target/arm: Implement SVE2 multiply-add long (indexed)
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (61 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 62/81] target/arm: Implement SVE2 saturating multiply high (indexed) Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 64/81] target/arm: Implement SVE2 complex integer multiply-add (indexed) Richard Henderson
` (19 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 17 +++++++++++++++++
target/arm/sve.decode | 18 ++++++++++++++++++
target/arm/sve_helper.c | 16 ++++++++++++++++
target/arm/translate-sve.c | 20 ++++++++++++++++++++
4 files changed, 71 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index ff85e1327b..90872fa0e6 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2698,3 +2698,20 @@ DEF_HELPER_FLAGS_4(sve2_sqdmull_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve2_sqdmull_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_smlal_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlal_idx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlsl_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlsl_idx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlal_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlal_idx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlsl_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlsl_idx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 80d76982e8..da77ad689f 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -825,6 +825,24 @@ SQDMLSLB_zzxw_d 01000100 .. 1 ..... 0011.0 ..... ..... @rrxw_d
SQDMLSLT_zzxw_s 01000100 .. 1 ..... 0011.1 ..... ..... @rrxw_s
SQDMLSLT_zzxw_d 01000100 .. 1 ..... 0011.1 ..... ..... @rrxw_d
+# SVE2 multiply-add long (indexed)
+SMLALB_zzxw_s 01000100 .. 1 ..... 1000.0 ..... ..... @rrxw_s
+SMLALB_zzxw_d 01000100 .. 1 ..... 1000.0 ..... ..... @rrxw_d
+SMLALT_zzxw_s 01000100 .. 1 ..... 1000.1 ..... ..... @rrxw_s
+SMLALT_zzxw_d 01000100 .. 1 ..... 1000.1 ..... ..... @rrxw_d
+UMLALB_zzxw_s 01000100 .. 1 ..... 1001.0 ..... ..... @rrxw_s
+UMLALB_zzxw_d 01000100 .. 1 ..... 1001.0 ..... ..... @rrxw_d
+UMLALT_zzxw_s 01000100 .. 1 ..... 1001.1 ..... ..... @rrxw_s
+UMLALT_zzxw_d 01000100 .. 1 ..... 1001.1 ..... ..... @rrxw_d
+SMLSLB_zzxw_s 01000100 .. 1 ..... 1010.0 ..... ..... @rrxw_s
+SMLSLB_zzxw_d 01000100 .. 1 ..... 1010.0 ..... ..... @rrxw_d
+SMLSLT_zzxw_s 01000100 .. 1 ..... 1010.1 ..... ..... @rrxw_s
+SMLSLT_zzxw_d 01000100 .. 1 ..... 1010.1 ..... ..... @rrxw_d
+UMLSLB_zzxw_s 01000100 .. 1 ..... 1011.0 ..... ..... @rrxw_s
+UMLSLB_zzxw_d 01000100 .. 1 ..... 1011.0 ..... ..... @rrxw_d
+UMLSLT_zzxw_s 01000100 .. 1 ..... 1011.1 ..... ..... @rrxw_s
+UMLSLT_zzxw_d 01000100 .. 1 ..... 1011.1 ..... ..... @rrxw_d
+
# SVE2 integer multiply long (indexed)
SMULLB_zzx_s 01000100 .. 1 ..... 1100.0 ..... ..... @rrxl_s
SMULLB_zzx_d 01000100 .. 1 ..... 1100.0 ..... ..... @rrxl_d
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 926fe98eb5..f71df89cb7 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1546,6 +1546,20 @@ void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
} \
}
+#define DO_MLA(N, M, A) (A + N * M)
+
+DO_ZZXW(sve2_smlal_idx_s, int32_t, int16_t, H1_4, H1_2, DO_MLA)
+DO_ZZXW(sve2_smlal_idx_d, int64_t, int32_t, , H1_4, DO_MLA)
+DO_ZZXW(sve2_umlal_idx_s, uint32_t, uint16_t, H1_4, H1_2, DO_MLA)
+DO_ZZXW(sve2_umlal_idx_d, uint64_t, uint32_t, , H1_4, DO_MLA)
+
+#define DO_MLS(N, M, A) (A - N * M)
+
+DO_ZZXW(sve2_smlsl_idx_s, int32_t, int16_t, H1_4, H1_2, DO_MLS)
+DO_ZZXW(sve2_smlsl_idx_d, int64_t, int32_t, , H1_4, DO_MLS)
+DO_ZZXW(sve2_umlsl_idx_s, uint32_t, uint16_t, H1_4, H1_2, DO_MLS)
+DO_ZZXW(sve2_umlsl_idx_d, uint64_t, uint32_t, , H1_4, DO_MLS)
+
#define DO_SQDMLAL_S(N, M, A) DO_SQADD_S(A, do_sqdmull_s(N, M))
#define DO_SQDMLAL_D(N, M, A) do_sqadd_d(A, do_sqdmull_d(N, M))
@@ -1558,6 +1572,8 @@ DO_ZZXW(sve2_sqdmlal_idx_d, int64_t, int32_t, , H1_4, DO_SQDMLAL_D)
DO_ZZXW(sve2_sqdmlsl_idx_s, int32_t, int16_t, H1_4, H1_2, DO_SQDMLSL_S)
DO_ZZXW(sve2_sqdmlsl_idx_d, int64_t, int32_t, , H1_4, DO_SQDMLSL_D)
+#undef DO_MLA
+#undef DO_MLS
#undef DO_ZZXW
#define DO_ZZX(NAME, TYPEW, TYPEN, HW, HN, OP) \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index bdee644c78..248fe4de42 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -3944,6 +3944,26 @@ DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, false)
DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, true)
DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, true)
+DO_SVE2_RRXR_TB(trans_SMLALB_zzxw_s, gen_helper_sve2_smlal_idx_s, false)
+DO_SVE2_RRXR_TB(trans_SMLALB_zzxw_d, gen_helper_sve2_smlal_idx_d, false)
+DO_SVE2_RRXR_TB(trans_SMLALT_zzxw_s, gen_helper_sve2_smlal_idx_s, true)
+DO_SVE2_RRXR_TB(trans_SMLALT_zzxw_d, gen_helper_sve2_smlal_idx_d, true)
+
+DO_SVE2_RRXR_TB(trans_UMLALB_zzxw_s, gen_helper_sve2_umlal_idx_s, false)
+DO_SVE2_RRXR_TB(trans_UMLALB_zzxw_d, gen_helper_sve2_umlal_idx_d, false)
+DO_SVE2_RRXR_TB(trans_UMLALT_zzxw_s, gen_helper_sve2_umlal_idx_s, true)
+DO_SVE2_RRXR_TB(trans_UMLALT_zzxw_d, gen_helper_sve2_umlal_idx_d, true)
+
+DO_SVE2_RRXR_TB(trans_SMLSLB_zzxw_s, gen_helper_sve2_smlsl_idx_s, false)
+DO_SVE2_RRXR_TB(trans_SMLSLB_zzxw_d, gen_helper_sve2_smlsl_idx_d, false)
+DO_SVE2_RRXR_TB(trans_SMLSLT_zzxw_s, gen_helper_sve2_smlsl_idx_s, true)
+DO_SVE2_RRXR_TB(trans_SMLSLT_zzxw_d, gen_helper_sve2_smlsl_idx_d, true)
+
+DO_SVE2_RRXR_TB(trans_UMLSLB_zzxw_s, gen_helper_sve2_umlsl_idx_s, false)
+DO_SVE2_RRXR_TB(trans_UMLSLB_zzxw_d, gen_helper_sve2_umlsl_idx_d, false)
+DO_SVE2_RRXR_TB(trans_UMLSLT_zzxw_s, gen_helper_sve2_umlsl_idx_s, true)
+DO_SVE2_RRXR_TB(trans_UMLSLT_zzxw_d, gen_helper_sve2_umlsl_idx_d, true)
+
#undef DO_SVE2_RRXR_TB
/*
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 64/81] target/arm: Implement SVE2 complex integer multiply-add (indexed)
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (62 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 63/81] target/arm: Implement SVE2 multiply-add long (indexed) Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 65/81] target/arm: Implement SVE mixed sign dot product (indexed) Richard Henderson
` (18 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 9 +++
target/arm/sve.decode | 12 ++++
target/arm/sve_helper.c | 142 +++++++++++++++++++++++++++++++------
target/arm/translate-sve.c | 38 +++++++---
4 files changed, 169 insertions(+), 32 deletions(-)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 90872fa0e6..671b3a8804 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2715,3 +2715,12 @@ DEF_HELPER_FLAGS_5(sve2_umlsl_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_umlsl_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_cmla_idx_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cmla_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index da77ad689f..e8011fe91b 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -825,6 +825,18 @@ SQDMLSLB_zzxw_d 01000100 .. 1 ..... 0011.0 ..... ..... @rrxw_d
SQDMLSLT_zzxw_s 01000100 .. 1 ..... 0011.1 ..... ..... @rrxw_s
SQDMLSLT_zzxw_d 01000100 .. 1 ..... 0011.1 ..... ..... @rrxw_d
+# SVE2 complex integer multiply-add (indexed)
+CMLA_zzxz_h 01000100 10 1 index:2 rm:3 0110 rot:2 rn:5 rd:5 \
+ ra=%reg_movprfx
+CMLA_zzxz_s 01000100 11 1 index:1 rm:4 0110 rot:2 rn:5 rd:5 \
+ ra=%reg_movprfx
+
+# SVE2 complex saturating integer multiply-add (indexed)
+SQRDCMLAH_zzxz_h 01000100 10 1 index:2 rm:3 0111 rot:2 rn:5 rd:5 \
+ ra=%reg_movprfx
+SQRDCMLAH_zzxz_s 01000100 11 1 index:1 rm:4 0111 rot:2 rn:5 rd:5 \
+ ra=%reg_movprfx
+
# SVE2 multiply-add long (indexed)
SMLALB_zzxw_s 01000100 .. 1 ..... 1000.0 ..... ..... @rrxw_s
SMLALB_zzxw_d 01000100 .. 1 ..... 1000.0 ..... ..... @rrxw_d
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index f71df89cb7..b2f24d5cfd 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1466,34 +1466,132 @@ void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
} \
}
-#define do_cmla(N, M, A, S) (A + (N * M) * (S ? -1 : 1))
+static int8_t do_cmla_b(int8_t n, int8_t m, int8_t a, bool sub)
+{
+ return n * m * (sub ? -1 : 1) + a;
+}
-DO_CMLA(sve2_cmla_zzzz_b, uint8_t, H1, do_cmla)
-DO_CMLA(sve2_cmla_zzzz_h, uint16_t, H2, do_cmla)
-DO_CMLA(sve2_cmla_zzzz_s, uint32_t, H4, do_cmla)
-DO_CMLA(sve2_cmla_zzzz_d, uint64_t, , do_cmla)
+static int16_t do_cmla_h(int16_t n, int16_t m, int16_t a, bool sub)
+{
+ return n * m * (sub ? -1 : 1) + a;
+}
-#define DO_SQRDMLAH_B(N, M, A, S) \
- do_sqrdmlah_b(N, M, A, S, true)
-#define DO_SQRDMLAH_H(N, M, A, S) \
- ({ uint32_t discard; do_sqrdmlah_h(N, M, A, S, true, &discard); })
-#define DO_SQRDMLAH_S(N, M, A, S) \
- ({ uint32_t discard; do_sqrdmlah_s(N, M, A, S, true, &discard); })
-#define DO_SQRDMLAH_D(N, M, A, S) \
- do_sqrdmlah_d(N, M, A, S, true)
+static int32_t do_cmla_s(int32_t n, int32_t m, int32_t a, bool sub)
+{
+ return n * m * (sub ? -1 : 1) + a;
+}
-DO_CMLA(sve2_sqrdcmlah_zzzz_b, int8_t, H1, DO_SQRDMLAH_B)
-DO_CMLA(sve2_sqrdcmlah_zzzz_h, int16_t, H2, DO_SQRDMLAH_H)
-DO_CMLA(sve2_sqrdcmlah_zzzz_s, int32_t, H4, DO_SQRDMLAH_S)
-DO_CMLA(sve2_sqrdcmlah_zzzz_d, int64_t, , DO_SQRDMLAH_D)
+static int64_t do_cmla_d(int64_t n, int64_t m, int64_t a, bool sub)
+{
+ return n * m * (sub ? -1 : 1) + a;
+}
+
+DO_CMLA(sve2_cmla_zzzz_b, uint8_t, H1, do_cmla_b)
+DO_CMLA(sve2_cmla_zzzz_h, uint16_t, H2, do_cmla_h)
+DO_CMLA(sve2_cmla_zzzz_s, uint32_t, H4, do_cmla_s)
+DO_CMLA(sve2_cmla_zzzz_d, uint64_t, , do_cmla_d)
+
+static int8_t do_sqrdcmlah_b(int8_t n, int8_t m, int8_t a, bool sub)
+{
+ return do_sqrdmlah_b(n, m, a, sub, true);
+}
+
+static int16_t do_sqrdcmlah_h(int16_t n, int16_t m, int16_t a, bool sub)
+{
+ uint32_t discard;
+ return do_sqrdmlah_h(n, m, a, sub, true, &discard);
+}
+
+static int32_t do_sqrdcmlah_s(int32_t n, int32_t m, int32_t a, bool sub)
+{
+ uint32_t discard;
+ return do_sqrdmlah_s(n, m, a, sub, true, &discard);
+}
+
+static int64_t do_sqrdcmlah_d(int64_t n, int64_t m, int64_t a, bool sub)
+{
+ return do_sqrdmlah_d(n, m, a, sub, true);
+}
+
+DO_CMLA(sve2_sqrdcmlah_zzzz_b, int8_t, H1, do_sqrdcmlah_b)
+DO_CMLA(sve2_sqrdcmlah_zzzz_h, int16_t, H2, do_sqrdcmlah_h)
+DO_CMLA(sve2_sqrdcmlah_zzzz_s, int32_t, H4, do_sqrdcmlah_s)
+DO_CMLA(sve2_sqrdcmlah_zzzz_d, int64_t, , do_sqrdcmlah_d)
-#undef DO_SQRDMLAH_B
-#undef DO_SQRDMLAH_H
-#undef DO_SQRDMLAH_S
-#undef DO_SQRDMLAH_D
-#undef do_cmla
#undef DO_CMLA
+static void do_cmla_idx_h(int16_t *d, int16_t *n, int16_t *m,
+ int16_t *a, uint32_t desc,
+ int16_t (*fn)(int16_t, int16_t, int16_t, bool))
+{
+ intptr_t i, j, oprsz = simd_oprsz(desc);
+ int rot = extract32(desc, SIMD_DATA_SHIFT, 2);
+ int idx = extract32(desc, SIMD_DATA_SHIFT + 2, 2) * 2;
+ int sel_a = rot & 1, sel_b = sel_a ^ 1;
+ bool sub_r = rot == 1 || rot == 2;
+ bool sub_i = rot >= 2;
+
+ for (i = 0; i < oprsz / 2; i += 16 / 2) {
+ int16_t elt2_a = m[H2(i + idx + sel_a)];
+ int16_t elt2_b = m[H2(i + idx + sel_b)];
+
+ for (j = 0; j < 16 / 2; j += 2) {
+ int16_t elt1_a = n[H2(i + j + sel_a)];
+
+ d[H2(i + j)] = fn(elt1_a, elt2_a, a[H2(i + j)], sub_r);
+ d[H2(i + j + 1)] = fn(elt1_a, elt2_b, a[H2(i + j + 1)], sub_i);
+ }
+ }
+}
+
+void HELPER(sve2_cmla_idx_h)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ do_cmla_idx_h(vd, vn, vm, va, desc, do_cmla_h);
+}
+
+void HELPER(sve2_sqrdcmlah_idx_h)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ do_cmla_idx_h(vd, vn, vm, va, desc, do_sqrdcmlah_h);
+}
+
+static void do_cmla_idx_s(int32_t *d, int32_t *n, int32_t *m,
+ int32_t *a, uint32_t desc,
+ int32_t (*fn)(int32_t, int32_t, int32_t, bool))
+{
+ intptr_t i, j, oprsz = simd_oprsz(desc);
+ int rot = extract32(desc, SIMD_DATA_SHIFT, 2);
+ int idx = extract32(desc, SIMD_DATA_SHIFT + 2, 2) * 2;
+ int sel_a = rot & 1, sel_b = sel_a ^ 1;
+ bool sub_r = rot == 1 || rot == 2;
+ bool sub_i = rot >= 2;
+
+ for (i = 0; i < oprsz / 4; i += 16 / 4) {
+ int32_t elt2_a = m[H4(i + idx + sel_a)];
+ int32_t elt2_b = m[H4(i + idx + sel_b)];
+
+ for (j = 0; j < 16 / 4; j += 2) {
+ int32_t elt1_a = n[H4(i + j + sel_a)];
+
+ d[H4(i + j)] = fn(elt1_a, elt2_a, a[H4(i + j)], sub_r);
+ d[H4(i + j + 1)] = fn(elt1_a, elt2_b, a[H4(i + j + 1)], sub_i);
+ }
+ }
+}
+
+void HELPER(sve2_cmla_idx_s)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ do_cmla_idx_s(vd, vn, vm, va, desc, do_cmla_s);
+}
+
+void HELPER(sve2_sqrdcmlah_idx_s)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ do_cmla_idx_s(vd, vn, vm, va, desc, do_sqrdcmlah_s);
+}
+
#define DO_ZZXZ(NAME, TYPE, H, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
{ \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 248fe4de42..4628198b76 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -3821,21 +3821,21 @@ static bool trans_DOT_zzzz(DisasContext *s, arg_DOT_zzzz *a)
* SVE Multiply - Indexed
*/
-static bool do_zzxz_data(DisasContext *s, arg_rrxr_esz *a,
+static bool do_zzxz_data(DisasContext *s, int rd, int rn, int rm, int ra,
gen_helper_gvec_4 *fn, int data)
{
if (fn == NULL) {
return false;
}
if (sve_access_check(s)) {
- gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data);
+ gen_gvec_ool_zzzz(s, fn, rd, rn, rm, ra, data);
}
return true;
}
#define DO_RRXR(NAME, FUNC) \
static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
- { return do_zzxz_data(s, a, FUNC, a->index); }
+ { return do_zzxz_data(s, a->rd, a->rn, a->rm, a->ra, FUNC, a->index); }
DO_RRXR(trans_SDOT_zzxw_s, gen_helper_gvec_sdot_idx_b)
DO_RRXR(trans_SDOT_zzxw_d, gen_helper_gvec_sdot_idx_h)
@@ -3899,18 +3899,18 @@ DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_d, gen_helper_sve2_sqdmull_idx_d, true)
#undef DO_SVE2_RRX_TB
-static bool do_sve2_zzxz_data(DisasContext *s, arg_rrxr_esz *a,
- gen_helper_gvec_4 *fn, int data)
+static bool do_sve2_zzxz_data(DisasContext *s, int rd, int rn, int rm,
+ int ra, gen_helper_gvec_4 *fn, int data)
{
if (!dc_isar_feature(aa64_sve2, s)) {
return false;
}
- return do_zzxz_data(s, a, fn, data);
+ return do_zzxz_data(s, rd, rn, rm, ra, fn, data);
}
#define DO_SVE2_RRXR(NAME, FUNC) \
- static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
- { return do_sve2_zzxz_data(s, a, FUNC, a->index); }
+static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
+{ return do_sve2_zzxz_data(s, a->rd, a->rn, a->rm, a->ra, FUNC, a->index); }
DO_SVE2_RRXR(trans_MLA_zzxz_h, gen_helper_gvec_mla_idx_h)
DO_SVE2_RRXR(trans_MLA_zzxz_s, gen_helper_gvec_mla_idx_s)
@@ -3931,8 +3931,11 @@ DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_d, gen_helper_sve2_sqrdmlsh_idx_d)
#undef DO_SVE2_RRXR
#define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
- static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
- { return do_sve2_zzxz_data(s, a, FUNC, (a->index << 1) | TOP); }
+static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
+{ \
+ return do_sve2_zzxz_data(s, a->rd, a->rn, a->rm, a->ra, \
+ FUNC, (a->index << 1) | TOP); \
+}
DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, false)
DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, false)
@@ -3966,6 +3969,21 @@ DO_SVE2_RRXR_TB(trans_UMLSLT_zzxw_d, gen_helper_sve2_umlsl_idx_d, true)
#undef DO_SVE2_RRXR_TB
+#define DO_SVE2_RRXR_ROT(NAME, FUNC) \
+static bool trans_##NAME(DisasContext *s, arg_##NAME *a) \
+{ \
+ return do_sve2_zzxz_data(s, a->rd, a->rn, a->rm, a->ra, \
+ FUNC, (a->index << 2) | a->rot); \
+}
+
+DO_SVE2_RRXR_ROT(CMLA_zzxz_h, gen_helper_sve2_cmla_idx_h)
+DO_SVE2_RRXR_ROT(CMLA_zzxz_s, gen_helper_sve2_cmla_idx_s)
+
+DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h, gen_helper_sve2_sqrdcmlah_idx_h)
+DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s, gen_helper_sve2_sqrdcmlah_idx_s)
+
+#undef DO_SVE2_RRXR_ROT
+
/*
*** SVE Floating Point Multiply-Add Indexed Group
*/
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 65/81] target/arm: Implement SVE mixed sign dot product (indexed)
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (63 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 64/81] target/arm: Implement SVE2 complex integer multiply-add (indexed) Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 66/81] target/arm: Implement SVE mixed sign dot product Richard Henderson
` (17 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/cpu.h | 5 +++
target/arm/helper.h | 4 +++
target/arm/sve.decode | 4 +++
target/arm/translate-sve.c | 18 +++++++++++
target/arm/vec_helper.c | 66 ++++++++++++++++++++++++++++++++++++++
5 files changed, 97 insertions(+)
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index aaefbf0167..b394e15c0c 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -3917,6 +3917,11 @@ static inline bool isar_feature_aa64_sve2_bitperm(const ARMISARegisters *id)
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, BITPERM) != 0;
}
+static inline bool isar_feature_aa64_sve2_i8mm(const ARMISARegisters *id)
+{
+ return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, I8MM) != 0;
+}
+
static inline bool isar_feature_aa64_sve2_f32mm(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, F32MM) != 0;
diff --git a/target/arm/helper.h b/target/arm/helper.h
index 828fcc0fc9..0f17b59f45 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -608,6 +608,10 @@ DEF_HELPER_FLAGS_5(gvec_sdot_idx_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_udot_idx_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_sudot_idx_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_usdot_idx_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_fcaddh, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index e8011fe91b..51acbfa797 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -815,6 +815,10 @@ SQRDMLSH_zzxz_h 01000100 .. 1 ..... 000101 ..... ..... @rrxr_h
SQRDMLSH_zzxz_s 01000100 .. 1 ..... 000101 ..... ..... @rrxr_s
SQRDMLSH_zzxz_d 01000100 .. 1 ..... 000101 ..... ..... @rrxr_d
+# SVE mixed sign dot product (indexed)
+USDOT_zzxw_s 01000100 .. 1 ..... 000110 ..... ..... @rrxr_s
+SUDOT_zzxw_s 01000100 .. 1 ..... 000111 ..... ..... @rrxr_s
+
# SVE2 saturating multiply-add (indexed)
SQDMLALB_zzxw_s 01000100 .. 1 ..... 0010.0 ..... ..... @rrxw_s
SQDMLALB_zzxw_d 01000100 .. 1 ..... 0010.0 ..... ..... @rrxw_d
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 4628198b76..ae61659b1d 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -3842,6 +3842,24 @@ DO_RRXR(trans_SDOT_zzxw_d, gen_helper_gvec_sdot_idx_h)
DO_RRXR(trans_UDOT_zzxw_s, gen_helper_gvec_udot_idx_b)
DO_RRXR(trans_UDOT_zzxw_d, gen_helper_gvec_udot_idx_h)
+static bool trans_SUDOT_zzxw_s(DisasContext *s, arg_rrxr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2_i8mm, s)) {
+ return false;
+ }
+ return do_zzxz_data(s, a->rd, a->rn, a->rm, a->ra,
+ gen_helper_gvec_sudot_idx_b, a->index);
+}
+
+static bool trans_USDOT_zzxw_s(DisasContext *s, arg_rrxr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2_i8mm, s)) {
+ return false;
+ }
+ return do_zzxz_data(s, a->rd, a->rn, a->rm, a->ra,
+ gen_helper_gvec_usdot_idx_b, a->index);
+}
+
#undef DO_RRXR
static bool do_sve2_zzx_data(DisasContext *s, arg_rrx_esz *a,
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index 339e722383..8dccc6789d 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -677,6 +677,72 @@ void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm,
clear_tail(d, opr_sz, simd_maxsz(desc));
}
+void HELPER(gvec_sudot_idx_b)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4;
+ intptr_t index = simd_data(desc);
+ int32_t *d = vd, *a = va;
+ int8_t *n = vn;
+ uint8_t *m_indexed = (uint8_t *)vm + index * 4;
+
+ /* Notice the special case of opr_sz == 8, from aa64/aa32 advsimd.
+ * Otherwise opr_sz is a multiple of 16.
+ */
+ segend = MIN(4, opr_sz_4);
+ i = 0;
+ do {
+ uint8_t m0 = m_indexed[i * 4 + 0];
+ uint8_t m1 = m_indexed[i * 4 + 1];
+ uint8_t m2 = m_indexed[i * 4 + 2];
+ uint8_t m3 = m_indexed[i * 4 + 3];
+
+ do {
+ d[i] = (a[i] +
+ n[i * 4 + 0] * m0 +
+ n[i * 4 + 1] * m1 +
+ n[i * 4 + 2] * m2 +
+ n[i * 4 + 3] * m3);
+ } while (++i < segend);
+ segend = i + 4;
+ } while (i < opr_sz_4);
+
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_usdot_idx_b)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4;
+ intptr_t index = simd_data(desc);
+ uint32_t *d = vd, *a = va;
+ uint8_t *n = vn;
+ int8_t *m_indexed = (int8_t *)vm + index * 4;
+
+ /* Notice the special case of opr_sz == 8, from aa64/aa32 advsimd.
+ * Otherwise opr_sz is a multiple of 16.
+ */
+ segend = MIN(4, opr_sz_4);
+ i = 0;
+ do {
+ int8_t m0 = m_indexed[i * 4 + 0];
+ int8_t m1 = m_indexed[i * 4 + 1];
+ int8_t m2 = m_indexed[i * 4 + 2];
+ int8_t m3 = m_indexed[i * 4 + 3];
+
+ do {
+ d[i] = (a[i] +
+ n[i * 4 + 0] * m0 +
+ n[i * 4 + 1] * m1 +
+ n[i * 4 + 2] * m2 +
+ n[i * 4 + 3] * m3);
+ } while (++i < segend);
+ segend = i + 4;
+ } while (i < opr_sz_4);
+
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm,
void *va, uint32_t desc)
{
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 66/81] target/arm: Implement SVE mixed sign dot product
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (64 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 65/81] target/arm: Implement SVE mixed sign dot product (indexed) Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 67/81] target/arm: Implement SVE2 crypto unary operations Richard Henderson
` (16 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper.h | 2 ++
target/arm/sve.decode | 4 ++++
target/arm/translate-sve.c | 16 ++++++++++++++++
target/arm/vec_helper.c | 18 ++++++++++++++++++
4 files changed, 40 insertions(+)
diff --git a/target/arm/helper.h b/target/arm/helper.h
index 0f17b59f45..868c2e9121 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -599,6 +599,8 @@ DEF_HELPER_FLAGS_5(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_udot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_usdot_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_sdot_idx_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 51acbfa797..0be8a020f6 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1523,6 +1523,10 @@ UMLSLT_zzzw 01000100 .. 0 ..... 010 111 ..... ..... @rda_rn_rm
CMLA_zzzz 01000100 esz:2 0 rm:5 0010 rot:2 rn:5 rd:5 ra=%reg_movprfx
SQRDCMLAH_zzzz 01000100 esz:2 0 rm:5 0011 rot:2 rn:5 rd:5 ra=%reg_movprfx
+## SVE mixed sign dot product
+
+USDOT_zzzz 01000100 .. 0 ..... 011 110 ..... ..... @rda_rn_rm
+
### SVE2 floating point matrix multiply accumulate
FMMLA 01100100 .. 1 ..... 111001 ..... ..... @rda_rn_rm
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index ae61659b1d..4341b83414 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -8111,3 +8111,19 @@ static bool trans_SQRDCMLAH_zzzz(DisasContext *s, arg_SQRDCMLAH_zzzz *a)
}
return true;
}
+
+static bool trans_USDOT_zzzz(DisasContext *s, arg_USDOT_zzzz *a)
+{
+ if (a->esz != 2 || !dc_isar_feature(aa64_sve2_i8mm, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ tcg_gen_gvec_4_ool(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ vec_full_reg_offset(s, a->ra),
+ vsz, vsz, 0, gen_helper_gvec_usdot_b);
+ }
+ return true;
+}
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index 8dccc6789d..c8d1656797 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -579,6 +579,24 @@ void HELPER(gvec_udot_b)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
clear_tail(d, opr_sz, simd_maxsz(desc));
}
+void HELPER(gvec_usdot_b)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ int32_t *d = vd, *a = va;
+ uint8_t *n = vn;
+ int8_t *m = vm;
+
+ for (i = 0; i < opr_sz / 4; ++i) {
+ d[i] = (a[i] +
+ n[i * 4 + 0] * m[i * 4 + 0] +
+ n[i * 4 + 1] * m[i * 4 + 1] +
+ n[i * 4 + 2] * m[i * 4 + 2] +
+ n[i * 4 + 3] * m[i * 4 + 3]);
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
void HELPER(gvec_sdot_h)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
{
intptr_t i, opr_sz = simd_oprsz(desc);
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 67/81] target/arm: Implement SVE2 crypto unary operations
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (65 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 66/81] target/arm: Implement SVE mixed sign dot product Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 68/81] target/arm: Implement SVE2 crypto destructive binary operations Richard Henderson
` (15 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve.decode | 6 ++++++
target/arm/translate-sve.c | 11 +++++++++++
2 files changed, 17 insertions(+)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 0be8a020f6..9b0d0f3a5d 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1551,3 +1551,9 @@ STNT1_zprz 1110010 .. 00 ..... 001 ... ..... ..... \
# SVE2 32-bit scatter non-temporal store (vector plus scalar)
STNT1_zprz 1110010 .. 10 ..... 001 ... ..... ..... \
@rprr_scatter_store xs=0 esz=2 scale=0
+
+### SVE2 Crypto Extensions
+
+# SVE2 crypto unary operations
+# AESMC and AESIMC
+AESMC 01000101 00 10000011100 decrypt:1 00000 rd:5
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 4341b83414..3c46fd7e2b 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -8127,3 +8127,14 @@ static bool trans_USDOT_zzzz(DisasContext *s, arg_USDOT_zzzz *a)
}
return true;
}
+
+static bool trans_AESMC(DisasContext *s, arg_AESMC *a)
+{
+ if (!dc_isar_feature(aa64_sve2_aes, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_ool_zz(s, gen_helper_crypto_aesmc, a->rd, a->rd, a->decrypt);
+ }
+ return true;
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 68/81] target/arm: Implement SVE2 crypto destructive binary operations
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (66 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 67/81] target/arm: Implement SVE2 crypto unary operations Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 69/81] target/arm: Implement SVE2 crypto constructive " Richard Henderson
` (14 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/cpu.h | 5 +++++
target/arm/sve.decode | 7 +++++++
target/arm/translate-sve.c | 38 ++++++++++++++++++++++++++++++++++++++
3 files changed, 50 insertions(+)
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index b394e15c0c..f5052a58a4 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -3917,6 +3917,11 @@ static inline bool isar_feature_aa64_sve2_bitperm(const ARMISARegisters *id)
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, BITPERM) != 0;
}
+static inline bool isar_feature_aa64_sve2_sm4(const ARMISARegisters *id)
+{
+ return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SM4) != 0;
+}
+
static inline bool isar_feature_aa64_sve2_i8mm(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, I8MM) != 0;
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 9b0d0f3a5d..2ebf65f376 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -118,6 +118,8 @@
@pd_pn_pm ........ esz:2 .. rm:4 ....... rn:4 . rd:4 &rrr_esz
@rdn_rm ........ esz:2 ...... ...... rm:5 rd:5 \
&rrr_esz rn=%reg_movprfx
+@rdn_rm_e0 ........ .. ...... ...... rm:5 rd:5 \
+ &rrr_esz rn=%reg_movprfx esz=0
@rdn_sh_i8u ........ esz:2 ...... ...... ..... rd:5 \
&rri_esz rn=%reg_movprfx imm=%sh8_i8u
@rdn_i8u ........ esz:2 ...... ... imm:8 rd:5 \
@@ -1557,3 +1559,8 @@ STNT1_zprz 1110010 .. 10 ..... 001 ... ..... ..... \
# SVE2 crypto unary operations
# AESMC and AESIMC
AESMC 01000101 00 10000011100 decrypt:1 00000 rd:5
+
+# SVE2 crypto destructive binary operations
+AESE 01000101 00 10001 0 11100 0 ..... ..... @rdn_rm_e0
+AESD 01000101 00 10001 0 11100 1 ..... ..... @rdn_rm_e0
+SM4E 01000101 00 10001 1 11100 0 ..... ..... @rdn_rm_e0
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 3c46fd7e2b..aa459ed30a 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -8138,3 +8138,41 @@ static bool trans_AESMC(DisasContext *s, arg_AESMC *a)
}
return true;
}
+
+static bool do_aese(DisasContext *s, arg_rrr_esz *a, bool decrypt)
+{
+ if (!dc_isar_feature(aa64_sve2_aes, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_ool_zzz(s, gen_helper_crypto_aese,
+ a->rd, a->rn, a->rm, decrypt);
+ }
+ return true;
+}
+
+static bool trans_AESE(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_aese(s, a, false);
+}
+
+static bool trans_AESD(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_aese(s, a, true);
+}
+
+static bool do_sm4(DisasContext *s, arg_rrr_esz *a, gen_helper_gvec_3 *fn)
+{
+ if (!dc_isar_feature(aa64_sve2_sm4, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, 0);
+ }
+ return true;
+}
+
+static bool trans_SM4E(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_sm4(s, a, gen_helper_crypto_sm4e);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 69/81] target/arm: Implement SVE2 crypto constructive binary operations
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (67 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 68/81] target/arm: Implement SVE2 crypto destructive binary operations Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 70/81] target/arm: Implement SVE2 TBL, TBX Richard Henderson
` (13 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/cpu.h | 5 +++++
target/arm/sve.decode | 4 ++++
target/arm/translate-sve.c | 16 ++++++++++++++++
3 files changed, 25 insertions(+)
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index f5052a58a4..67b2834035 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -3917,6 +3917,11 @@ static inline bool isar_feature_aa64_sve2_bitperm(const ARMISARegisters *id)
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, BITPERM) != 0;
}
+static inline bool isar_feature_aa64_sve2_sha3(const ARMISARegisters *id)
+{
+ return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SHA3) != 0;
+}
+
static inline bool isar_feature_aa64_sve2_sm4(const ARMISARegisters *id)
{
return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SM4) != 0;
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 2ebf65f376..5f2fad4754 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1564,3 +1564,7 @@ AESMC 01000101 00 10000011100 decrypt:1 00000 rd:5
AESE 01000101 00 10001 0 11100 0 ..... ..... @rdn_rm_e0
AESD 01000101 00 10001 0 11100 1 ..... ..... @rdn_rm_e0
SM4E 01000101 00 10001 1 11100 0 ..... ..... @rdn_rm_e0
+
+# SVE2 crypto constructive binary operations
+SM4EKEY 01000101 00 1 ..... 11110 0 ..... ..... @rd_rn_rm_e0
+RAX1 01000101 00 1 ..... 11110 1 ..... ..... @rd_rn_rm_e0
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index aa459ed30a..cb493d838d 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -8176,3 +8176,19 @@ static bool trans_SM4E(DisasContext *s, arg_rrr_esz *a)
{
return do_sm4(s, a, gen_helper_crypto_sm4e);
}
+
+static bool trans_SM4EKEY(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_sm4(s, a, gen_helper_crypto_sm4ekey);
+}
+
+static bool trans_RAX1(DisasContext *s, arg_rrr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2_sha3, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_fn_zzz(s, gen_gvec_rax1, MO_64, a->rd, a->rn, a->rm);
+ }
+ return true;
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 70/81] target/arm: Implement SVE2 TBL, TBX
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (68 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 69/81] target/arm: Implement SVE2 crypto constructive " Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 71/81] target/arm: Implement SVE2 FCVTNT Richard Henderson
` (12 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200428144352.9275-1-steplong@quicinc.com>
[rth: rearrange the macros a little and rebase]
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 10 +++++
target/arm/sve.decode | 5 +++
target/arm/sve_helper.c | 90 ++++++++++++++++++++++++++++++--------
target/arm/translate-sve.c | 33 ++++++++++++++
4 files changed, 119 insertions(+), 19 deletions(-)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 671b3a8804..b46723ebe3 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -661,6 +661,16 @@ DEF_HELPER_FLAGS_4(sve_tbl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_tbl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_tbl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_tbl_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_tbl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_tbl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_tbl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_tbx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_tbx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_tbx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_tbx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_3(sve_sunpk_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve_sunpk_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve_sunpk_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 5f2fad4754..609ecce520 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -560,6 +560,11 @@ TBL 00000101 .. 1 ..... 001100 ..... ..... @rd_rn_rm
# SVE unpack vector elements
UNPK 00000101 esz:2 1100 u:1 h:1 001110 rn:5 rd:5
+# SVE2 Table Lookup (three sources)
+
+TBL_sve2 00000101 .. 1 ..... 001010 ..... ..... @rd_rn_rm
+TBX 00000101 .. 1 ..... 001011 ..... ..... @rd_rn_rm
+
### SVE Permute - Predicates Group
# SVE permute predicate elements
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index b2f24d5cfd..e651c0e9f1 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -3034,28 +3034,80 @@ void HELPER(sve_rev_d)(void *vd, void *vn, uint32_t desc)
}
}
-#define DO_TBL(NAME, TYPE, H) \
-void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
-{ \
- intptr_t i, opr_sz = simd_oprsz(desc); \
- uintptr_t elem = opr_sz / sizeof(TYPE); \
- TYPE *d = vd, *n = vn, *m = vm; \
- ARMVectorReg tmp; \
- if (unlikely(vd == vn)) { \
- n = memcpy(&tmp, vn, opr_sz); \
- } \
- for (i = 0; i < elem; i++) { \
- TYPE j = m[H(i)]; \
- d[H(i)] = j < elem ? n[H(j)] : 0; \
- } \
+typedef void tb_impl_fn(void *, void *, void *, void *, uintptr_t, bool);
+
+static inline void do_tbl1(void *vd, void *vn, void *vm, uint32_t desc,
+ bool is_tbx, tb_impl_fn *fn)
+{
+ ARMVectorReg scratch;
+ uintptr_t oprsz = simd_oprsz(desc);
+
+ if (unlikely(vd == vn)) {
+ vn = memcpy(&scratch, vn, oprsz);
+ }
+
+ fn(vd, vn, NULL, vm, oprsz, is_tbx);
}
-DO_TBL(sve_tbl_b, uint8_t, H1)
-DO_TBL(sve_tbl_h, uint16_t, H2)
-DO_TBL(sve_tbl_s, uint32_t, H4)
-DO_TBL(sve_tbl_d, uint64_t, )
+static inline void do_tbl2(void *vd, void *vn0, void *vn1, void *vm,
+ uint32_t desc, bool is_tbx, tb_impl_fn *fn)
+{
+ ARMVectorReg scratch;
+ uintptr_t oprsz = simd_oprsz(desc);
-#undef TBL
+ if (unlikely(vd == vn0)) {
+ vn0 = memcpy(&scratch, vn0, oprsz);
+ if (vd == vn1) {
+ vn1 = vn0;
+ }
+ } else if (unlikely(vd == vn1)) {
+ vn1 = memcpy(&scratch, vn1, oprsz);
+ }
+
+ fn(vd, vn0, vn1, vm, oprsz, is_tbx);
+}
+
+#define DO_TB(SUFF, TYPE, H) \
+static inline void do_tb_##SUFF(void *vd, void *vt0, void *vt1, \
+ void *vm, uintptr_t oprsz, bool is_tbx) \
+{ \
+ TYPE *d = vd, *tbl0 = vt0, *tbl1 = vt1, *indexes = vm; \
+ uintptr_t i, nelem = oprsz / sizeof(TYPE); \
+ for (i = 0; i < nelem; ++i) { \
+ TYPE index = indexes[H1(i)], val = 0; \
+ if (index < nelem) { \
+ val = tbl0[H(index)]; \
+ } else { \
+ index -= nelem; \
+ if (tbl1 && index < nelem) { \
+ val = tbl1[H(index)]; \
+ } else if (is_tbx) { \
+ continue; \
+ } \
+ } \
+ d[H(i)] = val; \
+ } \
+} \
+void HELPER(sve_tbl_##SUFF)(void *vd, void *vn, void *vm, uint32_t desc) \
+{ \
+ do_tbl1(vd, vn, vm, desc, false, do_tb_##SUFF); \
+} \
+void HELPER(sve2_tbl_##SUFF)(void *vd, void *vn0, void *vn1, \
+ void *vm, uint32_t desc) \
+{ \
+ do_tbl2(vd, vn0, vn1, vm, desc, false, do_tb_##SUFF); \
+} \
+void HELPER(sve2_tbx_##SUFF)(void *vd, void *vn, void *vm, uint32_t desc) \
+{ \
+ do_tbl1(vd, vn, vm, desc, true, do_tb_##SUFF); \
+}
+
+DO_TB(b, uint8_t, H1)
+DO_TB(h, uint16_t, H2)
+DO_TB(s, uint32_t, H4)
+DO_TB(d, uint64_t, )
+
+#undef DO_TB
#define DO_UNPK(NAME, TYPED, TYPES, HD, HS) \
void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index cb493d838d..e8fef28750 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -2417,6 +2417,39 @@ static bool trans_TBL(DisasContext *s, arg_rrr_esz *a)
return true;
}
+static bool trans_TBL_sve2(DisasContext *s, arg_rrr_esz *a)
+{
+ static gen_helper_gvec_4 * const fns[4] = {
+ gen_helper_sve2_tbl_b, gen_helper_sve2_tbl_h,
+ gen_helper_sve2_tbl_s, gen_helper_sve2_tbl_d
+ };
+
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_ool_zzzz(s, fns[a->esz], a->rd, a->rn,
+ (a->rn + 1) % 32, a->rm, 0);
+ }
+ return true;
+}
+
+static bool trans_TBX(DisasContext *s, arg_rrr_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve2_tbx_b, gen_helper_sve2_tbx_h,
+ gen_helper_sve2_tbx_s, gen_helper_sve2_tbx_d
+ };
+
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_ool_zzz(s, fns[a->esz], a->rd, a->rn, a->rm, 0);
+ }
+ return true;
+}
+
static bool trans_UNPK(DisasContext *s, arg_UNPK *a)
{
static gen_helper_gvec_2 * const fns[4][2] = {
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 71/81] target/arm: Implement SVE2 FCVTNT
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (69 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 70/81] target/arm: Implement SVE2 TBL, TBX Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 72/81] target/arm: Implement SVE2 FCVTLT Richard Henderson
` (11 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200428174332.17162-2-steplong@quicinc.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 5 +++++
target/arm/sve.decode | 4 ++++
target/arm/sve_helper.c | 20 ++++++++++++++++++++
target/arm/translate-sve.c | 16 ++++++++++++++++
4 files changed, 45 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index b46723ebe3..0d8667a7d1 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2734,3 +2734,8 @@ DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_fcvtnt_sh, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_fcvtnt_ds, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 609ecce520..9ba4bb476e 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1573,3 +1573,7 @@ SM4E 01000101 00 10001 1 11100 0 ..... ..... @rdn_rm_e0
# SVE2 crypto constructive binary operations
SM4EKEY 01000101 00 1 ..... 11110 0 ..... ..... @rd_rn_rm_e0
RAX1 01000101 00 1 ..... 11110 1 ..... ..... @rd_rn_rm_e0
+
+### SVE2 floating-point convert precision odd elements
+FCVTNT_sh 01100100 10 0010 00 101 ... ..... ..... @rd_pg_rn_e0
+FCVTNT_ds 01100100 11 0010 10 101 ... ..... ..... @rd_pg_rn_e0
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index e651c0e9f1..21023c2f7f 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -7594,3 +7594,23 @@ void HELPER(fmmla_d)(void *vd, void *vn, void *vm, void *va,
d[3] = float64_add(a[3], float64_add(p0, p1, status), status);
}
}
+
+#define DO_FCVTNT(NAME, TYPEW, TYPEN, HW, HN, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vg, void *status, uint32_t desc) \
+{ \
+ intptr_t i = simd_oprsz(desc); \
+ uint64_t *g = vg; \
+ do { \
+ uint64_t pg = g[(i - 1) >> 6]; \
+ do { \
+ i -= sizeof(TYPEW); \
+ if (likely((pg >> (i & 63)) & 1)) { \
+ TYPEW nn = *(TYPEW *)(vn + HW(i)); \
+ *(TYPEN *)(vd + HN(i + sizeof(TYPEN))) = OP(nn, status); \
+ } \
+ } while (i & 63); \
+ } while (i != 0); \
+}
+
+DO_FCVTNT(sve2_fcvtnt_sh, uint32_t, uint16_t, H1_4, H1_2, sve_f32_to_f16)
+DO_FCVTNT(sve2_fcvtnt_ds, uint64_t, uint32_t, H1_4, H1_2, float64_to_float32)
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index e8fef28750..8a3a646bf5 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -8225,3 +8225,19 @@ static bool trans_RAX1(DisasContext *s, arg_rrr_esz *a)
}
return true;
}
+
+static bool trans_FCVTNT_sh(DisasContext *s, arg_rpr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve2_fcvtnt_sh);
+}
+
+static bool trans_FCVTNT_ds(DisasContext *s, arg_rpr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve2_fcvtnt_ds);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 72/81] target/arm: Implement SVE2 FCVTLT
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (70 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 71/81] target/arm: Implement SVE2 FCVTNT Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 73/81] target/arm: Implement SVE2 FCVTXNT, FCVTX Richard Henderson
` (10 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200428174332.17162-3-steplong@quicinc.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 5 +++++
target/arm/sve.decode | 2 ++
target/arm/sve_helper.c | 23 +++++++++++++++++++++++
target/arm/translate-sve.c | 16 ++++++++++++++++
4 files changed, 46 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 0d8667a7d1..df141aec41 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2739,3 +2739,8 @@ DEF_HELPER_FLAGS_5(sve2_fcvtnt_sh, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_fcvtnt_ds, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_fcvtlt_hs, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_fcvtlt_sd, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 9ba4bb476e..abe26f2424 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1576,4 +1576,6 @@ RAX1 01000101 00 1 ..... 11110 1 ..... ..... @rd_rn_rm_e0
### SVE2 floating-point convert precision odd elements
FCVTNT_sh 01100100 10 0010 00 101 ... ..... ..... @rd_pg_rn_e0
+FCVTLT_hs 01100100 10 0010 01 101 ... ..... ..... @rd_pg_rn_e0
FCVTNT_ds 01100100 11 0010 10 101 ... ..... ..... @rd_pg_rn_e0
+FCVTLT_sd 01100100 11 0010 11 101 ... ..... ..... @rd_pg_rn_e0
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 21023c2f7f..8539493c72 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -7614,3 +7614,26 @@ void HELPER(NAME)(void *vd, void *vn, void *vg, void *status, uint32_t desc) \
DO_FCVTNT(sve2_fcvtnt_sh, uint32_t, uint16_t, H1_4, H1_2, sve_f32_to_f16)
DO_FCVTNT(sve2_fcvtnt_ds, uint64_t, uint32_t, H1_4, H1_2, float64_to_float32)
+
+#define DO_FCVTLT(NAME, TYPEW, TYPEN, HW, HN, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vg, void *status, uint32_t desc) \
+{ \
+ intptr_t i = simd_oprsz(desc); \
+ uint64_t *g = vg; \
+ do { \
+ uint64_t pg = g[(i - 1) >> 6]; \
+ do { \
+ i -= sizeof(TYPEW); \
+ if (likely((pg >> (i & 63)) & 1)) { \
+ TYPEN nn = *(TYPEN *)(vn + HN(i + sizeof(TYPEN))); \
+ *(TYPEW *)(vd + HW(i)) = OP(nn, status); \
+ } \
+ } while (i & 63); \
+ } while (i != 0); \
+}
+
+DO_FCVTLT(sve2_fcvtlt_hs, uint32_t, uint16_t, H1_4, H1_2, sve_f16_to_f32)
+DO_FCVTLT(sve2_fcvtlt_sd, uint64_t, uint32_t, H1_4, H1_2, float32_to_float64)
+
+#undef DO_FCVTLT
+#undef DO_FCVTNT
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 8a3a646bf5..247288e164 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -8241,3 +8241,19 @@ static bool trans_FCVTNT_ds(DisasContext *s, arg_rpr_esz *a)
}
return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve2_fcvtnt_ds);
}
+
+static bool trans_FCVTLT_hs(DisasContext *s, arg_rpr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve2_fcvtlt_hs);
+}
+
+static bool trans_FCVTLT_sd(DisasContext *s, arg_rpr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve2_fcvtlt_sd);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 73/81] target/arm: Implement SVE2 FCVTXNT, FCVTX
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (71 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 72/81] target/arm: Implement SVE2 FCVTLT Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 74/81] target/arm: Implement SVE2 FLOGB Richard Henderson
` (9 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200428174332.17162-4-steplong@quicinc.com>
[rth: Use do_frint_mode, which avoids a specific runtime helper.]
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve.decode | 2 ++
target/arm/translate-sve.c | 49 ++++++++++++++++++++++++++++++--------
2 files changed, 41 insertions(+), 10 deletions(-)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index abe26f2424..6c0e39d553 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1575,6 +1575,8 @@ SM4EKEY 01000101 00 1 ..... 11110 0 ..... ..... @rd_rn_rm_e0
RAX1 01000101 00 1 ..... 11110 1 ..... ..... @rd_rn_rm_e0
### SVE2 floating-point convert precision odd elements
+FCVTXNT_ds 01100100 00 0010 10 101 ... ..... ..... @rd_pg_rn_e0
+FCVTX_ds 01100101 00 0010 10 101 ... ..... ..... @rd_pg_rn_e0
FCVTNT_sh 01100100 10 0010 00 101 ... ..... ..... @rd_pg_rn_e0
FCVTLT_hs 01100100 10 0010 01 101 ... ..... ..... @rd_pg_rn_e0
FCVTNT_ds 01100100 11 0010 10 101 ... ..... ..... @rd_pg_rn_e0
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 247288e164..7cdb128a5b 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -4769,11 +4769,9 @@ static bool trans_FRINTX(DisasContext *s, arg_rpr_esz *a)
return do_zpz_ptr(s, a->rd, a->rn, a->pg, a->esz == MO_16, fns[a->esz - 1]);
}
-static bool do_frint_mode(DisasContext *s, arg_rpr_esz *a, int mode)
+static bool do_frint_mode(DisasContext *s, arg_rpr_esz *a,
+ int mode, gen_helper_gvec_3_ptr *fn)
{
- if (a->esz == 0) {
- return false;
- }
if (sve_access_check(s)) {
unsigned vsz = vec_full_reg_size(s);
TCGv_i32 tmode = tcg_const_i32(mode);
@@ -4784,7 +4782,7 @@ static bool do_frint_mode(DisasContext *s, arg_rpr_esz *a, int mode)
tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
vec_full_reg_offset(s, a->rn),
pred_full_reg_offset(s, a->pg),
- status, vsz, vsz, 0, frint_fns[a->esz - 1]);
+ status, vsz, vsz, 0, fn);
gen_helper_set_rmode(tmode, tmode, status);
tcg_temp_free_i32(tmode);
@@ -4795,27 +4793,42 @@ static bool do_frint_mode(DisasContext *s, arg_rpr_esz *a, int mode)
static bool trans_FRINTN(DisasContext *s, arg_rpr_esz *a)
{
- return do_frint_mode(s, a, float_round_nearest_even);
+ if (a->esz == 0) {
+ return false;
+ }
+ return do_frint_mode(s, a, float_round_nearest_even, frint_fns[a->esz - 1]);
}
static bool trans_FRINTP(DisasContext *s, arg_rpr_esz *a)
{
- return do_frint_mode(s, a, float_round_up);
+ if (a->esz == 0) {
+ return false;
+ }
+ return do_frint_mode(s, a, float_round_up, frint_fns[a->esz - 1]);
}
static bool trans_FRINTM(DisasContext *s, arg_rpr_esz *a)
{
- return do_frint_mode(s, a, float_round_down);
+ if (a->esz == 0) {
+ return false;
+ }
+ return do_frint_mode(s, a, float_round_down, frint_fns[a->esz - 1]);
}
static bool trans_FRINTZ(DisasContext *s, arg_rpr_esz *a)
{
- return do_frint_mode(s, a, float_round_to_zero);
+ if (a->esz == 0) {
+ return false;
+ }
+ return do_frint_mode(s, a, float_round_to_zero, frint_fns[a->esz - 1]);
}
static bool trans_FRINTA(DisasContext *s, arg_rpr_esz *a)
{
- return do_frint_mode(s, a, float_round_ties_away);
+ if (a->esz == 0) {
+ return false;
+ }
+ return do_frint_mode(s, a, float_round_ties_away, frint_fns[a->esz - 1]);
}
static bool trans_FRECPX(DisasContext *s, arg_rpr_esz *a)
@@ -8257,3 +8270,19 @@ static bool trans_FCVTLT_sd(DisasContext *s, arg_rpr_esz *a)
}
return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve2_fcvtlt_sd);
}
+
+static bool trans_FCVTX_ds(DisasContext *s, arg_rpr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_frint_mode(s, a, float_round_to_odd, gen_helper_sve_fcvt_ds);
+}
+
+static bool trans_FCVTXNT_ds(DisasContext *s, arg_rpr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_frint_mode(s, a, float_round_to_odd, gen_helper_sve2_fcvtnt_ds);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 74/81] target/arm: Implement SVE2 FLOGB
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (72 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 73/81] target/arm: Implement SVE2 FCVTXNT, FCVTX Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 75/81] target/arm: Share table of sve load functions Richard Henderson
` (8 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200430191405.21641-1-steplong@quicinc.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
v2: Fixed esz index and c++ comments
v3: Fixed denormal arithmetic and raise invalid.
---
target/arm/helper-sve.h | 4 +++
target/arm/sve.decode | 3 +++
target/arm/sve_helper.c | 52 ++++++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 24 ++++++++++++++++++
4 files changed, 83 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index df141aec41..f25b403ddb 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2744,3 +2744,7 @@ DEF_HELPER_FLAGS_5(sve2_fcvtlt_hs, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_fcvtlt_sd, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(flogb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(flogb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(flogb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 6c0e39d553..6808ff4194 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1581,3 +1581,6 @@ FCVTNT_sh 01100100 10 0010 00 101 ... ..... ..... @rd_pg_rn_e0
FCVTLT_hs 01100100 10 0010 01 101 ... ..... ..... @rd_pg_rn_e0
FCVTNT_ds 01100100 11 0010 10 101 ... ..... ..... @rd_pg_rn_e0
FCVTLT_sd 01100100 11 0010 11 101 ... ..... ..... @rd_pg_rn_e0
+
+### SVE2 floating-point convert to integer
+FLOGB 01100101 00 011 esz:2 0101 pg:3 rn:5 rd:5 &rpr_esz
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 8539493c72..dc51cb75fc 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -4696,6 +4696,58 @@ DO_ZPZ_FP(sve_ucvt_dh, uint64_t, , uint64_to_float16)
DO_ZPZ_FP(sve_ucvt_ds, uint64_t, , uint64_to_float32)
DO_ZPZ_FP(sve_ucvt_dd, uint64_t, , uint64_to_float64)
+static int16_t do_float16_logb_as_int(float16 a, float_status *s)
+{
+ if (float16_is_normal(a)) {
+ return extract16(a, 10, 5) - 15;
+ } else if (float16_is_infinity(a)) {
+ return INT16_MAX;
+ } else if (float16_is_any_nan(a) || float16_is_zero(a)) {
+ float_raise(float_flag_invalid, s);
+ return INT16_MIN;
+ } else {
+ /*
+ * denormal: bias - fractional_zeros
+ * = bias + masked_zeros - uint32_zeros
+ */
+ return -15 + 22 - clz32(extract16(a, 0, 10));
+ }
+}
+
+static int32_t do_float32_logb_as_int(float32 a, float_status *s)
+{
+ if (float32_is_normal(a)) {
+ return extract32(a, 23, 8) - 127;
+ } else if (float32_is_infinity(a)) {
+ return INT32_MAX;
+ } else if (float32_is_any_nan(a) || float32_is_zero(a)) {
+ float_raise(float_flag_invalid, s);
+ return INT32_MIN;
+ } else {
+ /* denormal (see above) */
+ return -127 + 9 - clz32(extract32(a, 0, 23));
+ }
+}
+
+static int64_t do_float64_logb_as_int(float64 a, float_status *s)
+{
+ if (float64_is_normal(a)) {
+ return extract64(a, 52, 11) - 1023;
+ } else if (float64_is_infinity(a)) {
+ return INT64_MAX;
+ } else if (float64_is_any_nan(a) || float64_is_zero(a)) {
+ float_raise(float_flag_invalid, s);
+ return INT64_MIN;
+ } else {
+ /* denormal (see above) */
+ return -1023 + 12 - clz64(extract64(a, 0, 52));
+ }
+}
+
+DO_ZPZ_FP(flogb_h, float16, H1_2, do_float16_logb_as_int)
+DO_ZPZ_FP(flogb_s, float32, H1_4, do_float32_logb_as_int)
+DO_ZPZ_FP(flogb_d, float64, , do_float64_logb_as_int)
+
#undef DO_ZPZ_FP
static void do_fmla_zpzzz_h(void *vd, void *vn, void *vm, void *va, void *vg,
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 7cdb128a5b..d6719d99d5 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -8286,3 +8286,27 @@ static bool trans_FCVTXNT_ds(DisasContext *s, arg_rpr_esz *a)
}
return do_frint_mode(s, a, float_round_to_odd, gen_helper_sve2_fcvtnt_ds);
}
+
+static bool trans_FLOGB(DisasContext *s, arg_rpr_esz *a)
+{
+ static gen_helper_gvec_3_ptr * const fns[] = {
+ NULL, gen_helper_flogb_h,
+ gen_helper_flogb_s, gen_helper_flogb_d
+ };
+
+ if (!dc_isar_feature(aa64_sve2, s) || fns[a->esz] == NULL) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ TCGv_ptr status =
+ fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
+ unsigned vsz = vec_full_reg_size(s);
+
+ tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ pred_full_reg_offset(s, a->pg),
+ status, vsz, vsz, 0, fns[a->esz]);
+ tcg_temp_free_ptr(status);
+ }
+ return true;
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 75/81] target/arm: Share table of sve load functions
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (73 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 74/81] target/arm: Implement SVE2 FLOGB Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 76/81] target/arm: Implement SVE2 LD1RO Richard Henderson
` (7 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
The table used by do_ldrq is a subset of the table used by do_ld_zpa;
we can share them by passing dtype instead of msz to do_ldrq.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/translate-sve.c | 254 ++++++++++++++++++-------------------
1 file changed, 126 insertions(+), 128 deletions(-)
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index d6719d99d5..79af31f12e 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -5208,128 +5208,130 @@ static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
tcg_temp_free_i32(t_desc);
}
+/* Indexed by [mte][be][dtype][nreg] */
+static gen_helper_gvec_mem * const ldr_fns[2][2][16][4] = {
+ { /* mte inactive, little-endian */
+ { { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r,
+ gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r },
+ { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1sds_le_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hh_le_r, gen_helper_sve_ld2hh_le_r,
+ gen_helper_sve_ld3hh_le_r, gen_helper_sve_ld4hh_le_r },
+ { gen_helper_sve_ld1hsu_le_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hdu_le_r, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1hds_le_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hss_le_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1ss_le_r, gen_helper_sve_ld2ss_le_r,
+ gen_helper_sve_ld3ss_le_r, gen_helper_sve_ld4ss_le_r },
+ { gen_helper_sve_ld1sdu_le_r, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1bds_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bss_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1dd_le_r, gen_helper_sve_ld2dd_le_r,
+ gen_helper_sve_ld3dd_le_r, gen_helper_sve_ld4dd_le_r } },
+
+ /* mte inactive, big-endian */
+ { { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r,
+ gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r },
+ { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1sds_be_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hh_be_r, gen_helper_sve_ld2hh_be_r,
+ gen_helper_sve_ld3hh_be_r, gen_helper_sve_ld4hh_be_r },
+ { gen_helper_sve_ld1hsu_be_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hdu_be_r, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1hds_be_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hss_be_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1ss_be_r, gen_helper_sve_ld2ss_be_r,
+ gen_helper_sve_ld3ss_be_r, gen_helper_sve_ld4ss_be_r },
+ { gen_helper_sve_ld1sdu_be_r, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1bds_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bss_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1dd_be_r, gen_helper_sve_ld2dd_be_r,
+ gen_helper_sve_ld3dd_be_r, gen_helper_sve_ld4dd_be_r } } },
+
+ { /* mte active, little-endian */
+ { { gen_helper_sve_ld1bb_r_mte,
+ gen_helper_sve_ld2bb_r_mte,
+ gen_helper_sve_ld3bb_r_mte,
+ gen_helper_sve_ld4bb_r_mte },
+ { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1sds_le_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hh_le_r_mte,
+ gen_helper_sve_ld2hh_le_r_mte,
+ gen_helper_sve_ld3hh_le_r_mte,
+ gen_helper_sve_ld4hh_le_r_mte },
+ { gen_helper_sve_ld1hsu_le_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hdu_le_r_mte, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1hds_le_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hss_le_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1ss_le_r_mte,
+ gen_helper_sve_ld2ss_le_r_mte,
+ gen_helper_sve_ld3ss_le_r_mte,
+ gen_helper_sve_ld4ss_le_r_mte },
+ { gen_helper_sve_ld1sdu_le_r_mte, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1dd_le_r_mte,
+ gen_helper_sve_ld2dd_le_r_mte,
+ gen_helper_sve_ld3dd_le_r_mte,
+ gen_helper_sve_ld4dd_le_r_mte } },
+
+ /* mte active, big-endian */
+ { { gen_helper_sve_ld1bb_r_mte,
+ gen_helper_sve_ld2bb_r_mte,
+ gen_helper_sve_ld3bb_r_mte,
+ gen_helper_sve_ld4bb_r_mte },
+ { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1sds_be_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hh_be_r_mte,
+ gen_helper_sve_ld2hh_be_r_mte,
+ gen_helper_sve_ld3hh_be_r_mte,
+ gen_helper_sve_ld4hh_be_r_mte },
+ { gen_helper_sve_ld1hsu_be_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hdu_be_r_mte, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1hds_be_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hss_be_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1ss_be_r_mte,
+ gen_helper_sve_ld2ss_be_r_mte,
+ gen_helper_sve_ld3ss_be_r_mte,
+ gen_helper_sve_ld4ss_be_r_mte },
+ { gen_helper_sve_ld1sdu_be_r_mte, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL },
+ { gen_helper_sve_ld1dd_be_r_mte,
+ gen_helper_sve_ld2dd_be_r_mte,
+ gen_helper_sve_ld3dd_be_r_mte,
+ gen_helper_sve_ld4dd_be_r_mte } } },
+};
+
static void do_ld_zpa(DisasContext *s, int zt, int pg,
TCGv_i64 addr, int dtype, int nreg)
{
- static gen_helper_gvec_mem * const fns[2][2][16][4] = {
- { /* mte inactive, little-endian */
- { { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r,
- gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r },
- { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL },
-
- { gen_helper_sve_ld1sds_le_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1hh_le_r, gen_helper_sve_ld2hh_le_r,
- gen_helper_sve_ld3hh_le_r, gen_helper_sve_ld4hh_le_r },
- { gen_helper_sve_ld1hsu_le_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1hdu_le_r, NULL, NULL, NULL },
-
- { gen_helper_sve_ld1hds_le_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1hss_le_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1ss_le_r, gen_helper_sve_ld2ss_le_r,
- gen_helper_sve_ld3ss_le_r, gen_helper_sve_ld4ss_le_r },
- { gen_helper_sve_ld1sdu_le_r, NULL, NULL, NULL },
-
- { gen_helper_sve_ld1bds_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1bss_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1dd_le_r, gen_helper_sve_ld2dd_le_r,
- gen_helper_sve_ld3dd_le_r, gen_helper_sve_ld4dd_le_r } },
-
- /* mte inactive, big-endian */
- { { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r,
- gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r },
- { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL },
-
- { gen_helper_sve_ld1sds_be_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1hh_be_r, gen_helper_sve_ld2hh_be_r,
- gen_helper_sve_ld3hh_be_r, gen_helper_sve_ld4hh_be_r },
- { gen_helper_sve_ld1hsu_be_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1hdu_be_r, NULL, NULL, NULL },
-
- { gen_helper_sve_ld1hds_be_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1hss_be_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1ss_be_r, gen_helper_sve_ld2ss_be_r,
- gen_helper_sve_ld3ss_be_r, gen_helper_sve_ld4ss_be_r },
- { gen_helper_sve_ld1sdu_be_r, NULL, NULL, NULL },
-
- { gen_helper_sve_ld1bds_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1bss_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL },
- { gen_helper_sve_ld1dd_be_r, gen_helper_sve_ld2dd_be_r,
- gen_helper_sve_ld3dd_be_r, gen_helper_sve_ld4dd_be_r } } },
-
- { /* mte active, little-endian */
- { { gen_helper_sve_ld1bb_r_mte,
- gen_helper_sve_ld2bb_r_mte,
- gen_helper_sve_ld3bb_r_mte,
- gen_helper_sve_ld4bb_r_mte },
- { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL },
-
- { gen_helper_sve_ld1sds_le_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1hh_le_r_mte,
- gen_helper_sve_ld2hh_le_r_mte,
- gen_helper_sve_ld3hh_le_r_mte,
- gen_helper_sve_ld4hh_le_r_mte },
- { gen_helper_sve_ld1hsu_le_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1hdu_le_r_mte, NULL, NULL, NULL },
-
- { gen_helper_sve_ld1hds_le_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1hss_le_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1ss_le_r_mte,
- gen_helper_sve_ld2ss_le_r_mte,
- gen_helper_sve_ld3ss_le_r_mte,
- gen_helper_sve_ld4ss_le_r_mte },
- { gen_helper_sve_ld1sdu_le_r_mte, NULL, NULL, NULL },
-
- { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1dd_le_r_mte,
- gen_helper_sve_ld2dd_le_r_mte,
- gen_helper_sve_ld3dd_le_r_mte,
- gen_helper_sve_ld4dd_le_r_mte } },
-
- /* mte active, big-endian */
- { { gen_helper_sve_ld1bb_r_mte,
- gen_helper_sve_ld2bb_r_mte,
- gen_helper_sve_ld3bb_r_mte,
- gen_helper_sve_ld4bb_r_mte },
- { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL },
-
- { gen_helper_sve_ld1sds_be_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1hh_be_r_mte,
- gen_helper_sve_ld2hh_be_r_mte,
- gen_helper_sve_ld3hh_be_r_mte,
- gen_helper_sve_ld4hh_be_r_mte },
- { gen_helper_sve_ld1hsu_be_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1hdu_be_r_mte, NULL, NULL, NULL },
-
- { gen_helper_sve_ld1hds_be_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1hss_be_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1ss_be_r_mte,
- gen_helper_sve_ld2ss_be_r_mte,
- gen_helper_sve_ld3ss_be_r_mte,
- gen_helper_sve_ld4ss_be_r_mte },
- { gen_helper_sve_ld1sdu_be_r_mte, NULL, NULL, NULL },
-
- { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL },
- { gen_helper_sve_ld1dd_be_r_mte,
- gen_helper_sve_ld2dd_be_r_mte,
- gen_helper_sve_ld3dd_be_r_mte,
- gen_helper_sve_ld4dd_be_r_mte } } },
- };
gen_helper_gvec_mem *fn
- = fns[s->mte_active[0]][s->be_data == MO_BE][dtype][nreg];
+ = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][nreg];
/*
* While there are holes in the table, they are not
@@ -5567,14 +5569,8 @@ static bool trans_LDNF1_zpri(DisasContext *s, arg_rpri_load *a)
return true;
}
-static void do_ldrq(DisasContext *s, int zt, int pg, TCGv_i64 addr, int msz)
+static void do_ldrq(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype)
{
- static gen_helper_gvec_mem * const fns[2][4] = {
- { gen_helper_sve_ld1bb_r, gen_helper_sve_ld1hh_le_r,
- gen_helper_sve_ld1ss_le_r, gen_helper_sve_ld1dd_le_r },
- { gen_helper_sve_ld1bb_r, gen_helper_sve_ld1hh_be_r,
- gen_helper_sve_ld1ss_be_r, gen_helper_sve_ld1dd_be_r },
- };
unsigned vsz = vec_full_reg_size(s);
TCGv_ptr t_pg;
TCGv_i32 t_desc;
@@ -5606,7 +5602,9 @@ static void do_ldrq(DisasContext *s, int zt, int pg, TCGv_i64 addr, int msz)
t_pg = tcg_temp_new_ptr();
tcg_gen_addi_ptr(t_pg, cpu_env, poff);
- fns[s->be_data == MO_BE][msz](cpu_env, t_pg, addr, t_desc);
+ gen_helper_gvec_mem *fn
+ = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][0];
+ fn(cpu_env, t_pg, addr, t_desc);
tcg_temp_free_ptr(t_pg);
tcg_temp_free_i32(t_desc);
@@ -5628,7 +5626,7 @@ static bool trans_LD1RQ_zprr(DisasContext *s, arg_rprr_load *a)
TCGv_i64 addr = new_tmp_a64(s);
tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), msz);
tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
- do_ldrq(s, a->rd, a->pg, addr, msz);
+ do_ldrq(s, a->rd, a->pg, addr, a->dtype);
}
return true;
}
@@ -5638,7 +5636,7 @@ static bool trans_LD1RQ_zpri(DisasContext *s, arg_rpri_load *a)
if (sve_access_check(s)) {
TCGv_i64 addr = new_tmp_a64(s);
tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 16);
- do_ldrq(s, a->rd, a->pg, addr, dtype_msz(a->dtype));
+ do_ldrq(s, a->rd, a->pg, addr, a->dtype);
}
return true;
}
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 76/81] target/arm: Implement SVE2 LD1RO
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (74 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 75/81] target/arm: Share table of sve load functions Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 77/81] target/arm: Implement 128-bit ZIP, UZP, TRN Richard Henderson
` (6 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/sve.decode | 4 ++
target/arm/translate-sve.c | 97 ++++++++++++++++++++++++++++++++++++++
2 files changed, 101 insertions(+)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 6808ff4194..e0d093c5d7 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1119,11 +1119,15 @@ LD_zpri 1010010 .. nreg:2 0.... 111 ... ..... ..... @rpri_load_msz
# SVE load and broadcast quadword (scalar plus scalar)
LD1RQ_zprr 1010010 .. 00 ..... 000 ... ..... ..... \
@rprr_load_msz nreg=0
+LD1RO_zprr 1010010 .. 01 ..... 000 ... ..... ..... \
+ @rprr_load_msz nreg=0
# SVE load and broadcast quadword (scalar plus immediate)
# LD1RQB, LD1RQH, LD1RQS, LD1RQD
LD1RQ_zpri 1010010 .. 00 0.... 001 ... ..... ..... \
@rpri_load_msz nreg=0
+LD1RO_zpri 1010010 .. 01 0.... 001 ... ..... ..... \
+ @rpri_load_msz nreg=0
# SVE 32-bit gather prefetch (scalar plus 32-bit scaled offsets)
PRF 1000010 00 -1 ----- 0-- --- ----- 0 ----
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 79af31f12e..7c6a6a3bda 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -5641,6 +5641,103 @@ static bool trans_LD1RQ_zpri(DisasContext *s, arg_rpri_load *a)
return true;
}
+static void do_ldro(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype)
+{
+ unsigned vsz = vec_full_reg_size(s);
+ unsigned vsz_r32;
+ TCGv_ptr t_pg;
+ TCGv_i32 t_desc;
+ int desc, poff, doff;
+
+ if (vsz < 32) {
+ /*
+ * Note that this UNDEFINED check comes after CheckSVEEnabled()
+ * in the ARM pseudocode, which is the sve_access_check() done
+ * in our caller. We should not now return false from the caller.
+ */
+ unallocated_encoding(s);
+ return;
+ }
+
+ /* Load the first octaword using the normal predicated load helpers. */
+
+ poff = pred_full_reg_offset(s, pg);
+ if (vsz > 32) {
+ /*
+ * Zero-extend the first 32 bits of the predicate into a temporary.
+ * This avoids triggering an assert making sure we don't have bits
+ * set within a predicate beyond VQ, but we have lowered VQ to 2
+ * for this load operation.
+ */
+ TCGv_i64 tmp = tcg_temp_new_i64();
+#ifdef HOST_WORDS_BIGENDIAN
+ poff += 4;
+#endif
+ tcg_gen_ld32u_i64(tmp, cpu_env, poff);
+
+ poff = offsetof(CPUARMState, vfp.preg_tmp);
+ tcg_gen_st_i64(tmp, cpu_env, poff);
+ tcg_temp_free_i64(tmp);
+ }
+
+ t_pg = tcg_temp_new_ptr();
+ tcg_gen_addi_ptr(t_pg, cpu_env, poff);
+
+ desc = simd_desc(32, 32, zt);
+ t_desc = tcg_const_i32(desc);
+
+ gen_helper_gvec_mem *fn
+ = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][0];
+ fn(cpu_env, t_pg, addr, t_desc);
+
+ tcg_temp_free_ptr(t_pg);
+ tcg_temp_free_i32(t_desc);
+
+ /*
+ * Replicate that first octaword.
+ * The replication happens in units of 32; if the full vector size
+ * is not a multiple of 32, the final bits are zeroed.
+ */
+ doff = vec_full_reg_offset(s, zt);
+ vsz_r32 = QEMU_ALIGN_DOWN(vsz, 32);
+ if (vsz >= 64) {
+ tcg_gen_gvec_dup_mem(5, doff + 32, doff, vsz_r32 - 32, vsz - 32);
+ } else if (vsz > vsz_r32) {
+ /* Nop move, with side effect of clearing the tail. */
+ tcg_gen_gvec_mov(MO_64, doff, doff, vsz_r32, vsz);
+ }
+}
+
+static bool trans_LD1RO_zprr(DisasContext *s, arg_rprr_load *a)
+{
+ if (!dc_isar_feature(aa64_sve2_f64mm, s)) {
+ return false;
+ }
+ if (a->rm == 31) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ TCGv_i64 addr = new_tmp_a64(s);
+ tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype));
+ tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
+ do_ldro(s, a->rd, a->pg, addr, a->dtype);
+ }
+ return true;
+}
+
+static bool trans_LD1RO_zpri(DisasContext *s, arg_rpri_load *a)
+{
+ if (!dc_isar_feature(aa64_sve2_f64mm, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ TCGv_i64 addr = new_tmp_a64(s);
+ tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 32);
+ do_ldro(s, a->rd, a->pg, addr, a->dtype);
+ }
+ return true;
+}
+
/* Load and broadcast element. */
static bool trans_LD1R_zpri(DisasContext *s, arg_rpri_load *a)
{
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 77/81] target/arm: Implement 128-bit ZIP, UZP, TRN
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (75 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 76/81] target/arm: Implement SVE2 LD1RO Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 78/81] target/arm: Implement SVE2 bitwise shift immediate Richard Henderson
` (5 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 3 ++
target/arm/sve.decode | 8 ++++++
target/arm/sve_helper.c | 29 +++++++++++++------
target/arm/translate-sve.c | 58 ++++++++++++++++++++++++++++++++++++++
4 files changed, 90 insertions(+), 8 deletions(-)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index f25b403ddb..feed23bbd1 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -689,16 +689,19 @@ DEF_HELPER_FLAGS_4(sve_zip_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_zip_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_zip_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_zip_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_zip_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_uzp_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_uzp_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_uzp_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_uzp_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uzp_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_trn_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_trn_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_trn_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_trn_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_trn_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_compact_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_compact_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index e0d093c5d7..4e21274dc4 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -592,6 +592,14 @@ UZP2_z 00000101 .. 1 ..... 011 011 ..... ..... @rd_rn_rm
TRN1_z 00000101 .. 1 ..... 011 100 ..... ..... @rd_rn_rm
TRN2_z 00000101 .. 1 ..... 011 101 ..... ..... @rd_rn_rm
+# SVE2 permute vector segments
+ZIP1_q 00000101 10 1 ..... 000 000 ..... ..... @rd_rn_rm_e0
+ZIP2_q 00000101 10 1 ..... 000 001 ..... ..... @rd_rn_rm_e0
+UZP1_q 00000101 10 1 ..... 000 010 ..... ..... @rd_rn_rm_e0
+UZP2_q 00000101 10 1 ..... 000 011 ..... ..... @rd_rn_rm_e0
+TRN1_q 00000101 10 1 ..... 000 110 ..... ..... @rd_rn_rm_e0
+TRN2_q 00000101 10 1 ..... 000 111 ..... ..... @rd_rn_rm_e0
+
### SVE Permute - Predicated Group
# SVE compress active elements
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index dc51cb75fc..12d21dcd65 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -3457,36 +3457,45 @@ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
*(TYPE *)(vd + H(2 * i + 0)) = *(TYPE *)(vn + H(i)); \
*(TYPE *)(vd + H(2 * i + sizeof(TYPE))) = *(TYPE *)(vm + H(i)); \
} \
+ if (sizeof(TYPE) == 16 && unlikely(oprsz & 16)) { \
+ memset(vd + oprsz - 16, 0, 16); \
+ } \
}
DO_ZIP(sve_zip_b, uint8_t, H1)
DO_ZIP(sve_zip_h, uint16_t, H1_2)
DO_ZIP(sve_zip_s, uint32_t, H1_4)
DO_ZIP(sve_zip_d, uint64_t, )
+DO_ZIP(sve2_zip_q, Int128, )
#define DO_UZP(NAME, TYPE, H) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
{ \
intptr_t oprsz = simd_oprsz(desc); \
- intptr_t oprsz_2 = oprsz / 2; \
intptr_t odd_ofs = simd_data(desc); \
- intptr_t i; \
+ intptr_t i, p; \
ARMVectorReg tmp_m; \
if (unlikely((vm - vd) < (uintptr_t)oprsz)) { \
vm = memcpy(&tmp_m, vm, oprsz); \
} \
- for (i = 0; i < oprsz_2; i += sizeof(TYPE)) { \
- *(TYPE *)(vd + H(i)) = *(TYPE *)(vn + H(2 * i + odd_ofs)); \
- } \
- for (i = 0; i < oprsz_2; i += sizeof(TYPE)) { \
- *(TYPE *)(vd + H(oprsz_2 + i)) = *(TYPE *)(vm + H(2 * i + odd_ofs)); \
- } \
+ i = 0, p = odd_ofs; \
+ do { \
+ *(TYPE *)(vd + H(i)) = *(TYPE *)(vn + H(p)); \
+ i += sizeof(TYPE), p += 2 * sizeof(TYPE); \
+ } while (p < oprsz); \
+ p -= oprsz; \
+ do { \
+ *(TYPE *)(vd + H(i)) = *(TYPE *)(vm + H(p)); \
+ i += sizeof(TYPE), p += 2 * sizeof(TYPE); \
+ } while (p < oprsz); \
+ tcg_debug_assert(i == oprsz); \
}
DO_UZP(sve_uzp_b, uint8_t, H1)
DO_UZP(sve_uzp_h, uint16_t, H1_2)
DO_UZP(sve_uzp_s, uint32_t, H1_4)
DO_UZP(sve_uzp_d, uint64_t, )
+DO_UZP(sve2_uzp_q, Int128, )
#define DO_TRN(NAME, TYPE, H) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
@@ -3500,12 +3509,16 @@ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
*(TYPE *)(vd + H(i + 0)) = ae; \
*(TYPE *)(vd + H(i + sizeof(TYPE))) = be; \
} \
+ if (sizeof(TYPE) == 16 && unlikely(oprsz & 16)) { \
+ memset(vd + oprsz - 16, 0, 16); \
+ } \
}
DO_TRN(sve_trn_b, uint8_t, H1)
DO_TRN(sve_trn_h, uint16_t, H1_2)
DO_TRN(sve_trn_s, uint32_t, H1_4)
DO_TRN(sve_trn_d, uint64_t, )
+DO_TRN(sve2_trn_q, Int128, )
#undef DO_ZIP
#undef DO_UZP
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 7c6a6a3bda..214788ea23 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -2633,6 +2633,32 @@ static bool trans_ZIP2_z(DisasContext *s, arg_rrr_esz *a)
return do_zip(s, a, true);
}
+static bool do_zip_q(DisasContext *s, arg_rrr_esz *a, bool high)
+{
+ if (!dc_isar_feature(aa64_sve2_f64mm, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ unsigned high_ofs = high ? QEMU_ALIGN_DOWN(vsz, 32) / 2 : 0;
+ tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn) + high_ofs,
+ vec_full_reg_offset(s, a->rm) + high_ofs,
+ vsz, vsz, 0, gen_helper_sve2_zip_q);
+ }
+ return true;
+}
+
+static bool trans_ZIP1_q(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_zip_q(s, a, false);
+}
+
+static bool trans_ZIP2_q(DisasContext *s, arg_rrr_esz *a)
+{
+ return do_zip_q(s, a, true);
+}
+
static gen_helper_gvec_3 * const uzp_fns[4] = {
gen_helper_sve_uzp_b, gen_helper_sve_uzp_h,
gen_helper_sve_uzp_s, gen_helper_sve_uzp_d,
@@ -2648,6 +2674,22 @@ static bool trans_UZP2_z(DisasContext *s, arg_rrr_esz *a)
return do_zzz_data_ool(s, a, 1 << a->esz, uzp_fns[a->esz]);
}
+static bool trans_UZP1_q(DisasContext *s, arg_rrr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2_f64mm, s)) {
+ return false;
+ }
+ return do_zzz_data_ool(s, a, 0, gen_helper_sve2_uzp_q);
+}
+
+static bool trans_UZP2_q(DisasContext *s, arg_rrr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2_f64mm, s)) {
+ return false;
+ }
+ return do_zzz_data_ool(s, a, 16, gen_helper_sve2_uzp_q);
+}
+
static gen_helper_gvec_3 * const trn_fns[4] = {
gen_helper_sve_trn_b, gen_helper_sve_trn_h,
gen_helper_sve_trn_s, gen_helper_sve_trn_d,
@@ -2663,6 +2705,22 @@ static bool trans_TRN2_z(DisasContext *s, arg_rrr_esz *a)
return do_zzz_data_ool(s, a, 1 << a->esz, trn_fns[a->esz]);
}
+static bool trans_TRN1_q(DisasContext *s, arg_rrr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2_f64mm, s)) {
+ return false;
+ }
+ return do_zzz_data_ool(s, a, 0, gen_helper_sve2_trn_q);
+}
+
+static bool trans_TRN2_q(DisasContext *s, arg_rrr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve2_f64mm, s)) {
+ return false;
+ }
+ return do_zzz_data_ool(s, a, 16, gen_helper_sve2_trn_q);
+}
+
/*
*** SVE Permute Vector - Predicated Group
*/
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 78/81] target/arm: Implement SVE2 bitwise shift immediate
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (76 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 77/81] target/arm: Implement 128-bit ZIP, UZP, TRN Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 79/81] target/arm: Implement SVE2 fp multiply-add long Richard Henderson
` (4 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Implements SQSHL/UQSHL, SRSHR/URSHR, and SQSHLU
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200430194159.24064-1-steplong@quicinc.com>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 33 +++++++++++++++++++++
target/arm/sve.decode | 5 ++++
target/arm/sve_helper.c | 35 ++++++++++++++++++++++
target/arm/translate-sve.c | 60 ++++++++++++++++++++++++++++++++++++++
4 files changed, 133 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index feed23bbd1..9542f01c42 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2738,6 +2738,39 @@ DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_h, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqshl_zpzi_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqshl_zpzi_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqshl_zpzi_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqshl_zpzi_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_uqshl_zpzi_b, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uqshl_zpzi_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uqshl_zpzi_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uqshl_zpzi_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_srshr_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_srshr_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_srshr_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_srshr_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_urshr_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_urshr_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_urshr_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_urshr_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve2_sqshlu_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqshlu_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqshlu_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqshlu_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_5(sve2_fcvtnt_sh, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_fcvtnt_ds, TCG_CALL_NO_RWG,
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 4e21274dc4..d2f33d96f3 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -342,6 +342,11 @@ ASR_zpzi 00000100 .. 000 000 100 ... .. ... ..... @rdn_pg_tszimm_shr
LSR_zpzi 00000100 .. 000 001 100 ... .. ... ..... @rdn_pg_tszimm_shr
LSL_zpzi 00000100 .. 000 011 100 ... .. ... ..... @rdn_pg_tszimm_shl
ASRD 00000100 .. 000 100 100 ... .. ... ..... @rdn_pg_tszimm_shr
+SQSHL_zpzi 00000100 .. 000 110 100 ... .. ... ..... @rdn_pg_tszimm_shl
+UQSHL_zpzi 00000100 .. 000 111 100 ... .. ... ..... @rdn_pg_tszimm_shl
+SRSHR 00000100 .. 001 100 100 ... .. ... ..... @rdn_pg_tszimm_shr
+URSHR 00000100 .. 001 101 100 ... .. ... ..... @rdn_pg_tszimm_shr
+SQSHLU 00000100 .. 001 111 100 ... .. ... ..... @rdn_pg_tszimm_shl
# SVE bitwise shift by vector (predicated)
ASR_zpzz 00000100 .. 010 000 100 ... ..... ..... @rdn_pg_rm
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 12d21dcd65..ef04a0f95a 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -2203,6 +2203,41 @@ DO_ZPZI(sve_asrd_h, int16_t, H1_2, DO_ASRD)
DO_ZPZI(sve_asrd_s, int32_t, H1_4, DO_ASRD)
DO_ZPZI_D(sve_asrd_d, int64_t, DO_ASRD)
+/* SVE2 bitwise shift by immediate */
+DO_ZPZI(sve2_sqshl_zpzi_b, int8_t, H1, do_sqshl_b)
+DO_ZPZI(sve2_sqshl_zpzi_h, int16_t, H1_2, do_sqshl_h)
+DO_ZPZI(sve2_sqshl_zpzi_s, int32_t, H1_4, do_sqshl_s)
+DO_ZPZI_D(sve2_sqshl_zpzi_d, int64_t, do_sqshl_d)
+
+DO_ZPZI(sve2_uqshl_zpzi_b, uint8_t, H1, do_uqshl_b)
+DO_ZPZI(sve2_uqshl_zpzi_h, uint16_t, H1_2, do_uqshl_h)
+DO_ZPZI(sve2_uqshl_zpzi_s, uint32_t, H1_4, do_uqshl_s)
+DO_ZPZI_D(sve2_uqshl_zpzi_d, uint64_t, do_uqshl_d)
+
+DO_ZPZI(sve2_srshr_b, int8_t, H1, do_srshr)
+DO_ZPZI(sve2_srshr_h, int16_t, H1_2, do_srshr)
+DO_ZPZI(sve2_srshr_s, int32_t, H1_4, do_srshr)
+DO_ZPZI_D(sve2_srshr_d, int64_t, do_srshr)
+
+DO_ZPZI(sve2_urshr_b, uint8_t, H1, do_urshr)
+DO_ZPZI(sve2_urshr_h, uint16_t, H1_2, do_urshr)
+DO_ZPZI(sve2_urshr_s, uint32_t, H1_4, do_urshr)
+DO_ZPZI_D(sve2_urshr_d, uint64_t, do_urshr)
+
+#define do_suqrshl_b(n, m) \
+ ({ uint32_t discard; do_suqrshl_bhs(n, (int8_t)m, 8, false, &discard); })
+#define do_suqrshl_h(n, m) \
+ ({ uint32_t discard; do_suqrshl_bhs(n, (int16_t)m, 16, false, &discard); })
+#define do_suqrshl_s(n, m) \
+ ({ uint32_t discard; do_suqrshl_bhs(n, m, 32, false, &discard); })
+#define do_suqrshl_d(n, m) \
+ ({ uint32_t discard; do_suqrshl_d(n, m, false, &discard); })
+
+DO_ZPZI(sve2_sqshlu_b, int8_t, H1, do_suqrshl_b)
+DO_ZPZI(sve2_sqshlu_h, int16_t, H1_2, do_suqrshl_h)
+DO_ZPZI(sve2_sqshlu_s, int32_t, H1_4, do_suqrshl_s)
+DO_ZPZI_D(sve2_sqshlu_d, int64_t, do_suqrshl_d)
+
#undef DO_ASRD
#undef DO_ZPZI
#undef DO_ZPZI_D
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 214788ea23..acd6d53724 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -1044,6 +1044,66 @@ static bool trans_ASRD(DisasContext *s, arg_rpri_esz *a)
}
}
+static bool trans_SQSHL_zpzi(DisasContext *s, arg_rpri_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve2_sqshl_zpzi_b, gen_helper_sve2_sqshl_zpzi_h,
+ gen_helper_sve2_sqshl_zpzi_s, gen_helper_sve2_sqshl_zpzi_d,
+ };
+ if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_zpzi_ool(s, a, fns[a->esz]);
+}
+
+static bool trans_UQSHL_zpzi(DisasContext *s, arg_rpri_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve2_uqshl_zpzi_b, gen_helper_sve2_uqshl_zpzi_h,
+ gen_helper_sve2_uqshl_zpzi_s, gen_helper_sve2_uqshl_zpzi_d,
+ };
+ if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_zpzi_ool(s, a, fns[a->esz]);
+}
+
+static bool trans_SRSHR(DisasContext *s, arg_rpri_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve2_srshr_b, gen_helper_sve2_srshr_h,
+ gen_helper_sve2_srshr_s, gen_helper_sve2_srshr_d,
+ };
+ if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_zpzi_ool(s, a, fns[a->esz]);
+}
+
+static bool trans_URSHR(DisasContext *s, arg_rpri_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve2_urshr_b, gen_helper_sve2_urshr_h,
+ gen_helper_sve2_urshr_s, gen_helper_sve2_urshr_d,
+ };
+ if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_zpzi_ool(s, a, fns[a->esz]);
+}
+
+static bool trans_SQSHLU(DisasContext *s, arg_rpri_esz *a)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve2_sqshlu_b, gen_helper_sve2_sqshlu_h,
+ gen_helper_sve2_sqshlu_s, gen_helper_sve2_sqshlu_d,
+ };
+ if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ return do_zpzi_ool(s, a, fns[a->esz]);
+}
+
/*
*** SVE Bitwise Shift - Predicated Group
*/
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 79/81] target/arm: Implement SVE2 fp multiply-add long
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (77 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 78/81] target/arm: Implement SVE2 bitwise shift immediate Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 80/81] target/arm: Implement SVE2 complex integer dot product Richard Henderson
` (3 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee, Stephen Long
From: Stephen Long <steplong@quicinc.com>
Implements both vectored and indexed FMLALB, FMLALT, FMLSLB, FMLSLT
Signed-off-by: Stephen Long <steplong@quicinc.com>
Message-Id: <20200504171240.11220-1-steplong@quicinc.com>
[rth: Rearrange to use float16_to_float32_by_bits.]
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper.h | 5 +++
target/arm/sve.decode | 12 ++++++
target/arm/translate-sve.c | 75 ++++++++++++++++++++++++++++++++++++++
target/arm/vec_helper.c | 51 ++++++++++++++++++++++++++
4 files changed, 143 insertions(+)
diff --git a/target/arm/helper.h b/target/arm/helper.h
index 868c2e9121..41f114da2a 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -974,6 +974,11 @@ DEF_HELPER_FLAGS_4(sve2_sqrdmulh_idx_s, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_4(sve2_sqrdmulh_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fmlal_zzzw_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fmlal_zzxw_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_4(gvec_xar_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
#ifdef TARGET_AARCH64
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index d2f33d96f3..6708c048e0 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1601,3 +1601,15 @@ FCVTLT_sd 01100100 11 0010 11 101 ... ..... ..... @rd_pg_rn_e0
### SVE2 floating-point convert to integer
FLOGB 01100101 00 011 esz:2 0101 pg:3 rn:5 rd:5 &rpr_esz
+
+### SVE2 floating-point multiply-add long (vectors)
+FMLALB_zzzw 01100100 .. 1 ..... 10 0 00 0 ..... ..... @rda_rn_rm
+FMLALT_zzzw 01100100 .. 1 ..... 10 0 00 1 ..... ..... @rda_rn_rm
+FMLSLB_zzzw 01100100 .. 1 ..... 10 1 00 0 ..... ..... @rda_rn_rm
+FMLSLT_zzzw 01100100 .. 1 ..... 10 1 00 1 ..... ..... @rda_rn_rm
+
+### SVE2 floating-point multiply-add long (indexed)
+FMLALB_zzxw 01100100 .. 1 ..... 0100.0 ..... ..... @rrxw_s
+FMLALT_zzxw 01100100 .. 1 ..... 0100.1 ..... ..... @rrxw_s
+FMLSLB_zzxw 01100100 .. 1 ..... 0110.0 ..... ..... @rrxw_s
+FMLSLT_zzxw 01100100 .. 1 ..... 0110.1 ..... ..... @rrxw_s
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index acd6d53724..92d1297a5c 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -8523,3 +8523,78 @@ static bool trans_FLOGB(DisasContext *s, arg_rpr_esz *a)
}
return true;
}
+
+static bool do_FMLAL_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sub, bool sel)
+{
+ if (a->esz != MO_32 || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ vec_full_reg_offset(s, a->ra),
+ cpu_env, vsz, vsz, (sel << 1) | sub,
+ gen_helper_sve2_fmlal_zzzw_s);
+ }
+ return true;
+}
+
+static bool trans_FMLALB_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_FMLAL_zzzw(s, a, false, false);
+}
+
+static bool trans_FMLALT_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_FMLAL_zzzw(s, a, false, true);
+}
+
+static bool trans_FMLSLB_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_FMLAL_zzzw(s, a, true, false);
+}
+
+static bool trans_FMLSLT_zzzw(DisasContext *s, arg_rrrr_esz *a)
+{
+ return do_FMLAL_zzzw(s, a, true, true);
+}
+
+static bool do_FMLAL_zzxw(DisasContext *s, arg_rrxr_esz *a, bool sub, bool sel)
+{
+ if (a->esz != MO_32 || !dc_isar_feature(aa64_sve2, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ vec_full_reg_offset(s, a->ra),
+ cpu_env, vsz, vsz,
+ (a->index << 2) | (sel << 1) | sub,
+ gen_helper_sve2_fmlal_zzxw_s);
+ }
+ return true;
+}
+
+static bool trans_FMLALB_zzxw(DisasContext *s, arg_rrxr_esz *a)
+{
+ return do_FMLAL_zzxw(s, a, false, false);
+}
+
+static bool trans_FMLALT_zzxw(DisasContext *s, arg_rrxr_esz *a)
+{
+ return do_FMLAL_zzxw(s, a, false, true);
+}
+
+static bool trans_FMLSLB_zzxw(DisasContext *s, arg_rrxr_esz *a)
+{
+ return do_FMLAL_zzxw(s, a, true, false);
+}
+
+static bool trans_FMLSLT_zzxw(DisasContext *s, arg_rrxr_esz *a)
+{
+ return do_FMLAL_zzxw(s, a, true, true);
+}
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index c8d1656797..74874ae3c1 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -29,10 +29,14 @@
so addressing units smaller than that needs a host-endian fixup. */
#ifdef HOST_WORDS_BIGENDIAN
#define H1(x) ((x) ^ 7)
+#define H1_2(x) ((x) ^ 6)
+#define H1_4(x) ((x) ^ 4)
#define H2(x) ((x) ^ 3)
#define H4(x) ((x) ^ 1)
#else
#define H1(x) (x)
+#define H1_2(x) (x)
+#define H1_4(x) (x)
#define H2(x) (x)
#define H4(x) (x)
#endif
@@ -1905,6 +1909,27 @@ void HELPER(gvec_fmlal_a64)(void *vd, void *vn, void *vm,
get_flush_inputs_to_zero(&env->vfp.fp_status_f16));
}
+void HELPER(sve2_fmlal_zzzw_s)(void *vd, void *vn, void *vm, void *va,
+ void *venv, uint32_t desc)
+{
+ intptr_t i, oprsz = simd_oprsz(desc);
+ uint16_t negn = extract32(desc, SIMD_DATA_SHIFT, 1) << 15;
+ intptr_t sel = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(float16);
+ CPUARMState *env = venv;
+ float_status *status = &env->vfp.fp_status;
+ bool fz16 = get_flush_inputs_to_zero(&env->vfp.fp_status_f16);
+
+ for (i = 0; i < oprsz; i += sizeof(float32)) {
+ float16 nn_16 = *(float16 *)(vn + H1_2(i + sel)) ^ negn;
+ float16 mm_16 = *(float16 *)(vm + H1_2(i + sel));
+ float32 nn = float16_to_float32_by_bits(nn_16, fz16);
+ float32 mm = float16_to_float32_by_bits(mm_16, fz16);
+ float32 aa = *(float32 *)(va + H1_4(i));
+
+ *(float32 *)(vd + H1_4(i)) = float32_muladd(nn, mm, aa, 0, status);
+ }
+}
+
static void do_fmlal_idx(float32 *d, void *vn, void *vm, float_status *fpst,
uint32_t desc, bool fz16)
{
@@ -1949,6 +1974,32 @@ void HELPER(gvec_fmlal_idx_a64)(void *vd, void *vn, void *vm,
get_flush_inputs_to_zero(&env->vfp.fp_status_f16));
}
+void HELPER(sve2_fmlal_zzxw_s)(void *vd, void *vn, void *vm, void *va,
+ void *venv, uint32_t desc)
+{
+ intptr_t i, j, oprsz = simd_oprsz(desc);
+ uint16_t negn = extract32(desc, SIMD_DATA_SHIFT, 1) << 15;
+ intptr_t sel = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(float16);
+ intptr_t idx = extract32(desc, SIMD_DATA_SHIFT + 2, 3) * sizeof(float16);
+ CPUARMState *env = venv;
+ float_status *status = &env->vfp.fp_status;
+ bool fz16 = get_flush_inputs_to_zero(&env->vfp.fp_status_f16);
+
+ for (i = 0; i < oprsz; i += 16) {
+ float16 mm_16 = *(float16 *)(vm + i + idx);
+ float32 mm = float16_to_float32_by_bits(mm_16, fz16);
+
+ for (j = 0; j < 16; j += sizeof(float32)) {
+ float16 nn_16 = *(float16 *)(vn + H1_2(i + j + sel)) ^ negn;
+ float32 nn = float16_to_float32_by_bits(nn_16, fz16);
+ float32 aa = *(float32 *)(va + H1_4(i + j));
+
+ *(float32 *)(vd + H1_4(i + j)) =
+ float32_muladd(nn, mm, aa, 0, status);
+ }
+ }
+}
+
void HELPER(gvec_sshl_b)(void *vd, void *vn, void *vm, uint32_t desc)
{
intptr_t i, opr_sz = simd_oprsz(desc);
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 80/81] target/arm: Implement SVE2 complex integer dot product
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (78 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 79/81] target/arm: Implement SVE2 fp multiply-add long Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-09-18 18:37 ` [PATCH v3 81/81] target/arm: Enable SVE2 and some extensions Richard Henderson
` (2 subsequent siblings)
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-sve.h | 10 ++++
target/arm/sve.decode | 9 ++++
target/arm/sve_helper.c | 99 ++++++++++++++++++++++++++++++++++++++
target/arm/translate-sve.c | 17 +++++++
4 files changed, 135 insertions(+)
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 9542f01c42..4af5e1a5ce 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -2784,3 +2784,13 @@ DEF_HELPER_FLAGS_5(sve2_fcvtlt_sd, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(flogb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(flogb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(flogb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_cdot_zzzz_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cdot_zzzz_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_cdot_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cdot_idx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 6708c048e0..84232ff9e5 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -811,6 +811,9 @@ MUL_zzi 00100101 .. 110 000 110 ........ ..... @rdn_i8s
DOT_zzzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 \
ra=%reg_movprfx
+# SVE2 complex dot product (vectors)
+CDOT_zzzz 01000100 esz:2 0 rm:5 0001 rot:2 rn:5 rd:5 ra=%reg_movprfx
+
#### SVE Multiply - Indexed
# SVE integer dot product (indexed)
@@ -849,6 +852,12 @@ SQDMLSLB_zzxw_d 01000100 .. 1 ..... 0011.0 ..... ..... @rrxw_d
SQDMLSLT_zzxw_s 01000100 .. 1 ..... 0011.1 ..... ..... @rrxw_s
SQDMLSLT_zzxw_d 01000100 .. 1 ..... 0011.1 ..... ..... @rrxw_d
+# SVE2 complex integer dot product (indexed)
+CDOT_zzxw_s 01000100 10 1 index:2 rm:3 0100 rot:2 rn:5 rd:5 \
+ ra=%reg_movprfx
+CDOT_zzxw_d 01000100 11 1 index:1 rm:4 0100 rot:2 rn:5 rd:5 \
+ ra=%reg_movprfx
+
# SVE2 complex integer multiply-add (indexed)
CMLA_zzxz_h 01000100 10 1 index:2 rm:3 0110 rot:2 rn:5 rd:5 \
ra=%reg_movprfx
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index ef04a0f95a..0ff5e0baf8 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -1592,6 +1592,105 @@ void HELPER(sve2_sqrdcmlah_idx_s)(void *vd, void *vn, void *vm,
do_cmla_idx_s(vd, vn, vm, va, desc, do_sqrdcmlah_s);
}
+/* Note N and M are 4 elements bundled into one unit. */
+static int32_t do_cdot_s(uint32_t n, uint32_t m, int32_t a,
+ int sel_a, int sel_b, int sub_i)
+{
+ for (int i = 0; i <= 1; i++) {
+ int32_t elt1_r = (int8_t)(n >> (16 * i));
+ int32_t elt1_i = (int8_t)(n >> (16 * i + 8));
+ int32_t elt2_a = (int8_t)(m >> (16 * i + 8 * sel_a));
+ int32_t elt2_b = (int8_t)(m >> (16 * i + 8 * sel_b));
+
+ a += elt1_r * elt2_a + elt1_i * elt2_b * sub_i;
+ }
+ return a;
+}
+
+static int64_t do_cdot_d(uint64_t n, uint64_t m, int64_t a,
+ int sel_a, int sel_b, int sub_i)
+{
+ for (int i = 0; i <= 1; i++) {
+ int64_t elt1_r = (int16_t)(n >> (32 * i + 0));
+ int64_t elt1_i = (int16_t)(n >> (32 * i + 16));
+ int64_t elt2_a = (int16_t)(m >> (32 * i + 16 * sel_a));
+ int64_t elt2_b = (int16_t)(m >> (32 * i + 16 * sel_b));
+
+ a += elt1_r * elt2_a + elt1_i * elt2_b * sub_i;
+ }
+ return a;
+}
+
+void HELPER(sve2_cdot_zzzz_s)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ int opr_sz = simd_oprsz(desc);
+ int rot = simd_data(desc);
+ int sel_a = rot & 1;
+ int sel_b = sel_a ^ 1;
+ int sub_i = (rot == 0 || rot == 3 ? -1 : 1);
+ uint32_t *d = vd, *n = vn, *m = vm, *a = va;
+
+ for (int e = 0; e < opr_sz / 4; e++) {
+ d[e] = do_cdot_s(n[e], m[e], a[e], sel_a, sel_b, sub_i);
+ }
+}
+
+void HELPER(sve2_cdot_zzzz_d)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ int opr_sz = simd_oprsz(desc);
+ int rot = simd_data(desc);
+ int sel_a = rot & 1;
+ int sel_b = sel_a ^ 1;
+ int sub_i = (rot == 0 || rot == 3 ? -1 : 1);
+ uint64_t *d = vd, *n = vn, *m = vm, *a = va;
+
+ for (int e = 0; e < opr_sz / 8; e++) {
+ d[e] = do_cdot_d(n[e], m[e], a[e], sel_a, sel_b, sub_i);
+ }
+}
+
+void HELPER(sve2_cdot_idx_s)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ int opr_sz = simd_oprsz(desc);
+ int rot = extract32(desc, SIMD_DATA_SHIFT, 2);
+ int idx = H4(extract32(desc, SIMD_DATA_SHIFT + 2, 2));
+ int sel_a = rot & 1;
+ int sel_b = sel_a ^ 1;
+ int sub_i = (rot == 0 || rot == 3 ? -1 : 1);
+ uint32_t *d = vd, *n = vn, *m = vm, *a = va;
+
+ for (int seg = 0; seg < opr_sz / 4; seg += 4) {
+ uint32_t seg_m = m[seg + idx];
+ for (int e = 0; e < 4; e++) {
+ d[seg + e] = do_cdot_s(n[seg + e], seg_m, a[seg + e],
+ sel_a, sel_b, sub_i);
+ }
+ }
+}
+
+void HELPER(sve2_cdot_idx_d)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ int seg, opr_sz = simd_oprsz(desc);
+ int rot = extract32(desc, SIMD_DATA_SHIFT, 2);
+ int idx = extract32(desc, SIMD_DATA_SHIFT + 2, 2);
+ int sel_a = rot & 1;
+ int sel_b = sel_a ^ 1;
+ int sub_i = (rot == 0 || rot == 3 ? -1 : 1);
+ uint64_t *d = vd, *n = vn, *m = vm, *a = va;
+
+ for (seg = 0; seg < opr_sz / 8; seg += 2) {
+ uint64_t seg_m = m[seg + idx];
+ for (int e = 0; e < 2; e++) {
+ d[seg + e] = do_cdot_d(n[seg + e], seg_m, a[seg + e],
+ sel_a, sel_b, sub_i);
+ }
+ }
+}
+
#define DO_ZZXZ(NAME, TYPE, H, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
{ \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 92d1297a5c..4893968cdb 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -4151,6 +4151,9 @@ DO_SVE2_RRXR_ROT(CMLA_zzxz_s, gen_helper_sve2_cmla_idx_s)
DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h, gen_helper_sve2_sqrdcmlah_idx_h)
DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s, gen_helper_sve2_sqrdcmlah_idx_s)
+DO_SVE2_RRXR_ROT(CDOT_zzxw_s, gen_helper_sve2_cdot_idx_s)
+DO_SVE2_RRXR_ROT(CDOT_zzxw_d, gen_helper_sve2_cdot_idx_d)
+
#undef DO_SVE2_RRXR_ROT
/*
@@ -8355,6 +8358,20 @@ static bool trans_CMLA_zzzz(DisasContext *s, arg_CMLA_zzzz *a)
return true;
}
+static bool trans_CDOT_zzzz(DisasContext *s, arg_CMLA_zzzz *a)
+{
+ if (!dc_isar_feature(aa64_sve2, s) || a->esz < MO_32) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_helper_gvec_4 *fn = (a->esz == MO_32
+ ? gen_helper_sve2_cdot_zzzz_s
+ : gen_helper_sve2_cdot_zzzz_d);
+ gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->rot);
+ }
+ return true;
+}
+
static bool trans_SQRDCMLAH_zzzz(DisasContext *s, arg_SQRDCMLAH_zzzz *a)
{
static gen_helper_gvec_4 * const fns[] = {
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* [PATCH v3 81/81] target/arm: Enable SVE2 and some extensions
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (79 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 80/81] target/arm: Implement SVE2 complex integer dot product Richard Henderson
@ 2020-09-18 18:37 ` Richard Henderson
2020-11-10 19:55 ` [PATCH v3 00/81] target/arm: Implement SVE2 Stephen Long
2020-11-11 18:17 ` Stephen Long
82 siblings, 0 replies; 94+ messages in thread
From: Richard Henderson @ 2020-09-18 18:37 UTC (permalink / raw)
To: qemu-devel; +Cc: peter.maydell, alex.bennee
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/cpu64.c | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
index 3c2b3d9599..46b8a3908c 100644
--- a/target/arm/cpu64.c
+++ b/target/arm/cpu64.c
@@ -667,6 +667,17 @@ static void aarch64_max_initfn(Object *obj)
t = FIELD_DP64(t, ID_AA64MMFR2, CNP, 1); /* TTCNP */
cpu->isar.id_aa64mmfr2 = t;
+ t = cpu->isar.id_aa64zfr0;
+ t = FIELD_DP64(t, ID_AA64ZFR0, SVEVER, 1);
+ t = FIELD_DP64(t, ID_AA64ZFR0, AES, 2); /* PMULL */
+ t = FIELD_DP64(t, ID_AA64ZFR0, BITPERM, 1);
+ t = FIELD_DP64(t, ID_AA64ZFR0, SHA3, 1);
+ t = FIELD_DP64(t, ID_AA64ZFR0, SM4, 1);
+ t = FIELD_DP64(t, ID_AA64ZFR0, I8MM, 1);
+ t = FIELD_DP64(t, ID_AA64ZFR0, F32MM, 1);
+ t = FIELD_DP64(t, ID_AA64ZFR0, F64MM, 1);
+ cpu->isar.id_aa64zfr0 = t;
+
/* Replicate the same data to the 32-bit id registers. */
u = cpu->isar.id_isar5;
u = FIELD_DP32(u, ID_ISAR5, AES, 2); /* AES + PMULL */
--
2.25.1
^ permalink raw reply related [flat|nested] 94+ messages in thread* Re: [PATCH v3 00/81] target/arm: Implement SVE2
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (80 preceding siblings ...)
2020-09-18 18:37 ` [PATCH v3 81/81] target/arm: Enable SVE2 and some extensions Richard Henderson
@ 2020-11-10 19:55 ` Stephen Long
2020-11-12 21:06 ` Richard Henderson
2020-11-11 18:17 ` Stephen Long
82 siblings, 1 reply; 94+ messages in thread
From: Stephen Long @ 2020-11-10 19:55 UTC (permalink / raw)
To: qemu-devel
Hi Richard, what's the plan to get this patch series into master?
Thanks,
Stephen
^ permalink raw reply [flat|nested] 94+ messages in thread* Re: [PATCH v3 00/81] target/arm: Implement SVE2
2020-09-18 18:36 [PATCH v3 00/81] target/arm: Implement SVE2 Richard Henderson
` (81 preceding siblings ...)
2020-11-10 19:55 ` [PATCH v3 00/81] target/arm: Implement SVE2 Stephen Long
@ 2020-11-11 18:17 ` Stephen Long
82 siblings, 0 replies; 94+ messages in thread
From: Stephen Long @ 2020-11-11 18:17 UTC (permalink / raw)
To: qemu-devel; +Cc: richard.henderson
Hi Richard,
Apologies for forgetting to cc you in the previous reply.
What's the plan to get this patch series into master?
Thanks,
Stephen
^ permalink raw reply [flat|nested] 94+ messages in thread