From: Richard Henderson <richard.henderson@linaro.org>
To: qemu-devel@nongnu.org
Cc: Peter Maydell <peter.maydell@linaro.org>
Subject: [PULL 09/27] tcg/aarch64: Support 128-bit load/store
Date: Tue, 30 May 2023 11:59:31 -0700 [thread overview]
Message-ID: <20230530185949.410208-10-richard.henderson@linaro.org> (raw)
In-Reply-To: <20230530185949.410208-1-richard.henderson@linaro.org>
With FEAT_LSE2, LDP/STP suffices. Without FEAT_LSE2, use LDXP+STXP
16-byte atomicity is required and LDP/STP otherwise.
Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
tcg/aarch64/tcg-target-con-set.h | 2 +
tcg/aarch64/tcg-target.h | 11 ++-
tcg/aarch64/tcg-target.c.inc | 141 ++++++++++++++++++++++++++++++-
3 files changed, 151 insertions(+), 3 deletions(-)
diff --git a/tcg/aarch64/tcg-target-con-set.h b/tcg/aarch64/tcg-target-con-set.h
index 109f4ab97c..3fdee26a3d 100644
--- a/tcg/aarch64/tcg-target-con-set.h
+++ b/tcg/aarch64/tcg-target-con-set.h
@@ -13,6 +13,7 @@ C_O0_I1(r)
C_O0_I2(r, rA)
C_O0_I2(rZ, r)
C_O0_I2(w, r)
+C_O0_I3(rZ, rZ, r)
C_O1_I1(r, r)
C_O1_I1(w, r)
C_O1_I1(w, w)
@@ -31,4 +32,5 @@ C_O1_I2(w, w, wO)
C_O1_I2(w, w, wZ)
C_O1_I3(w, w, w, w)
C_O1_I4(r, r, rA, rZ, rZ)
+C_O2_I1(r, r, r)
C_O2_I4(r, r, rZ, rZ, rA, rMZ)
diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
index d5f7614880..192a2758c5 100644
--- a/tcg/aarch64/tcg-target.h
+++ b/tcg/aarch64/tcg-target.h
@@ -131,7 +131,16 @@ typedef enum {
#define TCG_TARGET_HAS_muluh_i64 1
#define TCG_TARGET_HAS_mulsh_i64 1
-#define TCG_TARGET_HAS_qemu_ldst_i128 0
+/*
+ * Without FEAT_LSE2, we must use LDXP+STXP to implement atomic 128-bit load,
+ * which requires writable pages. We must defer to the helper for user-only,
+ * but in system mode all ram is writable for the host.
+ */
+#ifdef CONFIG_USER_ONLY
+#define TCG_TARGET_HAS_qemu_ldst_i128 have_lse2
+#else
+#define TCG_TARGET_HAS_qemu_ldst_i128 1
+#endif
#define TCG_TARGET_HAS_v64 1
#define TCG_TARGET_HAS_v128 1
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index 00d9033e2f..261ad25210 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -385,6 +385,10 @@ typedef enum {
I3305_LDR_v64 = 0x5c000000,
I3305_LDR_v128 = 0x9c000000,
+ /* Load/store exclusive. */
+ I3306_LDXP = 0xc8600000,
+ I3306_STXP = 0xc8200000,
+
/* Load/store register. Described here as 3.3.12, but the helper
that emits them can transform to 3.3.10 or 3.3.13. */
I3312_STRB = 0x38000000 | LDST_ST << 22 | MO_8 << 30,
@@ -449,6 +453,9 @@ typedef enum {
I3406_ADR = 0x10000000,
I3406_ADRP = 0x90000000,
+ /* Add/subtract extended register instructions. */
+ I3501_ADD = 0x0b200000,
+
/* Add/subtract shifted register instructions (without a shift). */
I3502_ADD = 0x0b000000,
I3502_ADDS = 0x2b000000,
@@ -619,6 +626,12 @@ static void tcg_out_insn_3305(TCGContext *s, AArch64Insn insn,
tcg_out32(s, insn | (imm19 & 0x7ffff) << 5 | rt);
}
+static void tcg_out_insn_3306(TCGContext *s, AArch64Insn insn, TCGReg rs,
+ TCGReg rt, TCGReg rt2, TCGReg rn)
+{
+ tcg_out32(s, insn | rs << 16 | rt2 << 10 | rn << 5 | rt);
+}
+
static void tcg_out_insn_3201(TCGContext *s, AArch64Insn insn, TCGType ext,
TCGReg rt, int imm19)
{
@@ -701,6 +714,14 @@ static void tcg_out_insn_3406(TCGContext *s, AArch64Insn insn,
tcg_out32(s, insn | (disp & 3) << 29 | (disp & 0x1ffffc) << (5 - 2) | rd);
}
+static inline void tcg_out_insn_3501(TCGContext *s, AArch64Insn insn,
+ TCGType sf, TCGReg rd, TCGReg rn,
+ TCGReg rm, int opt, int imm3)
+{
+ tcg_out32(s, insn | sf << 31 | rm << 16 | opt << 13 |
+ imm3 << 10 | rn << 5 | rd);
+}
+
/* This function is for both 3.5.2 (Add/Subtract shifted register), for
the rare occasion when we actually want to supply a shift amount. */
static inline void tcg_out_insn_3502S(TCGContext *s, AArch64Insn insn,
@@ -1628,16 +1649,16 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h,
TCGType addr_type = s->addr_type;
TCGLabelQemuLdst *ldst = NULL;
MemOp opc = get_memop(oi);
+ MemOp s_bits = opc & MO_SIZE;
unsigned a_mask;
h->aa = atom_and_align_for_opc(s, opc,
have_lse2 ? MO_ATOM_WITHIN16
: MO_ATOM_IFALIGN,
- false);
+ s_bits == MO_128);
a_mask = (1 << h->aa.align) - 1;
#ifdef CONFIG_SOFTMMU
- unsigned s_bits = opc & MO_SIZE;
unsigned s_mask = (1u << s_bits) - 1;
unsigned mem_index = get_mmuidx(oi);
TCGReg addr_adj;
@@ -1818,6 +1839,108 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
}
}
+static void tcg_out_qemu_ldst_i128(TCGContext *s, TCGReg datalo, TCGReg datahi,
+ TCGReg addr_reg, MemOpIdx oi, bool is_ld)
+{
+ TCGLabelQemuLdst *ldst;
+ HostAddress h;
+ TCGReg base;
+ bool use_pair;
+
+ ldst = prepare_host_addr(s, &h, addr_reg, oi, is_ld);
+
+ /* Compose the final address, as LDP/STP have no indexing. */
+ if (h.index == TCG_REG_XZR) {
+ base = h.base;
+ } else {
+ base = TCG_REG_TMP2;
+ if (h.index_ext == TCG_TYPE_I32) {
+ /* add base, base, index, uxtw */
+ tcg_out_insn(s, 3501, ADD, TCG_TYPE_I64, base,
+ h.base, h.index, MO_32, 0);
+ } else {
+ /* add base, base, index */
+ tcg_out_insn(s, 3502, ADD, 1, base, h.base, h.index);
+ }
+ }
+
+ use_pair = h.aa.atom < MO_128 || have_lse2;
+
+ if (!use_pair) {
+ tcg_insn_unit *branch = NULL;
+ TCGReg ll, lh, sl, sh;
+
+ /*
+ * If we have already checked for 16-byte alignment, that's all
+ * we need. Otherwise we have determined that misaligned atomicity
+ * may be handled with two 8-byte loads.
+ */
+ if (h.aa.align < MO_128) {
+ /*
+ * TODO: align should be MO_64, so we only need test bit 3,
+ * which means we could use TBNZ instead of ANDS+B_C.
+ */
+ tcg_out_logicali(s, I3404_ANDSI, 0, TCG_REG_XZR, addr_reg, 15);
+ branch = s->code_ptr;
+ tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0);
+ use_pair = true;
+ }
+
+ if (is_ld) {
+ /*
+ * 16-byte atomicity without LSE2 requires LDXP+STXP loop:
+ * ldxp lo, hi, [base]
+ * stxp t0, lo, hi, [base]
+ * cbnz t0, .-8
+ * Require no overlap between data{lo,hi} and base.
+ */
+ if (datalo == base || datahi == base) {
+ tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_TMP2, base);
+ base = TCG_REG_TMP2;
+ }
+ ll = sl = datalo;
+ lh = sh = datahi;
+ } else {
+ /*
+ * 16-byte atomicity without LSE2 requires LDXP+STXP loop:
+ * 1: ldxp t0, t1, [base]
+ * stxp t0, lo, hi, [base]
+ * cbnz t0, 1b
+ */
+ tcg_debug_assert(base != TCG_REG_TMP0 && base != TCG_REG_TMP1);
+ ll = TCG_REG_TMP0;
+ lh = TCG_REG_TMP1;
+ sl = datalo;
+ sh = datahi;
+ }
+
+ tcg_out_insn(s, 3306, LDXP, TCG_REG_XZR, ll, lh, base);
+ tcg_out_insn(s, 3306, STXP, TCG_REG_TMP0, sl, sh, base);
+ tcg_out_insn(s, 3201, CBNZ, 0, TCG_REG_TMP0, -2);
+
+ if (use_pair) {
+ /* "b .+8", branching across the one insn of use_pair. */
+ tcg_out_insn(s, 3206, B, 2);
+ reloc_pc19(branch, tcg_splitwx_to_rx(s->code_ptr));
+ }
+ }
+
+ if (use_pair) {
+ if (is_ld) {
+ tcg_out_insn(s, 3314, LDP, datalo, datahi, base, 0, 1, 0);
+ } else {
+ tcg_out_insn(s, 3314, STP, datalo, datahi, base, 0, 1, 0);
+ }
+ }
+
+ if (ldst) {
+ ldst->type = TCG_TYPE_I128;
+ ldst->datalo_reg = datalo;
+ ldst->datahi_reg = datahi;
+ ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
+ }
+}
+
static const tcg_insn_unit *tb_ret_addr;
static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
@@ -2157,6 +2280,14 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
case INDEX_op_qemu_st_a64_i64:
tcg_out_qemu_st(s, REG0(0), a1, a2, ext);
break;
+ case INDEX_op_qemu_ld_a32_i128:
+ case INDEX_op_qemu_ld_a64_i128:
+ tcg_out_qemu_ldst_i128(s, a0, a1, a2, args[3], true);
+ break;
+ case INDEX_op_qemu_st_a32_i128:
+ case INDEX_op_qemu_st_a64_i128:
+ tcg_out_qemu_ldst_i128(s, REG0(0), REG0(1), a2, args[3], false);
+ break;
case INDEX_op_bswap64_i64:
tcg_out_rev(s, TCG_TYPE_I64, MO_64, a0, a1);
@@ -2796,11 +2927,17 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op)
case INDEX_op_qemu_ld_a32_i64:
case INDEX_op_qemu_ld_a64_i64:
return C_O1_I1(r, r);
+ case INDEX_op_qemu_ld_a32_i128:
+ case INDEX_op_qemu_ld_a64_i128:
+ return C_O2_I1(r, r, r);
case INDEX_op_qemu_st_a32_i32:
case INDEX_op_qemu_st_a64_i32:
case INDEX_op_qemu_st_a32_i64:
case INDEX_op_qemu_st_a64_i64:
return C_O0_I2(rZ, r);
+ case INDEX_op_qemu_st_a32_i128:
+ case INDEX_op_qemu_st_a64_i128:
+ return C_O0_I3(rZ, rZ, r);
case INDEX_op_deposit_i32:
case INDEX_op_deposit_i64:
--
2.34.1
next prev parent reply other threads:[~2023-05-30 19:06 UTC|newest]
Thread overview: 35+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-05-30 18:59 [PULL 00/27] tcg patch queue Richard Henderson
2023-05-30 18:59 ` [PULL 01/27] tcg: Fix register move type in tcg_out_ld_helper_ret Richard Henderson
2023-05-30 18:59 ` [PULL 02/27] accel/tcg: Fix check for page writeability in load_atomic16_or_exit Richard Henderson
2023-05-30 18:59 ` [PULL 03/27] meson: Split test for __int128_t type from __int128_t arithmetic Richard Henderson
2023-05-30 18:59 ` [PULL 04/27] qemu/atomic128: Add x86_64 atomic128-ldst.h Richard Henderson
2023-05-30 18:59 ` [PULL 05/27] tcg/i386: Support 128-bit load/store Richard Henderson
2023-05-30 18:59 ` [PULL 06/27] tcg/aarch64: Rename temporaries Richard Henderson
2023-05-30 18:59 ` [PULL 07/27] tcg/aarch64: Reserve TCG_REG_TMP1, TCG_REG_TMP2 Richard Henderson
2023-05-30 18:59 ` [PULL 08/27] tcg/aarch64: Simplify constraints on qemu_ld/st Richard Henderson
2023-05-30 18:59 ` Richard Henderson [this message]
2023-05-30 18:59 ` [PULL 10/27] tcg/ppc: Support 128-bit load/store Richard Henderson
2023-05-30 18:59 ` [PULL 11/27] tcg/s390x: " Richard Henderson
2023-07-10 8:58 ` TCG broken on s390x hosts (was: [PULL 11/27] tcg/s390x: Support 128-bit load/store) Thomas Huth
2023-07-10 9:31 ` Richard Henderson
2023-07-10 11:10 ` Thomas Huth
2023-05-30 18:59 ` [PULL 12/27] accel/tcg: Extract load_atom_extract_al16_or_al8 to host header Richard Henderson
2023-05-30 18:59 ` [PULL 13/27] accel/tcg: Extract store_atom_insert_al16 " Richard Henderson
2023-05-30 18:59 ` [PULL 14/27] accel/tcg: Add x86_64 load_atom_extract_al16_or_al8 Richard Henderson
2023-05-30 18:59 ` [PULL 15/27] accel/tcg: Add aarch64 lse2 load_atom_extract_al16_or_al8 Richard Henderson
2023-05-30 18:59 ` [PULL 16/27] accel/tcg: Add aarch64 store_atom_insert_al16 Richard Henderson
2023-05-30 18:59 ` [PULL 17/27] tcg: Remove TCG_TARGET_TLB_DISPLACEMENT_BITS Richard Henderson
2023-05-30 18:59 ` [PULL 18/27] decodetree: Add --test-for-error Richard Henderson
2023-05-30 18:59 ` [PULL 19/27] decodetree: Fix recursion in prop_format and build_tree Richard Henderson
2023-05-30 18:59 ` [PULL 20/27] decodetree: Diagnose empty pattern group Richard Henderson
2023-05-30 18:59 ` [PULL 21/27] decodetree: Do not remove output_file from /dev Richard Henderson
2023-05-30 18:59 ` [PULL 22/27] tests/decode: Convert tests to meson Richard Henderson
2023-05-30 18:59 ` [PULL 23/27] docs: Document decodetree named field syntax Richard Henderson
2023-05-30 18:59 ` [PULL 24/27] scripts/decodetree: Pass lvalue-formatter function to str_extract() Richard Henderson
2023-05-30 18:59 ` [PULL 25/27] scripts/decodetree: Implement a topological sort Richard Henderson
2023-05-30 18:59 ` [PULL 26/27] scripts/decodetree: Implement named field support Richard Henderson
2023-05-30 18:59 ` [PULL 27/27] tests/decode: Add tests for various named-field cases Richard Henderson
2023-05-31 1:08 ` [PULL 00/27] tcg patch queue Richard Henderson
2023-05-31 16:12 ` Thomas Huth
[not found] ` <227e27e0-4035-8e17-2259-3098340f716e@linaro.org>
2023-05-31 22:20 ` Richard Henderson
2023-06-01 10:52 ` Mark Cave-Ayland
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230530185949.410208-10-richard.henderson@linaro.org \
--to=richard.henderson@linaro.org \
--cc=peter.maydell@linaro.org \
--cc=qemu-devel@nongnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).