From: Richard Henderson <richard.henderson@linaro.org>
To: qemu-devel@nongnu.org
Cc: alex.bennee@linaro.org
Subject: [PATCH v5 16/36] tcg: Add tcg_gen_{non}atomic_cmpxchg_i128
Date: Wed, 25 Jan 2023 18:38:04 -1000 [thread overview]
Message-ID: <20230126043824.54819-17-richard.henderson@linaro.org> (raw)
In-Reply-To: <20230126043824.54819-1-richard.henderson@linaro.org>
This will allow targets to avoid rolling their own.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
accel/tcg/tcg-runtime.h | 11 +++++
include/tcg/tcg-op.h | 5 +++
tcg/tcg-op.c | 85 +++++++++++++++++++++++++++++++++++
accel/tcg/atomic_common.c.inc | 45 +++++++++++++++++++
4 files changed, 146 insertions(+)
diff --git a/accel/tcg/tcg-runtime.h b/accel/tcg/tcg-runtime.h
index 37cbd722bf..e141a6ab24 100644
--- a/accel/tcg/tcg-runtime.h
+++ b/accel/tcg/tcg-runtime.h
@@ -55,6 +55,17 @@ DEF_HELPER_FLAGS_5(atomic_cmpxchgq_be, TCG_CALL_NO_WG,
DEF_HELPER_FLAGS_5(atomic_cmpxchgq_le, TCG_CALL_NO_WG,
i64, env, tl, i64, i64, i32)
#endif
+#ifdef CONFIG_CMPXCHG128
+DEF_HELPER_FLAGS_5(atomic_cmpxchgo_be, TCG_CALL_NO_WG,
+ i128, env, tl, i128, i128, i32)
+DEF_HELPER_FLAGS_5(atomic_cmpxchgo_le, TCG_CALL_NO_WG,
+ i128, env, tl, i128, i128, i32)
+#endif
+
+DEF_HELPER_FLAGS_5(nonatomic_cmpxchgo_be, TCG_CALL_NO_WG,
+ i128, env, tl, i128, i128, i32)
+DEF_HELPER_FLAGS_5(nonatomic_cmpxchgo_le, TCG_CALL_NO_WG,
+ i128, env, tl, i128, i128, i32)
#ifdef CONFIG_ATOMIC64
#define GEN_ATOMIC_HELPERS(NAME) \
diff --git a/include/tcg/tcg-op.h b/include/tcg/tcg-op.h
index e5f5b63c37..31bf3d287e 100644
--- a/include/tcg/tcg-op.h
+++ b/include/tcg/tcg-op.h
@@ -907,6 +907,11 @@ void tcg_gen_atomic_cmpxchg_i32(TCGv_i32, TCGv, TCGv_i32, TCGv_i32,
TCGArg, MemOp);
void tcg_gen_atomic_cmpxchg_i64(TCGv_i64, TCGv, TCGv_i64, TCGv_i64,
TCGArg, MemOp);
+void tcg_gen_atomic_cmpxchg_i128(TCGv_i128, TCGv, TCGv_i128, TCGv_i128,
+ TCGArg, MemOp);
+
+void tcg_gen_nonatomic_cmpxchg_i128(TCGv_i128, TCGv, TCGv_i128, TCGv_i128,
+ TCGArg, MemOp);
void tcg_gen_atomic_xchg_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp);
void tcg_gen_atomic_xchg_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp);
diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c
index 33ef325f6e..5811ecd3e7 100644
--- a/tcg/tcg-op.c
+++ b/tcg/tcg-op.c
@@ -3295,6 +3295,8 @@ typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
TCGv_i32, TCGv_i32, TCGv_i32);
typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
TCGv_i64, TCGv_i64, TCGv_i32);
+typedef void (*gen_atomic_cx_i128)(TCGv_i128, TCGv_env, TCGv,
+ TCGv_i128, TCGv_i128, TCGv_i32);
typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
TCGv_i32, TCGv_i32);
typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
@@ -3305,6 +3307,11 @@ typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
#else
# define WITH_ATOMIC64(X)
#endif
+#ifdef CONFIG_CMPXCHG128
+# define WITH_ATOMIC128(X) X,
+#else
+# define WITH_ATOMIC128(X)
+#endif
static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {
[MO_8] = gen_helper_atomic_cmpxchgb,
@@ -3314,6 +3321,8 @@ static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {
[MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
+ WITH_ATOMIC128([MO_128 | MO_LE] = gen_helper_atomic_cmpxchgo_le)
+ WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_cmpxchgo_be)
};
void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
@@ -3412,6 +3421,82 @@ void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
}
}
+void tcg_gen_nonatomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv,
+ TCGv_i128 newv, TCGArg idx, MemOp memop)
+{
+ if (TCG_TARGET_REG_BITS == 32) {
+ /* Inline expansion below is simply too large for 32-bit hosts. */
+ gen_atomic_cx_i128 gen = ((memop & MO_BSWAP) == MO_LE
+ ? gen_helper_nonatomic_cmpxchgo_le
+ : gen_helper_nonatomic_cmpxchgo_be);
+ MemOpIdx oi = make_memop_idx(memop, idx);
+
+ tcg_debug_assert((memop & MO_SIZE) == MO_128);
+ tcg_debug_assert((memop & MO_SIGN) == 0);
+
+ gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
+ } else {
+ TCGv_i128 oldv = tcg_temp_new_i128();
+ TCGv_i128 tmpv = tcg_temp_new_i128();
+ TCGv_i64 t0 = tcg_temp_new_i64();
+ TCGv_i64 t1 = tcg_temp_new_i64();
+ TCGv_i64 z = tcg_constant_i64(0);
+
+ tcg_gen_qemu_ld_i128(oldv, addr, idx, memop);
+
+ /* Compare i128 */
+ tcg_gen_xor_i64(t0, TCGV128_LOW(oldv), TCGV128_LOW(cmpv));
+ tcg_gen_xor_i64(t1, TCGV128_HIGH(oldv), TCGV128_HIGH(cmpv));
+ tcg_gen_or_i64(t0, t0, t1);
+
+ /* tmpv = equal ? newv : oldv */
+ tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_LOW(tmpv), t0, z,
+ TCGV128_LOW(newv), TCGV128_LOW(oldv));
+ tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_HIGH(tmpv), t0, z,
+ TCGV128_HIGH(newv), TCGV128_HIGH(oldv));
+
+ /* Unconditional writeback. */
+ tcg_gen_qemu_st_i128(tmpv, addr, idx, memop);
+ tcg_gen_mov_i128(retv, oldv);
+
+ tcg_temp_free_i64(t0);
+ tcg_temp_free_i64(t1);
+ tcg_temp_free_i128(tmpv);
+ tcg_temp_free_i128(oldv);
+ }
+}
+
+void tcg_gen_atomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv,
+ TCGv_i128 newv, TCGArg idx, MemOp memop)
+{
+ gen_atomic_cx_i128 gen;
+
+ if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
+ tcg_gen_nonatomic_cmpxchg_i128(retv, addr, cmpv, newv, idx, memop);
+ return;
+ }
+
+ tcg_debug_assert((memop & MO_SIZE) == MO_128);
+ tcg_debug_assert((memop & MO_SIGN) == 0);
+ gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
+
+ if (gen) {
+ MemOpIdx oi = make_memop_idx(memop, idx);
+ gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
+ return;
+ }
+
+ gen_helper_exit_atomic(cpu_env);
+
+ /*
+ * Produce a result for a well-formed opcode stream. This satisfies
+ * liveness for set before used, which happens before this dead code
+ * is removed.
+ */
+ tcg_gen_movi_i64(TCGV128_LOW(retv), 0);
+ tcg_gen_movi_i64(TCGV128_HIGH(retv), 0);
+}
+
static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
TCGArg idx, MemOp memop, bool new_val,
void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
diff --git a/accel/tcg/atomic_common.c.inc b/accel/tcg/atomic_common.c.inc
index 6602d7689f..8f2ce43ee6 100644
--- a/accel/tcg/atomic_common.c.inc
+++ b/accel/tcg/atomic_common.c.inc
@@ -55,8 +55,53 @@ CMPXCHG_HELPER(cmpxchgq_be, uint64_t)
CMPXCHG_HELPER(cmpxchgq_le, uint64_t)
#endif
+#ifdef CONFIG_CMPXCHG128
+CMPXCHG_HELPER(cmpxchgo_be, Int128)
+CMPXCHG_HELPER(cmpxchgo_le, Int128)
+#endif
+
#undef CMPXCHG_HELPER
+Int128 HELPER(nonatomic_cmpxchgo_be)(CPUArchState *env, target_ulong addr,
+ Int128 cmpv, Int128 newv, uint32_t oi)
+{
+#if TCG_TARGET_REG_BITS == 32
+ uintptr_t ra = GETPC();
+ Int128 oldv;
+
+ oldv = cpu_ld16_be_mmu(env, addr, oi, ra);
+ if (int128_eq(oldv, cmpv)) {
+ cpu_st16_be_mmu(env, addr, newv, oi, ra);
+ } else {
+ /* Even with comparison failure, still need a write cycle. */
+ probe_write(env, addr, 16, get_mmuidx(oi), ra);
+ }
+ return oldv;
+#else
+ g_assert_not_reached();
+#endif
+}
+
+Int128 HELPER(nonatomic_cmpxchgo_le)(CPUArchState *env, target_ulong addr,
+ Int128 cmpv, Int128 newv, uint32_t oi)
+{
+#if TCG_TARGET_REG_BITS == 32
+ uintptr_t ra = GETPC();
+ Int128 oldv;
+
+ oldv = cpu_ld16_le_mmu(env, addr, oi, ra);
+ if (int128_eq(oldv, cmpv)) {
+ cpu_st16_le_mmu(env, addr, newv, oi, ra);
+ } else {
+ /* Even with comparison failure, still need a write cycle. */
+ probe_write(env, addr, 16, get_mmuidx(oi), ra);
+ }
+ return oldv;
+#else
+ g_assert_not_reached();
+#endif
+}
+
#define ATOMIC_HELPER(OP, TYPE) \
TYPE HELPER(glue(atomic_,OP))(CPUArchState *env, target_ulong addr, \
TYPE val, uint32_t oi) \
--
2.34.1
next prev parent reply other threads:[~2023-01-26 4:41 UTC|newest]
Thread overview: 63+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-01-26 4:37 [PATCH v5 00/36] tcg: Support for Int128 with helpers Richard Henderson
2023-01-26 4:37 ` [PATCH v5 01/36] tcg: Define TCG_TYPE_I128 and related helper macros Richard Henderson
2023-01-26 4:37 ` [PATCH v5 02/36] tcg: Handle dh_typecode_i128 with TCG_CALL_{RET, ARG}_NORMAL Richard Henderson
2023-01-26 4:37 ` [PATCH v5 03/36] tcg: Allocate objects contiguously in temp_allocate_frame Richard Henderson
2023-01-26 17:12 ` Alex Bennée
2023-01-26 19:48 ` Richard Henderson
2023-01-26 4:37 ` [PATCH v5 04/36] tcg: Introduce tcg_out_addi_ptr Richard Henderson
2023-01-26 4:37 ` [PATCH v5 05/36] tcg: Add TCG_CALL_{RET,ARG}_BY_REF Richard Henderson
2023-01-27 10:40 ` Alex Bennée
2023-01-27 18:48 ` Richard Henderson
2023-01-26 4:37 ` [PATCH v5 06/36] tcg: Introduce tcg_target_call_oarg_reg Richard Henderson
2023-01-26 4:37 ` [PATCH v5 07/36] tcg: Add TCG_CALL_RET_BY_VEC Richard Henderson
2023-01-26 4:37 ` [PATCH v5 08/36] include/qemu/int128: Use Int128 structure for TCI Richard Henderson
2023-01-27 13:51 ` Alex Bennée
2023-01-26 4:37 ` [PATCH v5 09/36] tcg/i386: Add TCG_TARGET_CALL_{RET,ARG}_I128 Richard Henderson
2023-01-27 13:52 ` Alex Bennée
2023-01-26 4:37 ` [PATCH v5 10/36] tcg/tci: Fix big-endian return register ordering Richard Henderson
2023-01-27 13:53 ` Alex Bennée
2023-01-26 4:37 ` [PATCH v5 11/36] tcg/tci: Add TCG_TARGET_CALL_{RET,ARG}_I128 Richard Henderson
2023-01-27 14:00 ` Alex Bennée
2023-01-27 18:55 ` Richard Henderson
2023-01-26 4:38 ` [PATCH v5 12/36] tcg: " Richard Henderson
2023-01-27 17:04 ` Alex Bennée
2023-01-26 4:38 ` [PATCH v5 13/36] tcg: Add temp allocation for TCGv_i128 Richard Henderson
2023-01-27 17:08 ` Alex Bennée
2023-01-27 18:56 ` Richard Henderson
2023-01-26 4:38 ` [PATCH v5 14/36] tcg: Add basic data movement " Richard Henderson
2023-01-27 18:23 ` Alex Bennée
2023-01-26 4:38 ` [PATCH v5 15/36] tcg: Add guest load/store primitives " Richard Henderson
2023-01-26 4:38 ` Richard Henderson [this message]
2023-01-27 0:45 ` [PATCH v5 16/36] tcg: Add tcg_gen_{non}atomic_cmpxchg_i128 Philippe Mathieu-Daudé
2023-01-27 6:39 ` Richard Henderson
2023-01-27 23:49 ` Philippe Mathieu-Daudé
2023-01-26 4:38 ` [PATCH v5 17/36] tcg: Split out tcg_gen_nonatomic_cmpxchg_i{32,64} Richard Henderson
2023-01-27 0:53 ` Philippe Mathieu-Daudé
2023-01-27 6:44 ` Richard Henderson
2023-01-26 4:38 ` [PATCH v5 18/36] target/arm: Use tcg_gen_atomic_cmpxchg_i128 for STXP Richard Henderson
2023-01-26 4:38 ` [PATCH v5 19/36] target/arm: Use tcg_gen_atomic_cmpxchg_i128 for CASP Richard Henderson
2023-01-26 4:38 ` [PATCH v5 20/36] target/ppc: Use tcg_gen_atomic_cmpxchg_i128 for STQCX Richard Henderson
2023-01-26 4:38 ` [PATCH v5 21/36] tests/tcg/s390x: Add div.c Richard Henderson
2023-01-26 4:38 ` [PATCH v5 22/36] tests/tcg/s390x: Add clst.c Richard Henderson
2023-01-26 4:38 ` [PATCH v5 23/36] tests/tcg/s390x: Add long-double.c Richard Henderson
2023-01-26 4:38 ` [PATCH v5 24/36] target/s390x: Use a single return for helper_divs32/u32 Richard Henderson
2023-01-26 9:58 ` David Hildenbrand
2023-01-27 0:57 ` Philippe Mathieu-Daudé
2023-01-26 4:38 ` [PATCH v5 25/36] target/s390x: Use a single return for helper_divs64/u64 Richard Henderson
2023-01-26 4:38 ` [PATCH v5 26/36] target/s390x: Use Int128 for return from CLST Richard Henderson
2023-01-26 4:38 ` [PATCH v5 27/36] target/s390x: Use Int128 for return from CKSM Richard Henderson
2023-01-26 4:38 ` [PATCH v5 28/36] target/s390x: Use Int128 for return from TRE Richard Henderson
2023-01-26 4:38 ` [PATCH v5 29/36] target/s390x: Copy wout_x1 to wout_x1_P Richard Henderson
2023-01-26 4:38 ` [PATCH v5 30/36] target/s390x: Use Int128 for returning float128 Richard Henderson
2023-01-26 10:06 ` David Hildenbrand
2023-01-26 4:38 ` [PATCH v5 31/36] target/s390x: Use Int128 for passing float128 Richard Henderson
2023-01-26 11:19 ` David Hildenbrand
2023-01-26 4:38 ` [PATCH v5 32/36] target/s390x: Use tcg_gen_atomic_cmpxchg_i128 for CDSG Richard Henderson
2023-01-26 11:27 ` David Hildenbrand
2023-01-26 21:01 ` Richard Henderson
2023-01-27 16:09 ` David Hildenbrand
2023-01-26 4:38 ` [PATCH v5 33/36] target/s390x: Implement CC_OP_NZ in gen_op_calc_cc Richard Henderson
2023-01-26 11:25 ` David Hildenbrand
2023-01-26 4:38 ` [PATCH v5 34/36] target/i386: Split out gen_cmpxchg8b, gen_cmpxchg16b Richard Henderson
2023-01-26 4:38 ` [PATCH v5 35/36] target/i386: Inline cmpxchg8b Richard Henderson
2023-01-26 4:38 ` [PATCH v5 36/36] target/i386: Inline cmpxchg16b Richard Henderson
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230126043824.54819-17-richard.henderson@linaro.org \
--to=richard.henderson@linaro.org \
--cc=alex.bennee@linaro.org \
--cc=qemu-devel@nongnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).