From: Richard Henderson <richard.henderson@linaro.org>
To: qemu-devel@nongnu.org
Cc: "Philippe Mathieu-Daudé" <philmd@linaro.org>
Subject: [PULL 18/22] tcg: Add host memory barriers to cpu_ldst.h interfaces
Date: Mon, 26 Jun 2023 17:39:41 +0200 [thread overview]
Message-ID: <20230626153945.76180-19-richard.henderson@linaro.org> (raw)
In-Reply-To: <20230626153945.76180-1-richard.henderson@linaro.org>
Bring the helpers into line with the rest of tcg in respecting
guest memory ordering.
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
accel/tcg/internal.h | 34 ++++++++++++++++++++++++++++++++++
accel/tcg/cputlb.c | 10 ++++++++++
accel/tcg/user-exec.c | 10 ++++++++++
3 files changed, 54 insertions(+)
diff --git a/accel/tcg/internal.h b/accel/tcg/internal.h
index 91f308bdfa..650c3ac53f 100644
--- a/accel/tcg/internal.h
+++ b/accel/tcg/internal.h
@@ -78,4 +78,38 @@ extern int64_t max_advance;
extern bool one_insn_per_tb;
+/**
+ * tcg_req_mo:
+ * @type: TCGBar
+ *
+ * Filter @type to the barrier that is required for the guest
+ * memory ordering vs the host memory ordering. A non-zero
+ * result indicates that some barrier is required.
+ *
+ * If TCG_GUEST_DEFAULT_MO is not defined, assume that the
+ * guest requires strict ordering.
+ *
+ * This is a macro so that it's constant even without optimization.
+ */
+#ifdef TCG_GUEST_DEFAULT_MO
+# define tcg_req_mo(type) \
+ ((type) & TCG_GUEST_DEFAULT_MO & ~TCG_TARGET_DEFAULT_MO)
+#else
+# define tcg_req_mo(type) ((type) & ~TCG_TARGET_DEFAULT_MO)
+#endif
+
+/**
+ * cpu_req_mo:
+ * @type: TCGBar
+ *
+ * If tcg_req_mo indicates a barrier for @type is required
+ * for the guest memory model, issue a host memory barrier.
+ */
+#define cpu_req_mo(type) \
+ do { \
+ if (tcg_req_mo(type)) { \
+ smp_mb(); \
+ } \
+ } while (0)
+
#endif /* ACCEL_TCG_INTERNAL_H */
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index e02cfc550e..5666a8e23a 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -2339,6 +2339,7 @@ static uint8_t do_ld1_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi,
MMULookupLocals l;
bool crosspage;
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
crosspage = mmu_lookup(env, addr, oi, ra, access_type, &l);
tcg_debug_assert(!crosspage);
@@ -2360,6 +2361,7 @@ static uint16_t do_ld2_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi,
uint16_t ret;
uint8_t a, b;
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
crosspage = mmu_lookup(env, addr, oi, ra, access_type, &l);
if (likely(!crosspage)) {
return do_ld_2(env, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
@@ -2390,6 +2392,7 @@ static uint32_t do_ld4_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi,
bool crosspage;
uint32_t ret;
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
crosspage = mmu_lookup(env, addr, oi, ra, access_type, &l);
if (likely(!crosspage)) {
return do_ld_4(env, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
@@ -2417,6 +2420,7 @@ static uint64_t do_ld8_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi,
bool crosspage;
uint64_t ret;
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
crosspage = mmu_lookup(env, addr, oi, ra, access_type, &l);
if (likely(!crosspage)) {
return do_ld_8(env, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
@@ -2469,6 +2473,7 @@ static Int128 do_ld16_mmu(CPUArchState *env, vaddr addr,
Int128 ret;
int first;
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD, &l);
if (likely(!crosspage)) {
/* Perform the load host endian. */
@@ -2802,6 +2807,7 @@ void helper_stb_mmu(CPUArchState *env, uint64_t addr, uint32_t val,
bool crosspage;
tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_8);
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE, &l);
tcg_debug_assert(!crosspage);
@@ -2815,6 +2821,7 @@ static void do_st2_mmu(CPUArchState *env, vaddr addr, uint16_t val,
bool crosspage;
uint8_t a, b;
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE, &l);
if (likely(!crosspage)) {
do_st_2(env, &l.page[0], val, l.mmu_idx, l.memop, ra);
@@ -2843,6 +2850,7 @@ static void do_st4_mmu(CPUArchState *env, vaddr addr, uint32_t val,
MMULookupLocals l;
bool crosspage;
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE, &l);
if (likely(!crosspage)) {
do_st_4(env, &l.page[0], val, l.mmu_idx, l.memop, ra);
@@ -2870,6 +2878,7 @@ static void do_st8_mmu(CPUArchState *env, vaddr addr, uint64_t val,
MMULookupLocals l;
bool crosspage;
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE, &l);
if (likely(!crosspage)) {
do_st_8(env, &l.page[0], val, l.mmu_idx, l.memop, ra);
@@ -2899,6 +2908,7 @@ static void do_st16_mmu(CPUArchState *env, vaddr addr, Int128 val,
uint64_t a, b;
int first;
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE, &l);
if (likely(!crosspage)) {
/* Swap to host endian if necessary, then store. */
diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c
index f8b16d6ab8..8fbcbf9771 100644
--- a/accel/tcg/user-exec.c
+++ b/accel/tcg/user-exec.c
@@ -914,6 +914,7 @@ static uint8_t do_ld1_mmu(CPUArchState *env, abi_ptr addr,
uint8_t ret;
tcg_debug_assert((mop & MO_SIZE) == MO_8);
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD);
ret = ldub_p(haddr);
clear_helper_retaddr();
@@ -947,6 +948,7 @@ static uint16_t do_ld2_mmu(CPUArchState *env, abi_ptr addr,
uint16_t ret;
tcg_debug_assert((mop & MO_SIZE) == MO_16);
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD);
ret = load_atom_2(env, ra, haddr, mop);
clear_helper_retaddr();
@@ -984,6 +986,7 @@ static uint32_t do_ld4_mmu(CPUArchState *env, abi_ptr addr,
uint32_t ret;
tcg_debug_assert((mop & MO_SIZE) == MO_32);
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD);
ret = load_atom_4(env, ra, haddr, mop);
clear_helper_retaddr();
@@ -1021,6 +1024,7 @@ static uint64_t do_ld8_mmu(CPUArchState *env, abi_ptr addr,
uint64_t ret;
tcg_debug_assert((mop & MO_SIZE) == MO_64);
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD);
ret = load_atom_8(env, ra, haddr, mop);
clear_helper_retaddr();
@@ -1052,6 +1056,7 @@ static Int128 do_ld16_mmu(CPUArchState *env, abi_ptr addr,
Int128 ret;
tcg_debug_assert((mop & MO_SIZE) == MO_128);
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD);
ret = load_atom_16(env, ra, haddr, mop);
clear_helper_retaddr();
@@ -1087,6 +1092,7 @@ static void do_st1_mmu(CPUArchState *env, abi_ptr addr, uint8_t val,
void *haddr;
tcg_debug_assert((mop & MO_SIZE) == MO_8);
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE);
stb_p(haddr, val);
clear_helper_retaddr();
@@ -1111,6 +1117,7 @@ static void do_st2_mmu(CPUArchState *env, abi_ptr addr, uint16_t val,
void *haddr;
tcg_debug_assert((mop & MO_SIZE) == MO_16);
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE);
if (mop & MO_BSWAP) {
@@ -1139,6 +1146,7 @@ static void do_st4_mmu(CPUArchState *env, abi_ptr addr, uint32_t val,
void *haddr;
tcg_debug_assert((mop & MO_SIZE) == MO_32);
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE);
if (mop & MO_BSWAP) {
@@ -1167,6 +1175,7 @@ static void do_st8_mmu(CPUArchState *env, abi_ptr addr, uint64_t val,
void *haddr;
tcg_debug_assert((mop & MO_SIZE) == MO_64);
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE);
if (mop & MO_BSWAP) {
@@ -1195,6 +1204,7 @@ static void do_st16_mmu(CPUArchState *env, abi_ptr addr, Int128 val,
void *haddr;
tcg_debug_assert((mop & MO_SIZE) == MO_128);
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE);
if (mop & MO_BSWAP) {
--
2.34.1
next prev parent reply other threads:[~2023-06-26 15:45 UTC|newest]
Thread overview: 26+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-06-26 15:39 [PULL 00/22] tcg patch queue Richard Henderson
2023-06-26 15:39 ` [PULL 01/22] accel: Replace target_ulong in tlb_*() Richard Henderson
2023-06-26 15:39 ` [PULL 02/22] accel/tcg/translate-all.c: Widen pc and cs_base Richard Henderson
2023-07-11 16:39 ` Peter Maydell
2023-06-26 15:39 ` [PULL 03/22] target: Widen pc/cs_base in cpu_get_tb_cpu_state Richard Henderson
2023-06-26 15:39 ` [PULL 04/22] accel/tcg/cputlb.c: Widen CPUTLBEntry access functions Richard Henderson
2023-06-26 15:39 ` [PULL 05/22] accel/tcg/cputlb.c: Widen addr in MMULookupPageData Richard Henderson
2023-06-26 15:39 ` [PULL 06/22] accel/tcg/cpu-exec.c: Widen pc to vaddr Richard Henderson
2023-07-11 16:40 ` Peter Maydell
2023-06-26 15:39 ` [PULL 07/22] accel/tcg: Widen pc to vaddr in CPUJumpCache Richard Henderson
2023-06-26 15:39 ` [PULL 08/22] accel: Replace target_ulong with vaddr in probe_*() Richard Henderson
2023-06-26 15:39 ` [PULL 09/22] accel/tcg: Replace target_ulong with vaddr in *_mmu_lookup() Richard Henderson
2023-06-26 15:39 ` [PULL 10/22] accel/tcg: Replace target_ulong with vaddr in translator_*() Richard Henderson
2023-06-26 15:39 ` [PULL 11/22] cpu: Replace target_ulong with hwaddr in tb_invalidate_phys_addr() Richard Henderson
2023-06-26 15:39 ` [PULL 12/22] softfloat: use QEMU_FLATTEN to avoid mistaken isra inlining Richard Henderson
2023-06-26 15:39 ` [PULL 13/22] tests/plugin: Remove duplicate insn log from libinsn.so Richard Henderson
2023-06-26 15:39 ` [PULL 14/22] accel/tcg: remove CONFIG_PROFILER Richard Henderson
2023-06-26 15:39 ` [PULL 15/22] tcg: Fix temporary variable in tcg_gen_gvec_andcs Richard Henderson
2023-06-26 15:39 ` [PULL 16/22] target/microblaze: Define TCG_GUEST_DEFAULT_MO Richard Henderson
2023-06-26 15:39 ` [PULL 17/22] tcg: Do not elide memory barriers for !CF_PARALLEL in system mode Richard Henderson
2023-06-26 15:39 ` Richard Henderson [this message]
2023-06-26 15:39 ` [PULL 19/22] accel/tcg: Remove check_tcg_memory_orders_compatible Richard Henderson
2023-06-26 15:39 ` [PULL 20/22] accel/tcg: Store some tlb flags in CPUTLBEntryFull Richard Henderson
2023-06-26 15:39 ` [PULL 21/22] accel/tcg: Move TLB_WATCHPOINT to TLB_SLOW_FLAGS_MASK Richard Henderson
2023-06-26 15:39 ` [PULL 22/22] accel/tcg: Renumber TLB_DISCARD_WRITE Richard Henderson
2023-06-26 18:11 ` [PULL 00/22] tcg patch queue Richard Henderson
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230626153945.76180-19-richard.henderson@linaro.org \
--to=richard.henderson@linaro.org \
--cc=philmd@linaro.org \
--cc=qemu-devel@nongnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).