From: "Alex Bennée" <alex.bennee@linaro.org>
To: Richard Henderson <rth@twiddle.net>
Cc: qemu-devel@nongnu.org, cota@braap.org
Subject: Re: [Qemu-devel] [PATCH v7 13/35] tcg: Add atomic helpers
Date: Thu, 20 Oct 2016 11:49:05 +0100 [thread overview]
Message-ID: <87bmyffg7y.fsf@linaro.org> (raw)
In-Reply-To: <1476897727-791-14-git-send-email-rth@twiddle.net>
Richard Henderson <rth@twiddle.net> writes:
> Add all of cmpxchg, op_fetch, fetch_op, and xchg.
> Handle both endian-ness, and sizes up to 8.
> Handle expanding non-atomically, when emulating in serial.
>
> Reviewed-by: Emilio G. Cota <cota@braap.org>
> Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
> Signed-off-by: Richard Henderson <rth@twiddle.net>
<snip>
>
> /* Provide shorter names for GCC atomic builtins, return old value */
> #define atomic_fetch_inc(ptr) __atomic_fetch_add(ptr, 1, __ATOMIC_SEQ_CST)
> @@ -308,8 +321,11 @@
> /* These will only be atomic if the processor does the fetch or store
> * in a single issue memory operation
> */
> -#define atomic_read(ptr) (*(__typeof__(*ptr) volatile*) (ptr))
> -#define atomic_set(ptr, i) ((*(__typeof__(*ptr) volatile*) (ptr)) = (i))
> +#define atomic_read__nocheck(p) (*(__typeof__(*ptr) volatile*) (p))
> +#define atomic_set__nocheck(p, i) ((*(__typeof__(*ptr) volatile*) (p)) = (i))
> +
> +#define atomic_read(ptr) atomic_read__nocheck(ptr)
> +#define atomic_set(ptr, i) atomic_set__nocheck(ptr,i)
s/ptr/p/ during the transposition:
/* These will only be atomic if the processor does the fetch or store
* in a single issue memory operation
*/
#define atomic_read__nocheck(p) (*(__typeof__(*p) volatile*) (p))
#define atomic_set__nocheck(p, i) ((*(__typeof__(*p) volatile*) (p)) = (i))
#define atomic_read(ptr) atomic_read__nocheck(ptr)
#define atomic_set(ptr, i) atomic_set__nocheck(ptr, i)
Otherwise fine.
>
> /**
> * atomic_rcu_read - reads a RCU-protected pointer to a local variable
> @@ -395,25 +411,27 @@
> #define atomic_xchg(ptr, i) (smp_mb(), __sync_lock_test_and_set(ptr, i))
> #endif
> #endif
> +#define atomic_xchg__nocheck atomic_xchg
>
> /* Provide shorter names for GCC atomic builtins. */
> #define atomic_fetch_inc(ptr) __sync_fetch_and_add(ptr, 1)
> #define atomic_fetch_dec(ptr) __sync_fetch_and_add(ptr, -1)
> -#define atomic_fetch_add __sync_fetch_and_add
> -#define atomic_fetch_sub __sync_fetch_and_sub
> -#define atomic_fetch_and __sync_fetch_and_and
> -#define atomic_fetch_or __sync_fetch_and_or
> -#define atomic_fetch_xor __sync_fetch_and_xor
> +#define atomic_fetch_add(ptr, n) __sync_fetch_and_add(ptr, n)
> +#define atomic_fetch_sub(ptr, n) __sync_fetch_and_sub(ptr, n)
> +#define atomic_fetch_and(ptr, n) __sync_fetch_and_and(ptr, n)
> +#define atomic_fetch_or(ptr, n) __sync_fetch_and_or(ptr, n)
> +#define atomic_fetch_xor(ptr, n) __sync_fetch_and_xor(ptr, n)
>
> #define atomic_inc_fetch(ptr) __sync_add_and_fetch(ptr, 1)
> #define atomic_dec_fetch(ptr) __sync_add_and_fetch(ptr, -1)
> -#define atomic_add_fetch __sync_add_and_fetch
> -#define atomic_sub_fetch __sync_sub_and_fetch
> -#define atomic_and_fetch __sync_and_and_fetch
> -#define atomic_or_fetch __sync_or_and_fetch
> -#define atomic_xor_fetch __sync_xor_and_fetch
> -
> -#define atomic_cmpxchg __sync_val_compare_and_swap
> +#define atomic_add_fetch(ptr, n) __sync_add_and_fetch(ptr, n)
> +#define atomic_sub_fetch(ptr, n) __sync_sub_and_fetch(ptr, n)
> +#define atomic_and_fetch(ptr, n) __sync_and_and_fetch(ptr, n)
> +#define atomic_or_fetch(ptr, n) __sync_or_and_fetch(ptr, n)
> +#define atomic_xor_fetch(ptr, n) __sync_xor_and_fetch(ptr, n)
> +
> +#define atomic_cmpxchg(ptr, old, new) __sync_val_compare_and_swap(ptr, old, new)
> +#define atomic_cmpxchg__nocheck(ptr, old, new) atomic_cmpxchg(ptr, old, new)
>
> /* And even shorter names that return void. */
> #define atomic_inc(ptr) ((void) __sync_fetch_and_add(ptr, 1))
> diff --git a/tcg-runtime.c b/tcg-runtime.c
> index ea2ad64..aa55d12 100644
> --- a/tcg-runtime.c
> +++ b/tcg-runtime.c
> @@ -23,17 +23,10 @@
> */
> #include "qemu/osdep.h"
> #include "qemu/host-utils.h"
> -
> -/* This file is compiled once, and thus we can't include the standard
> - "exec/helper-proto.h", which has includes that are target specific. */
> -
> -#include "exec/helper-head.h"
> -
> -#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \
> - dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2));
> -
> -#include "tcg-runtime.h"
> -
> +#include "cpu.h"
> +#include "exec/helper-proto.h"
> +#include "exec/cpu_ldst.h"
> +#include "exec/exec-all.h"
>
> /* 32-bit helpers */
>
> @@ -107,3 +100,37 @@ int64_t HELPER(mulsh_i64)(int64_t arg1, int64_t arg2)
> muls64(&l, &h, arg1, arg2);
> return h;
> }
> +
> +#ifndef CONFIG_SOFTMMU
> +/* The softmmu versions of these helpers are in cputlb.c. */
> +
> +/* Do not allow unaligned operations to proceed. Return the host address. */
> +static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
> + int size, uintptr_t retaddr)
> +{
> + /* Enforce qemu required alignment. */
> + if (unlikely(addr & (size - 1))) {
> + cpu_loop_exit_atomic(ENV_GET_CPU(env), retaddr);
> + }
> + return g2h(addr);
> +}
> +
> +/* Macro to call the above, with local variables from the use context. */
> +#define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, DATA_SIZE, GETPC())
> +
> +#define ATOMIC_NAME(X) HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
> +#define EXTRA_ARGS
> +
> +#define DATA_SIZE 1
> +#include "atomic_template.h"
> +
> +#define DATA_SIZE 2
> +#include "atomic_template.h"
> +
> +#define DATA_SIZE 4
> +#include "atomic_template.h"
> +
> +#define DATA_SIZE 8
> +#include "atomic_template.h"
> +
> +#endif /* !CONFIG_SOFTMMU */
> diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c
> index 291d50b..65e3663 100644
> --- a/tcg/tcg-op.c
> +++ b/tcg/tcg-op.c
> @@ -1975,3 +1975,331 @@ void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
> addr, trace_mem_get_info(memop, 1));
> gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
> }
> +
> +static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc)
> +{
> + switch (opc & MO_SSIZE) {
> + case MO_SB:
> + tcg_gen_ext8s_i32(ret, val);
> + break;
> + case MO_UB:
> + tcg_gen_ext8u_i32(ret, val);
> + break;
> + case MO_SW:
> + tcg_gen_ext16s_i32(ret, val);
> + break;
> + case MO_UW:
> + tcg_gen_ext16u_i32(ret, val);
> + break;
> + default:
> + tcg_gen_mov_i32(ret, val);
> + break;
> + }
> +}
> +
> +static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc)
> +{
> + switch (opc & MO_SSIZE) {
> + case MO_SB:
> + tcg_gen_ext8s_i64(ret, val);
> + break;
> + case MO_UB:
> + tcg_gen_ext8u_i64(ret, val);
> + break;
> + case MO_SW:
> + tcg_gen_ext16s_i64(ret, val);
> + break;
> + case MO_UW:
> + tcg_gen_ext16u_i64(ret, val);
> + break;
> + case MO_SL:
> + tcg_gen_ext32s_i64(ret, val);
> + break;
> + case MO_UL:
> + tcg_gen_ext32u_i64(ret, val);
> + break;
> + default:
> + tcg_gen_mov_i64(ret, val);
> + break;
> + }
> +}
> +
> +#ifdef CONFIG_SOFTMMU
> +typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
> + TCGv_i32, TCGv_i32, TCGv_i32);
> +typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
> + TCGv_i64, TCGv_i64, TCGv_i32);
> +typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
> + TCGv_i32, TCGv_i32);
> +typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
> + TCGv_i64, TCGv_i32);
> +#else
> +typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32);
> +typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64);
> +typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32);
> +typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64);
> +#endif
> +
> +static void * const table_cmpxchg[16] = {
> + [MO_8] = gen_helper_atomic_cmpxchgb,
> + [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
> + [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
> + [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
> + [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
> + [MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le,
> + [MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be,
> +};
> +
> +void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
> + TCGv_i32 newv, TCGArg idx, TCGMemOp memop)
> +{
> + memop = tcg_canonicalize_memop(memop, 0, 0);
> +
> + if (!parallel_cpus) {
> + TCGv_i32 t1 = tcg_temp_new_i32();
> + TCGv_i32 t2 = tcg_temp_new_i32();
> +
> + tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
> +
> + tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
> + tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
> + tcg_gen_qemu_st_i32(t2, addr, idx, memop);
> + tcg_temp_free_i32(t2);
> +
> + if (memop & MO_SIGN) {
> + tcg_gen_ext_i32(retv, t1, memop);
> + } else {
> + tcg_gen_mov_i32(retv, t1);
> + }
> + tcg_temp_free_i32(t1);
> + } else {
> + gen_atomic_cx_i32 gen;
> +
> + gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
> + tcg_debug_assert(gen != NULL);
> +
> +#ifdef CONFIG_SOFTMMU
> + {
> + TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
> + gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv, oi);
> + tcg_temp_free_i32(oi);
> + }
> +#else
> + gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv);
> +#endif
> +
> + if (memop & MO_SIGN) {
> + tcg_gen_ext_i32(retv, retv, memop);
> + }
> + }
> +}
> +
> +void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
> + TCGv_i64 newv, TCGArg idx, TCGMemOp memop)
> +{
> + memop = tcg_canonicalize_memop(memop, 1, 0);
> +
> + if (!parallel_cpus) {
> + TCGv_i64 t1 = tcg_temp_new_i64();
> + TCGv_i64 t2 = tcg_temp_new_i64();
> +
> + tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
> +
> + tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
> + tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
> + tcg_gen_qemu_st_i64(t2, addr, idx, memop);
> + tcg_temp_free_i64(t2);
> +
> + if (memop & MO_SIGN) {
> + tcg_gen_ext_i64(retv, t1, memop);
> + } else {
> + tcg_gen_mov_i64(retv, t1);
> + }
> + tcg_temp_free_i64(t1);
> + } else if ((memop & MO_SIZE) == MO_64) {
> + gen_atomic_cx_i64 gen;
> +
> + gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
> + tcg_debug_assert(gen != NULL);
> +
> +#ifdef CONFIG_SOFTMMU
> + {
> + TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx));
> + gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv, oi);
> + tcg_temp_free_i32(oi);
> + }
> +#else
> + gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv);
> +#endif
> + } else {
> + TCGv_i32 c32 = tcg_temp_new_i32();
> + TCGv_i32 n32 = tcg_temp_new_i32();
> + TCGv_i32 r32 = tcg_temp_new_i32();
> +
> + tcg_gen_extrl_i64_i32(c32, cmpv);
> + tcg_gen_extrl_i64_i32(n32, newv);
> + tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
> + tcg_temp_free_i32(c32);
> + tcg_temp_free_i32(n32);
> +
> + tcg_gen_extu_i32_i64(retv, r32);
> + tcg_temp_free_i32(r32);
> +
> + if (memop & MO_SIGN) {
> + tcg_gen_ext_i64(retv, retv, memop);
> + }
> + }
> +}
> +
> +static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
> + TCGArg idx, TCGMemOp memop, bool new_val,
> + void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
> +{
> + TCGv_i32 t1 = tcg_temp_new_i32();
> + TCGv_i32 t2 = tcg_temp_new_i32();
> +
> + memop = tcg_canonicalize_memop(memop, 0, 0);
> +
> + tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
> + gen(t2, t1, val);
> + tcg_gen_qemu_st_i32(t2, addr, idx, memop);
> +
> + tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
> + tcg_temp_free_i32(t1);
> + tcg_temp_free_i32(t2);
> +}
> +
> +static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
> + TCGArg idx, TCGMemOp memop, void * const table[])
> +{
> + gen_atomic_op_i32 gen;
> +
> + memop = tcg_canonicalize_memop(memop, 0, 0);
> +
> + gen = table[memop & (MO_SIZE | MO_BSWAP)];
> + tcg_debug_assert(gen != NULL);
> +
> +#ifdef CONFIG_SOFTMMU
> + {
> + TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
> + gen(ret, tcg_ctx.tcg_env, addr, val, oi);
> + tcg_temp_free_i32(oi);
> + }
> +#else
> + gen(ret, tcg_ctx.tcg_env, addr, val);
> +#endif
> +
> + if (memop & MO_SIGN) {
> + tcg_gen_ext_i32(ret, ret, memop);
> + }
> +}
> +
> +static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
> + TCGArg idx, TCGMemOp memop, bool new_val,
> + void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
> +{
> + TCGv_i64 t1 = tcg_temp_new_i64();
> + TCGv_i64 t2 = tcg_temp_new_i64();
> +
> + memop = tcg_canonicalize_memop(memop, 1, 0);
> +
> + tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
> + gen(t2, t1, val);
> + tcg_gen_qemu_st_i64(t2, addr, idx, memop);
> +
> + tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
> + tcg_temp_free_i64(t1);
> + tcg_temp_free_i64(t2);
> +}
> +
> +static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
> + TCGArg idx, TCGMemOp memop, void * const table[])
> +{
> + memop = tcg_canonicalize_memop(memop, 1, 0);
> +
> + if ((memop & MO_SIZE) == MO_64) {
> + gen_atomic_op_i64 gen;
> +
> + gen = table[memop & (MO_SIZE | MO_BSWAP)];
> + tcg_debug_assert(gen != NULL);
> +
> +#ifdef CONFIG_SOFTMMU
> + {
> + TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
> + gen(ret, tcg_ctx.tcg_env, addr, val, oi);
> + tcg_temp_free_i32(oi);
> + }
> +#else
> + gen(ret, tcg_ctx.tcg_env, addr, val);
> +#endif
> + } else {
> + TCGv_i32 v32 = tcg_temp_new_i32();
> + TCGv_i32 r32 = tcg_temp_new_i32();
> +
> + tcg_gen_extrl_i64_i32(v32, val);
> + do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
> + tcg_temp_free_i32(v32);
> +
> + tcg_gen_extu_i32_i64(ret, r32);
> + tcg_temp_free_i32(r32);
> +
> + if (memop & MO_SIGN) {
> + tcg_gen_ext_i64(ret, ret, memop);
> + }
> + }
> +}
> +
> +#define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
> +static void * const table_##NAME[16] = { \
> + [MO_8] = gen_helper_atomic_##NAME##b, \
> + [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
> + [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
> + [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
> + [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
> + [MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le, \
> + [MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be, \
> +}; \
> +void tcg_gen_atomic_##NAME##_i32 \
> + (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \
> +{ \
> + if (parallel_cpus) { \
> + do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
> + } else { \
> + do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
> + tcg_gen_##OP##_i32); \
> + } \
> +} \
> +void tcg_gen_atomic_##NAME##_i64 \
> + (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \
> +{ \
> + if (parallel_cpus) { \
> + do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
> + } else { \
> + do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
> + tcg_gen_##OP##_i64); \
> + } \
> +}
> +
> +GEN_ATOMIC_HELPER(fetch_add, add, 0)
> +GEN_ATOMIC_HELPER(fetch_and, and, 0)
> +GEN_ATOMIC_HELPER(fetch_or, or, 0)
> +GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
> +
> +GEN_ATOMIC_HELPER(add_fetch, add, 1)
> +GEN_ATOMIC_HELPER(and_fetch, and, 1)
> +GEN_ATOMIC_HELPER(or_fetch, or, 1)
> +GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
> +
> +static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
> +{
> + tcg_gen_mov_i32(r, b);
> +}
> +
> +static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
> +{
> + tcg_gen_mov_i64(r, b);
> +}
> +
> +GEN_ATOMIC_HELPER(xchg, mov2, 0)
> +
> +#undef GEN_ATOMIC_HELPER
> diff --git a/tcg/tcg-op.h b/tcg/tcg-op.h
> index 02cb376..89b59e8 100644
> --- a/tcg/tcg-op.h
> +++ b/tcg/tcg-op.h
> @@ -854,6 +854,30 @@ static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
> tcg_gen_qemu_st_i64(arg, addr, mem_index, MO_TEQ);
> }
>
> +void tcg_gen_atomic_cmpxchg_i32(TCGv_i32, TCGv, TCGv_i32, TCGv_i32,
> + TCGArg, TCGMemOp);
> +void tcg_gen_atomic_cmpxchg_i64(TCGv_i64, TCGv, TCGv_i64, TCGv_i64,
> + TCGArg, TCGMemOp);
> +
> +void tcg_gen_atomic_xchg_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_xchg_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_fetch_add_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_fetch_add_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_fetch_and_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_fetch_and_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_fetch_or_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_fetch_or_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_fetch_xor_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_fetch_xor_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_add_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_add_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_and_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_and_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_or_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_or_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_xor_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, TCGMemOp);
> +void tcg_gen_atomic_xor_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, TCGMemOp);
> +
> #if TARGET_LONG_BITS == 64
> #define tcg_gen_movi_tl tcg_gen_movi_i64
> #define tcg_gen_mov_tl tcg_gen_mov_i64
> @@ -932,6 +956,16 @@ static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
> #define tcg_gen_sub2_tl tcg_gen_sub2_i64
> #define tcg_gen_mulu2_tl tcg_gen_mulu2_i64
> #define tcg_gen_muls2_tl tcg_gen_muls2_i64
> +#define tcg_gen_atomic_cmpxchg_tl tcg_gen_atomic_cmpxchg_i64
> +#define tcg_gen_atomic_xchg_tl tcg_gen_atomic_xchg_i64
> +#define tcg_gen_atomic_fetch_add_tl tcg_gen_atomic_fetch_add_i64
> +#define tcg_gen_atomic_fetch_and_tl tcg_gen_atomic_fetch_and_i64
> +#define tcg_gen_atomic_fetch_or_tl tcg_gen_atomic_fetch_or_i64
> +#define tcg_gen_atomic_fetch_xor_tl tcg_gen_atomic_fetch_xor_i64
> +#define tcg_gen_atomic_add_fetch_tl tcg_gen_atomic_add_fetch_i64
> +#define tcg_gen_atomic_and_fetch_tl tcg_gen_atomic_and_fetch_i64
> +#define tcg_gen_atomic_or_fetch_tl tcg_gen_atomic_or_fetch_i64
> +#define tcg_gen_atomic_xor_fetch_tl tcg_gen_atomic_xor_fetch_i64
> #else
> #define tcg_gen_movi_tl tcg_gen_movi_i32
> #define tcg_gen_mov_tl tcg_gen_mov_i32
> @@ -1009,6 +1043,16 @@ static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
> #define tcg_gen_sub2_tl tcg_gen_sub2_i32
> #define tcg_gen_mulu2_tl tcg_gen_mulu2_i32
> #define tcg_gen_muls2_tl tcg_gen_muls2_i32
> +#define tcg_gen_atomic_cmpxchg_tl tcg_gen_atomic_cmpxchg_i32
> +#define tcg_gen_atomic_xchg_tl tcg_gen_atomic_xchg_i32
> +#define tcg_gen_atomic_fetch_add_tl tcg_gen_atomic_fetch_add_i32
> +#define tcg_gen_atomic_fetch_and_tl tcg_gen_atomic_fetch_and_i32
> +#define tcg_gen_atomic_fetch_or_tl tcg_gen_atomic_fetch_or_i32
> +#define tcg_gen_atomic_fetch_xor_tl tcg_gen_atomic_fetch_xor_i32
> +#define tcg_gen_atomic_add_fetch_tl tcg_gen_atomic_add_fetch_i32
> +#define tcg_gen_atomic_and_fetch_tl tcg_gen_atomic_and_fetch_i32
> +#define tcg_gen_atomic_or_fetch_tl tcg_gen_atomic_or_fetch_i32
> +#define tcg_gen_atomic_xor_fetch_tl tcg_gen_atomic_xor_fetch_i32
> #endif
>
> #if UINTPTR_MAX == UINT32_MAX
> diff --git a/tcg/tcg-runtime.h b/tcg/tcg-runtime.h
> index 23a0c37..22367aa 100644
> --- a/tcg/tcg-runtime.h
> +++ b/tcg/tcg-runtime.h
> @@ -14,3 +14,78 @@ DEF_HELPER_FLAGS_2(sar_i64, TCG_CALL_NO_RWG_SE, s64, s64, s64)
>
> DEF_HELPER_FLAGS_2(mulsh_i64, TCG_CALL_NO_RWG_SE, s64, s64, s64)
> DEF_HELPER_FLAGS_2(muluh_i64, TCG_CALL_NO_RWG_SE, i64, i64, i64)
> +
> +#ifdef CONFIG_SOFTMMU
> +
> +DEF_HELPER_FLAGS_5(atomic_cmpxchgb, TCG_CALL_NO_WG,
> + i32, env, tl, i32, i32, i32)
> +DEF_HELPER_FLAGS_5(atomic_cmpxchgw_be, TCG_CALL_NO_WG,
> + i32, env, tl, i32, i32, i32)
> +DEF_HELPER_FLAGS_5(atomic_cmpxchgl_be, TCG_CALL_NO_WG,
> + i32, env, tl, i32, i32, i32)
> +DEF_HELPER_FLAGS_5(atomic_cmpxchgq_be, TCG_CALL_NO_WG,
> + i64, env, tl, i64, i64, i32)
> +DEF_HELPER_FLAGS_5(atomic_cmpxchgw_le, TCG_CALL_NO_WG,
> + i32, env, tl, i32, i32, i32)
> +DEF_HELPER_FLAGS_5(atomic_cmpxchgl_le, TCG_CALL_NO_WG,
> + i32, env, tl, i32, i32, i32)
> +DEF_HELPER_FLAGS_5(atomic_cmpxchgq_le, TCG_CALL_NO_WG,
> + i64, env, tl, i64, i64, i32)
> +
> +#define GEN_ATOMIC_HELPERS(NAME) \
> + DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), b), \
> + TCG_CALL_NO_WG, i32, env, tl, i32, i32) \
> + DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), w_le), \
> + TCG_CALL_NO_WG, i32, env, tl, i32, i32) \
> + DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), w_be), \
> + TCG_CALL_NO_WG, i32, env, tl, i32, i32) \
> + DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), l_le), \
> + TCG_CALL_NO_WG, i32, env, tl, i32, i32) \
> + DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), l_be), \
> + TCG_CALL_NO_WG, i32, env, tl, i32, i32) \
> + DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), q_le), \
> + TCG_CALL_NO_WG, i64, env, tl, i64, i32) \
> + DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), q_be), \
> + TCG_CALL_NO_WG, i64, env, tl, i64, i32)
> +
> +#else
> +
> +DEF_HELPER_FLAGS_4(atomic_cmpxchgb, TCG_CALL_NO_WG, i32, env, tl, i32, i32)
> +DEF_HELPER_FLAGS_4(atomic_cmpxchgw_be, TCG_CALL_NO_WG, i32, env, tl, i32, i32)
> +DEF_HELPER_FLAGS_4(atomic_cmpxchgl_be, TCG_CALL_NO_WG, i32, env, tl, i32, i32)
> +DEF_HELPER_FLAGS_4(atomic_cmpxchgq_be, TCG_CALL_NO_WG, i64, env, tl, i64, i64)
> +DEF_HELPER_FLAGS_4(atomic_cmpxchgw_le, TCG_CALL_NO_WG, i32, env, tl, i32, i32)
> +DEF_HELPER_FLAGS_4(atomic_cmpxchgl_le, TCG_CALL_NO_WG, i32, env, tl, i32, i32)
> +DEF_HELPER_FLAGS_4(atomic_cmpxchgq_le, TCG_CALL_NO_WG, i64, env, tl, i64, i64)
> +
> +#define GEN_ATOMIC_HELPERS(NAME) \
> + DEF_HELPER_FLAGS_3(glue(glue(atomic_, NAME), b), \
> + TCG_CALL_NO_WG, i32, env, tl, i32) \
> + DEF_HELPER_FLAGS_3(glue(glue(atomic_, NAME), w_le), \
> + TCG_CALL_NO_WG, i32, env, tl, i32) \
> + DEF_HELPER_FLAGS_3(glue(glue(atomic_, NAME), w_be), \
> + TCG_CALL_NO_WG, i32, env, tl, i32) \
> + DEF_HELPER_FLAGS_3(glue(glue(atomic_, NAME), l_le), \
> + TCG_CALL_NO_WG, i32, env, tl, i32) \
> + DEF_HELPER_FLAGS_3(glue(glue(atomic_, NAME), l_be), \
> + TCG_CALL_NO_WG, i32, env, tl, i32) \
> + DEF_HELPER_FLAGS_3(glue(glue(atomic_, NAME), q_le), \
> + TCG_CALL_NO_WG, i64, env, tl, i64) \
> + DEF_HELPER_FLAGS_3(glue(glue(atomic_, NAME), q_be), \
> + TCG_CALL_NO_WG, i64, env, tl, i64)
> +
> +#endif /* CONFIG_SOFTMMU */
> +
> +GEN_ATOMIC_HELPERS(fetch_add)
> +GEN_ATOMIC_HELPERS(fetch_and)
> +GEN_ATOMIC_HELPERS(fetch_or)
> +GEN_ATOMIC_HELPERS(fetch_xor)
> +
> +GEN_ATOMIC_HELPERS(add_fetch)
> +GEN_ATOMIC_HELPERS(and_fetch)
> +GEN_ATOMIC_HELPERS(or_fetch)
> +GEN_ATOMIC_HELPERS(xor_fetch)
> +
> +GEN_ATOMIC_HELPERS(xchg)
> +
> +#undef GEN_ATOMIC_HELPERS
> diff --git a/tcg/tcg.h b/tcg/tcg.h
> index 3b21156..5931965 100644
> --- a/tcg/tcg.h
> +++ b/tcg/tcg.h
> @@ -1177,6 +1177,59 @@ uint64_t helper_be_ldq_cmmu(CPUArchState *env, target_ulong addr,
> # define helper_ret_ldq_cmmu helper_le_ldq_cmmu
> #endif
>
> +uint32_t helper_atomic_cmpxchgb_mmu(CPUArchState *env, target_ulong addr,
> + uint32_t cmpv, uint32_t newv,
> + TCGMemOpIdx oi, uintptr_t retaddr);
> +uint32_t helper_atomic_cmpxchgw_le_mmu(CPUArchState *env, target_ulong addr,
> + uint32_t cmpv, uint32_t newv,
> + TCGMemOpIdx oi, uintptr_t retaddr);
> +uint32_t helper_atomic_cmpxchgl_le_mmu(CPUArchState *env, target_ulong addr,
> + uint32_t cmpv, uint32_t newv,
> + TCGMemOpIdx oi, uintptr_t retaddr);
> +uint64_t helper_atomic_cmpxchgq_le_mmu(CPUArchState *env, target_ulong addr,
> + uint64_t cmpv, uint64_t newv,
> + TCGMemOpIdx oi, uintptr_t retaddr);
> +uint32_t helper_atomic_cmpxchgw_be_mmu(CPUArchState *env, target_ulong addr,
> + uint32_t cmpv, uint32_t newv,
> + TCGMemOpIdx oi, uintptr_t retaddr);
> +uint32_t helper_atomic_cmpxchgl_be_mmu(CPUArchState *env, target_ulong addr,
> + uint32_t cmpv, uint32_t newv,
> + TCGMemOpIdx oi, uintptr_t retaddr);
> +uint64_t helper_atomic_cmpxchgq_be_mmu(CPUArchState *env, target_ulong addr,
> + uint64_t cmpv, uint64_t newv,
> + TCGMemOpIdx oi, uintptr_t retaddr);
> +
> +#define GEN_ATOMIC_HELPER(NAME, TYPE, SUFFIX) \
> +TYPE helper_atomic_ ## NAME ## SUFFIX ## _mmu \
> + (CPUArchState *env, target_ulong addr, TYPE val, \
> + TCGMemOpIdx oi, uintptr_t retaddr);
> +
> +#define GEN_ATOMIC_HELPER_ALL(NAME) \
> + GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
> + GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
> + GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
> + GEN_ATOMIC_HELPER(NAME, uint64_t, q_le) \
> + GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
> + GEN_ATOMIC_HELPER(NAME, uint32_t, l_be) \
> + GEN_ATOMIC_HELPER(NAME, uint64_t, q_be)
> +
> +GEN_ATOMIC_HELPER_ALL(fetch_add)
> +GEN_ATOMIC_HELPER_ALL(fetch_sub)
> +GEN_ATOMIC_HELPER_ALL(fetch_and)
> +GEN_ATOMIC_HELPER_ALL(fetch_or)
> +GEN_ATOMIC_HELPER_ALL(fetch_xor)
> +
> +GEN_ATOMIC_HELPER_ALL(add_fetch)
> +GEN_ATOMIC_HELPER_ALL(sub_fetch)
> +GEN_ATOMIC_HELPER_ALL(and_fetch)
> +GEN_ATOMIC_HELPER_ALL(or_fetch)
> +GEN_ATOMIC_HELPER_ALL(xor_fetch)
> +
> +GEN_ATOMIC_HELPER_ALL(xchg)
> +
> +#undef GEN_ATOMIC_HELPER_ALL
> +#undef GEN_ATOMIC_HELPER
> +
> #endif /* CONFIG_SOFTMMU */
>
> #endif /* TCG_H */
--
Alex Bennée
next prev parent reply other threads:[~2016-10-20 10:49 UTC|newest]
Thread overview: 39+ messages / expand[flat|nested] mbox.gz Atom feed top
2016-10-19 17:21 [Qemu-devel] [PATCH v7 00/35] cmpxchg-based emulation of atomics Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 01/35] atomics: add atomic_xor Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 02/35] atomics: add atomic_op_fetch variants Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 03/35] exec: Avoid direct references to Int128 parts Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 04/35] int128: Use __int128 if available Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 05/35] int128: Add int128_make128 Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 07/35] linux-user: enable parallel code generation on clone Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 08/35] cputlb: Replace SHIFT with DATA_SIZE Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 09/35] cputlb: Move probe_write out of softmmu_template.h Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 10/35] cputlb: Remove includes from softmmu_template.h Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 11/35] cputlb: Move most of iotlb code out of line Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 12/35] cputlb: Tidy some macros Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 13/35] tcg: Add atomic helpers Richard Henderson
2016-10-20 10:49 ` Alex Bennée [this message]
2016-10-20 14:38 ` Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 14/35] tcg: Add atomic128 helpers Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 15/35] tcg: Add CONFIG_ATOMIC64 Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 16/35] tcg: Emit barriers with parallel_cpus Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 17/35] target-i386: emulate LOCK'ed cmpxchg using cmpxchg helpers Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 18/35] target-i386: emulate LOCK'ed OP instructions using atomic helpers Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 19/35] target-i386: emulate LOCK'ed INC using atomic helper Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 20/35] target-i386: emulate LOCK'ed NOT " Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 21/35] target-i386: emulate LOCK'ed NEG using cmpxchg helper Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 22/35] target-i386: emulate LOCK'ed XADD using atomic helper Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 23/35] target-i386: emulate LOCK'ed BTX ops using atomic helpers Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 24/35] target-i386: emulate XCHG using atomic helper Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 25/35] target-i386: remove helper_lock() Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 26/35] tests: add atomic_add-bench Richard Henderson
2016-10-19 17:21 ` [Qemu-devel] [PATCH v7 27/35] target-arm: Rearrange aa32 load and store functions Richard Henderson
2016-10-19 17:22 ` [Qemu-devel] [PATCH v7 28/35] target-arm: emulate LL/SC using cmpxchg helpers Richard Henderson
2016-10-19 17:22 ` [Qemu-devel] [PATCH v7 29/35] target-arm: emulate SWP with atomic_xchg helper Richard Henderson
2016-10-19 17:22 ` [Qemu-devel] [PATCH v7 30/35] target-arm: emulate aarch64's LL/SC using cmpxchg helpers Richard Henderson
2016-10-19 17:22 ` [Qemu-devel] [PATCH v7 31/35] linux-user: remove handling of ARM's EXCP_STREX Richard Henderson
2016-10-19 17:22 ` [Qemu-devel] [PATCH v7 32/35] linux-user: remove handling of aarch64's EXCP_STREX Richard Henderson
2016-10-19 17:22 ` [Qemu-devel] [PATCH v7 33/35] target-arm: remove EXCP_STREX + cpu_exclusive_{test, info} Richard Henderson
2016-10-19 17:22 ` [Qemu-devel] [PATCH v7 34/35] target-alpha: Introduce MMU_PHYS_IDX Richard Henderson
2016-10-19 17:22 ` [Qemu-devel] [PATCH v7 35/35] target-alpha: Emulate LL/SC using cmpxchg helpers Richard Henderson
2016-10-19 18:05 ` [Qemu-devel] [PATCH v7 06/35] tcg: Add EXCP_ATOMIC Richard Henderson
2016-10-20 8:50 ` [Qemu-devel] [PATCH v7 00/35] cmpxchg-based emulation of atomics Alex Bennée
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=87bmyffg7y.fsf@linaro.org \
--to=alex.bennee@linaro.org \
--cc=cota@braap.org \
--cc=qemu-devel@nongnu.org \
--cc=rth@twiddle.net \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.