[PATCH bpf] riscv, bpf: make some atomic operations fully ordered

[PATCH bpf] riscv, bpf: make some atomic operations fully ordered

[Puranjay Mohan]

The BPF atomic operations with the BPF_FETCH modifier along with
BPF_XCHG and BPF_CMPXCHG are fully ordered but the RISC-V JIT implements
all atomic operations except BPF_CMPXCHG with relaxed ordering.

Section 8.1 of the "The RISC-V Instruction Set Manual Volume I:
Unprivileged ISA" [1], titled, "Specifying Ordering of Atomic
Instructions" says:

| To provide more efficient support for release consistency [5], each
| atomic instruction has two bits, aq and rl, used to specify additional
| memory ordering constraints as viewed by other RISC-V harts.

and

| If only the aq bit is set, the atomic memory operation is treated as
| an acquire access.
| If only the rl bit is set, the atomic memory operation is treated as a
| release access.
|
| If both the aq and rl bits are set, the atomic memory operation is
| sequentially consistent.

Fix this by setting both aq and rl bits as 1 for operations with
BPF_FETCH and BPF_XCHG.

[1] https://riscv.org/wp-content/uploads/2017/05/riscv-spec-v2.2.pdf

Fixes: dd642ccb45ec ("riscv, bpf: Implement more atomic operations for RV64")
Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
---
 arch/riscv/net/bpf_jit_comp64.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c
index ec9d692838fc..fb5d1950042b 100644
--- a/arch/riscv/net/bpf_jit_comp64.c
+++ b/arch/riscv/net/bpf_jit_comp64.c
@@ -498,33 +498,33 @@ static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64,
 		break;
 	/* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */
 	case BPF_ADD | BPF_FETCH:
-		emit(is64 ? rv_amoadd_d(rs, rs, rd, 0, 0) :
-		     rv_amoadd_w(rs, rs, rd, 0, 0), ctx);
+		emit(is64 ? rv_amoadd_d(rs, rs, rd, 1, 1) :
+		     rv_amoadd_w(rs, rs, rd, 1, 1), ctx);
 		if (!is64)
 			emit_zextw(rs, rs, ctx);
 		break;
 	case BPF_AND | BPF_FETCH:
-		emit(is64 ? rv_amoand_d(rs, rs, rd, 0, 0) :
-		     rv_amoand_w(rs, rs, rd, 0, 0), ctx);
+		emit(is64 ? rv_amoand_d(rs, rs, rd, 1, 1) :
+		     rv_amoand_w(rs, rs, rd, 1, 1), ctx);
 		if (!is64)
 			emit_zextw(rs, rs, ctx);
 		break;
 	case BPF_OR | BPF_FETCH:
-		emit(is64 ? rv_amoor_d(rs, rs, rd, 0, 0) :
-		     rv_amoor_w(rs, rs, rd, 0, 0), ctx);
+		emit(is64 ? rv_amoor_d(rs, rs, rd, 1, 1) :
+		     rv_amoor_w(rs, rs, rd, 1, 1), ctx);
 		if (!is64)
 			emit_zextw(rs, rs, ctx);
 		break;
 	case BPF_XOR | BPF_FETCH:
-		emit(is64 ? rv_amoxor_d(rs, rs, rd, 0, 0) :
-		     rv_amoxor_w(rs, rs, rd, 0, 0), ctx);
+		emit(is64 ? rv_amoxor_d(rs, rs, rd, 1, 1) :
+		     rv_amoxor_w(rs, rs, rd, 1, 1), ctx);
 		if (!is64)
 			emit_zextw(rs, rs, ctx);
 		break;
 	/* src_reg = atomic_xchg(dst_reg + off16, src_reg); */
 	case BPF_XCHG:
-		emit(is64 ? rv_amoswap_d(rs, rs, rd, 0, 0) :
-		     rv_amoswap_w(rs, rs, rd, 0, 0), ctx);
+		emit(is64 ? rv_amoswap_d(rs, rs, rd, 1, 1) :
+		     rv_amoswap_w(rs, rs, rd, 1, 1), ctx);
 		if (!is64)
 			emit_zextw(rs, rs, ctx);
 		break;
-- 
2.40.1

Re: [PATCH bpf] riscv, bpf: make some atomic operations fully ordered

[Pu Lehui]

On 2024/5/6 4:16, Puranjay Mohan wrote:
> The BPF atomic operations with the BPF_FETCH modifier along with
> BPF_XCHG and BPF_CMPXCHG are fully ordered but the RISC-V JIT implements
> all atomic operations except BPF_CMPXCHG with relaxed ordering.
> 
> Section 8.1 of the "The RISC-V Instruction Set Manual Volume I:
> Unprivileged ISA" [1], titled, "Specifying Ordering of Atomic
> Instructions" says:
> 
> | To provide more efficient support for release consistency [5], each
> | atomic instruction has two bits, aq and rl, used to specify additional
> | memory ordering constraints as viewed by other RISC-V harts.
> 
> and
> 
> | If only the aq bit is set, the atomic memory operation is treated as
> | an acquire access.
> | If only the rl bit is set, the atomic memory operation is treated as a
> | release access.
> |
> | If both the aq and rl bits are set, the atomic memory operation is
> | sequentially consistent.
> 
> Fix this by setting both aq and rl bits as 1 for operations with
> BPF_FETCH and BPF_XCHG.
> 
> [1] https://riscv.org/wp-content/uploads/2017/05/riscv-spec-v2.2.pdf
> 
> Fixes: dd642ccb45ec ("riscv, bpf: Implement more atomic operations for RV64")
> Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
> ---
>   arch/riscv/net/bpf_jit_comp64.c | 20 ++++++++++----------
>   1 file changed, 10 insertions(+), 10 deletions(-)
> 
> diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c
> index ec9d692838fc..fb5d1950042b 100644
> --- a/arch/riscv/net/bpf_jit_comp64.c
> +++ b/arch/riscv/net/bpf_jit_comp64.c
> @@ -498,33 +498,33 @@ static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64,
>   		break;
>   	/* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */
>   	case BPF_ADD | BPF_FETCH:
> -		emit(is64 ? rv_amoadd_d(rs, rs, rd, 0, 0) :
> -		     rv_amoadd_w(rs, rs, rd, 0, 0), ctx);
> +		emit(is64 ? rv_amoadd_d(rs, rs, rd, 1, 1) :
> +		     rv_amoadd_w(rs, rs, rd, 1, 1), ctx);
>   		if (!is64)
>   			emit_zextw(rs, rs, ctx);
>   		break;
>   	case BPF_AND | BPF_FETCH:
> -		emit(is64 ? rv_amoand_d(rs, rs, rd, 0, 0) :
> -		     rv_amoand_w(rs, rs, rd, 0, 0), ctx);
> +		emit(is64 ? rv_amoand_d(rs, rs, rd, 1, 1) :
> +		     rv_amoand_w(rs, rs, rd, 1, 1), ctx);
>   		if (!is64)
>   			emit_zextw(rs, rs, ctx);
>   		break;
>   	case BPF_OR | BPF_FETCH:
> -		emit(is64 ? rv_amoor_d(rs, rs, rd, 0, 0) :
> -		     rv_amoor_w(rs, rs, rd, 0, 0), ctx);
> +		emit(is64 ? rv_amoor_d(rs, rs, rd, 1, 1) :
> +		     rv_amoor_w(rs, rs, rd, 1, 1), ctx);
>   		if (!is64)
>   			emit_zextw(rs, rs, ctx);
>   		break;
>   	case BPF_XOR | BPF_FETCH:
> -		emit(is64 ? rv_amoxor_d(rs, rs, rd, 0, 0) :
> -		     rv_amoxor_w(rs, rs, rd, 0, 0), ctx);
> +		emit(is64 ? rv_amoxor_d(rs, rs, rd, 1, 1) :
> +		     rv_amoxor_w(rs, rs, rd, 1, 1), ctx);
>   		if (!is64)
>   			emit_zextw(rs, rs, ctx);
>   		break;
>   	/* src_reg = atomic_xchg(dst_reg + off16, src_reg); */
>   	case BPF_XCHG:
> -		emit(is64 ? rv_amoswap_d(rs, rs, rd, 0, 0) :
> -		     rv_amoswap_w(rs, rs, rd, 0, 0), ctx);
> +		emit(is64 ? rv_amoswap_d(rs, rs, rd, 1, 1) :
> +		     rv_amoswap_w(rs, rs, rd, 1, 1), ctx);
>   		if (!is64)
>   			emit_zextw(rs, rs, ctx);
>   		break;

Reviewed-by: Pu Lehui <pulehui@huawei.com>

Re: [PATCH bpf] riscv, bpf: make some atomic operations fully ordered

[patchwork-bot+netdevbpf]

Hello:

This patch was applied to bpf/bpf-next.git (master)
by Alexei Starovoitov <ast@kernel.org>:

On Sun,  5 May 2024 20:16:33 +0000 you wrote:
> The BPF atomic operations with the BPF_FETCH modifier along with
> BPF_XCHG and BPF_CMPXCHG are fully ordered but the RISC-V JIT implements
> all atomic operations except BPF_CMPXCHG with relaxed ordering.
> 
> Section 8.1 of the "The RISC-V Instruction Set Manual Volume I:
> Unprivileged ISA" [1], titled, "Specifying Ordering of Atomic
> Instructions" says:
> 
> [...]

Here is the summary with links:
  - [bpf] riscv, bpf: make some atomic operations fully ordered
    https://git.kernel.org/bpf/bpf-next/c/20a759df3bba

You are awesome, thank you!
-- 
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html