[PATCH RFC v2 2/2] arm64: vdso: Implement __vdso_futex_robust_try_unlock()

["André Almeida" <andrealmeid@igalia.com>]

Based on the x86 implementation, implement the vDSO function for unlocking
a robust futex correctly.

Commit xxxxxxxxxxxx ("x86/vdso: Implement __vdso_futex_robust_try_unlock()") has
the full explanation about why this mechanism is needed.

The unlock assembly sequence for arm64 is:

	__futex_list64_try_unlock_cs_start:
		ldxr	x3, [x0] // Load the value at *futex
		cmp	x1, x3   // Compare with TID
		b.ne	__futex_list64_try_unlock_cs_end
		stlxr	w1, xzr, [x0] // Try to clear *futex
		cbnz	w1, __futex_list64_try_unlock_cs_start
	__futex_list64_try_unlock_cs_success:
		str	xzr, [x2] // After clearing *futex, clear *op_pending
	__futex_list64_try_unlock_cs_end:

The decision regarding if the pointer should be cleared or not lies on checking
the condition flag zero:

	return (regs->user_regs.pstate & PSR_Z_BIT) ?
		(void __user *) regs->user_regs.regs[2] : NULL;

If it's not zero, that means that the comparassion worked and the kernel should
clear op_pending (if userspace didn't managed to) stored at x2.

Signed-off-by: André Almeida <andrealmeid@igalia.com>
---
RFC:
 - Should I duplicate the explanation found in the x86 commit or can I just
 point to it?
 - Only LL/SC for now but I can add LSE later if this looks good
 - It the objdump I see that op_pending is store at x2. But how stable is this,
 how can I write it in a way that's always x2?
---
 arch/arm64/Kconfig                                 |  1 +
 arch/arm64/include/asm/futex_robust.h              | 35 +++++++++++++
 arch/arm64/kernel/vdso/Makefile                    |  9 +++-
 arch/arm64/kernel/vdso/vdso.lds.S                  |  4 ++
 .../kernel/vdso/vfutex_robust_list_try_unlock.c    | 59 ++++++++++++++++++++++
 5 files changed, 107 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 427151a9db7f..e10cb97a51c7 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -249,6 +249,7 @@ config ARM64
 	select HAVE_RELIABLE_STACKTRACE
 	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
 	select HAVE_FUNCTION_ARG_ACCESS_API
+	select HAVE_FUTEX_ROBUST_UNLOCK
 	select MMU_GATHER_RCU_TABLE_FREE
 	select HAVE_RSEQ
 	select HAVE_RUST if RUSTC_SUPPORTS_ARM64
diff --git a/arch/arm64/include/asm/futex_robust.h b/arch/arm64/include/asm/futex_robust.h
new file mode 100644
index 000000000000..f2b7a2b15cb5
--- /dev/null
+++ b/arch/arm64/include/asm/futex_robust.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_ARM64_FUTEX_ROBUST_H
+#define _ASM_ARM64_FUTEX_ROBUST_H
+
+#include <asm/ptrace.h>
+
+static __always_inline void __user *arm64_futex_robust_unlock_get_pop(struct pt_regs *regs)
+{
+	/*
+	 * RFC: According to the objdump bellow, x2 is the address of
+	 * op_pending. How stable is this?
+
+	 <__futex_list64_try_unlock_cs_start>:
+		ldxr	x3, [x0]
+		cmp	x1, x3
+		b.ne	d7c <__futex_list64_try_unlock_cs_end>  // b.any
+		stlxr	w1, xzr, [x0]
+		cbnz	w1, d64 <__futex_list64_try_unlock_cs_start>
+
+	<__futex_list64_try_unlock_cs_success>:
+		str	xzr, [x2]
+
+	<__futex_list64_try_unlock_cs_end>:
+		mov	w0, w3
+		ret
+	*/
+
+	return (regs->user_regs.pstate & PSR_Z_BIT) ? NULL
+		: (void __user *) regs->user_regs.regs[2];
+}
+
+#define arch_futex_robust_unlock_get_pop(regs)	\
+	arm64_futex_robust_unlock_get_pop(regs)
+
+#endif /* _ASM_ARM64_FUTEX_ROBUST_H */
diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile
index 7dec05dd33b7..a65893d8100e 100644
--- a/arch/arm64/kernel/vdso/Makefile
+++ b/arch/arm64/kernel/vdso/Makefile
@@ -9,7 +9,8 @@
 # Include the generic Makefile to check the built vdso.
 include $(srctree)/lib/vdso/Makefile.include
 
-obj-vdso := vgettimeofday.o note.o sigreturn.o vgetrandom.o vgetrandom-chacha.o
+obj-vdso := vgettimeofday.o note.o sigreturn.o vgetrandom.o vgetrandom-chacha.o \
+	    vfutex_robust_list_try_unlock.o
 
 # Build rules
 targets := $(obj-vdso) vdso.so vdso.so.dbg
@@ -45,9 +46,11 @@ CC_FLAGS_ADD_VDSO := -O2 -mcmodel=tiny -fasynchronous-unwind-tables
 
 CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_REMOVE_VDSO)
 CFLAGS_REMOVE_vgetrandom.o = $(CC_FLAGS_REMOVE_VDSO)
+CFLAGS_REMOVE_vfutex_robust_list_try_unlock.o = $(CC_FLAGS_REMOVE_VDSO)
 
 CFLAGS_vgettimeofday.o = $(CC_FLAGS_ADD_VDSO)
 CFLAGS_vgetrandom.o = $(CC_FLAGS_ADD_VDSO)
+CFLAGS_vfutex_robust_list_try_unlock.o = $(CC_FLAGS_ADD_VDSO)
 
 ifneq ($(c-gettimeofday-y),)
   CFLAGS_vgettimeofday.o += -include $(c-gettimeofday-y)
@@ -57,6 +60,10 @@ ifneq ($(c-getrandom-y),)
   CFLAGS_vgetrandom.o += -include $(c-getrandom-y)
 endif
 
+ifneq ($(c-vfutex_robust_list_try_unlock-y),)
+  CFLAGS_vfutex_robust_list_try_unlock.o += -include $(c-vfutex_robust_list_try_unlock-y)
+endif
+
 targets += vdso.lds
 CPPFLAGS_vdso.lds += -P -C -U$(ARCH)
 
diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S
index c532dae6ac8e..c324764cb6f9 100644
--- a/arch/arm64/kernel/vdso/vdso.lds.S
+++ b/arch/arm64/kernel/vdso/vdso.lds.S
@@ -104,6 +104,10 @@ VERSION
 		__kernel_clock_gettime;
 		__kernel_clock_getres;
 		__kernel_getrandom;
+		__vdso_futex_robust_list64_try_unlock;
+#ifdef CONFIG_COMPAT_VDSO
+		__vdso_futex_robust_list32_try_unlock;
+#endif
 	local: *;
 	};
 }
diff --git a/arch/arm64/kernel/vdso/vfutex_robust_list_try_unlock.c b/arch/arm64/kernel/vdso/vfutex_robust_list_try_unlock.c
new file mode 100644
index 000000000000..e8a8fb22a2fa
--- /dev/null
+++ b/arch/arm64/kernel/vdso/vfutex_robust_list_try_unlock.c
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <vdso/futex.h>
+#include <linux/stringify.h>
+
+#define LABEL(name, sz) __stringify(__futex_list##sz##_try_unlock_cs_##name)
+
+#define GLOBLS(sz) ".globl " LABEL(start, sz) ", " LABEL(success, sz) ", " LABEL(end, sz) "\n"
+
+__u32 __vdso_futex_robust_list64_try_unlock(__u32 *lock, __u32 tid, __u64 *pop)
+{
+	__u32 val, result;
+
+	asm volatile (
+		GLOBLS(64)
+		"	prfm pstl1strm, %[lock]			\n"
+		LABEL(start, 64)":				\n"
+		"	ldxr %[val], %[lock]			\n"
+		"	cmp %[tid], %[val]			\n"
+		"	bne " LABEL(end, 64)"			\n"
+		"	stlxr %w[result], xzr, %[lock]		\n"
+		"	cbnz %w[result], " LABEL(start, 64)"	\n"
+		LABEL(success, 64)":				\n"
+		"	str xzr, %[pop]				\n"
+		LABEL(end, 64)":				\n"
+
+		: [val] "=&r" (val), [result] "=r" (result)
+		: [tid] "r" (tid), [lock] "Q" (*lock), [pop] "Q" (*pop)
+		: "memory"
+	);
+
+	return val;
+}
+
+#ifdef CONFIG_COMPAT_VDSO
+__u32 __vdso_futex_robust_list32_try_unlock(__u32 *lock, __u32 tid, __u32 *pop)
+{
+	__u32 val, result;
+
+	asm volatile (
+		GLOBLS(32)
+		"	prfm pstl1strm, %[lock]			\n"
+		LABEL(start, 32)":				\n"
+		"	ldxr %w[val], %[lock]			\n"
+		"	cmp %w[tid], %w[val]			\n"
+		"	bne " LABEL(end, 32)"			\n"
+		"	stlxr %w[result], wzr, %w[lock]		\n"
+		"	cbnz %w[result], " LABEL(start, 32)"	\n"
+		LABEL(success, 32)":				\n"
+		"	str wzr, %w[pop]			\n"
+		LABEL(end, 32)":				\n"
+
+		: [val] "=&r" (val), [result] "=r" (result)
+		: [tid] "r" (tid), [lock] "Q" (*lock), [pop] "Q" (*pop)
+		: "memory"
+	);
+
+	return val;
+}
+#endif

-- 
2.53.0