[RFC][PATCH 21/22] x86,word-at-a-time: Remove .fixup usage

Peter Zijlstra
2021-11-04T17:01:19Z

Push the load_unaligned_zeropad() exception into exception context by
adding a new extable type. This however requires we have both the
address and the output register. Since we can only have a single
register argument, use the same for both.

This also means the source can no longer use "m" constraint.

XXX: I'm not really happy with this patch

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/x86/include/asm/extable_fixup_types.h | 2 ++
arch/x86/include/asm/word-at-a-time.h | 27 +++++++--------------------
arch/x86/mm/extable.c | 17 +++++++++++++++++
3 files changed, 26 insertions(+), 20 deletions(-)

--- a/arch/x86/include/asm/extable_fixup_types.h
+++ b/arch/x86/include/asm/extable_fixup_types.h
@@ -47,4 +47,6 @@
#define EX_TYPE_UACCESS_LEN4 (EX_TYPE_UACCESS_LEN | EX_TYPE_IMM(4))
#define EX_TYPE_UACCESS_LEN8 (EX_TYPE_UACCESS_LEN | EX_TYPE_IMM(8))

+#define EX_TYPE_LOAD_UNALIGNED 21 /* reg := (reg) */
+
#endif
--- a/arch/x86/include/asm/word-at-a-time.h
+++ b/arch/x86/include/asm/word-at-a-time.h
@@ -79,27 +79,14 @@ static inline unsigned long find_zero(un
*/
static inline unsigned long load_unaligned_zeropad(const void *addr)
{
- unsigned long ret, dummy;
+ unsigned long ret;
+
+ asm("1:\tmov (%0),%0\n"
+ "2:\n"
+ _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_LOAD_UNALIGNED, %0)
+ : "=&r" (ret)
+ : "0" ((unsigned long)addr));

- asm(
- "1:\tmov %2,%0\n"
- "2:\n"
- ".section .fixup,\"ax\"\n"
- "3:\t"
- "lea %2,%1\n\t"
- "and %3,%1\n\t"
- "mov (%1),%0\n\t"
- "leal %2,%%ecx\n\t"
- "andl %4,%%ecx\n\t"
- "shll $3,%%ecx\n\t"
- "shr %%cl,%0\n\t"
- "jmp 2b\n"
- ".previous\n"
- _ASM_EXTABLE(1b, 3b)
- :"=&r" (ret),"=&c" (dummy)
- :"m" (*(unsigned long *)addr),
- "i" (-sizeof(unsigned long)),
- "i" (sizeof(unsigned long)-1));
return ret;
}

--- a/arch/x86/mm/extable.c
+++ b/arch/x86/mm/extable.c
@@ -170,6 +170,21 @@ static bool ex_handler_uaccess_len(const
return ex_handler_uaccess(fixup, regs, trapnr);
}

+static bool ex_handler_load_unaligned_zeropad(const struct exception_table_entry *fixup,
+ struct pt_regs *regs, int reg)
+{
+ unsigned long addr, offset, data;
+
+ addr = *pt_regs_nr(regs, reg);
+ offset = addr & (sizeof(unsigned long) - 1);
+ addr &= ~(sizeof(unsigned long) - 1);
+ data = *(unsigned long *)addr;
+ data >>= offset*8;
+ *pt_regs_nr(regs, reg) = data;
+
+ return ex_handler_default(fixup, regs);
+}
+
#define EX_TYPE_MASK 0x000000FF
#define EX_REG_MASK 0x00000F00
#define EX_FLAG_MASK 0x0000F000
@@ -251,6 +266,8 @@ int fixup_exception(struct pt_regs *regs
return ex_handler_kvm_fastop(e, regs);
case EX_TYPE_UACCESS_LEN:
return ex_handler_uaccess_len(e, regs, trapnr, reg, imm);
+ case EX_TYPE_LOAD_UNALIGNED:
+ return ex_handler_load_unaligned_zeropad(e, regs, reg);
}
BUG();
}