[PATCH] x86: Optimize __get_user_asm() assembly

[Uros Bizjak <ubizjak@gmail.com>]

On x86, byte and word moves (MOVB, MOVW) write only to the low
portion of a 32-bit register, leaving the upper bits unchanged.
Modern compilers therefore prefer using MOVZBL and MOVZWL to load
8-bit and 16-bit values with zero-extension to the full register
width.

Update __get_user_asm() to follow this convention by explicitly
zero-extending 8-bit and 16-bit loads from memory.

An additional benefit of this change is that it enables the full
integer register set to be used for 8-bit loads. Also, it
eliminates the need for manual zero-extension of 8-bit values.

There is only a minimal increase in code size:

      text     data     bss      dec     hex filename
  28258526  4810554  740932 33810012  03e65c vmlinux-old.o
  28258550  4810554  740932 33810036  03e674 vmlinux-new.o

Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
-
---
 arch/x86/include/asm/uaccess.h | 40 +++++++++++++++-------------------
 1 file changed, 17 insertions(+), 23 deletions(-)

diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 367297b188c3..cb463c1301f6 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -260,30 +260,27 @@ do {									\
 	unsigned int __gu_low, __gu_high;				\
 	const unsigned int __user *__gu_ptr;				\
 	__gu_ptr = (const void __user *)(ptr);				\
-	__get_user_asm(__gu_low, __gu_ptr, "l", "=r", label);		\
-	__get_user_asm(__gu_high, __gu_ptr+1, "l", "=r", label);	\
+	__get_user_asm(__gu_low, __gu_ptr, "movl", "", label);		\
+	__get_user_asm(__gu_high, __gu_ptr+1, "movl", "", label);	\
 	(x) = ((unsigned long long)__gu_high << 32) | __gu_low;		\
 } while (0)
 #else
 #define __get_user_asm_u64(x, ptr, label)				\
-	__get_user_asm(x, ptr, "q", "=r", label)
+	__get_user_asm(x, ptr, "movq", "", label)
 #endif
 
 #define __get_user_size(x, ptr, size, label)				\
 do {									\
 	__chk_user_ptr(ptr);						\
 	switch (size) {							\
-	case 1:	{							\
-		unsigned char x_u8__;					\
-		__get_user_asm(x_u8__, ptr, "b", "=q", label);		\
-		(x) = x_u8__;						\
+	case 1:								\
+		__get_user_asm(x, ptr, "movzbl", "k", label);		\
 		break;							\
-	}								\
 	case 2:								\
-		__get_user_asm(x, ptr, "w", "=r", label);		\
+		__get_user_asm(x, ptr, "movzwl", "k", label);		\
 		break;							\
 	case 4:								\
-		__get_user_asm(x, ptr, "l", "=r", label);		\
+		__get_user_asm(x, ptr, "movl", "", label);		\
 		break;							\
 	case 8:								\
 		__get_user_asm_u64(x, ptr, label);			\
@@ -294,11 +291,11 @@ do {									\
 	instrument_get_user(x);						\
 } while (0)
 
-#define __get_user_asm(x, addr, itype, ltype, label)			\
+#define __get_user_asm(x, addr, insn, opmod, label)			\
 	asm_goto_output("\n"						\
-		     "1:	mov"itype" %[umem],%[output]\n"		\
+		     "1:	" insn " %[umem],%" opmod "[output]\n"	\
 		     _ASM_EXTABLE_UA(1b, %l2)				\
-		     : [output] ltype(x)				\
+		     : [output] "=r" (x)				\
 		     : [umem] "m" (__m(addr))				\
 		     : : label)
 
@@ -326,26 +323,23 @@ do {									\
 })
 
 #else
-#define __get_user_asm_u64(x, ptr, retval) \
-	 __get_user_asm(x, ptr, retval, "q")
+#define __get_user_asm_u64(x, ptr, retval)				\
+	__get_user_asm(x, ptr, "movq", "", retval)
 #endif
 
 #define __get_user_size(x, ptr, size, retval)				\
 do {									\
-	unsigned char x_u8__;						\
-									\
 	retval = 0;							\
 	__chk_user_ptr(ptr);						\
 	switch (size) {							\
 	case 1:								\
-		__get_user_asm(x_u8__, ptr, retval, "b");		\
-		(x) = x_u8__;						\
+		 __get_user_asm(x, ptr, "movzbl", "k", retval);		\
 		break;							\
 	case 2:								\
-		__get_user_asm(x, ptr, retval, "w");			\
+		__get_user_asm(x, ptr, "movzwl", "k", retval);		\
 		break;							\
 	case 4:								\
-		__get_user_asm(x, ptr, retval, "l");			\
+		__get_user_asm(x, ptr, "movl", "", retval);		\
 		break;							\
 	case 8:								\
 		__get_user_asm_u64(x, ptr, retval);			\
@@ -355,9 +349,9 @@ do {									\
 	}								\
 } while (0)
 
-#define __get_user_asm(x, addr, err, itype)				\
+#define __get_user_asm(x, addr, insn, opmod, err)			\
 	asm volatile("\n"						\
-		     "1:	mov"itype" %[umem],%[output]\n"		\
+		     "1:	" insn " %[umem],%" opmod "[output]\n"	\
 		     "2:\n"						\
 		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG | \
 					   EX_FLAG_CLEAR_AX,		\
-- 
2.52.0