[PATCH 3/6] arm64: io: replace NVIDIA Olympus erratum alternative with callback

[Ada Couprie Diaz <ada.coupriediaz@arm.com>]

`__raw_read{b,w,l,q}()` represent about 10k call sites
that need to be patched.
Commit 12aab25ca56e ("arm64: errata: work around NVIDIA Olympus
device store/load ordering") implements its workaround with alternative
instructions, adding 10k extra instructions growing the size of the image.

Implement and use `__io_nvidia_olympus_patch_dmb()` as
a callback alternative instead, saving close to 40kB of image size
with a defconfig.

As the alternative fix replaced a NOP with a DMB in all cases,
make use of `aarch64_insn_gen_dmb()` which was made alternative-safe
in previous commits.

The callback needs to be added to the KVM NVHE namespace as `readl()`
is used in the vgic-v2 driver.

Cc: Vikram Sethi <vsethi@nvidia.com>
Cc: Shanker Donthineni <sdonthineni@nvidia.com>
Signed-off-by: Ada Couprie Diaz <ada.coupriediaz@arm.com>
---
 arch/arm64/include/asm/io.h    | 24 ++++++++++++++++--------
 arch/arm64/kernel/image-vars.h |  1 +
 arch/arm64/kernel/io.c         | 16 ++++++++++++++++
 3 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h
index 49a7002661a92..0f1ca651c5c79 100644
--- a/arch/arm64/include/asm/io.h
+++ b/arch/arm64/include/asm/io.h
@@ -19,6 +19,10 @@
 #include <asm/cpufeature.h>
 #include <asm/rsi.h>
 
+/* IO-specific callbacks for alternative patching. */
+void __io_nvidia_olympus_patch_dmb(struct alt_instr *alt, __le32 *origptr,
+				   __le32 *updptr, int nr_inst);
+
 /*
  * Generic IO read/write.  These perform native-endian accesses.
  */
@@ -54,8 +58,9 @@ static __always_inline void __raw_writeq(u64 val, volatile void __iomem *addr)
 static __always_inline u8 __raw_readb(const volatile void __iomem *addr)
 {
 	u8 val;
-	asm volatile(ALTERNATIVE("nop", "dmb osh",
-				 ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027)
+	asm volatile(ALTERNATIVE_CB("nop",
+				    ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027,
+				    __io_nvidia_olympus_patch_dmb)
 		     ALTERNATIVE("ldrb %w0, [%1]",
 				 "ldarb %w0, [%1]",
 				 ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
@@ -68,8 +73,9 @@ static __always_inline u16 __raw_readw(const volatile void __iomem *addr)
 {
 	u16 val;
 
-	asm volatile(ALTERNATIVE("nop", "dmb osh",
-				 ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027)
+	asm volatile(ALTERNATIVE_CB("nop",
+				    ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027,
+				    __io_nvidia_olympus_patch_dmb)
 		     ALTERNATIVE("ldrh %w0, [%1]",
 				 "ldarh %w0, [%1]",
 				 ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
@@ -81,8 +87,9 @@ static __always_inline u16 __raw_readw(const volatile void __iomem *addr)
 static __always_inline u32 __raw_readl(const volatile void __iomem *addr)
 {
 	u32 val;
-	asm volatile(ALTERNATIVE("nop", "dmb osh",
-				 ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027)
+	asm volatile(ALTERNATIVE_CB("nop",
+				    ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027,
+				    __io_nvidia_olympus_patch_dmb)
 		     ALTERNATIVE("ldr %w0, [%1]",
 				 "ldar %w0, [%1]",
 				 ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
@@ -94,8 +101,9 @@ static __always_inline u32 __raw_readl(const volatile void __iomem *addr)
 static __always_inline u64 __raw_readq(const volatile void __iomem *addr)
 {
 	u64 val;
-	asm volatile(ALTERNATIVE("nop", "dmb osh",
-				 ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027)
+	asm volatile(ALTERNATIVE_CB("nop",
+				    ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027,
+				    __io_nvidia_olympus_patch_dmb)
 		     ALTERNATIVE("ldr %0, [%1]",
 				 "ldar %0, [%1]",
 				 ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index 14beb7b9d304c..a22519c5223b4 100644
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -94,6 +94,7 @@ KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb);
 KVM_NVHE_ALIAS(alt_cb_patch_nops);
 KVM_NVHE_ALIAS(kvm_compute_ich_hcr_trap_bits);
 KVM_NVHE_ALIAS(kvm_patch_ich_vtr_el2);
+KVM_NVHE_ALIAS(__io_nvidia_olympus_patch_dmb);
 
 /* Global kernel state accessed by nVHE hyp code. */
 KVM_NVHE_ALIAS(kvm_vgic_global_state);
diff --git a/arch/arm64/kernel/io.c b/arch/arm64/kernel/io.c
index fe86ada23c7d4..8ad3cd1773d20 100644
--- a/arch/arm64/kernel/io.c
+++ b/arch/arm64/kernel/io.c
@@ -9,6 +9,8 @@
 #include <linux/types.h>
 #include <linux/io.h>
 
+#include <asm/insn.h>
+
 /*
  * This generates a memcpy that works on a from/to address which is aligned to
  * bits. Count is in terms of the number of bits sized quantities to copy. It
@@ -50,3 +52,17 @@ void __iowrite32_copy_full(void __iomem *to, const void *from, size_t count)
 	dgh();
 }
 EXPORT_SYMBOL(__iowrite32_copy_full);
+
+noinstr void __io_nvidia_olympus_patch_dmb(struct alt_instr *alt, __le32 *origptr,
+					   __le32 *updptr, int nr_inst)
+{
+	u32 orinst;
+
+	BUG_ON(nr_inst != 1);
+
+	orinst = le32_to_cpu(origptr[0]);
+	BUG_ON(!aarch64_insn_is_nop(orinst));
+
+	updptr[0] = cpu_to_le32(aarch64_insn_gen_dmb(AARCH64_INSN_MB_OSH));
+}
+EXPORT_SYMBOL(__io_nvidia_olympus_patch_dmb);
-- 
2.43.0