[RFC PATCH 0/2] Risc-V Svinval support

[RFC PATCH 0/2] Risc-V Svinval support

[Mayuresh Chitale]

This patch adds support for the Svinval extension version 1.0 as defined in the
Risc V Privileged specification. It depends on and needs to be applied on the
following patchsets from Atish and Anup respectively:

https://patchwork.kernel.org/project/linux-riscv/list/?series=613234
https://patchwork.kernel.org/project/linux-riscv/list/?series=609361

The feature was tested with qemu from latest staging branch with following
additional patch:
https://lists.nongnu.org/archive/html/qemu-riscv/2022-02/msg00100.html 

Mayuresh Chitale (2):
  riscv: enum for svinval extension
  riscv: mm: use svinval instructions instead of sfence.vma

 arch/riscv/include/asm/hwcap.h    |  1 +
 arch/riscv/include/asm/tlbflush.h | 14 +++++++
 arch/riscv/kernel/cpu.c           |  1 +
 arch/riscv/kernel/setup.c         |  1 +
 arch/riscv/mm/Makefile            |  1 +
 arch/riscv/mm/tlb.S               | 53 +++++++++++++++++++++++
 arch/riscv/mm/tlbflush.c          | 70 ++++++++++++++++++++++++++++---
 7 files changed, 135 insertions(+), 6 deletions(-)
 create mode 100644 arch/riscv/mm/tlb.S

-- 
2.25.1

[RFC PATCH 1/2] riscv: enum for svinval extension

[Mayuresh Chitale]

Similar to the other ISA extensions, this patch enables
callers to check for the presence for the svinval extension.

Signed-off-by: Mayuresh Chitale <mchitale@ventanamicro.com>
---
 arch/riscv/include/asm/hwcap.h | 1 +
 arch/riscv/kernel/cpu.c        | 1 +
 2 files changed, 2 insertions(+)

diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h
index 691fc9c8099b..bbff7cb279ea 100644
--- a/arch/riscv/include/asm/hwcap.h
+++ b/arch/riscv/include/asm/hwcap.h
@@ -51,6 +51,7 @@ extern unsigned long elf_hwcap;
  * available logical extension id.
  */
 enum riscv_isa_ext_id {
+	RISCV_ISA_EXT_SVINVAL = RISCV_ISA_EXT_BASE,
 	RISCV_ISA_EXT_ID_MAX = RISCV_ISA_EXT_MAX,
 };
 
diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c
index ced7e5be8641..ff0613f8cc39 100644
--- a/arch/riscv/kernel/cpu.c
+++ b/arch/riscv/kernel/cpu.c
@@ -71,6 +71,7 @@ int riscv_of_parent_hartid(struct device_node *node)
 	}
 
 static struct riscv_isa_ext_data isa_ext_arr[] = {
+	__RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL),
 	__RISCV_ISA_EXT_DATA("", RISCV_ISA_EXT_MAX),
 };
 
-- 
2.25.1

Re: [RFC PATCH 1/2] riscv: enum for svinval extension

[Anup Patel]

On Wed, Feb 16, 2022 at 10:51 AM Mayuresh Chitale
<mchitale@ventanamicro.com> wrote:
>
> Similar to the other ISA extensions, this patch enables
> callers to check for the presence for the svinval extension.
>
> Signed-off-by: Mayuresh Chitale <mchitale@ventanamicro.com>

Please update riscv_fill_hwcap() in arch/riscv/kernel/cpufeature.c
to probe Svinval extension.

Regards,
Anup

> ---
>  arch/riscv/include/asm/hwcap.h | 1 +
>  arch/riscv/kernel/cpu.c        | 1 +
>  2 files changed, 2 insertions(+)
>
> diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h
> index 691fc9c8099b..bbff7cb279ea 100644
> --- a/arch/riscv/include/asm/hwcap.h
> +++ b/arch/riscv/include/asm/hwcap.h
> @@ -51,6 +51,7 @@ extern unsigned long elf_hwcap;
>   * available logical extension id.
>   */
>  enum riscv_isa_ext_id {
> +       RISCV_ISA_EXT_SVINVAL = RISCV_ISA_EXT_BASE,
>         RISCV_ISA_EXT_ID_MAX = RISCV_ISA_EXT_MAX,
>  };
>
> diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c
> index ced7e5be8641..ff0613f8cc39 100644
> --- a/arch/riscv/kernel/cpu.c
> +++ b/arch/riscv/kernel/cpu.c
> @@ -71,6 +71,7 @@ int riscv_of_parent_hartid(struct device_node *node)
>         }
>
>  static struct riscv_isa_ext_data isa_ext_arr[] = {
> +       __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL),
>         __RISCV_ISA_EXT_DATA("", RISCV_ISA_EXT_MAX),
>  };
>
> --
> 2.25.1
>

[RFC PATCH 2/2] riscv: mm: use svinval instructions instead of sfence.vma

[Mayuresh Chitale]

When svinval is supported the local_flush_tlb_page*
functions would prefer to use the following sequence
to optimize the tlb flushes instead of a simple sfence.vma:

sfence.w.inval
svinval.vma
  .
  .
svinval.vma
sfence.inval.ir

The maximum number of consecutive svinval.vma instructions
that can be executed in local_flush_tlb_page* functions is
limited to PTRS_PER_PTE. This is required to avoid soft
lockups and the approach is similar to that used in arm64.

Signed-off-by: Mayuresh Chitale <mchitale@ventanamicro.com>
---
 arch/riscv/include/asm/tlbflush.h | 14 +++++++
 arch/riscv/kernel/setup.c         |  1 +
 arch/riscv/mm/Makefile            |  1 +
 arch/riscv/mm/tlb.S               | 53 +++++++++++++++++++++++
 arch/riscv/mm/tlbflush.c          | 70 ++++++++++++++++++++++++++++---
 5 files changed, 133 insertions(+), 6 deletions(-)
 create mode 100644 arch/riscv/mm/tlb.S

diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h
index 801019381dea..9256a1c2ee03 100644
--- a/arch/riscv/include/asm/tlbflush.h
+++ b/arch/riscv/include/asm/tlbflush.h
@@ -22,9 +22,23 @@ static inline void local_flush_tlb_page(unsigned long addr)
 {
 	ALT_FLUSH_TLB_PAGE(__asm__ __volatile__ ("sfence.vma %0" : : "r" (addr) : "memory"));
 }
+
+void riscv_tlbflush_init(void);
+void __riscv_sfence_w_inval(void);
+void __riscv_sfence_inval_ir(void);
+void __riscv_sinval_vma(unsigned long addr);
+void __riscv_sinval_vma_asid(unsigned long addr, unsigned long asid);
+
+/* Check if we can use sinval for tlb flush */
+DECLARE_STATIC_KEY_FALSE(riscv_flush_tlb_svinval);
+#define riscv_use_flush_tlb_svinval() \
+	static_branch_unlikely(&riscv_flush_tlb_svinval)
+
 #else /* CONFIG_MMU */
 #define local_flush_tlb_all()			do { } while (0)
 #define local_flush_tlb_page(addr)		do { } while (0)
+#define riscv_use_flush_tlb_svinval()		do { } while (0)
+#define riscv_tlbflush_init()			do { } while (0)
 #endif /* CONFIG_MMU */
 
 #if defined(CONFIG_SMP) && defined(CONFIG_MMU)
diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
index b42bfdc67482..5dc79288b0ad 100644
--- a/arch/riscv/kernel/setup.c
+++ b/arch/riscv/kernel/setup.c
@@ -295,6 +295,7 @@ void __init setup_arch(char **cmdline_p)
 #endif
 
 	riscv_fill_hwcap();
+	riscv_tlbflush_init();
 }
 
 static int __init topology_init(void)
diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile
index 7ebaef10ea1b..d3a14d4d144e 100644
--- a/arch/riscv/mm/Makefile
+++ b/arch/riscv/mm/Makefile
@@ -16,6 +16,7 @@ obj-y += context.o
 
 ifeq ($(CONFIG_MMU),y)
 obj-$(CONFIG_SMP) += tlbflush.o
+obj-$(CONFIG_SMP) += tlb.o
 endif
 obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
 obj-$(CONFIG_PTDUMP_CORE) += ptdump.o
diff --git a/arch/riscv/mm/tlb.S b/arch/riscv/mm/tlb.S
new file mode 100644
index 000000000000..a530a9012c43
--- /dev/null
+++ b/arch/riscv/mm/tlb.S
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2022 Ventana Micro Sytems.
+ *
+ * Authors:
+ *     Mayuresh Chitale <mchitale@ventanamicro.com>
+ */
+
+#include <linux/linkage.h>
+#include <asm/asm.h>
+
+       .text
+       .altmacro
+       .option norelax
+
+
+ENTRY(__riscv_sfence_w_inval)
+       /*
+        * SFENCE.W.INVAL
+        * 0001100 00000 00000 000 00000 1110011
+        */
+       .word 0x18000073
+       ret
+ENDPROC(__riscv_sfence_w_inval)
+
+ENTRY(__riscv_sfence_inval_ir)
+       /*
+        * SFENCE.INVAL.IR
+        * 0001100 00001 00000 000 00000 1110011
+        */
+       .word 0x18100073
+       ret
+ENDPROC(__riscv_sfence_inval_ir)
+ENTRY(__riscv_sinval_vma_asid)
+       /*
+        * rs1 = VMA
+        * rs2 = asid
+        * SFENCE.W.INVAL
+        * 0001011 01011 01010 000 00000 1110011
+        */
+       .word 0x16B50073
+       ret
+ENDPROC(__riscv_sinval_vma_asid)
+ENTRY(__riscv_sinval_vma)
+       /*
+        * rs1 = vma
+        * rs2 = 0
+        * SFENCE.W.INVAL
+        * 0001011 00000 01010 000 00000 1110011
+        */
+       .word 0x16050073
+       ret
+ENDPROC(__riscv_sinval_vma)
diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c
index 27a7db8eb2c4..a4659f31b7a1 100644
--- a/arch/riscv/mm/tlbflush.c
+++ b/arch/riscv/mm/tlbflush.c
@@ -1,11 +1,14 @@
 // SPDX-License-Identifier: GPL-2.0
 
+#define pr_fmt(fmt) "riscv: " fmt
 #include <linux/mm.h>
 #include <linux/smp.h>
 #include <linux/sched.h>
 #include <asm/sbi.h>
 #include <asm/mmu_context.h>
 
+static unsigned long tlb_flush_all_threshold __read_mostly = PTRS_PER_PTE;
+
 static inline void local_flush_tlb_all_asid(unsigned long asid)
 {
 	__asm__ __volatile__ ("sfence.vma x0, %0"
@@ -26,19 +29,61 @@ static inline void local_flush_tlb_page_asid(unsigned long addr,
 static inline void local_flush_tlb_range(unsigned long start,
 		unsigned long size, unsigned long stride)
 {
-	if (size <= stride)
-		local_flush_tlb_page(start);
-	else
+	if ((size / stride) <= tlb_flush_all_threshold) {
+		if (riscv_use_flush_tlb_svinval()) {
+			__riscv_sfence_w_inval();
+			while (size) {
+				__riscv_sinval_vma(start);
+				start += stride;
+				if (size > stride)
+					size -= stride;
+				else
+					size = 0;
+			}
+			__riscv_sfence_inval_ir();
+		} else {
+			while (size) {
+				local_flush_tlb_page(start);
+				start += stride;
+				if (size > stride)
+					size -= stride;
+				else
+					size = 0;
+			}
+		}
+	} else {
 		local_flush_tlb_all();
+	}
 }
 
 static inline void local_flush_tlb_range_asid(unsigned long start,
 		unsigned long size, unsigned long stride, unsigned long asid)
 {
-	if (size <= stride)
-		local_flush_tlb_page_asid(start, asid);
-	else
+	if ((size / stride) <= tlb_flush_all_threshold) {
+		if (riscv_use_flush_tlb_svinval()) {
+			__riscv_sfence_w_inval();
+			while (size) {
+				__riscv_sinval_vma_asid(start, asid);
+				start += stride;
+				if (size > stride)
+					size -= stride;
+				else
+					size = 0;
+			}
+			__riscv_sfence_inval_ir();
+		} else {
+			while (size) {
+				local_flush_tlb_page_asid(start, asid);
+				start += stride;
+				if (size > stride)
+					size -= stride;
+				else
+					size = 0;
+			}
+		}
+	} else {
 		local_flush_tlb_all_asid(asid);
+	}
 }
 
 static void __ipi_flush_tlb_all(void *info)
@@ -149,3 +194,16 @@ void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
 	__flush_tlb_range(vma->vm_mm, start, end - start, PMD_SIZE);
 }
 #endif
+
+DEFINE_STATIC_KEY_FALSE(riscv_flush_tlb_svinval);
+EXPORT_SYMBOL_GPL(riscv_flush_tlb_svinval);
+
+void riscv_tlbflush_init(void)
+{
+	if (riscv_isa_extension_available(NULL, SVINVAL)) {
+		pr_info("Svinval extension supported\n");
+		static_branch_enable(&riscv_flush_tlb_svinval);
+	} else {
+		static_branch_disable(&riscv_flush_tlb_svinval);
+	}
+}
-- 
2.25.1