[PATCH v2 0/5] LoongArch/tcg: Add hardware page table walker support

[PATCH v2 0/5] LoongArch/tcg: Add hardware page table walker support

[Song Gao]

Loongson-3A6000 and newer processors have hardware page table walker
(PTW) support. PTW can handle all fastpaths of PIL/PIS/PIF/PIE
exceptions by hardware.

V2:
- Remove the '21' magic value, patch1;
- Add a flag is_debug for debug access, patch5;
- Use qatomic_cmpxchg to change the new pte_val, patch5.

Song Gao (5):
  target/loongarch: Add a new cpu_type la664
  target/loongarch: Add do_lddir/ldpte()
  target/loongarch: Add do_fill_tlb_entry()
  target/loongarch: Add get_random_tlb_index()
  target/loongarch/tcg: Add hardware page table walker support

 target/loongarch/cpu-csr.h        |   3 +
 target/loongarch/cpu.c            |  51 ++++--
 target/loongarch/cpu.h            |   1 +
 target/loongarch/cpu_helper.c     |  26 ++-
 target/loongarch/internals.h      |   4 +-
 target/loongarch/tcg/tlb_helper.c | 277 ++++++++++++++++++++++++------
 6 files changed, 293 insertions(+), 69 deletions(-)

-- 
2.33.0

[PATCH v2 1/5] target/loongarch: Add a new cpu_type la664

[Song Gao]

Add a new LoongArch cpu type la664. The la664 has many new features,
such as new atomic instructions, hardware page table walk, etc.
We will implement them later.

Signed-off-by: Song Gao <gaosong@loongson.cn>
---
 target/loongarch/cpu.c | 50 +++++++++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 15 deletions(-)

diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c
index 7212fb5f8f..d55b4110a9 100644
--- a/target/loongarch/cpu.c
+++ b/target/loongarch/cpu.c
@@ -374,20 +374,11 @@ static int loongarch_cpu_mmu_index(CPUState *cs, bool ifetch)
     return MMU_DA_IDX;
 }
 
-static void loongarch_la464_initfn(Object *obj)
+static void loongarch_common_initfn(CPULoongArchState *env, Object *obj)
 {
-    LoongArchCPU *cpu = LOONGARCH_CPU(obj);
-    CPULoongArchState *env = &cpu->env;
-    int i;
-
-    for (i = 0; i < 21; i++) {
-        env->cpucfg[i] = 0x0;
-    }
-
-    cpu->dtb_compatible = "loongarch,Loongson-3A5000";
-    env->cpucfg[0] = 0x14c010;  /* PRID */
+    uint32_t data;
 
-    uint32_t data = 0;
+    data = 0;
     data = FIELD_DP32(data, CPUCFG1, ARCH, 2);
     data = FIELD_DP32(data, CPUCFG1, PGMMU, 1);
     data = FIELD_DP32(data, CPUCFG1, IOCSR, 1);
@@ -472,14 +463,42 @@ static void loongarch_la464_initfn(Object *obj)
     loongarch_cpu_post_init(obj);
 }
 
-static void loongarch_la132_initfn(Object *obj)
+static void loongarch_la664_initfn(Object *obj)
 {
     LoongArchCPU *cpu = LOONGARCH_CPU(obj);
     CPULoongArchState *env = &cpu->env;
 
-    int i;
+    for (unsigned i = 0; i < ARRAY_SIZE(env->cpucfg); i++) {
+        env->cpucfg[i] = 0x0;
+    }
+
+    cpu->dtb_compatible = "loongarch,Loongson-3A6000";
+    env->cpucfg[0] = 0x14d000; /* PRID */
+
+    loongarch_common_initfn(env, obj);
+}
+
+static void loongarch_la464_initfn(Object *obj)
+{
+    LoongArchCPU *cpu = LOONGARCH_CPU(obj);
+    CPULoongArchState *env = &cpu->env;
+
+    for (unsigned i = 0; i < ARRAY_SIZE(env->cpucfg); i++) {
+        env->cpucfg[i] = 0x0;
+    }
+
+    cpu->dtb_compatible = "loongarch,Loongson-3A5000";
+    env->cpucfg[0] = 0x14c010;  /* PRID */
+
+    loongarch_common_initfn(env, obj);
+}
+
+static void loongarch_la132_initfn(Object *obj)
+{
+    LoongArchCPU *cpu = LOONGARCH_CPU(obj);
+    CPULoongArchState *env = &cpu->env;
 
-    for (i = 0; i < 21; i++) {
+    for (unsigned i = 0; i < ARRAY_SIZE(env->cpucfg); i++) {
         env->cpucfg[i] = 0x0;
     }
 
@@ -872,6 +891,7 @@ static const TypeInfo loongarch_cpu_type_infos[] = {
         .abstract = true,
         .class_init = loongarch64_cpu_class_init,
     },
+    DEFINE_LOONGARCH_CPU_TYPE(64, "la664", loongarch_la664_initfn),
     DEFINE_LOONGARCH_CPU_TYPE(64, "la464", loongarch_la464_initfn),
     DEFINE_LOONGARCH_CPU_TYPE(32, "la132", loongarch_la132_initfn),
     DEFINE_LOONGARCH_CPU_TYPE(64, "max", loongarch_max_initfn),
-- 
2.33.0

[PATCH v2 2/5] target/loongarch: Add do_lddir/ldpte()

[Song Gao]

do_lddir is used for accessing directory entries during page table
walking, do_ldpte is used for page table entry accesses during page
table walking.

Signed-off-by: Song Gao <gaosong@loongson.cn>
---
 target/loongarch/tcg/tlb_helper.c | 53 ++++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 19 deletions(-)

diff --git a/target/loongarch/tcg/tlb_helper.c b/target/loongarch/tcg/tlb_helper.c
index 97f38fc391..3c3452b316 100644
--- a/target/loongarch/tcg/tlb_helper.c
+++ b/target/loongarch/tcg/tlb_helper.c
@@ -507,11 +507,11 @@ bool loongarch_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
     cpu_loop_exit_restore(cs, retaddr);
 }
 
-target_ulong helper_lddir(CPULoongArchState *env, target_ulong base,
-                          target_ulong level, uint32_t mem_idx)
+static target_ulong do_lddir(CPULoongArchState *env, target_ulong base,
+                             target_ulong badvaddr, target_ulong level)
 {
     CPUState *cs = env_cpu(env);
-    target_ulong badvaddr, index, phys, ret;
+    target_ulong index, phys, ret;
     int shift;
     uint64_t dir_base, dir_width;
 
@@ -535,7 +535,6 @@ target_ulong helper_lddir(CPULoongArchState *env, target_ulong base,
         }
     }
 
-    badvaddr = env->CSR_TLBRBADV;
     base = base & TARGET_PHYS_MASK;
 
     /* 0:64bit, 1:128bit, 2:192bit, 3:256bit */
@@ -549,11 +548,18 @@ target_ulong helper_lddir(CPULoongArchState *env, target_ulong base,
     return ret;
 }
 
-void helper_ldpte(CPULoongArchState *env, target_ulong base, target_ulong odd,
-                  uint32_t mem_idx)
+target_ulong helper_lddir(CPULoongArchState *env, target_ulong base,
+                          target_ulong level, uint32_t mem_idx)
+{
+    return do_lddir(env, base, env->CSR_TLBRBADV, level);
+}
+
+static void do_ldpte(CPULoongArchState *env, target_ulong base,
+                     target_ulong badvaddr, target_ulong *ptval0,
+                     target_ulong *ptval1, target_ulong *ps)
 {
     CPUState *cs = env_cpu(env);
-    target_ulong phys, tmp0, ptindex, ptoffset0, ptoffset1, ps, badv;
+    target_ulong  ptindex, ptoffset0, ptoffset1, phys0, phys1;
     int shift;
     uint64_t ptbase = FIELD_EX64(env->CSR_PWCL, CSR_PWCL, PTBASE);
     uint64_t ptwidth = FIELD_EX64(env->CSR_PWCL, CSR_PWCL, PTWIDTH);
@@ -584,34 +590,43 @@ void helper_ldpte(CPULoongArchState *env, target_ulong base, target_ulong odd,
             base = FIELD_DP64(base, TLBENTRY, G, 1);
         }
 
-        ps = dir_base + dir_width - 1;
+        *ps = dir_base + dir_width - 1;
         /*
          * Huge pages are evenly split into parity pages
          * when loaded into the tlb,
          * so the tlb page size needs to be divided by 2.
          */
-        tmp0 = base;
-        if (odd) {
-            tmp0 += MAKE_64BIT_MASK(ps, 1);
-        }
+        *ptval0 = base;
+        *ptval1 = base + MAKE_64BIT_MASK(*ps, 1);
     } else {
         /* 0:64bit, 1:128bit, 2:192bit, 3:256bit */
         shift = FIELD_EX64(env->CSR_PWCL, CSR_PWCL, PTEWIDTH);
         shift = (shift + 1) * 3;
-        badv = env->CSR_TLBRBADV;
 
-        ptindex = (badv >> ptbase) & ((1 << ptwidth) - 1);
-        ptindex = ptindex & ~0x1;   /* clear bit 0 */
+        ptindex = (badvaddr >> ptbase) & ((1 << ptwidth) - 1);
+        ptindex = ptindex & ~0x1;  /* clear bit 0 */
         ptoffset0 = ptindex << shift;
         ptoffset1 = (ptindex + 1) << shift;
 
-        phys = base | (odd ? ptoffset1 : ptoffset0);
-        tmp0 = ldq_phys(cs->as, phys) & TARGET_PHYS_MASK;
-        ps = ptbase;
+        phys0 = base | ptoffset0;
+        phys1 = base | ptoffset1;
+        *ptval0 = ldq_phys(cs->as, phys0) & TARGET_PHYS_MASK;
+        *ptval1 = ldq_phys(cs->as, phys1) & TARGET_PHYS_MASK;
+        *ps = ptbase;
     }
 
+    return;
+}
+
+void helper_ldpte(CPULoongArchState *env, target_ulong base, target_ulong odd,
+                  uint32_t mem_idx)
+{
+    target_ulong tmp0, tmp1, ps;
+
+    do_ldpte(env, base, env->CSR_TLBRBADV, &tmp0, &tmp1, &ps);
+
     if (odd) {
-        env->CSR_TLBRELO1 = tmp0;
+        env->CSR_TLBRELO1 = tmp1;
     } else {
         env->CSR_TLBRELO0 = tmp0;
     }
-- 
2.33.0

[PATCH v2 3/5] target/loongarch: Add do_fill_tlb_entry()

[Song Gao]

do_fill_tlb_entry is used to fill a tlb entry.

Signed-off-by: Song Gao <gaosong@loongson.cn>
---
 target/loongarch/tcg/tlb_helper.c | 43 ++++++++++++++++++-------------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/target/loongarch/tcg/tlb_helper.c b/target/loongarch/tcg/tlb_helper.c
index 3c3452b316..bc6d708484 100644
--- a/target/loongarch/tcg/tlb_helper.c
+++ b/target/loongarch/tcg/tlb_helper.c
@@ -160,11 +160,33 @@ static void invalidate_tlb(CPULoongArchState *env, int index)
     invalidate_tlb_entry(env, index);
 }
 
-static void fill_tlb_entry(CPULoongArchState *env, int index)
+static void do_fill_tlb_entry(CPULoongArchState *env, uint64_t vppn,
+                              uint64_t lo0, uint64_t lo1, int index, uint8_t ps)
 {
     LoongArchTLB *tlb = &env->tlb[index];
+    uint16_t asid;
+
+    if (ps == 0) {
+        qemu_log_mask(CPU_LOG_MMU, "page size is 0\n");
+    }
+
+    /* Only MTLB has the ps fields */
+    if (index >= LOONGARCH_STLB) {
+        tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, PS, ps);
+    }
+
+    tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, VPPN, vppn);
+    tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, E, 1);
+    asid = FIELD_EX64(env->CSR_ASID, CSR_ASID, ASID);
+    tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, ASID, asid);
+
+    tlb->tlb_entry0 = lo0;
+    tlb->tlb_entry1 = lo1;
+}
+
+static void fill_tlb_entry(CPULoongArchState *env, int index)
+{
     uint64_t lo0, lo1, csr_vppn;
-    uint16_t csr_asid;
     uint8_t csr_ps;
 
     if (FIELD_EX64(env->CSR_TLBRERA, CSR_TLBRERA, ISTLBR)) {
@@ -187,22 +209,7 @@ static void fill_tlb_entry(CPULoongArchState *env, int index)
         lo1 = env->CSR_TLBELO1;
     }
 
-    if (csr_ps == 0) {
-        qemu_log_mask(CPU_LOG_MMU, "page size is 0\n");
-    }
-
-    /* Only MTLB has the ps fields */
-    if (index >= LOONGARCH_STLB) {
-        tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, PS, csr_ps);
-    }
-
-    tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, VPPN, csr_vppn);
-    tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, E, 1);
-    csr_asid = FIELD_EX64(env->CSR_ASID, CSR_ASID, ASID);
-    tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, ASID, csr_asid);
-
-    tlb->tlb_entry0 = lo0;
-    tlb->tlb_entry1 = lo1;
+    do_fill_tlb_entry(env, csr_vppn, lo0, lo1, index, csr_ps);
 }
 
 /* Return an random value between low and high */
-- 
2.33.0

[PATCH v2 4/5] target/loongarch: Add get_random_tlb_index()

[Song Gao]

get_random_tlb_index() is used to get a random tlb index.

Signed-off-by: Song Gao <gaosong@loongson.cn>
---
 target/loongarch/tcg/tlb_helper.c | 34 +++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/target/loongarch/tcg/tlb_helper.c b/target/loongarch/tcg/tlb_helper.c
index bc6d708484..463e9be7f2 100644
--- a/target/loongarch/tcg/tlb_helper.c
+++ b/target/loongarch/tcg/tlb_helper.c
@@ -291,19 +291,12 @@ void helper_tlbwr(CPULoongArchState *env)
     fill_tlb_entry(env, index);
 }
 
-void helper_tlbfill(CPULoongArchState *env)
+static int get_random_tlb_index(CPULoongArchState *env,
+                                uint64_t entryhi, uint16_t pagesize)
 {
-    uint64_t address, entryhi;
+    uint64_t address;
+    uint16_t stlb_ps;
     int index, set, stlb_idx;
-    uint16_t pagesize, stlb_ps;
-
-    if (FIELD_EX64(env->CSR_TLBRERA, CSR_TLBRERA, ISTLBR)) {
-        entryhi = env->CSR_TLBREHI;
-        pagesize = FIELD_EX64(env->CSR_TLBREHI, CSR_TLBREHI, PS);
-    } else {
-        entryhi = env->CSR_TLBEHI;
-        pagesize = FIELD_EX64(env->CSR_TLBIDX, CSR_TLBIDX, PS);
-    }
 
     stlb_ps = FIELD_EX64(env->CSR_STLBPS, CSR_STLBPS, PS);
 
@@ -323,6 +316,25 @@ void helper_tlbfill(CPULoongArchState *env)
         index = get_random_tlb(LOONGARCH_STLB, LOONGARCH_TLB_MAX - 1);
     }
 
+    return index;
+}
+
+void helper_tlbfill(CPULoongArchState *env)
+{
+    uint64_t entryhi;
+    uint16_t pagesize;
+    int index;
+
+    if (FIELD_EX64(env->CSR_TLBRERA, CSR_TLBRERA, ISTLBR)) {
+        entryhi = env->CSR_TLBREHI;
+        pagesize = FIELD_EX64(env->CSR_TLBREHI, CSR_TLBREHI, PS);
+    } else {
+        entryhi = env->CSR_TLBEHI;
+        pagesize = FIELD_EX64(env->CSR_TLBIDX, CSR_TLBIDX, PS);
+    }
+
+    index = get_random_tlb_index(env, entryhi, pagesize);
+
     invalidate_tlb(env, index);
     fill_tlb_entry(env, index);
 }
-- 
2.33.0

[PATCH v2 5/5] target/loongarch/tcg: Add hardware page table walker support

[Song Gao]

Add hardware page table walker (HPTW) feature for la664.
Set CPUCFG2.HPTW = 1 to indicate that HPTW is implemented on this CPU.
Set PWCH.HPTW_EN = 1 to enable HPTW.

Signed-off-by: Song Gao <gaosong@loongson.cn>
---
 target/loongarch/cpu-csr.h        |   3 +
 target/loongarch/cpu.c            |   1 +
 target/loongarch/cpu.h            |   1 +
 target/loongarch/cpu_helper.c     |  26 +++++-
 target/loongarch/internals.h      |   4 +-
 target/loongarch/tcg/tlb_helper.c | 147 +++++++++++++++++++++++++++++-
 6 files changed, 176 insertions(+), 6 deletions(-)

diff --git a/target/loongarch/cpu-csr.h b/target/loongarch/cpu-csr.h
index 0834e91f30..1aa015dc44 100644
--- a/target/loongarch/cpu-csr.h
+++ b/target/loongarch/cpu-csr.h
@@ -68,6 +68,8 @@ FIELD(TLBENTRY, PLV, 2, 2)
 FIELD(TLBENTRY, MAT, 4, 2)
 FIELD(TLBENTRY, G, 6, 1)
 FIELD(TLBENTRY, HUGE, 6, 1)
+FIELD(TLBENTRY, PRESENT, 7, 1)
+FIELD(TLBENTRY, WRITE, 8, 1)
 FIELD(TLBENTRY, HGLOBAL, 12, 1)
 FIELD(TLBENTRY, LEVEL, 13, 2)
 FIELD(TLBENTRY_32, PPN, 8, 24)
@@ -103,6 +105,7 @@ FIELD(CSR_PWCH, DIR3_BASE, 0, 6)
 FIELD(CSR_PWCH, DIR3_WIDTH, 6, 6)
 FIELD(CSR_PWCH, DIR4_BASE, 12, 6)
 FIELD(CSR_PWCH, DIR4_WIDTH, 18, 6)
+FIELD(CSR_PWCH, HPTW_EN, 24, 1)
 
 #define LOONGARCH_CSR_STLBPS         0x1e /* Stlb page size */
 FIELD(CSR_STLBPS, PS, 0, 5)
diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c
index d55b4110a9..9dbf2ce67b 100644
--- a/target/loongarch/cpu.c
+++ b/target/loongarch/cpu.c
@@ -476,6 +476,7 @@ static void loongarch_la664_initfn(Object *obj)
     env->cpucfg[0] = 0x14d000; /* PRID */
 
     loongarch_common_initfn(env, obj);
+    env->cpucfg[2] = FIELD_DP32(env->cpucfg[2], CPUCFG2, HPTW, 1);
 }
 
 static void loongarch_la464_initfn(Object *obj)
diff --git a/target/loongarch/cpu.h b/target/loongarch/cpu.h
index 6c41fafb70..84f92507d6 100644
--- a/target/loongarch/cpu.h
+++ b/target/loongarch/cpu.h
@@ -155,6 +155,7 @@ FIELD(CPUCFG2, LBT_ARM, 19, 1)
 FIELD(CPUCFG2, LBT_MIPS, 20, 1)
 FIELD(CPUCFG2, LSPW, 21, 1)
 FIELD(CPUCFG2, LAM, 22, 1)
+FIELD(CPUCFG2, HPTW, 24, 1)
 
 /* cpucfg[3] bits */
 FIELD(CPUCFG3, CCDMA, 0, 1)
diff --git a/target/loongarch/cpu_helper.c b/target/loongarch/cpu_helper.c
index 580362ac3e..35c95e9a27 100644
--- a/target/loongarch/cpu_helper.c
+++ b/target/loongarch/cpu_helper.c
@@ -178,10 +178,11 @@ static hwaddr dmw_va2pa(CPULoongArchState *env, target_ulong va,
 
 int get_physical_address(CPULoongArchState *env, hwaddr *physical,
                          int *prot, target_ulong address,
-                         MMUAccessType access_type, int mmu_idx)
+                         MMUAccessType access_type, int mmu_idx, bool is_debug)
 {
     int user_mode = mmu_idx == MMU_USER_IDX;
     int kernel_mode = mmu_idx == MMU_KERNEL_IDX;
+    int ret;
     uint32_t plv, base_c, base_v;
     int64_t addr_high;
     uint8_t da = FIELD_EX64(env->CSR_CRMD, CSR_CRMD, DA);
@@ -221,8 +222,25 @@ int get_physical_address(CPULoongArchState *env, hwaddr *physical,
     }
 
     /* Mapped address */
-    return loongarch_map_address(env, physical, prot, address,
-                                 access_type, mmu_idx);
+    ret = loongarch_map_address(env, physical, prot, address,
+                                access_type, mmu_idx);
+#ifdef CONFIG_TCG
+    if (!FIELD_EX32(env->cpucfg[2], CPUCFG2, HPTW)) {
+        return ret;
+    }
+
+    if (!FIELD_EX32(env->CSR_PWCH, CSR_PWCH, HPTW_EN)) {
+        return ret;
+    }
+
+    if (do_page_walk(env, address, access_type, ret, physical, is_debug)) {
+        if (!is_debug) {
+            ret = loongarch_map_address(env, physical, prot, address,
+                                        access_type, mmu_idx);
+        }
+    }
+#endif
+    return ret;
 }
 
 hwaddr loongarch_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
@@ -232,7 +250,7 @@ hwaddr loongarch_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
     int prot;
 
     if (get_physical_address(env, &phys_addr, &prot, addr, MMU_DATA_LOAD,
-                             cpu_mmu_index(cs, false)) != 0) {
+                             cpu_mmu_index(cs, false), true) != 0) {
         return -1;
     }
     return phys_addr;
diff --git a/target/loongarch/internals.h b/target/loongarch/internals.h
index 1a02427627..fc8357914b 100644
--- a/target/loongarch/internals.h
+++ b/target/loongarch/internals.h
@@ -56,13 +56,15 @@ bool loongarch_tlb_search(CPULoongArchState *env, target_ulong vaddr,
                           int *index);
 int get_physical_address(CPULoongArchState *env, hwaddr *physical,
                          int *prot, target_ulong address,
-                         MMUAccessType access_type, int mmu_idx);
+                         MMUAccessType access_type, int mmu_idx, bool is_debug);
 hwaddr loongarch_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
 
 #ifdef CONFIG_TCG
 bool loongarch_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
                             MMUAccessType access_type, int mmu_idx,
                             bool probe, uintptr_t retaddr);
+bool do_page_walk(CPULoongArchState *env, vaddr address,
+                  MMUAccessType, int tlb_error, hwaddr *physical, bool is_debug);
 #endif
 #endif /* !CONFIG_USER_ONLY */
 
diff --git a/target/loongarch/tcg/tlb_helper.c b/target/loongarch/tcg/tlb_helper.c
index 463e9be7f2..cecffde56e 100644
--- a/target/loongarch/tcg/tlb_helper.c
+++ b/target/loongarch/tcg/tlb_helper.c
@@ -8,6 +8,7 @@
 
 #include "qemu/osdep.h"
 #include "qemu/guest-random.h"
+#include "qemu/atomic.h"
 
 #include "cpu.h"
 #include "internals.h"
@@ -504,7 +505,7 @@ bool loongarch_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
 
     /* Data access */
     ret = get_physical_address(env, &physical, &prot, address,
-                               access_type, mmu_idx);
+                               access_type, mmu_idx, false);
 
     if (ret == TLBRET_MATCH) {
         tlb_set_page(cs, address & TARGET_PAGE_MASK,
@@ -651,3 +652,147 @@ void helper_ldpte(CPULoongArchState *env, target_ulong base, target_ulong odd,
     }
     env->CSR_TLBREHI = FIELD_DP64(env->CSR_TLBREHI, CSR_TLBREHI, PS, ps);
 }
+
+static target_ulong get_pte_base(CPULoongArchState *env, vaddr address)
+{
+    uint64_t dir_base, dir_width;
+    target_ulong base;
+    int i;
+
+    /* Get PGD */
+    base = ((address >> 63) & 0x1) ? env->CSR_PGDH : env->CSR_PGDL;
+
+    for (i = 4; i > 0; i--) {
+        get_dir_base_width(env, &dir_base, &dir_width, i);
+        /*
+         * LDDIR: level = 2 corresponds to Dir1 in PWCL.
+         * PWCL/PWCH: dir >= 1 && dir_width == 0 means no such level.
+         */
+        if (i >= 2 && dir_width == 0) {
+            continue;
+        }
+        base = do_lddir(env, base, address, i);
+    }
+
+    return base;
+}
+
+bool do_page_walk(CPULoongArchState *env, vaddr address,
+                  MMUAccessType access_type, int tlb_error,
+                  hwaddr *physical, bool is_debug)
+{
+    CPUState *cs = env_cpu(env);
+    target_ulong base, ps, tmp0, tmp1, ptindex, ptoffset, cur_val, new_val, old_val;
+    uint64_t entrylo0, entrylo1, tlbehi, vppn;
+    uint64_t ptbase = FIELD_EX64(env->CSR_PWCL, CSR_PWCL, PTBASE);
+    uint64_t ptwidth = FIELD_EX64(env->CSR_PWCL, CSR_PWCL, PTWIDTH);
+    int index, shift;
+    bool ret = false;
+
+    /*
+     * tlb error map :
+     * TLBRET_NOMATCH : tlb fill
+     * TLBRET_INVALID : access_type = 0/2  tlb_load
+     *                : access_type = 1    tlb_store
+     * TLBRET_DIRTY   : tlb_modify
+     */
+    switch (tlb_error) {
+    case TLBRET_NOMATCH:
+        base = get_pte_base(env, address);
+        if (base == 0) {
+            return ret;
+        }
+        do_ldpte(env, base, address, &tmp0, &tmp1, &ps);
+        entrylo0 = tmp0;
+        entrylo1 = tmp1;
+        tlbehi = address & (TARGET_PAGE_MASK << 1);
+        vppn = FIELD_EX64(tlbehi, CSR_TLBEHI_64, VPPN);
+        if (is_debug) {
+            uint64_t tlb_ppn, tlb_entry;
+            uint8_t n;
+            n = (address >> ps) & 0x1;/* Odd or even */
+
+            tlb_entry = n ? entrylo1: entrylo1;
+            tlb_ppn = FIELD_EX64(tlb_entry, TLBENTRY_64, PPN);
+            /* Remove sw bit between bit12 -- bit PS */
+            tlb_ppn = tlb_ppn & ~(((0x1UL << (ps - 12)) -1));
+            *physical = (tlb_ppn << R_TLBENTRY_64_PPN_SHIFT) |
+                        (address & MAKE_64BIT_MASK(0, ps));
+        } else {
+            index = get_random_tlb_index(env, tlbehi, ps);
+            invalidate_tlb(env, index);
+            do_fill_tlb_entry(env, vppn, entrylo0, entrylo1, index, ps);
+        }
+
+        ret = true;
+        break;
+    case TLBRET_DIRTY:
+    case TLBRET_INVALID:
+        base = get_pte_base(env, address);
+
+        /* 0:64bit, 1:128bit, 2:192bit, 3:256bit */
+        shift = FIELD_EX64(env->CSR_PWCL, CSR_PWCL, PTEWIDTH);
+        shift = (shift + 1) * 3;
+        ptindex = (address >> ptbase) & ((1 << ptwidth) -1);
+        ptoffset = ptindex << shift;
+        tmp0 = base | ptoffset;
+      retry:
+        old_val = ldq_phys(cs->as, tmp0) & TARGET_PHYS_MASK;
+
+        if (old_val == 0) {
+            return ret;
+        }
+
+        new_val = old_val;
+        /* Check pte val, and do tlb modify. */
+        if ((tlb_error == TLBRET_INVALID) &&
+            (access_type == MMU_INST_FETCH )) {
+            if (!(FIELD_EX64(new_val, TLBENTRY, PRESENT))) {
+                break;
+            }
+            new_val = FIELD_DP64(new_val, TLBENTRY, V, 1);
+        } else if ((tlb_error == TLBRET_INVALID) &&
+                   access_type == MMU_DATA_STORE) {
+            if (!((FIELD_EX64(new_val, TLBENTRY, PRESENT) &&
+                  (FIELD_EX64(new_val, TLBENTRY, WRITE))))){
+                break;
+            }
+            new_val = FIELD_DP64(new_val, TLBENTRY, V, 1);
+        } else if (tlb_error ==  TLBRET_DIRTY) {
+            if (!(FIELD_EX64(new_val, TLBENTRY, PRESENT) &&
+                 (FIELD_EX64(new_val, TLBENTRY, WRITE)))) {
+                break;
+            }
+            new_val = FIELD_DP64(new_val, TLBENTRY, V, 1);
+        }
+
+        if (old_val != new_val) {
+            cur_val = qatomic_cmpxchg((uint64_t *)tmp0, old_val, new_val);
+            if (cur_val == old_val) {
+                goto retry;
+            }
+        }
+
+        tmp0 = tmp0 & (~0x8);
+        entrylo0 = ldq_phys(cs->as, tmp0) & TARGET_PHYS_MASK;
+        entrylo1 = ldq_phys(cs->as, tmp0 | 0x8) & TARGET_PHYS_MASK;
+        tlbehi = address & (TARGET_PAGE_MASK << 1);
+        ps = FIELD_EX64(env->CSR_PWCL, CSR_PWCL, PTBASE);
+        vppn = FIELD_EX64(tlbehi, CSR_TLBEHI_64, VPPN);
+
+        /*
+         * srch tlb index with tlb entryhi
+         * if no match, we use get_random_tlb_index() to get random index.
+         */
+        if (!loongarch_tlb_search(env, tlbehi, &index)) {
+            index = get_random_tlb_index(env, tlbehi, ps);
+        }
+        invalidate_tlb(env, index);
+        do_fill_tlb_entry(env, vppn, entrylo0, entrylo1, index, ps);
+        ret = true;
+        break;
+    default:
+        ;
+    }
+    return ret;
+}
-- 
2.33.0

Re: [PATCH v2 0/5] LoongArch/tcg: Add hardware page table walker support

[gaosong]

Ping!

在 2024/10/10 下午2:35, Song Gao 写道:
> Loongson-3A6000 and newer processors have hardware page table walker
> (PTW) support. PTW can handle all fastpaths of PIL/PIS/PIF/PIE
> exceptions by hardware.
>
> V2:
> - Remove the '21' magic value, patch1;
> - Add a flag is_debug for debug access, patch5;
> - Use qatomic_cmpxchg to change the new pte_val, patch5.
>
> Song Gao (5):
>    target/loongarch: Add a new cpu_type la664
>    target/loongarch: Add do_lddir/ldpte()
>    target/loongarch: Add do_fill_tlb_entry()
>    target/loongarch: Add get_random_tlb_index()
>    target/loongarch/tcg: Add hardware page table walker support
>
>   target/loongarch/cpu-csr.h        |   3 +
>   target/loongarch/cpu.c            |  51 ++++--
>   target/loongarch/cpu.h            |   1 +
>   target/loongarch/cpu_helper.c     |  26 ++-
>   target/loongarch/internals.h      |   4 +-
>   target/loongarch/tcg/tlb_helper.c | 277 ++++++++++++++++++++++++------
>   6 files changed, 293 insertions(+), 69 deletions(-)
>

Re: [PATCH v2 5/5] target/loongarch/tcg: Add hardware page table walker support

[Richard Henderson]

On 10/10/24 07:35, Song Gao wrote:
> +        base = get_pte_base(env, address);
> +
> +        /* 0:64bit, 1:128bit, 2:192bit, 3:256bit */
> +        shift = FIELD_EX64(env->CSR_PWCL, CSR_PWCL, PTEWIDTH);
> +        shift = (shift + 1) * 3;
> +        ptindex = (address >> ptbase) & ((1 << ptwidth) -1);
> +        ptoffset = ptindex << shift;
> +        tmp0 = base | ptoffset;

This is a guest virtual address.

> +      retry:
> +        old_val = ldq_phys(cs->as, tmp0) & TARGET_PHYS_MASK;

Fine.

> +        if (old_val != new_val) {
> +            cur_val = qatomic_cmpxchg((uint64_t *)tmp0, old_val, new_val);

This uses a host address.  The cast, and the resulting reference, are incorrect.

This is why Arm and x86 structure things differently, using a different tlb index to 
resolve the host address.  This allows the result to be cached like any other address 
resolution.

Riscv does something a bit simpler, using address_space_translate to resolve the host address.

Most of the rest of this patch set is going to need review from loongson employees, since 
I've not seen public documentation in english for this feature.

r~

Re: [PATCH v2 5/5] target/loongarch/tcg: Add hardware page table walker support

[gaosong]

在 2024/11/5 下午10:27, Richard Henderson 写道:
> On 10/10/24 07:35, Song Gao wrote:
>> +        base = get_pte_base(env, address);
>> +
>> +        /* 0:64bit, 1:128bit, 2:192bit, 3:256bit */
>> +        shift = FIELD_EX64(env->CSR_PWCL, CSR_PWCL, PTEWIDTH);
>> +        shift = (shift + 1) * 3;
>> +        ptindex = (address >> ptbase) & ((1 << ptwidth) -1);
>> +        ptoffset = ptindex << shift;
>> +        tmp0 = base | ptoffset;
>
> This is a guest virtual address.
>
>> +      retry:
>> +        old_val = ldq_phys(cs->as, tmp0) & TARGET_PHYS_MASK;
>
> Fine.
>
>> +        if (old_val != new_val) {
>> +            cur_val = qatomic_cmpxchg((uint64_t *)tmp0, old_val, 
>> new_val);
>
> This uses a host address.  The cast, and the resulting reference, are 
> incorrect.
>
> This is why Arm and x86 structure things differently, using a 
> different tlb index to resolve the host address.  This allows the 
> result to be cached like any other address resolution.
>
> Riscv does something a bit simpler, using address_space_translate to 
> resolve the host address.
>
> Most of the rest of this patch set is going to need review from 
> loongson employees, since I've not seen public documentation in 
> english for this feature.
>
> r~

Hi, Yijun and Wangliupu

Could you guys help review this patch and provide some documentation 
about it?

Thanks.
Song Gao