* [PATCH 01/23] target/riscv: Split out vext_set_nf_elems_1s
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 02/23] target/riscv: Hoist vma check out of vext_set_tail_elems_1s Richard Henderson
` (21 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Provide a helper that 1's element I for all NF vectors.
For the callers, this performs the element mask test once
instead of NF times.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 69 ++++++++++++++++----------------
1 file changed, 34 insertions(+), 35 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index e28d8a3d9f..20a8f95efb 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -328,6 +328,15 @@ static void vext_set_tail_elems_1s(uint32_t vl, void *vd,
}
}
+static void vext_set_nf_elems_1s(void *vd, uint32_t i, uint32_t nf,
+ uint32_t esz, uint32_t max_elems)
+{
+ for (uint32_t k = 0; k < nf; ++k) {
+ vext_set_elems_1s(vd, true, (i + k * max_elems) * esz,
+ (i + k * max_elems + 1) * esz);
+ }
+}
+
/*
* stride: access vector element from strided memory
*/
@@ -337,7 +346,6 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, target_ulong stride,
vext_ldst_elem_fn_tlb *ldst_elem, uint32_t log2_esz,
uintptr_t ra)
{
- uint32_t i, k;
uint32_t nf = vext_nf(desc);
uint32_t max_elems = vext_max_elems(desc, log2_esz);
uint32_t esz = 1 << log2_esz;
@@ -345,23 +353,21 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, target_ulong stride,
VSTART_CHECK_EARLY_EXIT(env, env->vl);
- for (i = env->vstart; i < env->vl; env->vstart = ++i) {
- k = 0;
- while (k < nf) {
- if (!vm && !vext_elem_mask(v0, i)) {
- /* set masked-off elements to 1s */
- vext_set_elems_1s(vd, vma, (i + k * max_elems) * esz,
- (i + k * max_elems + 1) * esz);
- k++;
- continue;
+ for (uint32_t i = env->vstart; i < env->vl; env->vstart = ++i) {
+ if (!vm && !vext_elem_mask(v0, i)) {
+ if (vma) {
+ vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
+ continue;
+ }
+
+ for (uint32_t k = 0; k < nf; ++k) {
target_ulong addr = base + stride * i + (k << log2_esz);
ldst_elem(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
- k++;
}
}
- env->vstart = 0;
+ env->vstart = 0;
vext_set_tail_elems_1s(env->vl, vd, desc, nf, esz, max_elems);
}
@@ -632,7 +638,6 @@ vext_ldst_index(void *vd, void *v0, target_ulong base,
vext_ldst_elem_fn_tlb *ldst_elem,
uint32_t log2_esz, uintptr_t ra)
{
- uint32_t i, k;
uint32_t nf = vext_nf(desc);
uint32_t vm = vext_vm(desc);
uint32_t max_elems = vext_max_elems(desc, log2_esz);
@@ -641,24 +646,21 @@ vext_ldst_index(void *vd, void *v0, target_ulong base,
VSTART_CHECK_EARLY_EXIT(env, env->vl);
- /* load bytes from guest memory */
- for (i = env->vstart; i < env->vl; env->vstart = ++i) {
- k = 0;
- while (k < nf) {
- if (!vm && !vext_elem_mask(v0, i)) {
- /* set masked-off elements to 1s */
- vext_set_elems_1s(vd, vma, (i + k * max_elems) * esz,
- (i + k * max_elems + 1) * esz);
- k++;
- continue;
+ for (uint32_t i = env->vstart; i < env->vl; env->vstart = ++i) {
+ if (!vm && !vext_elem_mask(v0, i)) {
+ if (vma) {
+ vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
+ continue;
+ }
+
+ for (uint32_t k = 0; k < nf; ++k) {
abi_ptr addr = get_index_addr(base, i, vs2) + (k << log2_esz);
ldst_elem(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
- k++;
}
}
- env->vstart = 0;
+ env->vstart = 0;
vext_set_tail_elems_1s(env->vl, vd, desc, nf, esz, max_elems);
}
@@ -824,25 +826,22 @@ ProbeSuccess:
}
} else {
for (i = env->vstart; i < env->vl; i++) {
- k = 0;
- while (k < nf) {
- if (!vext_elem_mask(v0, i)) {
- /* set masked-off elements to 1s */
- vext_set_elems_1s(vd, vma, (i + k * max_elems) * esz,
- (i + k * max_elems + 1) * esz);
- k++;
- continue;
+ if (!vext_elem_mask(v0, i)) {
+ if (vma) {
+ vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
+ continue;
+ }
+ for (k = 0; k < nf; ++k) {
addr = base + ((i * nf + k) << log2_esz);
ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems,
vd, ra);
- k++;
}
}
}
}
- env->vstart = 0;
+ env->vstart = 0;
vext_set_tail_elems_1s(env->vl, vd, desc, nf, esz, max_elems);
}
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 02/23] target/riscv: Hoist vma check out of vext_set_tail_elems_1s
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
2026-08-15 19:45 ` [PATCH 01/23] target/riscv: Split out vext_set_nf_elems_1s Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-16 15:25 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 03/23] target/riscv: Split out vext_ldst_nf_tlb Richard Henderson
` (20 subsequent siblings)
22 siblings, 1 reply; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
In all cases we've already extracted it in the caller.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 35 +++++++++++++++++---------------
1 file changed, 19 insertions(+), 16 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 20a8f95efb..8f74162d50 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -311,19 +311,11 @@ vext_continuous_ldst_host(CPURISCVState *env, vext_ldst_elem_fn_host *ldst_host,
}
}
-static void vext_set_tail_elems_1s(uint32_t vl, void *vd,
- uint32_t desc, uint32_t nf,
+static void vext_set_tail_elems_1s(uint32_t vl, void *vd, uint32_t nf,
uint32_t esz, uint32_t max_elems)
{
- uint32_t vta = vext_vta(desc);
- int k;
-
- if (vta == 0) {
- return;
- }
-
- for (k = 0; k < nf; ++k) {
- vext_set_elems_1s(vd, vta, (k * max_elems + vl) * esz,
+ for (uint32_t k = 0; k < nf; ++k) {
+ vext_set_elems_1s(vd, true, (k * max_elems + vl) * esz,
(k * max_elems + max_elems) * esz);
}
}
@@ -368,7 +360,9 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, target_ulong stride,
}
env->vstart = 0;
- vext_set_tail_elems_1s(env->vl, vd, desc, nf, esz, max_elems);
+ if (vma) {
+ vext_set_tail_elems_1s(env->vl, vd, nf, esz, max_elems);
+ }
}
#define GEN_VEXT_LD_STRIDE(NAME, ETYPE, LOAD_FN) \
@@ -482,6 +476,7 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
uint32_t k;
target_ulong page_split, elems, addr;
uint32_t nf = vext_nf(desc);
+ uint32_t vma = vext_vma(desc);
uint32_t max_elems = vext_max_elems(desc, log2_esz);
uint32_t esz = 1 << log2_esz;
uint32_t msize = nf * esz;
@@ -500,7 +495,9 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
esz, is_load);
env->vstart = 0;
- vext_set_tail_elems_1s(evl, vd, desc, nf, esz, max_elems);
+ if (vma) {
+ vext_set_tail_elems_1s(evl, vd, nf, esz, max_elems);
+ }
return;
}
#endif
@@ -542,7 +539,9 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
}
env->vstart = 0;
- vext_set_tail_elems_1s(evl, vd, desc, nf, esz, max_elems);
+ if (vma) {
+ vext_set_tail_elems_1s(evl, vd, nf, esz, max_elems);
+ }
}
/*
@@ -661,7 +660,9 @@ vext_ldst_index(void *vd, void *v0, target_ulong base,
}
env->vstart = 0;
- vext_set_tail_elems_1s(env->vl, vd, desc, nf, esz, max_elems);
+ if (vma) {
+ vext_set_tail_elems_1s(env->vl, vd, nf, esz, max_elems);
+ }
}
#define GEN_VEXT_LD_INDEX(NAME, ETYPE, INDEX_FN, LOAD_FN) \
@@ -842,7 +843,9 @@ ProbeSuccess:
}
env->vstart = 0;
- vext_set_tail_elems_1s(env->vl, vd, desc, nf, esz, max_elems);
+ if (vma) {
+ vext_set_tail_elems_1s(env->vl, vd, nf, esz, max_elems);
+ }
}
#define GEN_VEXT_LDFF(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 03/23] target/riscv: Split out vext_ldst_nf_tlb
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
2026-08-15 19:45 ` [PATCH 01/23] target/riscv: Split out vext_set_nf_elems_1s Richard Henderson
2026-08-15 19:45 ` [PATCH 02/23] target/riscv: Hoist vma check out of vext_set_tail_elems_1s Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 04/23] target/riscv: Split out vext_ldst_nf_host Richard Henderson
` (19 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Provide a helper that calls ldst_tlb for all NF vectors.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 77 +++++++++++++++-----------------
1 file changed, 35 insertions(+), 42 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 8f74162d50..9c0cbcac16 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -329,6 +329,16 @@ static void vext_set_nf_elems_1s(void *vd, uint32_t i, uint32_t nf,
}
}
+static void vext_ldst_nf_tlb(CPURISCVState *env, void *vd, target_ulong addr,
+ uint32_t i, uint32_t nf,
+ uint32_t esz, uint32_t max_elems,
+ vext_ldst_elem_fn_tlb *ldst_tlb, uintptr_t ra)
+{
+ for (uint32_t k = 0; k < nf; k++, addr += esz) {
+ ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
+ }
+}
+
/*
* stride: access vector element from strided memory
*/
@@ -346,16 +356,11 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, target_ulong stride,
VSTART_CHECK_EARLY_EXIT(env, env->vl);
for (uint32_t i = env->vstart; i < env->vl; env->vstart = ++i) {
- if (!vm && !vext_elem_mask(v0, i)) {
- if (vma) {
- vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
- }
- continue;
- }
-
- for (uint32_t k = 0; k < nf; ++k) {
- target_ulong addr = base + stride * i + (k << log2_esz);
- ldst_elem(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
+ if (vm || vext_elem_mask(v0, i)) {
+ vext_ldst_nf_tlb(env, vd, base + stride * i, i, nf,
+ esz, max_elems, ldst_elem, ra);
+ } else if (vma) {
+ vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
}
@@ -411,6 +416,7 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
int i, k, flags;
uint32_t esz = 1 << log2_esz;
uint32_t size = (elems * nf) << log2_esz;
+ uint32_t msize = nf * esz;
uint32_t evl = env->vstart + elems;
MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE;
@@ -453,15 +459,10 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
vext_continuous_ldst_tlb(env, ldst_tlb, vd, evl, addr, env->vstart,
ra, esz, is_load);
} else {
- /* load bytes from guest memory */
for (i = env->vstart; i < evl; env->vstart = ++i) {
- k = 0;
- while (k < nf) {
- ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems,
- vd, ra);
- addr += esz;
- k++;
- }
+ vext_ldst_nf_tlb(env, vd, addr, i, nf, esz,
+ max_elems, ldst_tlb, ra);
+ addr += msize;
}
}
}
@@ -473,7 +474,6 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
uint32_t evl, uintptr_t ra, bool is_load)
{
- uint32_t k;
target_ulong page_split, elems, addr;
uint32_t nf = vext_nf(desc);
uint32_t vma = vext_vma(desc);
@@ -519,17 +519,16 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
/* Load/store elements in the second page */
if (unlikely(env->vstart < evl)) {
+ addr = base + env->vstart * msize;
+
/* Cross page element */
if (unlikely(page_split % msize)) {
- for (k = 0; k < nf; k++) {
- addr = base + ((env->vstart * nf + k) << log2_esz);
- ldst_tlb(env, adjust_addr(env, addr),
- env->vstart + k * max_elems, vd, ra);
- }
+ vext_ldst_nf_tlb(env, vd, addr, env->vstart,
+ nf, esz, max_elems, ldst_tlb, ra);
env->vstart++;
+ addr += msize;
}
- addr = base + ((env->vstart * nf) << log2_esz);
/* Get number of elements of second page */
elems = evl - env->vstart;
@@ -724,7 +723,7 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb,
vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz, uintptr_t ra)
{
- uint32_t i, k, vl = 0;
+ uint32_t i, vl = 0;
uint32_t nf = vext_nf(desc);
uint32_t vm = vext_vm(desc);
uint32_t max_elems = vext_max_elems(desc, log2_esz);
@@ -806,17 +805,16 @@ ProbeSuccess:
/* Load/store elements in the second page */
if (unlikely(env->vstart < env->vl)) {
+ addr = base + env->vstart * msize;
+
/* Cross page element */
if (unlikely(page_split % msize)) {
- for (k = 0; k < nf; k++) {
- addr = base + ((env->vstart * nf + k) << log2_esz);
- ldst_tlb(env, adjust_addr(env, addr),
- env->vstart + k * max_elems, vd, ra);
- }
+ vext_ldst_nf_tlb(env, vd, addr, env->vstart, nf,
+ esz, max_elems, ldst_tlb, ra);
env->vstart++;
+ addr += msize;
}
- addr = base + ((env->vstart * nf) << log2_esz);
/* Get number of elements of second page */
elems = env->vl - env->vstart;
@@ -827,16 +825,11 @@ ProbeSuccess:
}
} else {
for (i = env->vstart; i < env->vl; i++) {
- if (!vext_elem_mask(v0, i)) {
- if (vma) {
- vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
- }
- continue;
- }
- for (k = 0; k < nf; ++k) {
- addr = base + ((i * nf + k) << log2_esz);
- ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems,
- vd, ra);
+ if (vext_elem_mask(v0, i)) {
+ vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf,
+ esz, max_elems, ldst_tlb, ra);
+ } else if (vma) {
+ vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
}
}
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 04/23] target/riscv: Split out vext_ldst_nf_host
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (2 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 03/23] target/riscv: Split out vext_ldst_nf_tlb Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 05/23] target/riscv: Add evl argument to vext_ldst_elem_fn_host Richard Henderson
` (18 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Provide a helper that calls ldst_host for all NF vectors.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 23 ++++++++++++++---------
1 file changed, 14 insertions(+), 9 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 9c0cbcac16..661f1ab2f6 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -329,6 +329,15 @@ static void vext_set_nf_elems_1s(void *vd, uint32_t i, uint32_t nf,
}
}
+static void vext_ldst_nf_host(void *vd, void *host, uint32_t i, uint32_t nf,
+ uint32_t esz, uint32_t max_elems,
+ vext_ldst_elem_fn_host *ldst_host)
+{
+ for (uint32_t k = 0; k < nf; k++, host += esz) {
+ ldst_host(vd, i + k * max_elems, host);
+ }
+}
+
static void vext_ldst_nf_tlb(CPURISCVState *env, void *vd, target_ulong addr,
uint32_t i, uint32_t nf,
uint32_t esz, uint32_t max_elems,
@@ -413,7 +422,7 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
vext_ldst_elem_fn_host *ldst_host, uintptr_t ra)
{
void *host;
- int i, k, flags;
+ int flags;
uint32_t esz = 1 << log2_esz;
uint32_t size = (elems * nf) << log2_esz;
uint32_t msize = nf * esz;
@@ -444,13 +453,9 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
vext_continuous_ldst_host(env, ldst_host, vd, evl, env->vstart,
host, esz, is_load);
} else {
- for (i = env->vstart; i < evl; ++i) {
- k = 0;
- while (k < nf) {
- ldst_host(vd, i + k * max_elems, host);
- host += esz;
- k++;
- }
+ for (uint32_t i = env->vstart; i < evl; ++i) {
+ vext_ldst_nf_host(vd, host, i, nf, esz, max_elems, ldst_host);
+ host += msize;
}
}
env->vstart += elems;
@@ -459,7 +464,7 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
vext_continuous_ldst_tlb(env, ldst_tlb, vd, evl, addr, env->vstart,
ra, esz, is_load);
} else {
- for (i = env->vstart; i < evl; env->vstart = ++i) {
+ for (uint32_t i = env->vstart; i < evl; env->vstart = ++i) {
vext_ldst_nf_tlb(env, vd, addr, i, nf, esz,
max_elems, ldst_tlb, ra);
addr += msize;
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 05/23] target/riscv: Add evl argument to vext_ldst_elem_fn_host
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (3 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 04/23] target/riscv: Split out vext_ldst_nf_host Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-16 15:55 ` Richard Henderson
2026-08-25 18:30 ` Max Chou
2026-08-15 19:45 ` [PATCH 06/23] target/riscv: Drop is_load parameter from vext_continuous_ldst_tlb Richard Henderson
` (17 subsequent siblings)
22 siblings, 2 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
This merges vext_continuous_ldst_host into the ldst_host function.
We can then handle the byte little-endian optimization at compile-time.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 83 +++++++++++++++-----------------
1 file changed, 40 insertions(+), 43 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 661f1ab2f6..3786a17231 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -214,7 +214,8 @@ static inline MemOpIdx vext_make_memop_idx(CPURISCVState *env, size_t size)
/* elements operations for load and store */
typedef void vext_ldst_elem_fn_tlb(CPURISCVState *env, abi_ptr addr,
uint32_t idx, void *vd, uintptr_t retaddr);
-typedef void vext_ldst_elem_fn_host(void *vd, uint32_t idx, void *host);
+typedef void vext_ldst_elem_fn_host(void *vd, void *host,
+ uint32_t idx, uint32_t evl);
#define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF) \
static inline QEMU_ALWAYS_INLINE \
@@ -226,12 +227,15 @@ void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \
*cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr); \
} \
-#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \
-static inline QEMU_ALWAYS_INLINE \
-void NAME##_host(void *vd, uint32_t idx, void *host) \
-{ \
- ETYPE *cur = ((ETYPE *)vd + H(idx)); \
- *cur = (ETYPE)LDSUF##_p(host); \
+#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \
+static inline QEMU_ALWAYS_INLINE \
+void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl) \
+{ \
+ do { \
+ ETYPE *cur = (ETYPE *)vd + H(idx); \
+ *cur = LDSUF##_p(host); \
+ host += sizeof(ETYPE); \
+ } while (++idx < evl); \
}
GEN_VEXT_TLB_LD_ELEM(lde_b, uint8_t, H1, ldb)
@@ -239,7 +243,16 @@ GEN_VEXT_TLB_LD_ELEM(lde_h, uint16_t, H2, ldw)
GEN_VEXT_TLB_LD_ELEM(lde_w, uint32_t, H4, ldl)
GEN_VEXT_TLB_LD_ELEM(lde_d, uint64_t, H8, ldq)
+#if HOST_BIG_ENDIAN
GEN_VEXT_HOST_LD_ELEM(lde_b, uint8_t, H1, ldub)
+#else
+static inline QEMU_ALWAYS_INLINE
+void lde_b_host(void *vd, void *host, uint32_t idx, uint32_t evl)
+{
+ memcpy(vd + idx, host, evl - idx);
+}
+#endif
+
GEN_VEXT_HOST_LD_ELEM(lde_h, uint16_t, H2, lduw_le)
GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le)
GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le)
@@ -254,12 +267,15 @@ void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \
cpu_##STSUF##_mmu(env, addr, data, oi, retaddr); \
} \
-#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \
-static inline QEMU_ALWAYS_INLINE \
-void NAME##_host(void *vd, uint32_t idx, void *host) \
-{ \
- ETYPE data = *((ETYPE *)vd + H(idx)); \
- STSUF##_p(host, data); \
+#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \
+static inline QEMU_ALWAYS_INLINE \
+void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl) \
+{ \
+ do { \
+ ETYPE data = *((ETYPE *)vd + H(idx)); \
+ STSUF##_p(host, data); \
+ host += sizeof(ETYPE); \
+ } while (++idx < evl); \
}
GEN_VEXT_TLB_ST_ELEM(ste_b, uint8_t, H1, stb)
@@ -267,7 +283,16 @@ GEN_VEXT_TLB_ST_ELEM(ste_h, uint16_t, H2, stw)
GEN_VEXT_TLB_ST_ELEM(ste_w, uint32_t, H4, stl)
GEN_VEXT_TLB_ST_ELEM(ste_d, uint64_t, H8, stq)
+#if HOST_BIG_ENDIAN
GEN_VEXT_HOST_ST_ELEM(ste_b, uint8_t, H1, stb)
+#else
+static inline QEMU_ALWAYS_INLINE
+void ste_b_host(void *vd, void *host, uint32_t idx, uint32_t evl)
+{
+ memcpy(host, vd + idx, evl - idx);
+}
+#endif
+
GEN_VEXT_HOST_ST_ELEM(ste_h, uint16_t, H2, stw_le)
GEN_VEXT_HOST_ST_ELEM(ste_w, uint32_t, H4, stl_le)
GEN_VEXT_HOST_ST_ELEM(ste_d, uint64_t, H8, stq_le)
@@ -284,33 +309,6 @@ vext_continuous_ldst_tlb(CPURISCVState *env, vext_ldst_elem_fn_tlb *ldst_tlb,
}
}
-static inline QEMU_ALWAYS_INLINE void
-vext_continuous_ldst_host(CPURISCVState *env, vext_ldst_elem_fn_host *ldst_host,
- void *vd, uint32_t evl, uint32_t reg_start, void *host,
- uint32_t esz, bool is_load)
-{
- if (HOST_BIG_ENDIAN) {
- for (; reg_start < evl; reg_start++, host += esz) {
- ldst_host(vd, reg_start, host);
- }
- } else {
- if (esz == 1) {
- uint32_t byte_offset = reg_start * esz;
- uint32_t size = (evl - reg_start) * esz;
-
- if (is_load) {
- memcpy(vd + byte_offset, host, size);
- } else {
- memcpy(host, vd + byte_offset, size);
- }
- } else {
- for (; reg_start < evl; reg_start++, host += esz) {
- ldst_host(vd, reg_start, host);
- }
- }
- }
-}
-
static void vext_set_tail_elems_1s(uint32_t vl, void *vd, uint32_t nf,
uint32_t esz, uint32_t max_elems)
{
@@ -334,7 +332,7 @@ static void vext_ldst_nf_host(void *vd, void *host, uint32_t i, uint32_t nf,
vext_ldst_elem_fn_host *ldst_host)
{
for (uint32_t k = 0; k < nf; k++, host += esz) {
- ldst_host(vd, i + k * max_elems, host);
+ ldst_host(vd + k * max_elems, host, i, i + 1);
}
}
@@ -450,8 +448,7 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
*/
if (flags == 0 && (riscv_cpu_cfg(env)->ext_zicclsm || !misaligned)) {
if (nf == 1) {
- vext_continuous_ldst_host(env, ldst_host, vd, evl, env->vstart,
- host, esz, is_load);
+ ldst_host(vd, host, env->vstart, evl);
} else {
for (uint32_t i = env->vstart; i < evl; ++i) {
vext_ldst_nf_host(vd, host, i, nf, esz, max_elems, ldst_host);
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH 05/23] target/riscv: Add evl argument to vext_ldst_elem_fn_host
2026-08-15 19:45 ` [PATCH 05/23] target/riscv: Add evl argument to vext_ldst_elem_fn_host Richard Henderson
@ 2026-08-16 15:55 ` Richard Henderson
2026-08-25 18:30 ` Max Chou
1 sibling, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-16 15:55 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
On 8/15/26 12:45, Richard Henderson wrote:
> This merges vext_continuous_ldst_host into the ldst_host function.
> We can then handle the byte little-endian optimization at compile-time.
>
> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
> ---
> target/riscv/tcg/vector_helper.c | 83 +++++++++++++++-----------------
> 1 file changed, 40 insertions(+), 43 deletions(-)
In retrospect, this probably isn't a great idea. The load side is fine, but the store
side suffers from atomicity issues. I'll think of something else.
r~
>
> diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
> index 661f1ab2f6..3786a17231 100644
> --- a/target/riscv/tcg/vector_helper.c
> +++ b/target/riscv/tcg/vector_helper.c
> @@ -214,7 +214,8 @@ static inline MemOpIdx vext_make_memop_idx(CPURISCVState *env, size_t size)
> /* elements operations for load and store */
> typedef void vext_ldst_elem_fn_tlb(CPURISCVState *env, abi_ptr addr,
> uint32_t idx, void *vd, uintptr_t retaddr);
> -typedef void vext_ldst_elem_fn_host(void *vd, uint32_t idx, void *host);
> +typedef void vext_ldst_elem_fn_host(void *vd, void *host,
> + uint32_t idx, uint32_t evl);
>
> #define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF) \
> static inline QEMU_ALWAYS_INLINE \
> @@ -226,12 +227,15 @@ void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \
> *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr); \
> } \
>
> -#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \
> -static inline QEMU_ALWAYS_INLINE \
> -void NAME##_host(void *vd, uint32_t idx, void *host) \
> -{ \
> - ETYPE *cur = ((ETYPE *)vd + H(idx)); \
> - *cur = (ETYPE)LDSUF##_p(host); \
> +#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \
> +static inline QEMU_ALWAYS_INLINE \
> +void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl) \
> +{ \
> + do { \
> + ETYPE *cur = (ETYPE *)vd + H(idx); \
> + *cur = LDSUF##_p(host); \
> + host += sizeof(ETYPE); \
> + } while (++idx < evl); \
> }
>
> GEN_VEXT_TLB_LD_ELEM(lde_b, uint8_t, H1, ldb)
> @@ -239,7 +243,16 @@ GEN_VEXT_TLB_LD_ELEM(lde_h, uint16_t, H2, ldw)
> GEN_VEXT_TLB_LD_ELEM(lde_w, uint32_t, H4, ldl)
> GEN_VEXT_TLB_LD_ELEM(lde_d, uint64_t, H8, ldq)
>
> +#if HOST_BIG_ENDIAN
> GEN_VEXT_HOST_LD_ELEM(lde_b, uint8_t, H1, ldub)
> +#else
> +static inline QEMU_ALWAYS_INLINE
> +void lde_b_host(void *vd, void *host, uint32_t idx, uint32_t evl)
> +{
> + memcpy(vd + idx, host, evl - idx);
> +}
> +#endif
> +
> GEN_VEXT_HOST_LD_ELEM(lde_h, uint16_t, H2, lduw_le)
> GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le)
> GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le)
> @@ -254,12 +267,15 @@ void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \
> cpu_##STSUF##_mmu(env, addr, data, oi, retaddr); \
> } \
>
> -#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \
> -static inline QEMU_ALWAYS_INLINE \
> -void NAME##_host(void *vd, uint32_t idx, void *host) \
> -{ \
> - ETYPE data = *((ETYPE *)vd + H(idx)); \
> - STSUF##_p(host, data); \
> +#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \
> +static inline QEMU_ALWAYS_INLINE \
> +void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl) \
> +{ \
> + do { \
> + ETYPE data = *((ETYPE *)vd + H(idx)); \
> + STSUF##_p(host, data); \
> + host += sizeof(ETYPE); \
> + } while (++idx < evl); \
> }
>
> GEN_VEXT_TLB_ST_ELEM(ste_b, uint8_t, H1, stb)
> @@ -267,7 +283,16 @@ GEN_VEXT_TLB_ST_ELEM(ste_h, uint16_t, H2, stw)
> GEN_VEXT_TLB_ST_ELEM(ste_w, uint32_t, H4, stl)
> GEN_VEXT_TLB_ST_ELEM(ste_d, uint64_t, H8, stq)
>
> +#if HOST_BIG_ENDIAN
> GEN_VEXT_HOST_ST_ELEM(ste_b, uint8_t, H1, stb)
> +#else
> +static inline QEMU_ALWAYS_INLINE
> +void ste_b_host(void *vd, void *host, uint32_t idx, uint32_t evl)
> +{
> + memcpy(host, vd + idx, evl - idx);
> +}
> +#endif
> +
> GEN_VEXT_HOST_ST_ELEM(ste_h, uint16_t, H2, stw_le)
> GEN_VEXT_HOST_ST_ELEM(ste_w, uint32_t, H4, stl_le)
> GEN_VEXT_HOST_ST_ELEM(ste_d, uint64_t, H8, stq_le)
> @@ -284,33 +309,6 @@ vext_continuous_ldst_tlb(CPURISCVState *env, vext_ldst_elem_fn_tlb *ldst_tlb,
> }
> }
>
> -static inline QEMU_ALWAYS_INLINE void
> -vext_continuous_ldst_host(CPURISCVState *env, vext_ldst_elem_fn_host *ldst_host,
> - void *vd, uint32_t evl, uint32_t reg_start, void *host,
> - uint32_t esz, bool is_load)
> -{
> - if (HOST_BIG_ENDIAN) {
> - for (; reg_start < evl; reg_start++, host += esz) {
> - ldst_host(vd, reg_start, host);
> - }
> - } else {
> - if (esz == 1) {
> - uint32_t byte_offset = reg_start * esz;
> - uint32_t size = (evl - reg_start) * esz;
> -
> - if (is_load) {
> - memcpy(vd + byte_offset, host, size);
> - } else {
> - memcpy(host, vd + byte_offset, size);
> - }
> - } else {
> - for (; reg_start < evl; reg_start++, host += esz) {
> - ldst_host(vd, reg_start, host);
> - }
> - }
> - }
> -}
> -
> static void vext_set_tail_elems_1s(uint32_t vl, void *vd, uint32_t nf,
> uint32_t esz, uint32_t max_elems)
> {
> @@ -334,7 +332,7 @@ static void vext_ldst_nf_host(void *vd, void *host, uint32_t i, uint32_t nf,
> vext_ldst_elem_fn_host *ldst_host)
> {
> for (uint32_t k = 0; k < nf; k++, host += esz) {
> - ldst_host(vd, i + k * max_elems, host);
> + ldst_host(vd + k * max_elems, host, i, i + 1);
> }
> }
>
> @@ -450,8 +448,7 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
> */
> if (flags == 0 && (riscv_cpu_cfg(env)->ext_zicclsm || !misaligned)) {
> if (nf == 1) {
> - vext_continuous_ldst_host(env, ldst_host, vd, evl, env->vstart,
> - host, esz, is_load);
> + ldst_host(vd, host, env->vstart, evl);
> } else {
> for (uint32_t i = env->vstart; i < evl; ++i) {
> vext_ldst_nf_host(vd, host, i, nf, esz, max_elems, ldst_host);
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH 05/23] target/riscv: Add evl argument to vext_ldst_elem_fn_host
2026-08-15 19:45 ` [PATCH 05/23] target/riscv: Add evl argument to vext_ldst_elem_fn_host Richard Henderson
2026-08-16 15:55 ` Richard Henderson
@ 2026-08-25 18:30 ` Max Chou
1 sibling, 0 replies; 33+ messages in thread
From: Max Chou @ 2026-08-25 18:30 UTC (permalink / raw)
To: Richard Henderson; +Cc: qemu-devel, frank.chang, qemu-riscv
On 2026-08-15 12:45, Richard Henderson wrote:
> This merges vext_continuous_ldst_host into the ldst_host function.
> We can then handle the byte little-endian optimization at compile-time.
>
> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
> ---
> target/riscv/tcg/vector_helper.c | 83 +++++++++++++++-----------------
> 1 file changed, 40 insertions(+), 43 deletions(-)
>
> diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
> index 661f1ab2f6..3786a17231 100644
> --- a/target/riscv/tcg/vector_helper.c
> +++ b/target/riscv/tcg/vector_helper.c
> @@ -214,7 +214,8 @@ static inline MemOpIdx vext_make_memop_idx(CPURISCVState *env, size_t size)
> /* elements operations for load and store */
> typedef void vext_ldst_elem_fn_tlb(CPURISCVState *env, abi_ptr addr,
> uint32_t idx, void *vd, uintptr_t retaddr);
> -typedef void vext_ldst_elem_fn_host(void *vd, uint32_t idx, void *host);
> +typedef void vext_ldst_elem_fn_host(void *vd, void *host,
> + uint32_t idx, uint32_t evl);
>
> #define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF) \
> static inline QEMU_ALWAYS_INLINE \
> @@ -226,12 +227,15 @@ void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \
> *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr); \
> } \
>
> -#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \
> -static inline QEMU_ALWAYS_INLINE \
> -void NAME##_host(void *vd, uint32_t idx, void *host) \
> -{ \
> - ETYPE *cur = ((ETYPE *)vd + H(idx)); \
> - *cur = (ETYPE)LDSUF##_p(host); \
> +#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \
> +static inline QEMU_ALWAYS_INLINE \
> +void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl) \
> +{ \
> + do { \
> + ETYPE *cur = (ETYPE *)vd + H(idx); \
> + *cur = LDSUF##_p(host); \
> + host += sizeof(ETYPE); \
> + } while (++idx < evl); \
> }
>
> GEN_VEXT_TLB_LD_ELEM(lde_b, uint8_t, H1, ldb)
> @@ -239,7 +243,16 @@ GEN_VEXT_TLB_LD_ELEM(lde_h, uint16_t, H2, ldw)
> GEN_VEXT_TLB_LD_ELEM(lde_w, uint32_t, H4, ldl)
> GEN_VEXT_TLB_LD_ELEM(lde_d, uint64_t, H8, ldq)
>
> +#if HOST_BIG_ENDIAN
> GEN_VEXT_HOST_LD_ELEM(lde_b, uint8_t, H1, ldub)
> +#else
> +static inline QEMU_ALWAYS_INLINE
> +void lde_b_host(void *vd, void *host, uint32_t idx, uint32_t evl)
> +{
> + memcpy(vd + idx, host, evl - idx);
> +}
> +#endif
> +
> GEN_VEXT_HOST_LD_ELEM(lde_h, uint16_t, H2, lduw_le)
> GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le)
> GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le)
> @@ -254,12 +267,15 @@ void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \
> cpu_##STSUF##_mmu(env, addr, data, oi, retaddr); \
> } \
>
> -#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \
> -static inline QEMU_ALWAYS_INLINE \
> -void NAME##_host(void *vd, uint32_t idx, void *host) \
> -{ \
> - ETYPE data = *((ETYPE *)vd + H(idx)); \
> - STSUF##_p(host, data); \
> +#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \
> +static inline QEMU_ALWAYS_INLINE \
> +void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl) \
> +{ \
> + do { \
> + ETYPE data = *((ETYPE *)vd + H(idx)); \
> + STSUF##_p(host, data); \
> + host += sizeof(ETYPE); \
> + } while (++idx < evl); \
> }
>
> GEN_VEXT_TLB_ST_ELEM(ste_b, uint8_t, H1, stb)
> @@ -267,7 +283,16 @@ GEN_VEXT_TLB_ST_ELEM(ste_h, uint16_t, H2, stw)
> GEN_VEXT_TLB_ST_ELEM(ste_w, uint32_t, H4, stl)
> GEN_VEXT_TLB_ST_ELEM(ste_d, uint64_t, H8, stq)
>
> +#if HOST_BIG_ENDIAN
> GEN_VEXT_HOST_ST_ELEM(ste_b, uint8_t, H1, stb)
> +#else
> +static inline QEMU_ALWAYS_INLINE
> +void ste_b_host(void *vd, void *host, uint32_t idx, uint32_t evl)
> +{
> + memcpy(host, vd + idx, evl - idx);
> +}
> +#endif
> +
> GEN_VEXT_HOST_ST_ELEM(ste_h, uint16_t, H2, stw_le)
> GEN_VEXT_HOST_ST_ELEM(ste_w, uint32_t, H4, stl_le)
> GEN_VEXT_HOST_ST_ELEM(ste_d, uint64_t, H8, stq_le)
> @@ -284,33 +309,6 @@ vext_continuous_ldst_tlb(CPURISCVState *env, vext_ldst_elem_fn_tlb *ldst_tlb,
> }
> }
>
> -static inline QEMU_ALWAYS_INLINE void
> -vext_continuous_ldst_host(CPURISCVState *env, vext_ldst_elem_fn_host *ldst_host,
> - void *vd, uint32_t evl, uint32_t reg_start, void *host,
> - uint32_t esz, bool is_load)
> -{
> - if (HOST_BIG_ENDIAN) {
> - for (; reg_start < evl; reg_start++, host += esz) {
> - ldst_host(vd, reg_start, host);
> - }
> - } else {
> - if (esz == 1) {
> - uint32_t byte_offset = reg_start * esz;
> - uint32_t size = (evl - reg_start) * esz;
> -
> - if (is_load) {
> - memcpy(vd + byte_offset, host, size);
> - } else {
> - memcpy(host, vd + byte_offset, size);
> - }
> - } else {
> - for (; reg_start < evl; reg_start++, host += esz) {
> - ldst_host(vd, reg_start, host);
> - }
> - }
> - }
> -}
> -
> static void vext_set_tail_elems_1s(uint32_t vl, void *vd, uint32_t nf,
> uint32_t esz, uint32_t max_elems)
> {
> @@ -334,7 +332,7 @@ static void vext_ldst_nf_host(void *vd, void *host, uint32_t i, uint32_t nf,
> vext_ldst_elem_fn_host *ldst_host)
> {
> for (uint32_t k = 0; k < nf; k++, host += esz) {
> - ldst_host(vd, i + k * max_elems, host);
> + ldst_host(vd + k * max_elems, host, i, i + 1);
The type of ldst_host vd parameter is void *, so vd + k * max_elems
advances in bytes, not the target element width.
I think here should be
> + ldst_host(vd, host, i + k * max_elems, i + k * max_elems + 1);
rnax
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH 06/23] target/riscv: Drop is_load parameter from vext_continuous_ldst_tlb
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (4 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 05/23] target/riscv: Add evl argument to vext_ldst_elem_fn_host Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 07/23] target/riscv: Remove vext_continuous_ldst_tlb Richard Henderson
` (16 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
This parameter is unused.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 3786a17231..a571982388 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -299,9 +299,8 @@ GEN_VEXT_HOST_ST_ELEM(ste_d, uint64_t, H8, stq_le)
static inline QEMU_ALWAYS_INLINE void
vext_continuous_ldst_tlb(CPURISCVState *env, vext_ldst_elem_fn_tlb *ldst_tlb,
- void *vd, uint32_t evl, target_ulong addr,
- uint32_t reg_start, uintptr_t ra, uint32_t esz,
- bool is_load)
+ void *vd, uint32_t evl, target_ulong addr,
+ uint32_t reg_start, uintptr_t ra, uint32_t esz)
{
uint32_t i;
for (i = env->vstart; i < evl; env->vstart = ++i, addr += esz) {
@@ -458,8 +457,8 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
env->vstart += elems;
} else {
if (nf == 1) {
- vext_continuous_ldst_tlb(env, ldst_tlb, vd, evl, addr, env->vstart,
- ra, esz, is_load);
+ vext_continuous_ldst_tlb(env, ldst_tlb, vd, evl, addr,
+ env->vstart, ra, esz);
} else {
for (uint32_t i = env->vstart; i < evl; env->vstart = ++i) {
vext_ldst_nf_tlb(env, vd, addr, i, nf, esz,
@@ -493,9 +492,8 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
*/
if (nf == 1 && (evl << log2_esz) <= 6) {
addr = base + (env->vstart << log2_esz);
- vext_continuous_ldst_tlb(env, ldst_tlb, vd, evl, addr, env->vstart, ra,
- esz, is_load);
-
+ vext_continuous_ldst_tlb(env, ldst_tlb, vd, evl, addr,
+ env->vstart, ra, esz);
env->vstart = 0;
if (vma) {
vext_set_tail_elems_1s(evl, vd, nf, esz, max_elems);
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 07/23] target/riscv: Remove vext_continuous_ldst_tlb
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (5 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 06/23] target/riscv: Drop is_load parameter from vext_continuous_ldst_tlb Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 08/23] target/riscv: Move misalignment check out of vext_page_ldst_us Richard Henderson
` (15 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Drop the usage in vext_page_ldst_us, merging the nf == 1
case into the vext_ldst_nf_tlb loop.
Inline it into the last use in vext_ldst_us.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 34 ++++++++++----------------------
1 file changed, 10 insertions(+), 24 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index a571982388..79c1d77a14 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -297,17 +297,6 @@ GEN_VEXT_HOST_ST_ELEM(ste_h, uint16_t, H2, stw_le)
GEN_VEXT_HOST_ST_ELEM(ste_w, uint32_t, H4, stl_le)
GEN_VEXT_HOST_ST_ELEM(ste_d, uint64_t, H8, stq_le)
-static inline QEMU_ALWAYS_INLINE void
-vext_continuous_ldst_tlb(CPURISCVState *env, vext_ldst_elem_fn_tlb *ldst_tlb,
- void *vd, uint32_t evl, target_ulong addr,
- uint32_t reg_start, uintptr_t ra, uint32_t esz)
-{
- uint32_t i;
- for (i = env->vstart; i < evl; env->vstart = ++i, addr += esz) {
- ldst_tlb(env, adjust_addr(env, addr), i, vd, ra);
- }
-}
-
static void vext_set_tail_elems_1s(uint32_t vl, void *vd, uint32_t nf,
uint32_t esz, uint32_t max_elems)
{
@@ -456,15 +445,10 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
}
env->vstart += elems;
} else {
- if (nf == 1) {
- vext_continuous_ldst_tlb(env, ldst_tlb, vd, evl, addr,
- env->vstart, ra, esz);
- } else {
- for (uint32_t i = env->vstart; i < evl; env->vstart = ++i) {
- vext_ldst_nf_tlb(env, vd, addr, i, nf, esz,
- max_elems, ldst_tlb, ra);
- addr += msize;
- }
+ for (uint32_t i = env->vstart; i < evl; env->vstart = ++i) {
+ vext_ldst_nf_tlb(env, vd, addr, i, nf, esz,
+ max_elems, ldst_tlb, ra);
+ addr += msize;
}
}
}
@@ -487,13 +471,15 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
#if defined(CONFIG_USER_ONLY)
/*
- * For data sizes <= 6 bytes we get better performance by simply calling
- * vext_continuous_ldst_tlb
+ * For data sizes <= 6 bytes we get better performance
+ * by simply calling ldst_tlb.
*/
if (nf == 1 && (evl << log2_esz) <= 6) {
addr = base + (env->vstart << log2_esz);
- vext_continuous_ldst_tlb(env, ldst_tlb, vd, evl, addr,
- env->vstart, ra, esz);
+ for (uint32_t i = env->vstart; i < evl;
+ env->vstart = ++i, addr += esz) {
+ ldst_tlb(env, adjust_addr(env, addr), i, vd, ra);
+ }
env->vstart = 0;
if (vma) {
vext_set_tail_elems_1s(evl, vd, nf, esz, max_elems);
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 08/23] target/riscv: Move misalignment check out of vext_page_ldst_us
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (6 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 07/23] target/riscv: Remove vext_continuous_ldst_tlb Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 09/23] target/riscv: Split out vext_page_ldst_us_{host,tlb} Richard Henderson
` (14 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Alignment faults generally have precedence over page faults,
therefore we need to test for that before probing pages.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 62 +++++++++++++++++++++++++-------
1 file changed, 50 insertions(+), 12 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 79c1d77a14..12ef020840 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -31,6 +31,9 @@
#include "tcg/tcg-gvec-desc.h"
#include "internals.h"
#include "vector_internals.h"
+#ifdef CONFIG_USER_ONLY
+#include "user/cpu_loop.h"
+#endif
#include <math.h>
static target_ulong vtype_reserved(CPURISCVState *env, target_ulong vtype)
@@ -425,16 +428,7 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
probe_pages(env, addr, size, ra, access_type, mmu_index, &host, &flags,
true);
- bool misaligned = addr & (esz - 1);
-
- /*
- * Allow the host fast-pash when:
- * 1. Page permission/pmp/watchpoint are checked and we have a contigous
- * host mapping.
- * 2. Zicclsm is enabled or load/store is not a misaligned access.
- * Otherwise, we will fall back to the slow TLB-path.
- */
- if (flags == 0 && (riscv_cpu_cfg(env)->ext_zicclsm || !misaligned)) {
+ if (flags == 0) {
if (nf == 1) {
ldst_host(vd, host, env->vstart, evl);
} else {
@@ -453,6 +447,20 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
}
}
+static void vext_test_alignment(CPURISCVState *env, vaddr addr, uint32_t esz,
+ MMUAccessType access_type, int mmu_index,
+ uintptr_t ra)
+{
+ if (!riscv_cpu_cfg(env)->ext_zicclsm && (addr & (esz - 1)) != 0) {
+#ifdef CONFIG_USER_ONLY
+ cpu_loop_exit_sigbus(env_cpu(env), addr, access_type, ra);
+#else
+ riscv_cpu_do_unaligned_access(env_cpu(env), addr, access_type,
+ mmu_index, ra);
+#endif
+ }
+}
+
static inline QEMU_ALWAYS_INLINE void
vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
vext_ldst_elem_fn_tlb *ldst_tlb,
@@ -466,16 +474,21 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
uint32_t esz = 1 << log2_esz;
uint32_t msize = nf * esz;
int mmu_index = riscv_env_mmu_index(env, false);
+ MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE;
VSTART_CHECK_EARLY_EXIT(env, evl);
+ addr = base + env->vstart * msize;
+
+ /* Recognize alignment fault before memory protection fault. */
+ vext_test_alignment(env, addr, esz, access_type, mmu_index, ra);
+
#if defined(CONFIG_USER_ONLY)
/*
* For data sizes <= 6 bytes we get better performance
* by simply calling ldst_tlb.
*/
if (nf == 1 && (evl << log2_esz) <= 6) {
- addr = base + (env->vstart << log2_esz);
for (uint32_t i = env->vstart; i < evl;
env->vstart = ++i, addr += esz) {
ldst_tlb(env, adjust_addr(env, addr), i, vd, ra);
@@ -489,7 +502,6 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
#endif
/* Calculate the page range of first page */
- addr = base + ((env->vstart * nf) << log2_esz);
page_split = -(addr | TARGET_PAGE_MASK);
/* Get number of elements */
elems = page_split / msize;
@@ -723,7 +735,27 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
VSTART_CHECK_EARLY_EXIT(env, env->vl);
+ /* Search for the first active element. */
+ if (!vm) {
+ for (i = env->vstart; i < env->vl; ++i) {
+ if (vext_elem_mask(v0, i)) {
+ break;
+ }
+ if (vma) {
+ vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
+ }
+ }
+ if (i == env->vl) {
+ goto tail;
+ }
+ env->vstart = i;
+ }
+
addr = base + ((env->vstart * nf) << log2_esz);
+
+ /* Recognize alignment fault before memory protection fault. */
+ vext_test_alignment(env, addr, esz, MMU_DATA_LOAD, mmu_index, ra);
+
page_split = -(addr | TARGET_PAGE_MASK);
/* Get number of elements */
elems = page_split / msize;
@@ -821,6 +853,7 @@ ProbeSuccess:
}
}
+ tail:
env->vstart = 0;
if (vma) {
vext_set_tail_elems_1s(env->vl, vd, nf, esz, max_elems);
@@ -866,9 +899,14 @@ vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
uint32_t evl = nf * max_elems;
uint32_t esz = 1 << log2_esz;
int mmu_index = riscv_env_mmu_index(env, false);
+ MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE;
/* Calculate the page range of first page */
addr = base + (env->vstart << log2_esz);
+
+ /* Recognize alignment fault before memory protection fault. */
+ vext_test_alignment(env, addr, esz, access_type, mmu_index, ra);
+
page_split = -(addr | TARGET_PAGE_MASK);
/* Get number of elements */
elems = page_split / esz;
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 09/23] target/riscv: Split out vext_page_ldst_us_{host,tlb}
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (7 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 08/23] target/riscv: Move misalignment check out of vext_page_ldst_us Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 10/23] target/riscv: Rewrite vext_ldst_us Richard Henderson
` (13 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Split out the tail of vext_page_ldst_us, after probing.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 60 ++++++++++++++++++++------------
1 file changed, 38 insertions(+), 22 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 12ef020840..7f721dc24d 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -403,6 +403,39 @@ GEN_VEXT_ST_STRIDE(vsse64_v, int64_t, ste_d_tlb)
*/
/* unmasked unit-stride load and store operation */
+static inline QEMU_ALWAYS_INLINE void
+vext_page_ldst_us_host(void *vd, void *host, uint32_t i, uint32_t evl,
+ uint32_t nf, uint32_t log2_esz, uint32_t max_elems,
+ vext_ldst_elem_fn_host *ldst_host)
+{
+ if (nf == 1) {
+ ldst_host(vd, host, i, evl);
+ } else {
+ uint32_t esz = 1 << log2_esz;
+ uint32_t msize = nf << log2_esz;
+ do {
+ vext_ldst_nf_host(vd, host, i, nf, esz, max_elems, ldst_host);
+ host += msize;
+ } while (++i < evl);
+ }
+}
+
+static inline QEMU_ALWAYS_INLINE void
+vext_page_ldst_us_tlb(CPURISCVState *env, void *vd, target_ulong addr,
+ uint32_t i, uint32_t evl, uint32_t nf,
+ uint32_t log2_esz, uint32_t max_elems,
+ vext_ldst_elem_fn_tlb *ldst_tlb,
+ int mmu_index, uintptr_t ra)
+{
+ uint32_t esz = 1 << log2_esz;
+ uint32_t msize = nf << log2_esz;
+ do {
+ vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, ldst_tlb, ra);
+ addr += msize;
+ env->vstart = ++i;
+ } while (i < evl);
+}
+
static inline QEMU_ALWAYS_INLINE void
vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
uint32_t elems, uint32_t nf, uint32_t max_elems,
@@ -412,38 +445,21 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
{
void *host;
int flags;
- uint32_t esz = 1 << log2_esz;
uint32_t size = (elems * nf) << log2_esz;
- uint32_t msize = nf * esz;
- uint32_t evl = env->vstart + elems;
+ uint32_t i = env->vstart;
MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE;
- /*
- * Maximum vector length is VLMAX == 2^16 == LMUL * VL / SEW, and
- * occurs for LMUL == 8, SEW == 8, VL == 2^16.
- */
- g_assert(env->vstart < UINT16_MAX && UINT16_MAX - env->vstart >= elems);
-
/* Check page permission/pmp/watchpoint/etc. */
probe_pages(env, addr, size, ra, access_type, mmu_index, &host, &flags,
true);
if (flags == 0) {
- if (nf == 1) {
- ldst_host(vd, host, env->vstart, evl);
- } else {
- for (uint32_t i = env->vstart; i < evl; ++i) {
- vext_ldst_nf_host(vd, host, i, nf, esz, max_elems, ldst_host);
- host += msize;
- }
- }
+ vext_page_ldst_us_host(vd, host, i, i + elems, nf,
+ log2_esz, max_elems, ldst_host);
env->vstart += elems;
} else {
- for (uint32_t i = env->vstart; i < evl; env->vstart = ++i) {
- vext_ldst_nf_tlb(env, vd, addr, i, nf, esz,
- max_elems, ldst_tlb, ra);
- addr += msize;
- }
+ vext_page_ldst_us_tlb(env, vd, addr, i, i + elems, nf, log2_esz,
+ max_elems, ldst_tlb, mmu_index, ra);
}
}
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 10/23] target/riscv: Rewrite vext_ldst_us
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (8 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 09/23] target/riscv: Split out vext_page_ldst_us_{host,tlb} Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-25 19:00 ` Max Chou
2026-08-15 19:45 ` [PATCH 11/23] target/riscv: Rewrite vext_ldff Richard Henderson
` (12 subsequent siblings)
22 siblings, 1 reply; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
We already computed the page split here, which means we
were quite ready to call probe_access directly. Do so,
then use vext_ldst_nf_{host,tlb} to load the elements.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 92 ++++++++++++++++++++------------
1 file changed, 57 insertions(+), 35 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 7f721dc24d..5b03e23fc3 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -483,7 +483,7 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
uint32_t evl, uintptr_t ra, bool is_load)
{
- target_ulong page_split, elems, addr;
+ target_ulong elems, addr, last, last_in_page, page_split;
uint32_t nf = vext_nf(desc);
uint32_t vma = vext_vma(desc);
uint32_t max_elems = vext_max_elems(desc, log2_esz);
@@ -491,10 +491,12 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
uint32_t msize = nf * esz;
int mmu_index = riscv_env_mmu_index(env, false);
MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE;
+ uint32_t i = env->vstart;
+ void *host;
VSTART_CHECK_EARLY_EXIT(env, evl);
- addr = base + env->vstart * msize;
+ addr = base + i * msize;
/* Recognize alignment fault before memory protection fault. */
vext_test_alignment(env, addr, esz, access_type, mmu_index, ra);
@@ -505,52 +507,72 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
* by simply calling ldst_tlb.
*/
if (nf == 1 && (evl << log2_esz) <= 6) {
- for (uint32_t i = env->vstart; i < evl;
- env->vstart = ++i, addr += esz) {
+ for (; i < evl; env->vstart = ++i, addr += esz) {
ldst_tlb(env, adjust_addr(env, addr), i, vd, ra);
}
- env->vstart = 0;
- if (vma) {
- vext_set_tail_elems_1s(evl, vd, nf, esz, max_elems);
- }
- return;
+ goto tail;
}
#endif
- /* Calculate the page range of first page */
- page_split = -(addr | TARGET_PAGE_MASK);
- /* Get number of elements */
- elems = page_split / msize;
- if (unlikely(env->vstart + elems >= evl)) {
- elems = evl - env->vstart;
- }
+ /* Calculate the page range of first page. */
+ last = base + evl * msize - 1;
+ last_in_page = addr | ~TARGET_PAGE_MASK;
+ page_split = last_in_page - addr;
+
+ /* Validate the first page is accessible. */
+ host = probe_access(env, adjust_addr(env, addr),
+ MIN(last, last_in_page) - addr + 1,
+ access_type, mmu_index, ra);
+
+ /* Get number of complete elements in the first page. */
+ elems = MIN(page_split / msize, evl - i);
/* Load/store elements in the first page */
if (likely(elems)) {
- vext_page_ldst_us(env, vd, addr, elems, nf, max_elems, log2_esz,
- is_load, mmu_index, ldst_tlb, ldst_host, ra);
- }
+ uint32_t page_evl = i + elems;
- /* Load/store elements in the second page */
- if (unlikely(env->vstart < evl)) {
- addr = base + env->vstart * msize;
-
- /* Cross page element */
- if (unlikely(page_split % msize)) {
- vext_ldst_nf_tlb(env, vd, addr, env->vstart,
- nf, esz, max_elems, ldst_tlb, ra);
- env->vstart++;
- addr += msize;
+ if (host) {
+ vext_page_ldst_us_host(vd, host, i, page_evl, nf,
+ log2_esz, max_elems, ldst_host);
+ } else {
+ vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf,
+ log2_esz, max_elems,
+ ldst_tlb, mmu_index, ra);
}
- /* Get number of elements of second page */
- elems = evl - env->vstart;
-
- /* Load/store elements in the second page */
- vext_page_ldst_us(env, vd, addr, elems, nf, max_elems, log2_esz,
- is_load, mmu_index, ldst_tlb, ldst_host, ra);
+ /* Usually the first page contains the entire vector. */
+ if (likely(page_evl == evl)) {
+ goto tail;
+ }
+ env->vstart = i = page_evl;
+ addr += elems * msize;
}
+ /* Cross page element */
+ if (unlikely(page_split % msize)) {
+ vext_ldst_nf_tlb(env, vd, addr, i++, nf, esz, max_elems, ldst_tlb, ra);
+ if (i == evl) {
+ goto tail;
+ }
+ env->vstart = i;
+ addr += msize;
+ }
+
+ /* Validate the second page is accessible. */
+ assert(i < evl);
+ elems = evl - i;
+ host = probe_access(env, adjust_addr(env, addr), elems * msize,
+ access_type, mmu_index, ra);
+
+ if (host) {
+ vext_page_ldst_us_host(vd, host, i, evl, nf,
+ log2_esz, max_elems, ldst_host);
+ } else {
+ vext_page_ldst_us_tlb(env, vd, addr, i, evl, nf, log2_esz, max_elems,
+ ldst_tlb, mmu_index, ra);
+ }
+
+ tail:
env->vstart = 0;
if (vma) {
vext_set_tail_elems_1s(evl, vd, nf, esz, max_elems);
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH 10/23] target/riscv: Rewrite vext_ldst_us
2026-08-15 19:45 ` [PATCH 10/23] target/riscv: Rewrite vext_ldst_us Richard Henderson
@ 2026-08-25 19:00 ` Max Chou
0 siblings, 0 replies; 33+ messages in thread
From: Max Chou @ 2026-08-25 19:00 UTC (permalink / raw)
To: Richard Henderson; +Cc: qemu-devel, frank.chang, qemu-riscv
On 2026-08-15 12:45, Richard Henderson wrote:
> We already computed the page split here, which means we
> were quite ready to call probe_access directly. Do so,
> then use vext_ldst_nf_{host,tlb} to load the elements.
>
> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
> ---
> target/riscv/tcg/vector_helper.c | 92 ++++++++++++++++++++------------
> 1 file changed, 57 insertions(+), 35 deletions(-)
>
> diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
> index 7f721dc24d..5b03e23fc3 100644
> --- a/target/riscv/tcg/vector_helper.c
> +++ b/target/riscv/tcg/vector_helper.c
> @@ -483,7 +483,7 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
> vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
> uint32_t evl, uintptr_t ra, bool is_load)
> {
> - target_ulong page_split, elems, addr;
> + target_ulong elems, addr, last, last_in_page, page_split;
> uint32_t nf = vext_nf(desc);
> uint32_t vma = vext_vma(desc);
> uint32_t max_elems = vext_max_elems(desc, log2_esz);
> @@ -491,10 +491,12 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
> uint32_t msize = nf * esz;
> int mmu_index = riscv_env_mmu_index(env, false);
> MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE;
> + uint32_t i = env->vstart;
> + void *host;
>
> VSTART_CHECK_EARLY_EXIT(env, evl);
>
> - addr = base + env->vstart * msize;
> + addr = base + i * msize;
>
> /* Recognize alignment fault before memory protection fault. */
> vext_test_alignment(env, addr, esz, access_type, mmu_index, ra);
> @@ -505,52 +507,72 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
> * by simply calling ldst_tlb.
> */
> if (nf == 1 && (evl << log2_esz) <= 6) {
> - for (uint32_t i = env->vstart; i < evl;
> - env->vstart = ++i, addr += esz) {
> + for (; i < evl; env->vstart = ++i, addr += esz) {
> ldst_tlb(env, adjust_addr(env, addr), i, vd, ra);
> }
> - env->vstart = 0;
> - if (vma) {
> - vext_set_tail_elems_1s(evl, vd, nf, esz, max_elems);
> - }
> - return;
> + goto tail;
> }
> #endif
>
> - /* Calculate the page range of first page */
> - page_split = -(addr | TARGET_PAGE_MASK);
> - /* Get number of elements */
> - elems = page_split / msize;
> - if (unlikely(env->vstart + elems >= evl)) {
> - elems = evl - env->vstart;
> - }
> + /* Calculate the page range of first page. */
> + last = base + evl * msize - 1;
> + last_in_page = addr | ~TARGET_PAGE_MASK;
> + page_split = last_in_page - addr;
> +
> + /* Validate the first page is accessible. */
> + host = probe_access(env, adjust_addr(env, addr),
> + MIN(last, last_in_page) - addr + 1,
> + access_type, mmu_index, ra);
> +
> + /* Get number of complete elements in the first page. */
> + elems = MIN(page_split / msize, evl - i);
>
The page_split may excludes the byte at last_in_page, which the probe
size includes it. That elems will undercount a complete element ending
at the page boundary.
Then the following cross page element and second page probe will
be affected.
Maybe we could fix it by replacing the page_split with something like
probe_size below:
+ target_unlong probe_bytes = MIN(last, last_in_page) - addr + 1;
/* Validate the first page is accessible. */
- host = probe_access(env, adjust_addr(env, addr),
- MIN(last, last_in_page) - addr + 1,
+ host = probe_access(env, adjust_addr(env, addr), probe_bytes,
access_type, mmu_index, ra);
/* Get number of complete elements in the first page. */
- elems = MIN(page_split / msize, evl - i);
+ elems = MIN(probe_bytes / msize, evl - i);
> + /* Cross page element */
> + if (unlikely(page_split % msize)) {
+ if (unlikely(probe_bytes % msize)) {
> + vext_ldst_nf_tlb(env, vd, addr, i++, nf, esz, max_elems, ldst_tlb, ra);
> + if (i == evl) {
> + goto tail;
> + }
> + env->vstart = i;
> + addr += msize;
> + }
> +
> + /* Validate the second page is accessible. */
> + assert(i < evl);
> + elems = evl - i;
> + host = probe_access(env, adjust_addr(env, addr), elems * msize,
+ probe_bytes = last - addr + 1;
+ host = probe_access(env, adjust_addr(env, addr), probe_bytes,
> + access_type, mmu_index, ra);
rnax
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH 11/23] target/riscv: Rewrite vext_ldff
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (9 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 10/23] target/riscv: Rewrite vext_ldst_us Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-25 17:58 ` Max Chou
2026-08-15 19:45 ` [PATCH 12/23] target/riscv: Split out vext_ldst_us_desc Richard Henderson
` (11 subsequent siblings)
22 siblings, 1 reply; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Do not call probe_pages for every active element.
We can make do with no more than 2 such calls for
the two pages the insn might reference.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 258 ++++++++++++++++++++-----------
1 file changed, 172 insertions(+), 86 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 5b03e23fc3..b482537f40 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -759,135 +759,221 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb,
vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz, uintptr_t ra)
{
- uint32_t i, vl = 0;
+ uint32_t i = env->vstart, vl = env->vl;
uint32_t nf = vext_nf(desc);
uint32_t vm = vext_vm(desc);
uint32_t max_elems = vext_max_elems(desc, log2_esz);
uint32_t esz = 1 << log2_esz;
uint32_t msize = nf * esz;
uint32_t vma = vext_vma(desc);
- target_ulong addr, addr_i, offset, remain, page_split, elems;
+ target_ulong addr, last, last_in_page, page_split, elems;
int mmu_index = riscv_env_mmu_index(env, false);
+ bool first_active;
int flags;
void *host;
VSTART_CHECK_EARLY_EXIT(env, env->vl);
- /* Search for the first active element. */
- if (!vm) {
- for (i = env->vstart; i < env->vl; ++i) {
+ /* Skip forward to the first active element. */
+ if (vm) {
+ first_active = i == 0;
+ } else if (i == 0 && vext_elem_mask(v0, 0)) {
+ first_active = true;
+ } else {
+ first_active = false;
+ while (1) {
if (vext_elem_mask(v0, i)) {
break;
}
if (vma) {
vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
+ if (++i == vl) {
+ goto tail;
+ }
}
- if (i == env->vl) {
- goto tail;
- }
- env->vstart = i;
}
-
- addr = base + ((env->vstart * nf) << log2_esz);
+ addr = base + i * msize;
/* Recognize alignment fault before memory protection fault. */
vext_test_alignment(env, addr, esz, MMU_DATA_LOAD, mmu_index, ra);
- page_split = -(addr | TARGET_PAGE_MASK);
- /* Get number of elements */
- elems = page_split / msize;
- if (unlikely(env->vstart + elems >= env->vl)) {
- elems = env->vl - env->vstart;
+ /* Calculate the page range of first page. */
+ last = base + vl * msize - 1;
+ last_in_page = addr | ~TARGET_PAGE_MASK;
+ page_split = last_in_page - addr;
+
+ /*
+ * If the first element is active, it must succeed.
+ * If it also crosses the page, just go ahead and load it.
+ */
+ if (unlikely(page_split < msize) && first_active) {
+ vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz, max_elems, ldst_tlb, ra);
+ env->vstart = i = 1;
+ addr += msize;
+ page_split = -(addr | TARGET_PAGE_MASK);
+ first_active = false;
}
- /* Check page permission/pmp/watchpoint/etc. */
- probe_pages(env, addr, (env->vl - env->vstart) * msize, ra, MMU_DATA_LOAD,
- mmu_index, &host, &flags, true);
+ /*
+ * Test whether the first page is accessible.
+ * If the first element is active, it must succeed.
+ */
+ flags = probe_access_flags(env, adjust_addr(env, addr),
+ MIN(last, last_in_page) - addr + 1,
+ MMU_DATA_LOAD, mmu_index, !first_active,
+ &host, ra);
- if (flags & ~TLB_WATCHPOINT) {
- /* probe every access */
- for (i = env->vstart; i < env->vl; i++) {
- if (!vm && !vext_elem_mask(v0, i)) {
- continue;
- }
- addr_i = adjust_addr(env, base + i * (nf << log2_esz));
- if (i == 0) {
- /* Allow fault on first element. */
- probe_pages(env, addr_i, nf << log2_esz, ra, MMU_DATA_LOAD,
- mmu_index, &host, NULL, false);
+ /* Get number of complete elements in the first page. */
+ elems = MIN(page_split / msize, vl - i);
+
+ /* Load complete elements from the first page. */
+ if (likely(elems)) {
+ uint32_t page_evl = i + elems;
+
+ if (flags == 0) {
+ if (vm) {
+ vext_page_ldst_us_host(vd, host, i, page_evl, nf,
+ log2_esz, max_elems, ldst_host);
} else {
- remain = nf << log2_esz;
- while (remain > 0) {
- offset = -(addr_i | TARGET_PAGE_MASK);
-
- /* Probe nonfault on subsequent elements. */
- probe_pages(env, addr_i, offset, 0, MMU_DATA_LOAD,
- mmu_index, &host, &flags, true);
-
- /*
- * Stop if invalid (unmapped) or mmio (transaction may
- * fail). Do not stop if watchpoint, as the spec says that
- * first-fault should continue to access the same
- * elements regardless of any watchpoint.
- */
- if (flags & ~TLB_WATCHPOINT) {
- vl = i;
- goto ProbeSuccess;
+ host -= addr - base;
+ do {
+ if (vext_elem_mask(v0, i)) {
+ vext_ldst_nf_host(vd, host + i * msize, i, nf, esz,
+ max_elems, ldst_host);
+ } else if (vma) {
+ vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
- if (remain <= offset) {
- break;
+ } while (++i < page_evl);
+ }
+ } else {
+ /*
+ * If the first element is active, it must succeed.
+ * This will load from MMIO or fault from INVALID.
+ */
+ if (first_active) {
+ vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz,
+ max_elems, ldst_tlb, ra);
+ i = 1;
+ addr += msize;
+ }
+
+ /* Stop if invalid (unmapped) or mmio (transaction may fail). */
+ if (flags & (TLB_INVALID_MASK | TLB_MMIO)) {
+ env->vl = i;
+ goto tail;
+ }
+
+ /* None of these ldst_tlb calls may fault. */
+ if (vm) {
+ vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf,
+ log2_esz, max_elems,
+ ldst_tlb, mmu_index, ra);
+ } else {
+ do {
+ if (vext_elem_mask(v0, i)) {
+ vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf,
+ esz, max_elems, ldst_tlb, ra);
+ } else if (vma) {
+ vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
- remain -= offset;
- addr_i = adjust_addr(env, addr_i + offset);
- }
+ } while (++i < page_evl);
+ }
+ }
+
+ /* Usually the first page contains the entire vector. */
+ if (likely(page_evl == vl)) {
+ goto tail;
+ }
+ i = page_evl;
+ }
+
+ /* Skip forward to the next active element. */
+ if (!vm) {
+ while (1) {
+ if (vext_elem_mask(v0, i)) {
+ break;
+ }
+ if (vma) {
+ vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
+ }
+ if (++i == vl) {
+ goto tail;
}
}
}
-ProbeSuccess:
- /* load bytes from guest memory */
- if (vl != 0) {
- env->vl = vl;
+
+ addr = base + i * msize;
+ page_split = -(addr | TARGET_PAGE_MASK);
+
+ /* Validate the second page is accessible. */
+ if (unlikely(page_split < msize)) {
+ /*
+ * Cross page element which isn't first.
+ * We have not yet advanced addr to the next page.
+ */
+ target_ulong next_page = addr + page_split;
+ flags |= probe_access_flags(env, adjust_addr(env, next_page),
+ last - next_page + 1, MMU_DATA_LOAD,
+ mmu_index, true, &host, ra);
+
+ /* Stop if invalid (unmapped) or mmio (transaction may fail). */
+ if (flags & (TLB_INVALID_MASK | TLB_MMIO)) {
+ env->vl = i;
+ goto tail;
+ }
+
+ vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, ldst_tlb, ra);
+ if (++i == vl) {
+ goto tail;
+ }
+ addr += msize;
+ if (host) {
+ host += addr - next_page;
+ }
+ } else {
+ flags = probe_access_flags(env, adjust_addr(env, addr),
+ last - addr + 1, MMU_DATA_LOAD,
+ mmu_index, true, &host, ra);
+
+ /* Stop if invalid (unmapped) or mmio (transaction may fail). */
+ if (flags & (TLB_INVALID_MASK | TLB_MMIO)) {
+ env->vl = i;
+ goto tail;
+ }
}
- if (env->vstart < env->vl) {
+ /* Load complete elements from the second page. */
+ if (flags == 0) {
if (vm) {
- /* Load/store elements in the first page */
- if (likely(elems)) {
- vext_page_ldst_us(env, vd, addr, elems, nf, max_elems,
- log2_esz, true, mmu_index, ldst_tlb,
- ldst_host, ra);
- }
-
- /* Load/store elements in the second page */
- if (unlikely(env->vstart < env->vl)) {
- addr = base + env->vstart * msize;
-
- /* Cross page element */
- if (unlikely(page_split % msize)) {
- vext_ldst_nf_tlb(env, vd, addr, env->vstart, nf,
- esz, max_elems, ldst_tlb, ra);
- env->vstart++;
- addr += msize;
- }
-
- /* Get number of elements of second page */
- elems = env->vl - env->vstart;
-
- /* Load/store elements in the second page */
- vext_page_ldst_us(env, vd, addr, elems, nf, max_elems,
- log2_esz, true, mmu_index, ldst_tlb,
- ldst_host, ra);
- }
+ vext_page_ldst_us_host(vd, host, i, vl, nf,
+ log2_esz, max_elems, ldst_host);
} else {
- for (i = env->vstart; i < env->vl; i++) {
+ host -= addr - base;
+ do {
+ if (vext_elem_mask(v0, i)) {
+ vext_ldst_nf_host(vd, host + i * msize, i, nf, esz,
+ max_elems, ldst_host);
+ } else if (vma) {
+ vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
+ }
+ } while (++i < vl);
+ }
+ } else {
+ /* None of these ldst_tlb calls may fault. */
+ if (vm) {
+ vext_page_ldst_us_tlb(env, vd, addr, i, vl, nf,
+ log2_esz, max_elems,
+ ldst_tlb, mmu_index, ra);
+ } else {
+ do {
if (vext_elem_mask(v0, i)) {
vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf,
esz, max_elems, ldst_tlb, ra);
} else if (vma) {
vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
- }
+ } while (++i < vl);
}
}
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH 11/23] target/riscv: Rewrite vext_ldff
2026-08-15 19:45 ` [PATCH 11/23] target/riscv: Rewrite vext_ldff Richard Henderson
@ 2026-08-25 17:58 ` Max Chou
2026-08-25 20:15 ` Richard Henderson
0 siblings, 1 reply; 33+ messages in thread
From: Max Chou @ 2026-08-25 17:58 UTC (permalink / raw)
To: Richard Henderson; +Cc: qemu-devel, frank.chang, qemu-riscv
On 2026-08-15 12:45, Richard Henderson wrote:
> Do not call probe_pages for every active element.
> We can make do with no more than 2 such calls for
> the two pages the insn might reference.
>
> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
> + /*
> + * Test whether the first page is accessible.
> + * If the first element is active, it must succeed.
> + */
> + flags = probe_access_flags(env, adjust_addr(env, addr),
> + MIN(last, last_in_page) - addr + 1,
> + MMU_DATA_LOAD, mmu_index, !first_active,
> + &host, ra);
>
Hi Richard,
This probe traverses every byte from the initial active element to the
end of the page, not just the bytes of active elements. In the masked
case, the range may encompass a masked-off element, and masked-off body
elements do not perform memory accesses.
I think it may causes unexpected vl.
> + /* Get number of complete elements in the first page. */
> + elems = MIN(page_split / msize, vl - i);
> +
> + /* Load complete elements from the first page. */
> + if (likely(elems)) {
> + uint32_t page_evl = i + elems;
> +
> + if (flags == 0) {
...
> + } else {
> + /*
> + * If the first element is active, it must succeed.
> + * This will load from MMIO or fault from INVALID.
> + */
> + if (first_active) {
> + vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz,
> + max_elems, ldst_tlb, ra);
> + i = 1;
> + addr += msize;
> + }
> +
> + /* Stop if invalid (unmapped) or mmio (transaction may fail). */
> + if (flags & (TLB_INVALID_MASK | TLB_MMIO)) {
> + env->vl = i;
> + goto tail;
> + }
> +
For an example, assume
- vl = 3
- vstart = 0
- the mask be [1, 0, 1]
- assume element 0 and element 2 be readable, but deny the byte at element 1
- all three elements are in the same target page.
In theory, the element 1 is masked off and doesn’t perform any memory
access, so the value of vl remains 3.
But the previous probe covers element 1 to 2 and the flags will be non
zero due to the denied masked-off element 2. Then the vl will set to 1
here.
Maybe we could switch to per element prob when vm is 0 and flags is not 0?
rnax
> + /* None of these ldst_tlb calls may fault. */
> + if (vm) {
> + vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf,
> + log2_esz, max_elems,
> + ldst_tlb, mmu_index, ra);
> + } else {
> + do {
> + if (vext_elem_mask(v0, i)) {
> + vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf,
> + esz, max_elems, ldst_tlb, ra);
> + } else if (vma) {
> + vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
> }
> - remain -= offset;
> - addr_i = adjust_addr(env, addr_i + offset);
> - }
> + } while (++i < page_evl);
> + }
> + }
> +
> + /* Usually the first page contains the entire vector. */
> + if (likely(page_evl == vl)) {
> + goto tail;
> + }
> + i = page_evl;
> + }
> +
> + /* Skip forward to the next active element. */
> + if (!vm) {
> + while (1) {
> + if (vext_elem_mask(v0, i)) {
> + break;
> + }
> + if (vma) {
> + vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
> + }
> + if (++i == vl) {
> + goto tail;
> }
> }
> }
> -ProbeSuccess:
> - /* load bytes from guest memory */
> - if (vl != 0) {
> - env->vl = vl;
> +
> + addr = base + i * msize;
> + page_split = -(addr | TARGET_PAGE_MASK);
> +
> + /* Validate the second page is accessible. */
> + if (unlikely(page_split < msize)) {
> + /*
> + * Cross page element which isn't first.
> + * We have not yet advanced addr to the next page.
> + */
> + target_ulong next_page = addr + page_split;
> + flags |= probe_access_flags(env, adjust_addr(env, next_page),
> + last - next_page + 1, MMU_DATA_LOAD,
> + mmu_index, true, &host, ra);
> +
> + /* Stop if invalid (unmapped) or mmio (transaction may fail). */
> + if (flags & (TLB_INVALID_MASK | TLB_MMIO)) {
> + env->vl = i;
> + goto tail;
> + }
> +
> + vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, ldst_tlb, ra);
> + if (++i == vl) {
> + goto tail;
> + }
> + addr += msize;
> + if (host) {
> + host += addr - next_page;
> + }
> + } else {
> + flags = probe_access_flags(env, adjust_addr(env, addr),
> + last - addr + 1, MMU_DATA_LOAD,
> + mmu_index, true, &host, ra);
> +
> + /* Stop if invalid (unmapped) or mmio (transaction may fail). */
> + if (flags & (TLB_INVALID_MASK | TLB_MMIO)) {
> + env->vl = i;
> + goto tail;
> + }
> }
>
> - if (env->vstart < env->vl) {
> + /* Load complete elements from the second page. */
> + if (flags == 0) {
> if (vm) {
> - /* Load/store elements in the first page */
> - if (likely(elems)) {
> - vext_page_ldst_us(env, vd, addr, elems, nf, max_elems,
> - log2_esz, true, mmu_index, ldst_tlb,
> - ldst_host, ra);
> - }
> -
> - /* Load/store elements in the second page */
> - if (unlikely(env->vstart < env->vl)) {
> - addr = base + env->vstart * msize;
> -
> - /* Cross page element */
> - if (unlikely(page_split % msize)) {
> - vext_ldst_nf_tlb(env, vd, addr, env->vstart, nf,
> - esz, max_elems, ldst_tlb, ra);
> - env->vstart++;
> - addr += msize;
> - }
> -
> - /* Get number of elements of second page */
> - elems = env->vl - env->vstart;
> -
> - /* Load/store elements in the second page */
> - vext_page_ldst_us(env, vd, addr, elems, nf, max_elems,
> - log2_esz, true, mmu_index, ldst_tlb,
> - ldst_host, ra);
> - }
> + vext_page_ldst_us_host(vd, host, i, vl, nf,
> + log2_esz, max_elems, ldst_host);
> } else {
> - for (i = env->vstart; i < env->vl; i++) {
> + host -= addr - base;
> + do {
> + if (vext_elem_mask(v0, i)) {
> + vext_ldst_nf_host(vd, host + i * msize, i, nf, esz,
> + max_elems, ldst_host);
> + } else if (vma) {
> + vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
> + }
> + } while (++i < vl);
> + }
> + } else {
> + /* None of these ldst_tlb calls may fault. */
> + if (vm) {
> + vext_page_ldst_us_tlb(env, vd, addr, i, vl, nf,
> + log2_esz, max_elems,
> + ldst_tlb, mmu_index, ra);
> + } else {
> + do {
> if (vext_elem_mask(v0, i)) {
> vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf,
> esz, max_elems, ldst_tlb, ra);
> } else if (vma) {
> vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
> }
> - }
> + } while (++i < vl);
> }
> }
>
> --
> 2.43.0
>
>
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH 11/23] target/riscv: Rewrite vext_ldff
2026-08-25 17:58 ` Max Chou
@ 2026-08-25 20:15 ` Richard Henderson
2026-08-26 18:40 ` Max Chou
0 siblings, 1 reply; 33+ messages in thread
From: Richard Henderson @ 2026-08-25 20:15 UTC (permalink / raw)
To: Max Chou; +Cc: qemu-devel, frank.chang, qemu-riscv
On 8/25/26 10:58, Max Chou wrote:
> On 2026-08-15 12:45, Richard Henderson wrote:
>> Do not call probe_pages for every active element.
>> We can make do with no more than 2 such calls for
>> the two pages the insn might reference.
>>
>> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
>> + /*
>> + * Test whether the first page is accessible.
>> + * If the first element is active, it must succeed.
>> + */
>> + flags = probe_access_flags(env, adjust_addr(env, addr),
>> + MIN(last, last_in_page) - addr + 1,
>> + MMU_DATA_LOAD, mmu_index, !first_active,
>> + &host, ra);
>>
> Hi Richard,
>
> This probe traverses every byte from the initial active element to the
> end of the page, not just the bytes of active elements. In the masked
> case, the range may encompass a masked-off element, and masked-off body
> elements do not perform memory accesses.
> I think it may causes unexpected vl.
>
>> + /* Get number of complete elements in the first page. */
>> + elems = MIN(page_split / msize, vl - i);
>> +
>> + /* Load complete elements from the first page. */
>> + if (likely(elems)) {
>> + uint32_t page_evl = i + elems;
>> +
>> + if (flags == 0) {
> ...
>> + } else {
>> + /*
>> + * If the first element is active, it must succeed.
>> + * This will load from MMIO or fault from INVALID.
>> + */
>> + if (first_active) {
>> + vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz,
>> + max_elems, ldst_tlb, ra);
>> + i = 1;
>> + addr += msize;
>> + }
>> +
>> + /* Stop if invalid (unmapped) or mmio (transaction may fail). */
>> + if (flags & (TLB_INVALID_MASK | TLB_MMIO)) {
>> + env->vl = i;
>> + goto tail;
>> + }
>> +
> For an example, assume
> - vl = 3
> - vstart = 0
> - the mask be [1, 0, 1]
> - assume element 0 and element 2 be readable, but deny the byte at element 1
> - all three elements are in the same target page.
>
> In theory, the element 1 is masked off and doesn’t perform any memory
> access, so the value of vl remains 3.
>
> But the previous probe covers element 1 to 2 and the flags will be non
> zero due to the denied masked-off element 2. Then the vl will set to 1
> here.
>
> Maybe we could switch to per element prob when vm is 0 and flags is not 0?
How are you going to deny the byte at element 1 to be unreadable? Are
you expecting this to be some PMP thing, with a 1 byte range?
How I expect things to work is that:
(1) We scan forward for the first active element, set vstart.
(2) Probe the page for that element. If that faults, vstart is
visible to the trap handler.
(3) Discounting MMIO, all following elements in the same page cannot
fault, and we can process them immediately. Obviously inactive elements
get vma handling not loads.
I'm not sure I'm understanding your question properly...
r~
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH 11/23] target/riscv: Rewrite vext_ldff
2026-08-25 20:15 ` Richard Henderson
@ 2026-08-26 18:40 ` Max Chou
2026-08-26 21:33 ` Richard Henderson
0 siblings, 1 reply; 33+ messages in thread
From: Max Chou @ 2026-08-26 18:40 UTC (permalink / raw)
To: Richard Henderson; +Cc: qemu-devel, frank.chang, qemu-riscv
On 2026-08-25 13:15, Richard Henderson wrote:
> On 8/25/26 10:58, Max Chou wrote:
>
> > On 2026-08-15 12:45, Richard Henderson wrote:
> > > Do not call probe_pages for every active element.
> > > We can make do with no more than 2 such calls for
> > > the two pages the insn might reference.
> > >
> > > Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
> > > + /*
> > > + * Test whether the first page is accessible.
> > > + * If the first element is active, it must succeed.
> > > + */
> > > + flags = probe_access_flags(env, adjust_addr(env, addr),
> > > + MIN(last, last_in_page) - addr + 1,
> > > + MMU_DATA_LOAD, mmu_index, !first_active,
> > > + &host, ra);
> > Hi Richard,
> >
> > This probe traverses every byte from the initial active element to the
> > end of the page, not just the bytes of active elements. In the masked
> > case, the range may encompass a masked-off element, and masked-off body
> > elements do not perform memory accesses.
> > I think it may causes unexpected vl.
> >
> > > + /* Get number of complete elements in the first page. */
> > > + elems = MIN(page_split / msize, vl - i);
> > > +
> > > + /* Load complete elements from the first page. */
> > > + if (likely(elems)) {
> > > + uint32_t page_evl = i + elems;
> > > +
> > > + if (flags == 0) {
> > ...
> > > + } else {
> > > + /*
> > > + * If the first element is active, it must succeed.
> > > + * This will load from MMIO or fault from INVALID.
> > > + */
> > > + if (first_active) {
> > > + vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz,
> > > + max_elems, ldst_tlb, ra);
> > > + i = 1;
> > > + addr += msize;
> > > + }
> > > +
> > > + /* Stop if invalid (unmapped) or mmio (transaction may fail). */
> > > + if (flags & (TLB_INVALID_MASK | TLB_MMIO)) {
> > > + env->vl = i;
> > > + goto tail;
> > > + }
> > > +
> > For an example, assume
> > - vl = 3
> > - vstart = 0
> > - the mask be [1, 0, 1]
> > - assume element 0 and element 2 be readable, but deny the byte at element 1
> > - all three elements are in the same target page.
> >
> > In theory, the element 1 is masked off and doesn’t perform any memory
> > access, so the value of vl remains 3.
> >
> > But the previous probe covers element 1 to 2 and the flags will be non
> > zero due to the denied masked-off element 2. Then the vl will set to 1
> > here.
> >
> > Maybe we could switch to per element prob when vm is 0 and flags is not 0?
>
> How are you going to deny the byte at element 1 to be unreadable? Are you
> expecting this to be some PMP thing, with a 1 byte range?
>
> How I expect things to work is that:
>
> (1) We scan forward for the first active element, set vstart.
>
> (2) Probe the page for that element. If that faults, vstart is visible to
> the trap handler.
>
> (3) Discounting MMIO, all following elements in the same page cannot fault,
> and we can process them immediately. Obviously inactive elements get vma
> handling not loads.
>
Hi Richard,
Sorry, my previous example was unclear. It does not need a 1-byte PMP
region: RISC-V PMP definition allows the minimum 4-byte PMP region,
which exactly matches one e32 element.
For example, with vl=3, vstart=0, e32, and v0.mask = [1, 0, 1],
configure locked PMP entries as follows:
PMP0: NA4 [base + 4, base + 7], L, --- # element 1, no read
PMP1: NAPOT target page, L, R # lower-priority page allow
bytes: base base + 4 base + 8
+-----+ +-----+ +-----+
element: 0 1 2
mask: 1 0 1
access: active inactive active
PMP: PMP1 R PMP0 --- PMP1 R
The RISC-V spec defines that masked vector loads access memory and raise
exceptions only for active elements.
Therefore this situation has reads only for elements 0 and 2; PMP0 must
not create an access-fault condition for this instruction.
My concern is that the page probe may be unsafe in some situations
because PMP permissions can change at 4-byte granularity inside a
target page, while the probe includes bytes which are not part of
active vector memory operations.
Maybe we could use the page probe only when it returns flags == 0,
and otherwise fall back to checking only active elements or active
runs?
I'm trying to create masked fauly-only-first + PMP test for this.
Thanks,
rnax
> I'm not sure I'm understanding your question properly...
>
>
> r~
>
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH 11/23] target/riscv: Rewrite vext_ldff
2026-08-26 18:40 ` Max Chou
@ 2026-08-26 21:33 ` Richard Henderson
2026-08-27 12:48 ` Max Chou
0 siblings, 1 reply; 33+ messages in thread
From: Richard Henderson @ 2026-08-26 21:33 UTC (permalink / raw)
To: Max Chou; +Cc: qemu-devel, frank.chang, qemu-riscv
On 8/26/26 11:40, Max Chou wrote:
> Sorry, my previous example was unclear. It does not need a 1-byte PMP
> region: RISC-V PMP definition allows the minimum 4-byte PMP region,
> which exactly matches one e32 element.
>
> For example, with vl=3, vstart=0, e32, and v0.mask = [1, 0, 1],
> configure locked PMP entries as follows:
>
> PMP0: NA4 [base + 4, base + 7], L, --- # element 1, no read
> PMP1: NAPOT target page, L, R # lower-priority page allow
>
> bytes: base base + 4 base + 8
> +-----+ +-----+ +-----+
> element: 0 1 2
> mask: 1 0 1
> access: active inactive active
> PMP: PMP1 R PMP0 --- PMP1 R
>
> The RISC-V spec defines that masked vector loads access memory and raise
> exceptions only for active elements.
> Therefore this situation has reads only for elements 0 and 2; PMP0 must
> not create an access-fault condition for this instruction.
>
> My concern is that the page probe may be unsafe in some situations
> because PMP permissions can change at 4-byte granularity inside a
> target page, while the probe includes bytes which are not part of
> active vector memory operations.
>
> Maybe we could use the page probe only when it returns flags == 0,
> and otherwise fall back to checking only active elements or active
> runs?
Ok, thanks for the clear example. flags == 0 does not help with this case.
What we want is to probe the first active element, with
probe_access_full. For this particular case, where a PMP rule matches
something smaller than the page, we will have set lg_page_size <
TARGET_PAGE_BITS. So we get the fault (or not) for the first element
exactly, and can then tell via flags and lg_page_size whether the rest
of the page can use the fast or slow paths.
> I'm trying to create masked fauly-only-first + PMP test for this.
Thanks, that would be helpful. Similarly for masked not-first-fault,
please?
r~
^ permalink raw reply [flat|nested] 33+ messages in thread
* Re: [PATCH 11/23] target/riscv: Rewrite vext_ldff
2026-08-26 21:33 ` Richard Henderson
@ 2026-08-27 12:48 ` Max Chou
0 siblings, 0 replies; 33+ messages in thread
From: Max Chou @ 2026-08-27 12:48 UTC (permalink / raw)
To: Richard Henderson; +Cc: qemu-devel, frank.chang, qemu-riscv
[-- Attachment #1: Type: text/plain, Size: 2421 bytes --]
On Thu, Aug 27, 2026 at 05:33 Richard Henderson <
richard.henderson@linaro.org> wrote:
> On 8/26/26 11:40, Max Chou wrote:
> > Sorry, my previous example was unclear. It does not need a 1-byte PMP
> > region: RISC-V PMP definition allows the minimum 4-byte PMP region,
> > which exactly matches one e32 element.
> >
> > For example, with vl=3, vstart=0, e32, and v0.mask = [1, 0, 1],
> > configure locked PMP entries as follows:
> >
> > PMP0: NA4 [base + 4, base + 7], L, --- # element 1, no read
> > PMP1: NAPOT target page, L, R # lower-priority page
> allow
> >
> > bytes: base base + 4 base + 8
> > +-----+ +-----+ +-----+
> > element: 0 1 2
> > mask: 1 0 1
> > access: active inactive active
> > PMP: PMP1 R PMP0 --- PMP1 R
> >
> > The RISC-V spec defines that masked vector loads access memory and raise
> > exceptions only for active elements.
> > Therefore this situation has reads only for elements 0 and 2; PMP0 must
> > not create an access-fault condition for this instruction.
> >
> > My concern is that the page probe may be unsafe in some situations
> > because PMP permissions can change at 4-byte granularity inside a
> > target page, while the probe includes bytes which are not part of
> > active vector memory operations.
> >
> > Maybe we could use the page probe only when it returns flags == 0,
> > and otherwise fall back to checking only active elements or active
> > runs?
>
> Ok, thanks for the clear example. flags == 0 does not help with this case.
>
> What we want is to probe the first active element, with
> probe_access_full. For this particular case, where a PMP rule matches
> something smaller than the page, we will have set lg_page_size <
> TARGET_PAGE_BITS. So we get the fault (or not) for the first element
> exactly, and can then tell via flags and lg_page_size whether the rest
> of the page can use the fast or slow paths.
>
> > I'm trying to create masked fauly-only-first + PMP test for this.
>
> Thanks, that would be helpful. Similarly for masked not-first-fault,
> please?
>
I am glad to help with this. I will send the test cases for both the fault
at the first element and the no-fault at the first element scenarios for
masked vector fault-only-first instructions to this list.
rnax
>
> r~
>
>
[-- Attachment #2: Type: text/html, Size: 5302 bytes --]
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH 12/23] target/riscv: Split out vext_ldst_us_desc
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (10 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 11/23] target/riscv: Rewrite vext_ldff Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 13/23] target/riscv: Use vext_ldst_us in vext_ldst_whole Richard Henderson
` (10 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Rename vext_ldst_us to vext_ldst_us_desc and split out
a new vext_ldst_us that doesn't use 'desc'.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 38 ++++++++++++++++++++------------
1 file changed, 24 insertions(+), 14 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index b482537f40..abb1ebcb43 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -478,15 +478,14 @@ static void vext_test_alignment(CPURISCVState *env, vaddr addr, uint32_t esz,
}
static inline QEMU_ALWAYS_INLINE void
-vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
+vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env,
+ uint32_t log2_esz, uint32_t nf, uint32_t evl,
+ uint32_t max_elems, bool vma,
vext_ldst_elem_fn_tlb *ldst_tlb,
- vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
- uint32_t evl, uintptr_t ra, bool is_load)
+ vext_ldst_elem_fn_host *ldst_host,
+ uintptr_t ra, bool is_load)
{
target_ulong elems, addr, last, last_in_page, page_split;
- uint32_t nf = vext_nf(desc);
- uint32_t vma = vext_vma(desc);
- uint32_t max_elems = vext_max_elems(desc, log2_esz);
uint32_t esz = 1 << log2_esz;
uint32_t msize = nf * esz;
int mmu_index = riscv_env_mmu_index(env, false);
@@ -579,6 +578,17 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
}
}
+static inline QEMU_ALWAYS_INLINE void
+vext_ldst_us_desc(void *vd, target_ulong base, CPURISCVState *env,
+ uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb,
+ vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
+ uint32_t evl, uintptr_t ra, bool is_load)
+{
+ vext_ldst_us(vd, base, env, log2_esz, vext_nf(desc), evl,
+ vext_max_elems(desc, log2_esz), vext_vma(desc),
+ ldst_tlb, ldst_host, ra, is_load);
+}
+
/*
* masked unit-stride load and store operation will be a special case of
* stride, stride = NF * sizeof (ETYPE)
@@ -596,8 +606,8 @@ void HELPER(NAME##_mask)(void *vd, void *v0, target_ulong base, \
void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
CPURISCVState *env, uint32_t desc) \
{ \
- vext_ldst_us(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST, \
- ctzl(sizeof(ETYPE)), env->vl, GETPC(), true); \
+ vext_ldst_us_desc(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST, \
+ ctzl(sizeof(ETYPE)), env->vl, GETPC(), true); \
}
GEN_VEXT_LD_US(vle8_v, int8_t, lde_b_tlb, lde_b_host)
@@ -617,8 +627,8 @@ void HELPER(NAME##_mask)(void *vd, void *v0, target_ulong base, \
void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
CPURISCVState *env, uint32_t desc) \
{ \
- vext_ldst_us(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST, \
- ctzl(sizeof(ETYPE)), env->vl, GETPC(), false); \
+ vext_ldst_us_desc(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST, \
+ ctzl(sizeof(ETYPE)), env->vl, GETPC(), false); \
}
GEN_VEXT_ST_US(vse8_v, int8_t, ste_b_tlb, ste_b_host)
@@ -634,8 +644,8 @@ void HELPER(vlm_v)(void *vd, void *v0, target_ulong base,
{
/* evl = ceil(vl/8) */
uint8_t evl = (env->vl + 7) >> 3;
- vext_ldst_us(vd, base, env, desc, lde_b_tlb, lde_b_host,
- 0, evl, GETPC(), true);
+ vext_ldst_us_desc(vd, base, env, desc, lde_b_tlb, lde_b_host,
+ 0, evl, GETPC(), true);
}
void HELPER(vsm_v)(void *vd, void *v0, target_ulong base,
@@ -643,8 +653,8 @@ void HELPER(vsm_v)(void *vd, void *v0, target_ulong base,
{
/* evl = ceil(vl/8) */
uint8_t evl = (env->vl + 7) >> 3;
- vext_ldst_us(vd, base, env, desc, ste_b_tlb, ste_b_host,
- 0, evl, GETPC(), false);
+ vext_ldst_us_desc(vd, base, env, desc, ste_b_tlb, ste_b_host,
+ 0, evl, GETPC(), false);
}
/*
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 13/23] target/riscv: Use vext_ldst_us in vext_ldst_whole
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (11 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 12/23] target/riscv: Split out vext_ldst_us_desc Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 14/23] target/riscv: Mark VSTART_CHECK_EARLY_EXIT unlikely Richard Henderson
` (9 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
This eliminates the last use of probe_pages.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 120 +------------------------------
1 file changed, 3 insertions(+), 117 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index abb1ebcb43..178ed32429 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -149,50 +149,6 @@ static inline uint32_t vext_max_elems(uint32_t desc, uint32_t log2_esz)
return scale < 0 ? vlenb >> -scale : vlenb << scale;
}
-/*
- * This function checks watchpoint before real load operation.
- *
- * In system mode, the TLB API probe_access is enough for watchpoint check.
- * In user mode, there is no watchpoint support now.
- *
- * It will trigger an exception if there is no mapping in TLB
- * and page table walk can't fill the TLB entry. Then the guest
- * software can return here after process the exception or never return.
- *
- * This function can also be used when direct access to probe_access_flags is
- * needed in order to access the flags. If a pointer to a flags operand is
- * provided the function will call probe_access_flags instead, use nonfault
- * and update host and flags.
- */
-static void probe_pages(CPURISCVState *env, target_ulong addr, target_ulong len,
- uintptr_t ra, MMUAccessType access_type, int mmu_index,
- void **host, int *flags, bool nonfault)
-{
- target_ulong pagelen = -(addr | TARGET_PAGE_MASK);
- target_ulong curlen = MIN(pagelen, len);
-
- if (flags != NULL) {
- *flags = probe_access_flags(env, adjust_addr(env, addr), curlen,
- access_type, mmu_index, nonfault, host, ra);
- } else {
- probe_access(env, adjust_addr(env, addr), curlen, access_type,
- mmu_index, ra);
- }
-
- if (len > curlen) {
- addr += curlen;
- curlen = len - curlen;
- if (flags != NULL) {
- *flags |= probe_access_flags(env, adjust_addr(env, addr), curlen,
- access_type, mmu_index, nonfault,
- host, ra);
- } else {
- probe_access(env, adjust_addr(env, addr), curlen, access_type,
- mmu_index, ra);
- }
- }
-}
-
static inline void vext_set_elem_mask(void *v0, int index,
uint8_t value)
{
@@ -436,33 +392,6 @@ vext_page_ldst_us_tlb(CPURISCVState *env, void *vd, target_ulong addr,
} while (i < evl);
}
-static inline QEMU_ALWAYS_INLINE void
-vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
- uint32_t elems, uint32_t nf, uint32_t max_elems,
- uint32_t log2_esz, bool is_load, int mmu_index,
- vext_ldst_elem_fn_tlb *ldst_tlb,
- vext_ldst_elem_fn_host *ldst_host, uintptr_t ra)
-{
- void *host;
- int flags;
- uint32_t size = (elems * nf) << log2_esz;
- uint32_t i = env->vstart;
- MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE;
-
- /* Check page permission/pmp/watchpoint/etc. */
- probe_pages(env, addr, size, ra, access_type, mmu_index, &host, &flags,
- true);
-
- if (flags == 0) {
- vext_page_ldst_us_host(vd, host, i, i + elems, nf,
- log2_esz, max_elems, ldst_host);
- env->vstart += elems;
- } else {
- vext_page_ldst_us_tlb(env, vd, addr, i, i + elems, nf, log2_esz,
- max_elems, ldst_tlb, mmu_index, ra);
- }
-}
-
static void vext_test_alignment(CPURISCVState *env, vaddr addr, uint32_t esz,
MMUAccessType access_type, int mmu_index,
uintptr_t ra)
@@ -1026,53 +955,10 @@ vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
uintptr_t ra, bool is_load)
{
- target_ulong page_split, elems, addr;
- uint32_t nf = vext_nf(desc);
- uint32_t vlenb = riscv_cpu_cfg(env)->vlenb;
- uint32_t max_elems = vlenb >> log2_esz;
- uint32_t evl = nf * max_elems;
- uint32_t esz = 1 << log2_esz;
- int mmu_index = riscv_env_mmu_index(env, false);
- MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE;
+ uint32_t evl = (riscv_cpu_cfg(env)->vlenb * vext_nf(desc)) >> log2_esz;
- /* Calculate the page range of first page */
- addr = base + (env->vstart << log2_esz);
-
- /* Recognize alignment fault before memory protection fault. */
- vext_test_alignment(env, addr, esz, access_type, mmu_index, ra);
-
- page_split = -(addr | TARGET_PAGE_MASK);
- /* Get number of elements */
- elems = page_split / esz;
- if (unlikely(env->vstart + elems >= evl)) {
- elems = evl - env->vstart;
- }
-
- /* Load/store elements in the first page */
- if (likely(elems)) {
- vext_page_ldst_us(env, vd, addr, elems, 1, max_elems, log2_esz,
- is_load, mmu_index, ldst_tlb, ldst_host, ra);
- }
-
- /* Load/store elements in the second page */
- if (unlikely(env->vstart < evl)) {
- /* Cross page element */
- if (unlikely(page_split % esz)) {
- addr = base + (env->vstart << log2_esz);
- ldst_tlb(env, adjust_addr(env, addr), env->vstart, vd, ra);
- env->vstart++;
- }
-
- addr = base + (env->vstart << log2_esz);
- /* Get number of elements of second page */
- elems = evl - env->vstart;
-
- /* Load/store elements in the second page */
- vext_page_ldst_us(env, vd, addr, elems, 1, max_elems, log2_esz,
- is_load, mmu_index, ldst_tlb, ldst_host, ra);
- }
-
- env->vstart = 0;
+ vext_ldst_us(vd, base, env, log2_esz, 1, evl, evl, false,
+ ldst_tlb, ldst_host, ra, is_load);
}
#define GEN_VEXT_LD_WHOLE(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 14/23] target/riscv: Mark VSTART_CHECK_EARLY_EXIT unlikely
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (12 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 13/23] target/riscv: Use vext_ldst_us in vext_ldst_whole Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 15/23] target/riscv: Remove unused VDATA,WD Richard Henderson
` (8 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_internals.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/target/riscv/tcg/vector_internals.h b/target/riscv/tcg/vector_internals.h
index 5681b81815..d081397f65 100644
--- a/target/riscv/tcg/vector_internals.h
+++ b/target/riscv/tcg/vector_internals.h
@@ -26,7 +26,7 @@
#include "internals.h"
#define VSTART_CHECK_EARLY_EXIT(env, vl) do { \
- if (env->vstart >= vl) { \
+ if (unlikely(env->vstart >= vl)) { \
env->vstart = 0; \
return; \
} \
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 15/23] target/riscv: Remove unused VDATA,WD
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (13 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 14/23] target/riscv: Mark VSTART_CHECK_EARLY_EXIT unlikely Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 16/23] target/riscv: Add MEM_IDX, BSWAP, ALIGN to VDATA Richard Henderson
` (7 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/internals.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/target/riscv/internals.h b/target/riscv/internals.h
index 5d84e4de96..9e4da39ea0 100644
--- a/target/riscv/internals.h
+++ b/target/riscv/internals.h
@@ -94,7 +94,6 @@ FIELD(VDATA, VTA, 4, 1)
FIELD(VDATA, VTA_ALL_1S, 5, 1)
FIELD(VDATA, VMA, 6, 1)
FIELD(VDATA, NF, 7, 4)
-FIELD(VDATA, WD, 7, 1)
/* float point classify helpers */
target_ulong fclass_h_bf16(uint64_t frs1);
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 16/23] target/riscv: Add MEM_IDX, BSWAP, ALIGN to VDATA
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (14 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 15/23] target/riscv: Remove unused VDATA,WD Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 17/23] target/riscv: Pass MemOpIdx to vext_ldst_us Richard Henderson
` (6 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Populate it in all vector ldst operations.
Not yet used within the helpers.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/internals.h | 3 +++
target/riscv/tcg/insn_trans/trans_rvv.c.inc | 25 ++++++++++++++++-----
2 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/target/riscv/internals.h b/target/riscv/internals.h
index 9e4da39ea0..b7ffb6098c 100644
--- a/target/riscv/internals.h
+++ b/target/riscv/internals.h
@@ -94,6 +94,9 @@ FIELD(VDATA, VTA, 4, 1)
FIELD(VDATA, VTA_ALL_1S, 5, 1)
FIELD(VDATA, VMA, 6, 1)
FIELD(VDATA, NF, 7, 4)
+FIELD(VDATA, MEM_IDX, 11, 3)
+FIELD(VDATA, BSWAP, 14, 1)
+FIELD(VDATA, ALIGN, 15, 1)
/* float point classify helpers */
target_ulong fclass_h_bf16(uint64_t frs1);
diff --git a/target/riscv/tcg/insn_trans/trans_rvv.c.inc b/target/riscv/tcg/insn_trans/trans_rvv.c.inc
index a22e2cae6c..3916236762 100644
--- a/target/riscv/tcg/insn_trans/trans_rvv.c.inc
+++ b/target/riscv/tcg/insn_trans/trans_rvv.c.inc
@@ -693,6 +693,14 @@ static uint8_t vext_get_emul(DisasContext *s, uint8_t eew)
typedef void gen_helper_ldst_us(TCGv_ptr, TCGv_ptr, TCGv,
TCGv_env, TCGv_i32);
+static uint32_t vdata_add_memopidx(DisasContext *s, uint32_t data)
+{
+ data = FIELD_DP32(data, VDATA, MEM_IDX, s->mem_idx);
+ data = FIELD_DP32(data, VDATA, BSWAP, s->mo_endianness != 0);
+ data = FIELD_DP32(data, VDATA, ALIGN, !s->cfg_ptr->ext_zicclsm);
+ return data;
+}
+
static bool ldst_us_trans(uint32_t vd, uint32_t rs1, uint32_t data,
gen_helper_ldst_us *fn, DisasContext *s,
bool is_store)
@@ -712,6 +720,7 @@ static bool ldst_us_trans(uint32_t vd, uint32_t rs1, uint32_t data,
* The first part is vlen in bytes (vlenb), encoded in maxsz of simd_desc.
* The second part is lmul, encoded in data of simd_desc.
*/
+ data = vdata_add_memopidx(s, data);
desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
s->cfg_ptr->vlenb, data));
@@ -890,6 +899,8 @@ static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2,
mask = tcg_temp_new_ptr();
base = get_gpr(s, rs1, EXT_NONE);
stride = get_gpr(s, rs2, EXT_NONE);
+
+ data = vdata_add_memopidx(s, data);
desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
s->cfg_ptr->vlenb, data));
@@ -991,6 +1002,8 @@ static bool ldst_index_trans(uint32_t vd, uint32_t rs1, uint32_t vs2,
mask = tcg_temp_new_ptr();
index = tcg_temp_new_ptr();
base = get_gpr(s, rs1, EXT_NONE);
+
+ data = vdata_add_memopidx(s, data);
desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
s->cfg_ptr->vlenb, data));
@@ -1127,6 +1140,8 @@ static bool ldff_trans(uint32_t vd, uint32_t rs1, uint32_t data,
dest = tcg_temp_new_ptr();
mask = tcg_temp_new_ptr();
base = get_gpr(s, rs1, EXT_NONE);
+
+ data = vdata_add_memopidx(s, data);
desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
s->cfg_ptr->vlenb, data));
@@ -1231,15 +1246,15 @@ static bool ldst_whole_trans(uint32_t vd, uint32_t rs1, uint32_t nf,
}
}
} else {
- TCGv_ptr dest;
- TCGv base;
+ TCGv_ptr dest = tcg_temp_new_ptr();
+ TCGv base = get_gpr(s, rs1, EXT_NONE);
TCGv_i32 desc;
uint32_t data = FIELD_DP32(0, VDATA, NF, nf);
data = FIELD_DP32(data, VDATA, VM, 1);
- dest = tcg_temp_new_ptr();
+ data = vdata_add_memopidx(s, data);
+
desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
- s->cfg_ptr->vlenb, data));
- base = get_gpr(s, rs1, EXT_NONE);
+ s->cfg_ptr->vlenb, data));
tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd));
fn(dest, base, tcg_env, desc);
}
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 17/23] target/riscv: Pass MemOpIdx to vext_ldst_us
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (15 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 16/23] target/riscv: Add MEM_IDX, BSWAP, ALIGN to VDATA Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 18/23] target/riscv: Build MemOpIdx to vext_ldff Richard Henderson
` (5 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Use mmu data passed in with VDATA.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 23 +++++++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 178ed32429..5ead4cd6fe 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -158,6 +158,19 @@ static inline void vext_set_elem_mask(void *v0, int index,
((uint64_t *)v0)[idx] = deposit64(old, pos, 1, value);
}
+static inline MemOpIdx vext_oi(uint32_t desc, MemOp memop)
+{
+ uint32_t data = simd_data(desc);
+
+ if (FIELD_EX32(simd_data(desc), VDATA, BSWAP)) {
+ memop |= MO_BSWAP;
+ }
+ if (FIELD_EX32(simd_data(desc), VDATA, ALIGN)) {
+ memop |= MO_ALIGN;
+ }
+ return make_memop_idx(memop, FIELD_EX32(data, VDATA, MEM_IDX));
+}
+
static inline MemOpIdx vext_make_memop_idx(CPURISCVState *env, size_t size)
{
int mmu_idx = riscv_env_mmu_index(env, false);
@@ -412,12 +425,12 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env,
uint32_t max_elems, bool vma,
vext_ldst_elem_fn_tlb *ldst_tlb,
vext_ldst_elem_fn_host *ldst_host,
- uintptr_t ra, bool is_load)
+ uintptr_t ra, bool is_load, MemOpIdx oi)
{
target_ulong elems, addr, last, last_in_page, page_split;
uint32_t esz = 1 << log2_esz;
uint32_t msize = nf * esz;
- int mmu_index = riscv_env_mmu_index(env, false);
+ int mmu_index = get_mmuidx(oi);
MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE;
uint32_t i = env->vstart;
void *host;
@@ -515,7 +528,8 @@ vext_ldst_us_desc(void *vd, target_ulong base, CPURISCVState *env,
{
vext_ldst_us(vd, base, env, log2_esz, vext_nf(desc), evl,
vext_max_elems(desc, log2_esz), vext_vma(desc),
- ldst_tlb, ldst_host, ra, is_load);
+ ldst_tlb, ldst_host, ra, is_load,
+ vext_oi(desc, log2_esz));
}
/*
@@ -958,7 +972,8 @@ vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
uint32_t evl = (riscv_cpu_cfg(env)->vlenb * vext_nf(desc)) >> log2_esz;
vext_ldst_us(vd, base, env, log2_esz, 1, evl, evl, false,
- ldst_tlb, ldst_host, ra, is_load);
+ ldst_tlb, ldst_host, ra, is_load,
+ vext_oi(desc, log2_esz));
}
#define GEN_VEXT_LD_WHOLE(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 18/23] target/riscv: Build MemOpIdx to vext_ldff
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (16 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 17/23] target/riscv: Pass MemOpIdx to vext_ldst_us Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 19/23] target/riscv: Pass MemOpIdx to vext_ldst_elem_fn_tlb Richard Henderson
` (4 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Use mmu data passed in with VDATA.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 5ead4cd6fe..9c643f60ec 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -720,7 +720,8 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
uint32_t msize = nf * esz;
uint32_t vma = vext_vma(desc);
target_ulong addr, last, last_in_page, page_split, elems;
- int mmu_index = riscv_env_mmu_index(env, false);
+ MemOpIdx oi = vext_oi(desc, log2_esz);
+ int mmu_index = get_mmuidx(oi);
bool first_active;
int flags;
void *host;
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 19/23] target/riscv: Pass MemOpIdx to vext_ldst_elem_fn_tlb
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (17 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 18/23] target/riscv: Build MemOpIdx to vext_ldff Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 20/23] target/riscv: Use FLATTEN rather than ALWAYS_INLINE for vector ldst Richard Henderson
` (3 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Avoids continually looking up mmu_index with every operation.
Allows for big-endian support with no extra effort.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 91 +++++++++++++++-----------------
1 file changed, 43 insertions(+), 48 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 9c643f60ec..473b112426 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -171,33 +171,21 @@ static inline MemOpIdx vext_oi(uint32_t desc, MemOp memop)
return make_memop_idx(memop, FIELD_EX32(data, VDATA, MEM_IDX));
}
-static inline MemOpIdx vext_make_memop_idx(CPURISCVState *env, size_t size)
-{
- int mmu_idx = riscv_env_mmu_index(env, false);
- MemOp memop = size_memop(size) | mo_endian_env(env);
-
- if (!riscv_cpu_cfg(env)->ext_zicclsm) {
- memop |= MO_ALIGN;
- }
-
- return make_memop_idx(memop, mmu_idx);
-}
-
/* elements operations for load and store */
typedef void vext_ldst_elem_fn_tlb(CPURISCVState *env, abi_ptr addr,
- uint32_t idx, void *vd, uintptr_t retaddr);
+ uint32_t idx, void *vd,
+ MemOpIdx oi, uintptr_t retaddr);
typedef void vext_ldst_elem_fn_host(void *vd, void *host,
uint32_t idx, uint32_t evl);
-#define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF) \
-static inline QEMU_ALWAYS_INLINE \
-void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \
- uint32_t idx, void *vd, uintptr_t retaddr) \
-{ \
- ETYPE *cur = ((ETYPE *)vd + H(idx)); \
- MemOpIdx oi = vext_make_memop_idx(env, sizeof(ETYPE)); \
- *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr); \
-} \
+#define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF) \
+static inline QEMU_ALWAYS_INLINE \
+void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, \
+ void *vd, MemOpIdx oi, uintptr_t retaddr) \
+{ \
+ ETYPE *cur = ((ETYPE *)vd + H(idx)); \
+ *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr); \
+}
#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \
static inline QEMU_ALWAYS_INLINE \
@@ -229,15 +217,14 @@ GEN_VEXT_HOST_LD_ELEM(lde_h, uint16_t, H2, lduw_le)
GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le)
GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le)
-#define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, STSUF) \
-static inline QEMU_ALWAYS_INLINE \
-void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \
- uint32_t idx, void *vd, uintptr_t retaddr) \
-{ \
- ETYPE data = *((ETYPE *)vd + H(idx)); \
- MemOpIdx oi = vext_make_memop_idx(env, sizeof(ETYPE)); \
- cpu_##STSUF##_mmu(env, addr, data, oi, retaddr); \
-} \
+#define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, STSUF) \
+static inline QEMU_ALWAYS_INLINE \
+void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, \
+ void *vd, MemOpIdx oi, uintptr_t retaddr) \
+{ \
+ ETYPE data = *((ETYPE *)vd + H(idx)); \
+ cpu_##STSUF##_mmu(env, addr, data, oi, retaddr); \
+}
#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \
static inline QEMU_ALWAYS_INLINE \
@@ -299,10 +286,11 @@ static void vext_ldst_nf_host(void *vd, void *host, uint32_t i, uint32_t nf,
static void vext_ldst_nf_tlb(CPURISCVState *env, void *vd, target_ulong addr,
uint32_t i, uint32_t nf,
uint32_t esz, uint32_t max_elems,
- vext_ldst_elem_fn_tlb *ldst_tlb, uintptr_t ra)
+ vext_ldst_elem_fn_tlb *ldst_tlb,
+ MemOpIdx oi, uintptr_t ra)
{
for (uint32_t k = 0; k < nf; k++, addr += esz) {
- ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
+ ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems, vd, oi, ra);
}
}
@@ -319,13 +307,14 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, target_ulong stride,
uint32_t max_elems = vext_max_elems(desc, log2_esz);
uint32_t esz = 1 << log2_esz;
uint32_t vma = vext_vma(desc);
+ MemOpIdx oi = vext_oi(desc, log2_esz);
VSTART_CHECK_EARLY_EXIT(env, env->vl);
for (uint32_t i = env->vstart; i < env->vl; env->vstart = ++i) {
if (vm || vext_elem_mask(v0, i)) {
vext_ldst_nf_tlb(env, vd, base + stride * i, i, nf,
- esz, max_elems, ldst_elem, ra);
+ esz, max_elems, ldst_elem, oi, ra);
} else if (vma) {
vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
@@ -394,12 +383,13 @@ vext_page_ldst_us_tlb(CPURISCVState *env, void *vd, target_ulong addr,
uint32_t i, uint32_t evl, uint32_t nf,
uint32_t log2_esz, uint32_t max_elems,
vext_ldst_elem_fn_tlb *ldst_tlb,
- int mmu_index, uintptr_t ra)
+ MemOpIdx oi, uintptr_t ra)
{
uint32_t esz = 1 << log2_esz;
uint32_t msize = nf << log2_esz;
do {
- vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, ldst_tlb, ra);
+ vext_ldst_nf_tlb(env, vd, addr, i, nf, esz,
+ max_elems, ldst_tlb, oi, ra);
addr += msize;
env->vstart = ++i;
} while (i < evl);
@@ -449,7 +439,7 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env,
*/
if (nf == 1 && (evl << log2_esz) <= 6) {
for (; i < evl; env->vstart = ++i, addr += esz) {
- ldst_tlb(env, adjust_addr(env, addr), i, vd, ra);
+ ldst_tlb(env, adjust_addr(env, addr), i, vd, oi, ra);
}
goto tail;
}
@@ -478,7 +468,7 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env,
} else {
vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf,
log2_esz, max_elems,
- ldst_tlb, mmu_index, ra);
+ ldst_tlb, oi, ra);
}
/* Usually the first page contains the entire vector. */
@@ -491,7 +481,8 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env,
/* Cross page element */
if (unlikely(page_split % msize)) {
- vext_ldst_nf_tlb(env, vd, addr, i++, nf, esz, max_elems, ldst_tlb, ra);
+ vext_ldst_nf_tlb(env, vd, addr, i++, nf, esz,
+ max_elems, ldst_tlb, oi, ra);
if (i == evl) {
goto tail;
}
@@ -510,7 +501,7 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env,
log2_esz, max_elems, ldst_host);
} else {
vext_page_ldst_us_tlb(env, vd, addr, i, evl, nf, log2_esz, max_elems,
- ldst_tlb, mmu_index, ra);
+ ldst_tlb, oi, ra);
}
tail:
@@ -630,6 +621,7 @@ vext_ldst_index(void *vd, void *v0, target_ulong base,
uint32_t max_elems = vext_max_elems(desc, log2_esz);
uint32_t esz = 1 << log2_esz;
uint32_t vma = vext_vma(desc);
+ MemOpIdx oi = vext_oi(desc, log2_esz);
VSTART_CHECK_EARLY_EXIT(env, env->vl);
@@ -643,7 +635,8 @@ vext_ldst_index(void *vd, void *v0, target_ulong base,
for (uint32_t k = 0; k < nf; ++k) {
abi_ptr addr = get_index_addr(base, i, vs2) + (k << log2_esz);
- ldst_elem(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
+ ldst_elem(env, adjust_addr(env, addr),
+ i + k * max_elems, vd, oi, ra);
}
}
@@ -762,7 +755,8 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
* If it also crosses the page, just go ahead and load it.
*/
if (unlikely(page_split < msize) && first_active) {
- vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz, max_elems, ldst_tlb, ra);
+ vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz,
+ max_elems, ldst_tlb, oi, ra);
env->vstart = i = 1;
addr += msize;
page_split = -(addr | TARGET_PAGE_MASK);
@@ -807,7 +801,7 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
*/
if (first_active) {
vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz,
- max_elems, ldst_tlb, ra);
+ max_elems, ldst_tlb, oi, ra);
i = 1;
addr += msize;
}
@@ -822,12 +816,12 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
if (vm) {
vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf,
log2_esz, max_elems,
- ldst_tlb, mmu_index, ra);
+ ldst_tlb, oi, ra);
} else {
do {
if (vext_elem_mask(v0, i)) {
vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf,
- esz, max_elems, ldst_tlb, ra);
+ esz, max_elems, ldst_tlb, oi, ra);
} else if (vma) {
vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
@@ -877,7 +871,8 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
goto tail;
}
- vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, ldst_tlb, ra);
+ vext_ldst_nf_tlb(env, vd, addr, i, nf, esz,
+ max_elems, ldst_tlb, oi, ra);
if (++i == vl) {
goto tail;
}
@@ -918,12 +913,12 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
if (vm) {
vext_page_ldst_us_tlb(env, vd, addr, i, vl, nf,
log2_esz, max_elems,
- ldst_tlb, mmu_index, ra);
+ ldst_tlb, oi, ra);
} else {
do {
if (vext_elem_mask(v0, i)) {
vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf,
- esz, max_elems, ldst_tlb, ra);
+ esz, max_elems, ldst_tlb, oi, ra);
} else if (vma) {
vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 20/23] target/riscv: Use FLATTEN rather than ALWAYS_INLINE for vector ldst
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (18 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 19/23] target/riscv: Pass MemOpIdx to vext_ldst_elem_fn_tlb Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 21/23] target/riscv: Drop v0 argument from gen_helper_ldst_us Richard Henderson
` (2 subsequent siblings)
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
It turns out there are fewer places that need annotation if we
just flatten from the outer helper macros instead of everything
along the call stack.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/tcg/vector_helper.c | 143 +++++++++++++++----------------
1 file changed, 67 insertions(+), 76 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 473b112426..2a24e15a31 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -158,7 +158,7 @@ static inline void vext_set_elem_mask(void *v0, int index,
((uint64_t *)v0)[idx] = deposit64(old, pos, 1, value);
}
-static inline MemOpIdx vext_oi(uint32_t desc, MemOp memop)
+static MemOpIdx vext_oi(uint32_t desc, MemOp memop)
{
uint32_t data = simd_data(desc);
@@ -179,17 +179,15 @@ typedef void vext_ldst_elem_fn_host(void *vd, void *host,
uint32_t idx, uint32_t evl);
#define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF) \
-static inline QEMU_ALWAYS_INLINE \
-void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, \
- void *vd, MemOpIdx oi, uintptr_t retaddr) \
+static void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, \
+ void *vd, MemOpIdx oi, uintptr_t retaddr) \
{ \
ETYPE *cur = ((ETYPE *)vd + H(idx)); \
*cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr); \
}
#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \
-static inline QEMU_ALWAYS_INLINE \
-void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl) \
+static void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl) \
{ \
do { \
ETYPE *cur = (ETYPE *)vd + H(idx); \
@@ -206,8 +204,7 @@ GEN_VEXT_TLB_LD_ELEM(lde_d, uint64_t, H8, ldq)
#if HOST_BIG_ENDIAN
GEN_VEXT_HOST_LD_ELEM(lde_b, uint8_t, H1, ldub)
#else
-static inline QEMU_ALWAYS_INLINE
-void lde_b_host(void *vd, void *host, uint32_t idx, uint32_t evl)
+static void lde_b_host(void *vd, void *host, uint32_t idx, uint32_t evl)
{
memcpy(vd + idx, host, evl - idx);
}
@@ -218,8 +215,7 @@ GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le)
GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le)
#define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, STSUF) \
-static inline QEMU_ALWAYS_INLINE \
-void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, \
+static void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, \
void *vd, MemOpIdx oi, uintptr_t retaddr) \
{ \
ETYPE data = *((ETYPE *)vd + H(idx)); \
@@ -227,8 +223,7 @@ void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, \
}
#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \
-static inline QEMU_ALWAYS_INLINE \
-void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl) \
+static void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl) \
{ \
do { \
ETYPE data = *((ETYPE *)vd + H(idx)); \
@@ -245,8 +240,7 @@ GEN_VEXT_TLB_ST_ELEM(ste_d, uint64_t, H8, stq)
#if HOST_BIG_ENDIAN
GEN_VEXT_HOST_ST_ELEM(ste_b, uint8_t, H1, stb)
#else
-static inline QEMU_ALWAYS_INLINE
-void ste_b_host(void *vd, void *host, uint32_t idx, uint32_t evl)
+static void ste_b_host(void *vd, void *host, uint32_t idx, uint32_t evl)
{
memcpy(host, vd + idx, evl - idx);
}
@@ -361,7 +355,7 @@ GEN_VEXT_ST_STRIDE(vsse64_v, int64_t, ste_d_tlb)
*/
/* unmasked unit-stride load and store operation */
-static inline QEMU_ALWAYS_INLINE void
+static void
vext_page_ldst_us_host(void *vd, void *host, uint32_t i, uint32_t evl,
uint32_t nf, uint32_t log2_esz, uint32_t max_elems,
vext_ldst_elem_fn_host *ldst_host)
@@ -378,7 +372,7 @@ vext_page_ldst_us_host(void *vd, void *host, uint32_t i, uint32_t evl,
}
}
-static inline QEMU_ALWAYS_INLINE void
+static void
vext_page_ldst_us_tlb(CPURISCVState *env, void *vd, target_ulong addr,
uint32_t i, uint32_t evl, uint32_t nf,
uint32_t log2_esz, uint32_t max_elems,
@@ -409,7 +403,7 @@ static void vext_test_alignment(CPURISCVState *env, vaddr addr, uint32_t esz,
}
}
-static inline QEMU_ALWAYS_INLINE void
+static void
vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env,
uint32_t log2_esz, uint32_t nf, uint32_t evl,
uint32_t max_elems, bool vma,
@@ -511,7 +505,7 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env,
}
}
-static inline QEMU_ALWAYS_INLINE void
+static void
vext_ldst_us_desc(void *vd, target_ulong base, CPURISCVState *env,
uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb,
vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
@@ -528,20 +522,19 @@ vext_ldst_us_desc(void *vd, target_ulong base, CPURISCVState *env,
* stride, stride = NF * sizeof (ETYPE)
*/
-#define GEN_VEXT_LD_US(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \
-void HELPER(NAME##_mask)(void *vd, void *v0, target_ulong base, \
- CPURISCVState *env, uint32_t desc) \
-{ \
- uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE)); \
- vext_ldst_stride(vd, v0, base, stride, env, desc, false, \
- LOAD_FN_TLB, ctzl(sizeof(ETYPE)), GETPC()); \
-} \
- \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
- CPURISCVState *env, uint32_t desc) \
-{ \
- vext_ldst_us_desc(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST, \
- ctzl(sizeof(ETYPE)), env->vl, GETPC(), true); \
+#define GEN_VEXT_LD_US(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \
+QEMU_FLATTEN void HELPER(NAME##_mask)(void *vd, void *v0, \
+ target_ulong base, CPURISCVState *env, uint32_t desc) \
+{ \
+ uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE)); \
+ vext_ldst_stride(vd, v0, base, stride, env, desc, false, \
+ LOAD_FN_TLB, ctzl(sizeof(ETYPE)), GETPC()); \
+} \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
+ CPURISCVState *env, uint32_t desc) \
+{ \
+ vext_ldst_us_desc(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST, \
+ ctzl(sizeof(ETYPE)), env->vl, GETPC(), true); \
}
GEN_VEXT_LD_US(vle8_v, int8_t, lde_b_tlb, lde_b_host)
@@ -549,20 +542,19 @@ GEN_VEXT_LD_US(vle16_v, int16_t, lde_h_tlb, lde_h_host)
GEN_VEXT_LD_US(vle32_v, int32_t, lde_w_tlb, lde_w_host)
GEN_VEXT_LD_US(vle64_v, int64_t, lde_d_tlb, lde_d_host)
-#define GEN_VEXT_ST_US(NAME, ETYPE, STORE_FN_TLB, STORE_FN_HOST) \
-void HELPER(NAME##_mask)(void *vd, void *v0, target_ulong base, \
- CPURISCVState *env, uint32_t desc) \
-{ \
- uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE)); \
- vext_ldst_stride(vd, v0, base, stride, env, desc, false, \
- STORE_FN_TLB, ctzl(sizeof(ETYPE)), GETPC()); \
-} \
- \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
- CPURISCVState *env, uint32_t desc) \
-{ \
- vext_ldst_us_desc(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST, \
- ctzl(sizeof(ETYPE)), env->vl, GETPC(), false); \
+#define GEN_VEXT_ST_US(NAME, ETYPE, STORE_FN_TLB, STORE_FN_HOST) \
+QEMU_FLATTEN void HELPER(NAME##_mask)(void *vd, void *v0, \
+ target_ulong base, CPURISCVState *env, uint32_t desc) \
+{ \
+ uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE)); \
+ vext_ldst_stride(vd, v0, base, stride, env, desc, false, \
+ STORE_FN_TLB, ctzl(sizeof(ETYPE)), GETPC()); \
+} \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
+ CPURISCVState *env, uint32_t desc) \
+{ \
+ vext_ldst_us_desc(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST, \
+ ctzl(sizeof(ETYPE)), env->vl, GETPC(), false); \
}
GEN_VEXT_ST_US(vse8_v, int8_t, ste_b_tlb, ste_b_host)
@@ -573,8 +565,8 @@ GEN_VEXT_ST_US(vse64_v, int64_t, ste_d_tlb, ste_d_host)
/*
* unit stride mask load and store, EEW = 1
*/
-void HELPER(vlm_v)(void *vd, void *v0, target_ulong base,
- CPURISCVState *env, uint32_t desc)
+QEMU_FLATTEN void HELPER(vlm_v)(void *vd, void *v0, target_ulong base,
+ CPURISCVState *env, uint32_t desc)
{
/* evl = ceil(vl/8) */
uint8_t evl = (env->vl + 7) >> 3;
@@ -582,8 +574,8 @@ void HELPER(vlm_v)(void *vd, void *v0, target_ulong base,
0, evl, GETPC(), true);
}
-void HELPER(vsm_v)(void *vd, void *v0, target_ulong base,
- CPURISCVState *env, uint32_t desc)
+QEMU_FLATTEN void HELPER(vsm_v)(void *vd, void *v0, target_ulong base,
+ CPURISCVState *env, uint32_t desc)
{
/* evl = ceil(vl/8) */
uint8_t evl = (env->vl + 7) >> 3;
@@ -609,7 +601,7 @@ GEN_VEXT_GET_INDEX_ADDR(idx_h, uint16_t, H2)
GEN_VEXT_GET_INDEX_ADDR(idx_w, uint32_t, H4)
GEN_VEXT_GET_INDEX_ADDR(idx_d, uint64_t, H8)
-static inline void
+static void
vext_ldst_index(void *vd, void *v0, target_ulong base,
void *vs2, CPURISCVState *env, uint32_t desc,
vext_get_index_addr get_index_addr,
@@ -646,12 +638,12 @@ vext_ldst_index(void *vd, void *v0, target_ulong base,
}
}
-#define GEN_VEXT_LD_INDEX(NAME, ETYPE, INDEX_FN, LOAD_FN) \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
- void *vs2, CPURISCVState *env, uint32_t desc) \
-{ \
- vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN, \
- LOAD_FN, ctzl(sizeof(ETYPE)), GETPC()); \
+#define GEN_VEXT_LD_INDEX(NAME, ETYPE, INDEX_FN, LOAD_FN) \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
+ void *vs2, CPURISCVState *env, uint32_t desc) \
+{ \
+ vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN, \
+ LOAD_FN, ctzl(sizeof(ETYPE)), GETPC()); \
}
GEN_VEXT_LD_INDEX(vlxei8_8_v, int8_t, idx_b, lde_b_tlb)
@@ -671,13 +663,12 @@ GEN_VEXT_LD_INDEX(vlxei64_16_v, int16_t, idx_d, lde_h_tlb)
GEN_VEXT_LD_INDEX(vlxei64_32_v, int32_t, idx_d, lde_w_tlb)
GEN_VEXT_LD_INDEX(vlxei64_64_v, int64_t, idx_d, lde_d_tlb)
-#define GEN_VEXT_ST_INDEX(NAME, ETYPE, INDEX_FN, STORE_FN) \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
- void *vs2, CPURISCVState *env, uint32_t desc) \
-{ \
- vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN, \
- STORE_FN, ctzl(sizeof(ETYPE)), \
- GETPC()); \
+#define GEN_VEXT_ST_INDEX(NAME, ETYPE, INDEX_FN, STORE_FN) \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
+ void *vs2, CPURISCVState *env, uint32_t desc) \
+{ \
+ vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN, \
+ STORE_FN, ctzl(sizeof(ETYPE)), GETPC()); \
}
GEN_VEXT_ST_INDEX(vsxei8_8_v, int8_t, idx_b, ste_b_tlb)
@@ -700,7 +691,7 @@ GEN_VEXT_ST_INDEX(vsxei64_64_v, int64_t, idx_d, ste_d_tlb)
/*
* unit-stride fault-only-fisrt load instructions
*/
-static inline void
+static void
vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb,
vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz, uintptr_t ra)
@@ -933,12 +924,12 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
}
}
-#define GEN_VEXT_LDFF(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
- CPURISCVState *env, uint32_t desc) \
-{ \
- vext_ldff(vd, v0, base, env, desc, LOAD_FN_TLB, \
- LOAD_FN_HOST, ctzl(sizeof(ETYPE)), GETPC()); \
+#define GEN_VEXT_LDFF(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
+ CPURISCVState *env, uint32_t desc) \
+{ \
+ vext_ldff(vd, v0, base, env, desc, LOAD_FN_TLB, \
+ LOAD_FN_HOST, ctzl(sizeof(ETYPE)), GETPC()); \
}
GEN_VEXT_LDFF(vle8ff_v, int8_t, lde_b_tlb, lde_b_host)
@@ -959,7 +950,7 @@ GEN_VEXT_LDFF(vle64ff_v, int64_t, lde_d_tlb, lde_d_host)
/*
* load and store whole register instructions
*/
-static inline QEMU_ALWAYS_INLINE void
+static void
vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
vext_ldst_elem_fn_tlb *ldst_tlb,
vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
@@ -973,8 +964,8 @@ vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
}
#define GEN_VEXT_LD_WHOLE(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \
-void HELPER(NAME)(void *vd, target_ulong base, CPURISCVState *env, \
- uint32_t desc) \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, target_ulong base, \
+ CPURISCVState *env, uint32_t desc) \
{ \
vext_ldst_whole(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST, \
ctzl(sizeof(ETYPE)), GETPC(), true); \
@@ -998,8 +989,8 @@ GEN_VEXT_LD_WHOLE(vl8re32_v, int32_t, lde_w_tlb, lde_w_host)
GEN_VEXT_LD_WHOLE(vl8re64_v, int64_t, lde_d_tlb, lde_d_host)
#define GEN_VEXT_ST_WHOLE(NAME, ETYPE, STORE_FN_TLB, STORE_FN_HOST) \
-void HELPER(NAME)(void *vd, target_ulong base, CPURISCVState *env, \
- uint32_t desc) \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, target_ulong base, \
+ CPURISCVState *env, uint32_t desc) \
{ \
vext_ldst_whole(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST, \
ctzl(sizeof(ETYPE)), GETPC(), false); \
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 21/23] target/riscv: Drop v0 argument from gen_helper_ldst_us
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (19 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 20/23] target/riscv: Use FLATTEN rather than ALWAYS_INLINE for vector ldst Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 22/23] target/riscv: Drop v0 argument from gen_helper_ldst_stride Richard Henderson
2026-08-15 19:45 ` [PATCH 23/23] target/riscv: Drop v0 argument from gen_helper_ldst_index Richard Henderson
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Since we're also passing env, v0 is always env->vreg.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/helper.h | 44 ++++++++++-----------
target/riscv/tcg/vector_helper.c | 36 +++++++++--------
target/riscv/tcg/insn_trans/trans_rvv.c.inc | 25 ++++--------
3 files changed, 48 insertions(+), 57 deletions(-)
diff --git a/target/riscv/helper.h b/target/riscv/helper.h
index 542b7c264f..ffd759aef3 100644
--- a/target/riscv/helper.h
+++ b/target/riscv/helper.h
@@ -162,24 +162,24 @@ DEF_HELPER_FLAGS_3(hyp_hsv_d, TCG_CALL_NO_WG, void, env, tl, tl)
/* Vector functions */
DEF_HELPER_4(vsetvl, tl, env, tl, tl, tl)
-DEF_HELPER_5(vle8_v, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vle16_v, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vle32_v, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vle64_v, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vle8_v_mask, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vle16_v_mask, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vle32_v_mask, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vle64_v_mask, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vse8_v, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vse16_v, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vse32_v, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vse64_v, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vse8_v_mask, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vse16_v_mask, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vse32_v_mask, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vse64_v_mask, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vlm_v, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vsm_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_4(vle8_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vle16_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vle32_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vle64_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vle8_v_mask, void, ptr, tl, env, i32)
+DEF_HELPER_4(vle16_v_mask, void, ptr, tl, env, i32)
+DEF_HELPER_4(vle32_v_mask, void, ptr, tl, env, i32)
+DEF_HELPER_4(vle64_v_mask, void, ptr, tl, env, i32)
+DEF_HELPER_4(vse8_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vse16_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vse32_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vse64_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vse8_v_mask, void, ptr, tl, env, i32)
+DEF_HELPER_4(vse16_v_mask, void, ptr, tl, env, i32)
+DEF_HELPER_4(vse32_v_mask, void, ptr, tl, env, i32)
+DEF_HELPER_4(vse64_v_mask, void, ptr, tl, env, i32)
+DEF_HELPER_4(vlm_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vsm_v, void, ptr, tl, env, i32)
DEF_HELPER_6(vlse8_v, void, ptr, ptr, tl, tl, env, i32)
DEF_HELPER_6(vlse16_v, void, ptr, ptr, tl, tl, env, i32)
DEF_HELPER_6(vlse32_v, void, ptr, ptr, tl, tl, env, i32)
@@ -220,10 +220,10 @@ DEF_HELPER_6(vsxei64_8_v, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vsxei64_16_v, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vsxei64_32_v, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vsxei64_64_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_5(vle8ff_v, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vle16ff_v, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vle32ff_v, void, ptr, ptr, tl, env, i32)
-DEF_HELPER_5(vle64ff_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_4(vle8ff_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vle16ff_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vle32ff_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vle64ff_v, void, ptr, tl, env, i32)
DEF_HELPER_4(vl1re8_v, void, ptr, tl, env, i32)
DEF_HELPER_4(vl1re16_v, void, ptr, tl, env, i32)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 2a24e15a31..2548ad9c2c 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -292,11 +292,12 @@ static void vext_ldst_nf_tlb(CPURISCVState *env, void *vd, target_ulong addr,
* stride: access vector element from strided memory
*/
static void
-vext_ldst_stride(void *vd, void *v0, target_ulong base, target_ulong stride,
+vext_ldst_stride(void *vd, target_ulong base, target_ulong stride,
CPURISCVState *env, uint32_t desc, uint32_t vm,
vext_ldst_elem_fn_tlb *ldst_elem, uint32_t log2_esz,
uintptr_t ra)
{
+ void *v0 = env->vreg;
uint32_t nf = vext_nf(desc);
uint32_t max_elems = vext_max_elems(desc, log2_esz);
uint32_t esz = 1 << log2_esz;
@@ -326,7 +327,7 @@ void HELPER(NAME)(void *vd, void * v0, target_ulong base, \
uint32_t desc) \
{ \
uint32_t vm = vext_vm(desc); \
- vext_ldst_stride(vd, v0, base, stride, env, desc, vm, LOAD_FN, \
+ vext_ldst_stride(vd, base, stride, env, desc, vm, LOAD_FN, \
ctzl(sizeof(ETYPE)), GETPC()); \
}
@@ -341,7 +342,7 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
uint32_t desc) \
{ \
uint32_t vm = vext_vm(desc); \
- vext_ldst_stride(vd, v0, base, stride, env, desc, vm, STORE_FN, \
+ vext_ldst_stride(vd, base, stride, env, desc, vm, STORE_FN, \
ctzl(sizeof(ETYPE)), GETPC()); \
}
@@ -523,14 +524,14 @@ vext_ldst_us_desc(void *vd, target_ulong base, CPURISCVState *env,
*/
#define GEN_VEXT_LD_US(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \
-QEMU_FLATTEN void HELPER(NAME##_mask)(void *vd, void *v0, \
- target_ulong base, CPURISCVState *env, uint32_t desc) \
+QEMU_FLATTEN void HELPER(NAME##_mask)(void *vd, target_ulong base, \
+ CPURISCVState *env, uint32_t desc) \
{ \
uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE)); \
- vext_ldst_stride(vd, v0, base, stride, env, desc, false, \
+ vext_ldst_stride(vd, base, stride, env, desc, false, \
LOAD_FN_TLB, ctzl(sizeof(ETYPE)), GETPC()); \
} \
-QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, target_ulong base, \
CPURISCVState *env, uint32_t desc) \
{ \
vext_ldst_us_desc(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST, \
@@ -543,15 +544,15 @@ GEN_VEXT_LD_US(vle32_v, int32_t, lde_w_tlb, lde_w_host)
GEN_VEXT_LD_US(vle64_v, int64_t, lde_d_tlb, lde_d_host)
#define GEN_VEXT_ST_US(NAME, ETYPE, STORE_FN_TLB, STORE_FN_HOST) \
-QEMU_FLATTEN void HELPER(NAME##_mask)(void *vd, void *v0, \
- target_ulong base, CPURISCVState *env, uint32_t desc) \
+QEMU_FLATTEN void HELPER(NAME##_mask)(void *vd, target_ulong base, \
+ CPURISCVState *env, uint32_t desc) \
{ \
uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE)); \
- vext_ldst_stride(vd, v0, base, stride, env, desc, false, \
+ vext_ldst_stride(vd, base, stride, env, desc, false, \
STORE_FN_TLB, ctzl(sizeof(ETYPE)), GETPC()); \
} \
-QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
- CPURISCVState *env, uint32_t desc) \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, target_ulong base, \
+ CPURISCVState *env, uint32_t desc) \
{ \
vext_ldst_us_desc(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST, \
ctzl(sizeof(ETYPE)), env->vl, GETPC(), false); \
@@ -565,7 +566,7 @@ GEN_VEXT_ST_US(vse64_v, int64_t, ste_d_tlb, ste_d_host)
/*
* unit stride mask load and store, EEW = 1
*/
-QEMU_FLATTEN void HELPER(vlm_v)(void *vd, void *v0, target_ulong base,
+QEMU_FLATTEN void HELPER(vlm_v)(void *vd, target_ulong base,
CPURISCVState *env, uint32_t desc)
{
/* evl = ceil(vl/8) */
@@ -574,7 +575,7 @@ QEMU_FLATTEN void HELPER(vlm_v)(void *vd, void *v0, target_ulong base,
0, evl, GETPC(), true);
}
-QEMU_FLATTEN void HELPER(vsm_v)(void *vd, void *v0, target_ulong base,
+QEMU_FLATTEN void HELPER(vsm_v)(void *vd, target_ulong base,
CPURISCVState *env, uint32_t desc)
{
/* evl = ceil(vl/8) */
@@ -692,10 +693,11 @@ GEN_VEXT_ST_INDEX(vsxei64_64_v, int64_t, idx_d, ste_d_tlb)
* unit-stride fault-only-fisrt load instructions
*/
static void
-vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
+vext_ldff(void *vd, target_ulong base, CPURISCVState *env,
uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb,
vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz, uintptr_t ra)
{
+ void *v0 = env->vreg;
uint32_t i = env->vstart, vl = env->vl;
uint32_t nf = vext_nf(desc);
uint32_t vm = vext_vm(desc);
@@ -925,10 +927,10 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
}
#define GEN_VEXT_LDFF(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \
-QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, target_ulong base, \
CPURISCVState *env, uint32_t desc) \
{ \
- vext_ldff(vd, v0, base, env, desc, LOAD_FN_TLB, \
+ vext_ldff(vd, base, env, desc, LOAD_FN_TLB, \
LOAD_FN_HOST, ctzl(sizeof(ETYPE)), GETPC()); \
}
diff --git a/target/riscv/tcg/insn_trans/trans_rvv.c.inc b/target/riscv/tcg/insn_trans/trans_rvv.c.inc
index 3916236762..3d32564660 100644
--- a/target/riscv/tcg/insn_trans/trans_rvv.c.inc
+++ b/target/riscv/tcg/insn_trans/trans_rvv.c.inc
@@ -690,8 +690,7 @@ static uint8_t vext_get_emul(DisasContext *s, uint8_t eew)
/*
*** unit stride load and store
*/
-typedef void gen_helper_ldst_us(TCGv_ptr, TCGv_ptr, TCGv,
- TCGv_env, TCGv_i32);
+typedef void gen_helper_ldst_us(TCGv_ptr, TCGv, TCGv_env, TCGv_i32);
static uint32_t vdata_add_memopidx(DisasContext *s, uint32_t data)
{
@@ -705,14 +704,10 @@ static bool ldst_us_trans(uint32_t vd, uint32_t rs1, uint32_t data,
gen_helper_ldst_us *fn, DisasContext *s,
bool is_store)
{
- TCGv_ptr dest, mask;
- TCGv base;
+ TCGv_ptr dest = tcg_temp_new_ptr();
+ TCGv base = get_gpr(s, rs1, EXT_NONE);
TCGv_i32 desc;
- dest = tcg_temp_new_ptr();
- mask = tcg_temp_new_ptr();
- base = get_gpr(s, rs1, EXT_NONE);
-
/*
* As simd_desc supports at most 2048 bytes, and in this implementation,
* the max vector group length is 4096 bytes. So split it into two parts.
@@ -725,7 +720,6 @@ static bool ldst_us_trans(uint32_t vd, uint32_t rs1, uint32_t data,
s->cfg_ptr->vlenb, data));
tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd));
- tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0));
/*
* According to the specification
@@ -745,7 +739,7 @@ static bool ldst_us_trans(uint32_t vd, uint32_t rs1, uint32_t data,
mark_vs_dirty(s);
- fn(dest, mask, base, tcg_env, desc);
+ fn(dest, base, tcg_env, desc);
if (!is_store && s->ztso) {
tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
@@ -1133,22 +1127,17 @@ GEN_VEXT_TRANS(vsxei64_v, MO_64, rnfvm, st_index_op, st_index_check)
static bool ldff_trans(uint32_t vd, uint32_t rs1, uint32_t data,
gen_helper_ldst_us *fn, DisasContext *s)
{
- TCGv_ptr dest, mask;
- TCGv base;
+ TCGv_ptr dest = tcg_temp_new_ptr();
+ TCGv base = get_gpr(s, rs1, EXT_NONE);
TCGv_i32 desc;
- dest = tcg_temp_new_ptr();
- mask = tcg_temp_new_ptr();
- base = get_gpr(s, rs1, EXT_NONE);
-
data = vdata_add_memopidx(s, data);
desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
s->cfg_ptr->vlenb, data));
tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd));
- tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0));
- fn(dest, mask, base, tcg_env, desc);
+ fn(dest, base, tcg_env, desc);
finalize_rvv_inst(s);
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 22/23] target/riscv: Drop v0 argument from gen_helper_ldst_stride
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (20 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 21/23] target/riscv: Drop v0 argument from gen_helper_ldst_us Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
2026-08-15 19:45 ` [PATCH 23/23] target/riscv: Drop v0 argument from gen_helper_ldst_index Richard Henderson
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Since we're also passing env, v0 is always env->vreg.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/helper.h | 16 ++++++++--------
target/riscv/tcg/vector_helper.c | 10 ++++------
target/riscv/tcg/insn_trans/trans_rvv.c.inc | 16 +++++-----------
3 files changed, 17 insertions(+), 25 deletions(-)
diff --git a/target/riscv/helper.h b/target/riscv/helper.h
index ffd759aef3..82a478962b 100644
--- a/target/riscv/helper.h
+++ b/target/riscv/helper.h
@@ -180,14 +180,14 @@ DEF_HELPER_4(vse32_v_mask, void, ptr, tl, env, i32)
DEF_HELPER_4(vse64_v_mask, void, ptr, tl, env, i32)
DEF_HELPER_4(vlm_v, void, ptr, tl, env, i32)
DEF_HELPER_4(vsm_v, void, ptr, tl, env, i32)
-DEF_HELPER_6(vlse8_v, void, ptr, ptr, tl, tl, env, i32)
-DEF_HELPER_6(vlse16_v, void, ptr, ptr, tl, tl, env, i32)
-DEF_HELPER_6(vlse32_v, void, ptr, ptr, tl, tl, env, i32)
-DEF_HELPER_6(vlse64_v, void, ptr, ptr, tl, tl, env, i32)
-DEF_HELPER_6(vsse8_v, void, ptr, ptr, tl, tl, env, i32)
-DEF_HELPER_6(vsse16_v, void, ptr, ptr, tl, tl, env, i32)
-DEF_HELPER_6(vsse32_v, void, ptr, ptr, tl, tl, env, i32)
-DEF_HELPER_6(vsse64_v, void, ptr, ptr, tl, tl, env, i32)
+DEF_HELPER_5(vlse8_v, void, ptr, tl, tl, env, i32)
+DEF_HELPER_5(vlse16_v, void, ptr, tl, tl, env, i32)
+DEF_HELPER_5(vlse32_v, void, ptr, tl, tl, env, i32)
+DEF_HELPER_5(vlse64_v, void, ptr, tl, tl, env, i32)
+DEF_HELPER_5(vsse8_v, void, ptr, tl, tl, env, i32)
+DEF_HELPER_5(vsse16_v, void, ptr, tl, tl, env, i32)
+DEF_HELPER_5(vsse32_v, void, ptr, tl, tl, env, i32)
+DEF_HELPER_5(vsse64_v, void, ptr, tl, tl, env, i32)
DEF_HELPER_6(vlxei8_8_v, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vlxei8_16_v, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vlxei8_32_v, void, ptr, ptr, tl, ptr, env, i32)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 2548ad9c2c..354c0b0da5 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -322,9 +322,8 @@ vext_ldst_stride(void *vd, target_ulong base, target_ulong stride,
}
#define GEN_VEXT_LD_STRIDE(NAME, ETYPE, LOAD_FN) \
-void HELPER(NAME)(void *vd, void * v0, target_ulong base, \
- target_ulong stride, CPURISCVState *env, \
- uint32_t desc) \
+void HELPER(NAME)(void *vd, target_ulong base, target_ulong stride, \
+ CPURISCVState *env, uint32_t desc) \
{ \
uint32_t vm = vext_vm(desc); \
vext_ldst_stride(vd, base, stride, env, desc, vm, LOAD_FN, \
@@ -337,9 +336,8 @@ GEN_VEXT_LD_STRIDE(vlse32_v, int32_t, lde_w_tlb)
GEN_VEXT_LD_STRIDE(vlse64_v, int64_t, lde_d_tlb)
#define GEN_VEXT_ST_STRIDE(NAME, ETYPE, STORE_FN) \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
- target_ulong stride, CPURISCVState *env, \
- uint32_t desc) \
+void HELPER(NAME)(void *vd, target_ulong base, target_ulong stride, \
+ CPURISCVState *env, uint32_t desc) \
{ \
uint32_t vm = vext_vm(desc); \
vext_ldst_stride(vd, base, stride, env, desc, vm, STORE_FN, \
diff --git a/target/riscv/tcg/insn_trans/trans_rvv.c.inc b/target/riscv/tcg/insn_trans/trans_rvv.c.inc
index 3d32564660..5e352dbf41 100644
--- a/target/riscv/tcg/insn_trans/trans_rvv.c.inc
+++ b/target/riscv/tcg/insn_trans/trans_rvv.c.inc
@@ -878,32 +878,26 @@ GEN_VEXT_TRANS(vsm_v, MO_8, vsm_v, st_us_mask_op, st_us_mask_check)
/*
*** stride load and store
*/
-typedef void gen_helper_ldst_stride(TCGv_ptr, TCGv_ptr, TCGv,
- TCGv, TCGv_env, TCGv_i32);
+typedef void gen_helper_ldst_stride(TCGv_ptr, TCGv, TCGv, TCGv_env, TCGv_i32);
static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2,
uint32_t data, gen_helper_ldst_stride *fn,
DisasContext *s)
{
- TCGv_ptr dest, mask;
- TCGv base, stride;
+ TCGv_ptr dest = tcg_temp_new_ptr();
+ TCGv base = get_gpr(s, rs1, EXT_NONE);
+ TCGv stride = get_gpr(s, rs2, EXT_NONE);
TCGv_i32 desc;
- dest = tcg_temp_new_ptr();
- mask = tcg_temp_new_ptr();
- base = get_gpr(s, rs1, EXT_NONE);
- stride = get_gpr(s, rs2, EXT_NONE);
-
data = vdata_add_memopidx(s, data);
desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
s->cfg_ptr->vlenb, data));
tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd));
- tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0));
mark_vs_dirty(s);
- fn(dest, mask, base, stride, tcg_env, desc);
+ fn(dest, base, stride, tcg_env, desc);
finalize_rvv_inst(s);
return true;
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH 23/23] target/riscv: Drop v0 argument from gen_helper_ldst_index
2026-08-15 19:45 [PATCH 00/23] target/riscv: Reorg vector load/store Richard Henderson
` (21 preceding siblings ...)
2026-08-15 19:45 ` [PATCH 22/23] target/riscv: Drop v0 argument from gen_helper_ldst_stride Richard Henderson
@ 2026-08-15 19:45 ` Richard Henderson
22 siblings, 0 replies; 33+ messages in thread
From: Richard Henderson @ 2026-08-15 19:45 UTC (permalink / raw)
To: qemu-devel; +Cc: frank.chang, max.chou, qemu-riscv
Since we're also passing env, v0 is always env->vreg.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/riscv/helper.h | 64 ++++++++++-----------
target/riscv/tcg/vector_helper.c | 15 ++---
target/riscv/tcg/insn_trans/trans_rvv.c.inc | 17 ++----
3 files changed, 46 insertions(+), 50 deletions(-)
diff --git a/target/riscv/helper.h b/target/riscv/helper.h
index 82a478962b..96fb2347df 100644
--- a/target/riscv/helper.h
+++ b/target/riscv/helper.h
@@ -188,38 +188,38 @@ DEF_HELPER_5(vsse8_v, void, ptr, tl, tl, env, i32)
DEF_HELPER_5(vsse16_v, void, ptr, tl, tl, env, i32)
DEF_HELPER_5(vsse32_v, void, ptr, tl, tl, env, i32)
DEF_HELPER_5(vsse64_v, void, ptr, tl, tl, env, i32)
-DEF_HELPER_6(vlxei8_8_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei8_16_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei8_32_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei8_64_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei16_8_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei16_16_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei16_32_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei16_64_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei32_8_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei32_16_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei32_32_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei32_64_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei64_8_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei64_16_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei64_32_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vlxei64_64_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei8_8_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei8_16_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei8_32_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei8_64_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei16_8_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei16_16_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei16_32_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei16_64_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei32_8_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei32_16_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei32_32_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei32_64_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei64_8_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei64_16_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei64_32_v, void, ptr, ptr, tl, ptr, env, i32)
-DEF_HELPER_6(vsxei64_64_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei8_8_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei8_16_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei8_32_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei8_64_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei16_8_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei16_16_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei16_32_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei16_64_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei32_8_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei32_16_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei32_32_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei32_64_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei64_8_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei64_16_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei64_32_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vlxei64_64_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei8_8_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei8_16_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei8_32_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei8_64_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei16_8_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei16_16_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei16_32_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei16_64_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei32_8_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei32_16_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei32_32_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei32_64_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei64_8_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei64_16_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei64_32_v, void, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vsxei64_64_v, void, ptr, tl, ptr, env, i32)
DEF_HELPER_4(vle8ff_v, void, ptr, tl, env, i32)
DEF_HELPER_4(vle16ff_v, void, ptr, tl, env, i32)
DEF_HELPER_4(vle32ff_v, void, ptr, tl, env, i32)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 354c0b0da5..b7b3805a6d 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -601,12 +601,13 @@ GEN_VEXT_GET_INDEX_ADDR(idx_w, uint32_t, H4)
GEN_VEXT_GET_INDEX_ADDR(idx_d, uint64_t, H8)
static void
-vext_ldst_index(void *vd, void *v0, target_ulong base,
+vext_ldst_index(void *vd, target_ulong base,
void *vs2, CPURISCVState *env, uint32_t desc,
vext_get_index_addr get_index_addr,
vext_ldst_elem_fn_tlb *ldst_elem,
uint32_t log2_esz, uintptr_t ra)
{
+ void *v0 = env->vreg;
uint32_t nf = vext_nf(desc);
uint32_t vm = vext_vm(desc);
uint32_t max_elems = vext_max_elems(desc, log2_esz);
@@ -638,10 +639,10 @@ vext_ldst_index(void *vd, void *v0, target_ulong base,
}
#define GEN_VEXT_LD_INDEX(NAME, ETYPE, INDEX_FN, LOAD_FN) \
-QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
- void *vs2, CPURISCVState *env, uint32_t desc) \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, target_ulong base, void *vs2, \
+ CPURISCVState *env, uint32_t desc) \
{ \
- vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN, \
+ vext_ldst_index(vd, base, vs2, env, desc, INDEX_FN, \
LOAD_FN, ctzl(sizeof(ETYPE)), GETPC()); \
}
@@ -663,10 +664,10 @@ GEN_VEXT_LD_INDEX(vlxei64_32_v, int32_t, idx_d, lde_w_tlb)
GEN_VEXT_LD_INDEX(vlxei64_64_v, int64_t, idx_d, lde_d_tlb)
#define GEN_VEXT_ST_INDEX(NAME, ETYPE, INDEX_FN, STORE_FN) \
-QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \
- void *vs2, CPURISCVState *env, uint32_t desc) \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, target_ulong base, void *vs2, \
+ CPURISCVState *env, uint32_t desc) \
{ \
- vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN, \
+ vext_ldst_index(vd, base, vs2, env, desc, INDEX_FN, \
STORE_FN, ctzl(sizeof(ETYPE)), GETPC()); \
}
diff --git a/target/riscv/tcg/insn_trans/trans_rvv.c.inc b/target/riscv/tcg/insn_trans/trans_rvv.c.inc
index 5e352dbf41..4556c644cc 100644
--- a/target/riscv/tcg/insn_trans/trans_rvv.c.inc
+++ b/target/riscv/tcg/insn_trans/trans_rvv.c.inc
@@ -975,33 +975,28 @@ GEN_VEXT_TRANS(vsse64_v, MO_64, rnfvm, st_stride_op, st_stride_check)
/*
*** index load and store
*/
-typedef void gen_helper_ldst_index(TCGv_ptr, TCGv_ptr, TCGv,
- TCGv_ptr, TCGv_env, TCGv_i32);
+typedef void gen_helper_ldst_index(TCGv_ptr, TCGv, TCGv_ptr,
+ TCGv_env, TCGv_i32);
static bool ldst_index_trans(uint32_t vd, uint32_t rs1, uint32_t vs2,
uint32_t data, gen_helper_ldst_index *fn,
DisasContext *s)
{
- TCGv_ptr dest, mask, index;
- TCGv base;
+ TCGv_ptr dest = tcg_temp_new_ptr();
+ TCGv_ptr index = tcg_temp_new_ptr();
+ TCGv base = get_gpr(s, rs1, EXT_NONE);
TCGv_i32 desc;
- dest = tcg_temp_new_ptr();
- mask = tcg_temp_new_ptr();
- index = tcg_temp_new_ptr();
- base = get_gpr(s, rs1, EXT_NONE);
-
data = vdata_add_memopidx(s, data);
desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
s->cfg_ptr->vlenb, data));
tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd));
tcg_gen_addi_ptr(index, tcg_env, vreg_ofs(s, vs2));
- tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0));
mark_vs_dirty(s);
- fn(dest, mask, base, index, tcg_env, desc);
+ fn(dest, base, index, tcg_env, desc);
finalize_rvv_inst(s);
return true;
--
2.43.0
^ permalink raw reply related [flat|nested] 33+ messages in thread