* [PATCH 2/2] fib: gather entries at their own width with RVV
2026-10-05 7:04 [PATCH 0/2] fib: fix and speed up RVV lookup Sun Yuechi
2026-10-05 7:04 ` [PATCH 1/2] fib: fix lookup in network order with RVV Sun Yuechi
@ 2026-10-05 7:04 ` Sun Yuechi
1 sibling, 0 replies; 3+ messages in thread
From: Sun Yuechi @ 2026-10-05 7:04 UTC (permalink / raw)
To: Vladimir Medvedkin; +Cc: dev, Sun Yuechi
The lookup loaded 64-bit words and shifted the entry out of them. That
takes LMUL=8 values whatever the next hop size, one register group more
than there is, so a group was spilled inside the loop. Gather the
entries at the size they are stored with instead.
Signed-off-by: Sun Yuechi <sunyuechi@iscas.ac.cn>
---
lib/fib/dir24_8_rvv.c | 85 ++++++++++++++++++++++++-------------------
1 file changed, 47 insertions(+), 38 deletions(-)
diff --git a/lib/fib/dir24_8_rvv.c b/lib/fib/dir24_8_rvv.c
index 9c14ca0481..9b784bb3c5 100644
--- a/lib/fib/dir24_8_rvv.c
+++ b/lib/fib/dir24_8_rvv.c
@@ -10,55 +10,64 @@
#include "dir24_8.h"
#include "dir24_8_rvv.h"
-#define DECLARE_VECTOR_FN(SFX, NH_SZ) \
+/* Byte offset of entry number idx, for each next hop size. */
+#define OFS_1b(idx, vl) ((void)(vl), (idx))
+#define OFS_2b(idx, vl) __riscv_vsll_vx_u32m4(idx, 1, vl)
+#define OFS_4b(idx, vl) __riscv_vsll_vx_u32m4(idx, 2, vl)
+#define OFS_8b(idx, vl) __riscv_vsll_vx_u32m4(idx, 3, vl)
+
+/* Entry without its low bit, as 32-bit tbl8 group number. */
+#define GRP_1b(ent, vl) \
+ __riscv_vzext_vf4_u32m4(__riscv_vsrl_vx_u8m1(ent, 1, vl), vl)
+#define GRP_2b(ent, vl) \
+ __riscv_vzext_vf2_u32m4(__riscv_vsrl_vx_u16m2(ent, 1, vl), vl)
+#define GRP_4b(ent, vl) __riscv_vsrl_vx_u32m4(ent, 1, vl)
+#define GRP_8b(ent, vl) __riscv_vnsrl_wx_u32m4(ent, 1, vl)
+
+/* Entry without its low bit, as 64-bit next hop. */
+#define NH_1b(ent, vl) \
+ __riscv_vzext_vf8_u64m8(__riscv_vsrl_vx_u8m1(ent, 1, vl), vl)
+#define NH_2b(ent, vl) \
+ __riscv_vzext_vf4_u64m8(__riscv_vsrl_vx_u16m2(ent, 1, vl), vl)
+#define NH_4b(ent, vl) \
+ __riscv_vzext_vf2_u64m8(__riscv_vsrl_vx_u32m4(ent, 1, vl), vl)
+#define NH_8b(ent, vl) __riscv_vsrl_vx_u64m8(ent, 1, vl)
+
+/* Entries are gathered at their own width, with 32-bit byte offsets. */
+#define DECLARE_VECTOR_FN(SFX, TYPE, BITS, LMUL) \
void \
rte_dir24_8_vec_lookup_bulk_##SFX(void *p, \
const uint32_t *ips, uint64_t *next_hops, unsigned int n) \
{ \
- const uint8_t idx_bits = 3 - NH_SZ; \
- const uint32_t idx_mask = (1u << (3 - NH_SZ)) - 1u; \
- const uint64_t e_mask = ~0ULL >> (64 - (8u << NH_SZ)); \
- struct dir24_8_tbl *tbl = (struct dir24_8_tbl *)p; \
- const uint64_t *tbl24 = tbl->tbl24; \
+ const struct dir24_8_tbl *tbl = (const struct dir24_8_tbl *)p; \
+ const TYPE *tbl24 = (const TYPE *)tbl->tbl24; \
+ const TYPE *tbl8 = (const TYPE *)tbl->tbl8; \
size_t vl; \
for (unsigned int i = 0; i < n; i += vl) { \
vl = __riscv_vsetvl_e32m4(n - i); \
vuint32m4_t v_ips = __riscv_vle32_v_u32m4(&ips[i], vl); \
- vuint64m8_t vtbl_word = __riscv_vluxei32_v_u64m8(tbl24, \
- __riscv_vsll_vx_u32m4( \
- __riscv_vsrl_vx_u32m4(v_ips, idx_bits + 8, vl), 3, vl), vl); \
- vuint32m4_t v_tbl_index = __riscv_vsrl_vx_u32m4(v_ips, 8, vl); \
- vuint32m4_t v_entry_idx = __riscv_vand_vx_u32m4(v_tbl_index, idx_mask, vl); \
- vuint32m4_t v_shift = __riscv_vsll_vx_u32m4(v_entry_idx, 3 + NH_SZ, vl); \
- vuint64m8_t vtbl_entry = __riscv_vand_vx_u64m8( \
- __riscv_vsrl_vv_u64m8(vtbl_word, \
- __riscv_vwcvtu_x_x_v_u64m8(v_shift, vl), vl), e_mask, vl); \
- vbool8_t mask = __riscv_vmseq_vx_u64m8_b8( \
- __riscv_vand_vx_u64m8(vtbl_entry, 1, vl), 1, vl); \
- if (__riscv_vcpop_m_b8(mask, vl)) { \
- const uint64_t *tbl8 = tbl->tbl8; \
- v_tbl_index = __riscv_vadd_vv_u32m4_mu(mask, v_tbl_index, \
- __riscv_vsll_vx_u32m4( \
- __riscv_vnsrl_wx_u32m4(vtbl_entry, 1, vl), 8, vl), \
- __riscv_vand_vx_u32m4(v_ips, 0xFF, vl), vl); \
- vtbl_word = __riscv_vluxei32_v_u64m8_mu(mask, vtbl_word, tbl8, \
- __riscv_vsll_vx_u32m4( \
- __riscv_vsrl_vx_u32m4(v_tbl_index, idx_bits, vl), 3, vl), \
- vl); \
- v_entry_idx = __riscv_vand_vx_u32m4(v_tbl_index, idx_mask, vl); \
- v_shift = __riscv_vsll_vx_u32m4(v_entry_idx, 3 + NH_SZ, vl); \
- vtbl_entry = __riscv_vand_vx_u64m8( \
- __riscv_vsrl_vv_u64m8(vtbl_word, \
- __riscv_vwcvtu_x_x_v_u64m8(v_shift, vl), vl), e_mask, vl); \
+ vuint##BITS##m##LMUL##_t v_ent = \
+ __riscv_vluxei32_v_u##BITS##m##LMUL(tbl24, \
+ OFS_##SFX(__riscv_vsrl_vx_u32m4(v_ips, 8, vl), \
+ vl), vl); \
+ vbool8_t mask = __riscv_vmsne_vx_u##BITS##m##LMUL##_b8( \
+ __riscv_vand_vx_u##BITS##m##LMUL(v_ent, \
+ DIR24_8_EXT_ENT, vl), 0, vl); \
+ if (unlikely(__riscv_vfirst_m_b8(mask, vl) >= 0)) { \
+ vuint32m4_t v_idx = __riscv_vadd_vv_u32m4( \
+ __riscv_vsll_vx_u32m4(GRP_##SFX(v_ent, vl), \
+ 8, vl), \
+ __riscv_vand_vx_u32m4(v_ips, 0xFF, vl), vl); \
+ v_ent = __riscv_vluxei32_v_u##BITS##m##LMUL##_mu(mask, \
+ v_ent, tbl8, OFS_##SFX(v_idx, vl), vl); \
} \
- __riscv_vse64_v_u64m8(&next_hops[i], \
- __riscv_vsrl_vx_u64m8(vtbl_entry, 1, vl), vl); \
+ __riscv_vse64_v_u64m8(&next_hops[i], NH_##SFX(v_ent, vl), vl); \
} \
}
-DECLARE_VECTOR_FN(1b, 0)
-DECLARE_VECTOR_FN(2b, 1)
-DECLARE_VECTOR_FN(4b, 2)
-DECLARE_VECTOR_FN(8b, 3)
+DECLARE_VECTOR_FN(1b, uint8_t, 8, 1)
+DECLARE_VECTOR_FN(2b, uint16_t, 16, 2)
+DECLARE_VECTOR_FN(4b, uint32_t, 32, 4)
+DECLARE_VECTOR_FN(8b, uint64_t, 64, 8)
#endif
--
2.56.0
^ permalink raw reply related [flat|nested] 3+ messages in thread