[PATCH 0/2] fib: fix and speed up RVV lookup

[PATCH 0/2] fib: fix and speed up RVV lookup

[Sun Yuechi]

Patch 1 fixes wrong next hops from a FIB created with
RTE_FIB_F_LOOKUP_NETWORK_ORDER on RISC-V: the vector lookup read the
addresses in host order.

Patch 2 removes a register spill from the vector lookup loop by
gathering the entries at the width they are stored with.

Patch 2, ns per lookup, standalone benchmark, tables in cache, -O3:

                    1B     2B     4B     8B
  SpacemiT X60
    before         31.1   32.1   30.6   29.7
    after          27.2   27.9   27.6   26.4
  Sophgo SG2044
    before          5.6    5.7    5.9    7.5
    after           4.4    4.1    5.0    6.0
  SpacemiT X100
    before          4.7    4.8    4.8    5.9
    after           4.1    4.2    4.3    5.1

fib_perf_autotest, whose addresses miss the cache, gains 0 to 14%.

Sun Yuechi (2):
  fib: fix lookup in network order with RVV
  fib: gather entries at their own width with RVV

 lib/fib/dir24_8.c     |  4 +-
 lib/fib/dir24_8_rvv.c | 85 ++++++++++++++++++++++++-------------------
 2 files changed, 49 insertions(+), 40 deletions(-)

-- 
2.56.0

[PATCH 1/2] fib: fix lookup in network order with RVV

[Sun Yuechi]

A FIB created with RTE_FIB_F_LOOKUP_NETWORK_ORDER was given the vector
lookup, which reads the addresses in host order. Leave such a FIB to
the scalar functions.

Fixes: f2ccf5fc334b ("fib: lookup with RISC-V vector extension")
Cc: stable@dpdk.org
Signed-off-by: Sun Yuechi <sunyuechi@iscas.ac.cn>
---
 lib/fib/dir24_8.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/fib/dir24_8.c b/lib/fib/dir24_8.c
index 489d2ef427..c135285a65 100644
--- a/lib/fib/dir24_8.c
+++ b/lib/fib/dir24_8.c
@@ -94,8 +94,8 @@ get_vector_fn(enum rte_fib_dir24_8_nh_sz nh_sz, bool be_addr)
 		return NULL;
 	}
 #elif defined(RTE_RISCV_FEATURE_V)
-	RTE_SET_USED(be_addr);
-	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_RISCV_ISA_V) <= 0)
+	/* the vector functions take addresses in host order only */
+	if (be_addr || rte_cpu_get_flag_enabled(RTE_CPUFLAG_RISCV_ISA_V) <= 0)
 		return NULL;
 	switch (nh_sz) {
 	case RTE_FIB_DIR24_8_1B:
-- 
2.56.0

[PATCH 2/2] fib: gather entries at their own width with RVV

[Sun Yuechi]

The lookup loaded 64-bit words and shifted the entry out of them. That
takes LMUL=8 values whatever the next hop size, one register group more
than there is, so a group was spilled inside the loop. Gather the
entries at the size they are stored with instead.

Signed-off-by: Sun Yuechi <sunyuechi@iscas.ac.cn>
---
 lib/fib/dir24_8_rvv.c | 85 ++++++++++++++++++++++++-------------------
 1 file changed, 47 insertions(+), 38 deletions(-)

diff --git a/lib/fib/dir24_8_rvv.c b/lib/fib/dir24_8_rvv.c
index 9c14ca0481..9b784bb3c5 100644
--- a/lib/fib/dir24_8_rvv.c
+++ b/lib/fib/dir24_8_rvv.c
@@ -10,55 +10,64 @@
 #include "dir24_8.h"
 #include "dir24_8_rvv.h"
 
-#define DECLARE_VECTOR_FN(SFX, NH_SZ) \
+/* Byte offset of entry number idx, for each next hop size. */
+#define OFS_1b(idx, vl) ((void)(vl), (idx))
+#define OFS_2b(idx, vl) __riscv_vsll_vx_u32m4(idx, 1, vl)
+#define OFS_4b(idx, vl) __riscv_vsll_vx_u32m4(idx, 2, vl)
+#define OFS_8b(idx, vl) __riscv_vsll_vx_u32m4(idx, 3, vl)
+
+/* Entry without its low bit, as 32-bit tbl8 group number. */
+#define GRP_1b(ent, vl) \
+	__riscv_vzext_vf4_u32m4(__riscv_vsrl_vx_u8m1(ent, 1, vl), vl)
+#define GRP_2b(ent, vl) \
+	__riscv_vzext_vf2_u32m4(__riscv_vsrl_vx_u16m2(ent, 1, vl), vl)
+#define GRP_4b(ent, vl) __riscv_vsrl_vx_u32m4(ent, 1, vl)
+#define GRP_8b(ent, vl) __riscv_vnsrl_wx_u32m4(ent, 1, vl)
+
+/* Entry without its low bit, as 64-bit next hop. */
+#define NH_1b(ent, vl) \
+	__riscv_vzext_vf8_u64m8(__riscv_vsrl_vx_u8m1(ent, 1, vl), vl)
+#define NH_2b(ent, vl) \
+	__riscv_vzext_vf4_u64m8(__riscv_vsrl_vx_u16m2(ent, 1, vl), vl)
+#define NH_4b(ent, vl) \
+	__riscv_vzext_vf2_u64m8(__riscv_vsrl_vx_u32m4(ent, 1, vl), vl)
+#define NH_8b(ent, vl) __riscv_vsrl_vx_u64m8(ent, 1, vl)
+
+/* Entries are gathered at their own width, with 32-bit byte offsets. */
+#define DECLARE_VECTOR_FN(SFX, TYPE, BITS, LMUL) \
 void \
 rte_dir24_8_vec_lookup_bulk_##SFX(void *p, \
 		const uint32_t *ips, uint64_t *next_hops, unsigned int n) \
 { \
-	const uint8_t  idx_bits = 3 - NH_SZ; \
-	const uint32_t idx_mask = (1u << (3 - NH_SZ)) - 1u; \
-	const uint64_t e_mask   = ~0ULL >> (64 - (8u << NH_SZ)); \
-	struct dir24_8_tbl *tbl = (struct dir24_8_tbl *)p; \
-	const uint64_t *tbl24   = tbl->tbl24; \
+	const struct dir24_8_tbl *tbl = (const struct dir24_8_tbl *)p; \
+	const TYPE *tbl24 = (const TYPE *)tbl->tbl24; \
+	const TYPE *tbl8 = (const TYPE *)tbl->tbl8; \
 	size_t vl; \
 	for (unsigned int i = 0; i < n; i += vl) { \
 		vl = __riscv_vsetvl_e32m4(n - i); \
 		vuint32m4_t v_ips = __riscv_vle32_v_u32m4(&ips[i], vl); \
-		vuint64m8_t vtbl_word = __riscv_vluxei32_v_u64m8(tbl24, \
-				__riscv_vsll_vx_u32m4( \
-				__riscv_vsrl_vx_u32m4(v_ips, idx_bits + 8, vl), 3, vl), vl); \
-		vuint32m4_t v_tbl_index = __riscv_vsrl_vx_u32m4(v_ips, 8, vl); \
-		vuint32m4_t v_entry_idx = __riscv_vand_vx_u32m4(v_tbl_index, idx_mask, vl); \
-		vuint32m4_t v_shift     = __riscv_vsll_vx_u32m4(v_entry_idx, 3 + NH_SZ, vl); \
-		vuint64m8_t vtbl_entry  = __riscv_vand_vx_u64m8( \
-				__riscv_vsrl_vv_u64m8(vtbl_word, \
-					__riscv_vwcvtu_x_x_v_u64m8(v_shift, vl), vl), e_mask, vl); \
-		vbool8_t mask = __riscv_vmseq_vx_u64m8_b8( \
-				__riscv_vand_vx_u64m8(vtbl_entry, 1, vl), 1, vl); \
-		if (__riscv_vcpop_m_b8(mask, vl)) { \
-			const uint64_t *tbl8 = tbl->tbl8; \
-			v_tbl_index = __riscv_vadd_vv_u32m4_mu(mask, v_tbl_index, \
-					__riscv_vsll_vx_u32m4( \
-						__riscv_vnsrl_wx_u32m4(vtbl_entry, 1, vl), 8, vl), \
-						__riscv_vand_vx_u32m4(v_ips, 0xFF, vl), vl); \
-			vtbl_word = __riscv_vluxei32_v_u64m8_mu(mask, vtbl_word, tbl8, \
-					__riscv_vsll_vx_u32m4( \
-					__riscv_vsrl_vx_u32m4(v_tbl_index, idx_bits, vl), 3, vl), \
-						vl); \
-			v_entry_idx = __riscv_vand_vx_u32m4(v_tbl_index, idx_mask, vl); \
-			v_shift     = __riscv_vsll_vx_u32m4(v_entry_idx, 3 + NH_SZ, vl); \
-			vtbl_entry  = __riscv_vand_vx_u64m8( \
-					__riscv_vsrl_vv_u64m8(vtbl_word, \
-					__riscv_vwcvtu_x_x_v_u64m8(v_shift, vl), vl), e_mask, vl); \
+		vuint##BITS##m##LMUL##_t v_ent = \
+			__riscv_vluxei32_v_u##BITS##m##LMUL(tbl24, \
+				OFS_##SFX(__riscv_vsrl_vx_u32m4(v_ips, 8, vl), \
+					vl), vl); \
+		vbool8_t mask = __riscv_vmsne_vx_u##BITS##m##LMUL##_b8( \
+			__riscv_vand_vx_u##BITS##m##LMUL(v_ent, \
+				DIR24_8_EXT_ENT, vl), 0, vl); \
+		if (unlikely(__riscv_vfirst_m_b8(mask, vl) >= 0)) { \
+			vuint32m4_t v_idx = __riscv_vadd_vv_u32m4( \
+				__riscv_vsll_vx_u32m4(GRP_##SFX(v_ent, vl), \
+					8, vl), \
+				__riscv_vand_vx_u32m4(v_ips, 0xFF, vl), vl); \
+			v_ent = __riscv_vluxei32_v_u##BITS##m##LMUL##_mu(mask, \
+				v_ent, tbl8, OFS_##SFX(v_idx, vl), vl); \
 		} \
-		__riscv_vse64_v_u64m8(&next_hops[i], \
-				__riscv_vsrl_vx_u64m8(vtbl_entry, 1, vl), vl); \
+		__riscv_vse64_v_u64m8(&next_hops[i], NH_##SFX(v_ent, vl), vl); \
 	} \
 }
 
-DECLARE_VECTOR_FN(1b, 0)
-DECLARE_VECTOR_FN(2b, 1)
-DECLARE_VECTOR_FN(4b, 2)
-DECLARE_VECTOR_FN(8b, 3)
+DECLARE_VECTOR_FN(1b, uint8_t, 8, 1)
+DECLARE_VECTOR_FN(2b, uint16_t, 16, 2)
+DECLARE_VECTOR_FN(4b, uint32_t, 32, 4)
+DECLARE_VECTOR_FN(8b, uint64_t, 64, 8)
 
 #endif
-- 
2.56.0