* [PATCH] net/crc: make NEON CRC folding logic similar to x86 SSE
@ 2026-10-06 15:18 Shreesh Adiga
2026-10-09 3:24 ` Wathsala Vithanage
0 siblings, 1 reply; 3+ messages in thread
From: Shreesh Adiga @ 2026-10-06 15:18 UTC (permalink / raw)
To: Wathsala Vithanage; +Cc: dev
This patch includes the following minor changes:
1) Use TBL based shift left for variable len shift instead of using
vshift_bytes_left which adds a switch case for 16 len cases.
2) Align the fold constants to use PMULL and PMULL2 instructions
instead of using cross lane operands which requires additional shift.
3) Remove unnecessary static arrays used in crc32_reduce_64_to_32 by
adapting x86 SSE logic.
4) Use x86 SSE logic for processing partial bytes sequence to avoid
using multiple variable len shifts which inserts switch case.
Signed-off-by: Shreesh Adiga <16567adigashreesh@gmail.com>
---
lib/net/net_crc_neon.c | 71 +++++++++++++++++++++++++++---------------
1 file changed, 46 insertions(+), 25 deletions(-)
diff --git a/lib/net/net_crc_neon.c b/lib/net/net_crc_neon.c
index d924aed25d..3833fcba6e 100644
--- a/lib/net/net_crc_neon.c
+++ b/lib/net/net_crc_neon.c
@@ -24,6 +24,31 @@ struct crc_pmull_ctx {
alignas(16) struct crc_pmull_ctx crc32_eth_pmull;
alignas(16) struct crc_pmull_ctx crc16_ccitt_pmull;
+static const alignas(16) uint8_t crc_neon_shift_tab[32] = {
+ 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8,
+ 0xf7, 0xf6, 0xf5, 0xf4, 0xf3, 0xf2, 0xf1, 0xf0,
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
+};
+
+/**
+ * Shifts left 128 bit register by specified number of bytes
+ *
+ * @param reg
+ * 128 bit value
+ * @param num
+ * number of bytes to shift left reg by (0-16)
+ *
+ * @return
+ * reg << (num * 8)
+ */
+static inline uint64x2_t
+neon_shift_left(uint64x2_t reg, const unsigned int num)
+{
+ uint8x16_t tbl = vld1q_u8(crc_neon_shift_tab + 16 - num);
+ return vreinterpretq_u64_u8(vqtbl1q_u8(vreinterpretq_u8_u64(reg), tbl));
+}
+
/**
* @brief Performs one folding round
*
@@ -46,12 +71,12 @@ crcr32_folding_round(uint64x2_t data_block, uint64x2_t precomp,
uint64x2_t fold)
{
uint64x2_t tmp0 = vreinterpretq_u64_p128(vmull_p64(
- vgetq_lane_p64(vreinterpretq_p64_u64(fold), 1),
+ vgetq_lane_p64(vreinterpretq_p64_u64(fold), 0),
vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 0)));
- uint64x2_t tmp1 = vreinterpretq_u64_p128(vmull_p64(
- vgetq_lane_p64(vreinterpretq_p64_u64(fold), 0),
- vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 1)));
+ uint64x2_t tmp1 = vreinterpretq_u64_p128(vmull_high_p64(
+ vreinterpretq_p64_u64(fold),
+ vreinterpretq_p64_u64(precomp)));
return veorq_u64(tmp1, veorq_u64(data_block, tmp0));
}
@@ -98,26 +123,19 @@ static inline uint32_t
crcr32_reduce_64_to_32(uint64x2_t data64,
uint64x2_t precomp)
{
- static alignas(16) uint32_t mask1[4] = {
- 0xffffffff, 0xffffffff, 0x00000000, 0x00000000
- };
- static alignas(16) uint32_t mask2[4] = {
- 0x00000000, 0xffffffff, 0xffffffff, 0xffffffff
- };
uint64x2_t tmp0, tmp1, tmp2;
- tmp0 = vandq_u64(data64, vld1q_u64((uint64_t *)mask2));
+ tmp0 = vreinterpretq_u64_u32(
+ vsetq_lane_u32(0, vreinterpretq_u32_u64(data64), 0));
tmp1 = vreinterpretq_u64_p128(vmull_p64(
vgetq_lane_p64(vreinterpretq_p64_u64(tmp0), 0),
vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 0)));
tmp1 = veorq_u64(tmp1, tmp0);
- tmp1 = vandq_u64(tmp1, vld1q_u64((uint64_t *)mask1));
tmp2 = vreinterpretq_u64_p128(vmull_p64(
vgetq_lane_p64(vreinterpretq_p64_u64(tmp1), 0),
vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 1)));
- tmp2 = veorq_u64(tmp2, tmp1);
tmp2 = veorq_u64(tmp2, tmp0);
return vgetq_lane_u32(vreinterpretq_u32_u64(tmp2), 2);
@@ -177,10 +195,10 @@ crc32_eth_calc_pmull(
fold = vld1q_u64((uint64_t *)buffer);
fold = veorq_u64(fold, temp);
if (unlikely(data_len < 4)) {
- fold = vshift_bytes_left(fold, 8 - data_len);
+ fold = neon_shift_left(fold, 8 - data_len);
goto barret_reduction;
}
- fold = vshift_bytes_left(fold, 16 - data_len);
+ fold = neon_shift_left(fold, 16 - data_len);
goto reduction_128_64;
}
@@ -201,14 +219,17 @@ crc32_eth_calc_pmull(
/** Partial bytes - process last <16 bytes */
if (likely(n < data_len)) {
- uint64x2_t last16, a, b, mask;
+ uint8x16_t last16, t1, t2;
+ uint64x2_t a, b;
uint32_t rem = data_len & 15;
- last16 = vld1q_u64((const uint64_t *)&data[data_len - 16]);
- a = vshift_bytes_left(fold, 16 - rem);
- b = vshift_bytes_right(fold, rem);
- mask = vshift_bytes_left(vdupq_n_u64(-1), 16 - rem);
- b = vorrq_u64(b, vandq_u64(mask, last16));
+ last16 = vld1q_u8((const uint8_t *)&data[data_len - 16]);
+ t1 = vld1q_u8(crc_neon_shift_tab + rem);
+ a = vreinterpretq_u64_u8(vqtbl1q_u8(vreinterpretq_u8_u64(fold), t1));
+ t2 = vmvnq_u8(t1);
+ t2 = vqtbl1q_u8(vreinterpretq_u8_u64(fold), t2);
+ t1 = vcgezq_s8(vreinterpretq_s8_u8(t1));
+ b = vreinterpretq_u64_u8(vbslq_u8(t1, last16, t2));
/* k = rk3 & rk4 */
fold = crcr32_folding_round(b, k, a);
@@ -230,14 +251,14 @@ void
rte_net_crc_neon_init(void)
{
/* Initialize CRC16 data */
- uint64_t ccitt_k1_k2[2] = {0x14ff2LLU, 0x19a3cLLU};
- uint64_t ccitt_k3_k4[2] = {0x189aeLLU, 0x8e10LLU};
+ uint64_t ccitt_k1_k2[2] = {0x19a3cLLU, 0x14ff2LLU};
+ uint64_t ccitt_k3_k4[2] = {0x8e10LLU, 0x189aeLLU};
uint64_t ccitt_k5_k6[2] = {0x189aeLLU, 0x114aaLLU};
uint64_t ccitt_k7_k8[2] = {0x11c581910LLU, 0x10811LLU};
/* Initialize CRC32 data */
- uint64_t eth_k1_k2[2] = {0x1c6e41596LLU, 0x154442bd4LLU};
- uint64_t eth_k3_k4[2] = {0xccaa009eLLU, 0x1751997d0LLU};
+ uint64_t eth_k1_k2[2] = {0x154442bd4LLU, 0x1c6e41596LLU};
+ uint64_t eth_k3_k4[2] = {0x1751997d0LLU, 0xccaa009eLLU};
uint64_t eth_k5_k6[2] = {0xccaa009eLLU, 0x163cd6124LLU};
uint64_t eth_k7_k8[2] = {0x1f7011640LLU, 0x1db710641LLU};
--
2.55.0
^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [PATCH] net/crc: make NEON CRC folding logic similar to x86 SSE
2026-10-06 15:18 [PATCH] net/crc: make NEON CRC folding logic similar to x86 SSE Shreesh Adiga
@ 2026-10-09 3:24 ` Wathsala Vithanage
2026-10-09 3:41 ` Shreesh Adiga
0 siblings, 1 reply; 3+ messages in thread
From: Wathsala Vithanage @ 2026-10-09 3:24 UTC (permalink / raw)
To: Shreesh Adiga; +Cc: dev
Hi Shreesh,
Looks correct. Can you share before and after performance data in the
commit message covering short inputs below 16 bytes, lengths exercising
the different nonzero remainders modulo 16, and larger buffers to show
the effect on the folding loop?
Thanks
--wathsala
On 10/6/26 10:18, Shreesh Adiga wrote:
> This patch includes the following minor changes:
> 1) Use TBL based shift left for variable len shift instead of using
> vshift_bytes_left which adds a switch case for 16 len cases.
> 2) Align the fold constants to use PMULL and PMULL2 instructions
> instead of using cross lane operands which requires additional shift.
> 3) Remove unnecessary static arrays used in crc32_reduce_64_to_32 by
> adapting x86 SSE logic.
> 4) Use x86 SSE logic for processing partial bytes sequence to avoid
> using multiple variable len shifts which inserts switch case.
>
> Signed-off-by: Shreesh Adiga <16567adigashreesh@gmail.com>
> ---
> lib/net/net_crc_neon.c | 71 +++++++++++++++++++++++++++---------------
> 1 file changed, 46 insertions(+), 25 deletions(-)
>
> diff --git a/lib/net/net_crc_neon.c b/lib/net/net_crc_neon.c
> index d924aed25d..3833fcba6e 100644
> --- a/lib/net/net_crc_neon.c
> +++ b/lib/net/net_crc_neon.c
> @@ -24,6 +24,31 @@ struct crc_pmull_ctx {
> alignas(16) struct crc_pmull_ctx crc32_eth_pmull;
> alignas(16) struct crc_pmull_ctx crc16_ccitt_pmull;
>
> +static const alignas(16) uint8_t crc_neon_shift_tab[32] = {
> + 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8,
> + 0xf7, 0xf6, 0xf5, 0xf4, 0xf3, 0xf2, 0xf1, 0xf0,
> + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
> + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
> +};
> +
> +/**
> + * Shifts left 128 bit register by specified number of bytes
> + *
> + * @param reg
> + * 128 bit value
> + * @param num
> + * number of bytes to shift left reg by (0-16)
> + *
> + * @return
> + * reg << (num * 8)
> + */
> +static inline uint64x2_t
> +neon_shift_left(uint64x2_t reg, const unsigned int num)
> +{
> + uint8x16_t tbl = vld1q_u8(crc_neon_shift_tab + 16 - num);
> + return vreinterpretq_u64_u8(vqtbl1q_u8(vreinterpretq_u8_u64(reg), tbl));
> +}
> +
> /**
> * @brief Performs one folding round
> *
> @@ -46,12 +71,12 @@ crcr32_folding_round(uint64x2_t data_block, uint64x2_t precomp,
> uint64x2_t fold)
> {
> uint64x2_t tmp0 = vreinterpretq_u64_p128(vmull_p64(
> - vgetq_lane_p64(vreinterpretq_p64_u64(fold), 1),
> + vgetq_lane_p64(vreinterpretq_p64_u64(fold), 0),
> vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 0)));
>
> - uint64x2_t tmp1 = vreinterpretq_u64_p128(vmull_p64(
> - vgetq_lane_p64(vreinterpretq_p64_u64(fold), 0),
> - vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 1)));
> + uint64x2_t tmp1 = vreinterpretq_u64_p128(vmull_high_p64(
> + vreinterpretq_p64_u64(fold),
> + vreinterpretq_p64_u64(precomp)));
>
> return veorq_u64(tmp1, veorq_u64(data_block, tmp0));
> }
> @@ -98,26 +123,19 @@ static inline uint32_t
> crcr32_reduce_64_to_32(uint64x2_t data64,
> uint64x2_t precomp)
> {
> - static alignas(16) uint32_t mask1[4] = {
> - 0xffffffff, 0xffffffff, 0x00000000, 0x00000000
> - };
> - static alignas(16) uint32_t mask2[4] = {
> - 0x00000000, 0xffffffff, 0xffffffff, 0xffffffff
> - };
> uint64x2_t tmp0, tmp1, tmp2;
>
> - tmp0 = vandq_u64(data64, vld1q_u64((uint64_t *)mask2));
> + tmp0 = vreinterpretq_u64_u32(
> + vsetq_lane_u32(0, vreinterpretq_u32_u64(data64), 0));
>
> tmp1 = vreinterpretq_u64_p128(vmull_p64(
> vgetq_lane_p64(vreinterpretq_p64_u64(tmp0), 0),
> vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 0)));
> tmp1 = veorq_u64(tmp1, tmp0);
> - tmp1 = vandq_u64(tmp1, vld1q_u64((uint64_t *)mask1));
>
> tmp2 = vreinterpretq_u64_p128(vmull_p64(
> vgetq_lane_p64(vreinterpretq_p64_u64(tmp1), 0),
> vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 1)));
> - tmp2 = veorq_u64(tmp2, tmp1);
> tmp2 = veorq_u64(tmp2, tmp0);
>
> return vgetq_lane_u32(vreinterpretq_u32_u64(tmp2), 2);
> @@ -177,10 +195,10 @@ crc32_eth_calc_pmull(
> fold = vld1q_u64((uint64_t *)buffer);
> fold = veorq_u64(fold, temp);
> if (unlikely(data_len < 4)) {
> - fold = vshift_bytes_left(fold, 8 - data_len);
> + fold = neon_shift_left(fold, 8 - data_len);
> goto barret_reduction;
> }
> - fold = vshift_bytes_left(fold, 16 - data_len);
> + fold = neon_shift_left(fold, 16 - data_len);
> goto reduction_128_64;
> }
>
> @@ -201,14 +219,17 @@ crc32_eth_calc_pmull(
>
> /** Partial bytes - process last <16 bytes */
> if (likely(n < data_len)) {
> - uint64x2_t last16, a, b, mask;
> + uint8x16_t last16, t1, t2;
> + uint64x2_t a, b;
> uint32_t rem = data_len & 15;
>
> - last16 = vld1q_u64((const uint64_t *)&data[data_len - 16]);
> - a = vshift_bytes_left(fold, 16 - rem);
> - b = vshift_bytes_right(fold, rem);
> - mask = vshift_bytes_left(vdupq_n_u64(-1), 16 - rem);
> - b = vorrq_u64(b, vandq_u64(mask, last16));
> + last16 = vld1q_u8((const uint8_t *)&data[data_len - 16]);
> + t1 = vld1q_u8(crc_neon_shift_tab + rem);
> + a = vreinterpretq_u64_u8(vqtbl1q_u8(vreinterpretq_u8_u64(fold), t1));
> + t2 = vmvnq_u8(t1);
> + t2 = vqtbl1q_u8(vreinterpretq_u8_u64(fold), t2);
> + t1 = vcgezq_s8(vreinterpretq_s8_u8(t1));
> + b = vreinterpretq_u64_u8(vbslq_u8(t1, last16, t2));
>
> /* k = rk3 & rk4 */
> fold = crcr32_folding_round(b, k, a);
> @@ -230,14 +251,14 @@ void
> rte_net_crc_neon_init(void)
> {
> /* Initialize CRC16 data */
> - uint64_t ccitt_k1_k2[2] = {0x14ff2LLU, 0x19a3cLLU};
> - uint64_t ccitt_k3_k4[2] = {0x189aeLLU, 0x8e10LLU};
> + uint64_t ccitt_k1_k2[2] = {0x19a3cLLU, 0x14ff2LLU};
> + uint64_t ccitt_k3_k4[2] = {0x8e10LLU, 0x189aeLLU};
> uint64_t ccitt_k5_k6[2] = {0x189aeLLU, 0x114aaLLU};
> uint64_t ccitt_k7_k8[2] = {0x11c581910LLU, 0x10811LLU};
>
> /* Initialize CRC32 data */
> - uint64_t eth_k1_k2[2] = {0x1c6e41596LLU, 0x154442bd4LLU};
> - uint64_t eth_k3_k4[2] = {0xccaa009eLLU, 0x1751997d0LLU};
> + uint64_t eth_k1_k2[2] = {0x154442bd4LLU, 0x1c6e41596LLU};
> + uint64_t eth_k3_k4[2] = {0x1751997d0LLU, 0xccaa009eLLU};
> uint64_t eth_k5_k6[2] = {0xccaa009eLLU, 0x163cd6124LLU};
> uint64_t eth_k7_k8[2] = {0x1f7011640LLU, 0x1db710641LLU};
>
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [PATCH] net/crc: make NEON CRC folding logic similar to x86 SSE
2026-10-09 3:24 ` Wathsala Vithanage
@ 2026-10-09 3:41 ` Shreesh Adiga
0 siblings, 0 replies; 3+ messages in thread
From: Shreesh Adiga @ 2026-10-09 3:41 UTC (permalink / raw)
To: Wathsala Vithanage; +Cc: dev
[-- Attachment #1: Type: text/plain, Size: 8038 bytes --]
Hi Wathsala,
I tested this on QEMU for correctness. I don't have access to any ARM
environment for precise benchmarking.
The only device I have is an Android phone where I can't compile DPDK
either.
I can isolate this code and use libbenchmark via Termux if that is
needed, but I don't expect to see large measurable benefits.
This patch is based on my previous work on FFmpeg's av_crc optimization for
aarch64 which you can
refer here: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21651
Thanks,
Shreesh
On Fri, Oct 9, 2026 at 8:54 AM Wathsala Vithanage <
wathsala.vithanage@arm.com> wrote:
> Hi Shreesh,
>
> Looks correct. Can you share before and after performance data in the
> commit message covering short inputs below 16 bytes, lengths exercising
> the different nonzero remainders modulo 16, and larger buffers to show
> the effect on the folding loop?
>
> Thanks
>
> --wathsala
>
> On 10/6/26 10:18, Shreesh Adiga wrote:
> > This patch includes the following minor changes:
> > 1) Use TBL based shift left for variable len shift instead of using
> > vshift_bytes_left which adds a switch case for 16 len cases.
> > 2) Align the fold constants to use PMULL and PMULL2 instructions
> > instead of using cross lane operands which requires additional shift.
> > 3) Remove unnecessary static arrays used in crc32_reduce_64_to_32 by
> > adapting x86 SSE logic.
> > 4) Use x86 SSE logic for processing partial bytes sequence to avoid
> > using multiple variable len shifts which inserts switch case.
> >
> > Signed-off-by: Shreesh Adiga <16567adigashreesh@gmail.com>
> > ---
> > lib/net/net_crc_neon.c | 71 +++++++++++++++++++++++++++---------------
> > 1 file changed, 46 insertions(+), 25 deletions(-)
> >
> > diff --git a/lib/net/net_crc_neon.c b/lib/net/net_crc_neon.c
> > index d924aed25d..3833fcba6e 100644
> > --- a/lib/net/net_crc_neon.c
> > +++ b/lib/net/net_crc_neon.c
> > @@ -24,6 +24,31 @@ struct crc_pmull_ctx {
> > alignas(16) struct crc_pmull_ctx crc32_eth_pmull;
> > alignas(16) struct crc_pmull_ctx crc16_ccitt_pmull;
> >
> > +static const alignas(16) uint8_t crc_neon_shift_tab[32] = {
> > + 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8,
> > + 0xf7, 0xf6, 0xf5, 0xf4, 0xf3, 0xf2, 0xf1, 0xf0,
> > + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
> > + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
> > +};
> > +
> > +/**
> > + * Shifts left 128 bit register by specified number of bytes
> > + *
> > + * @param reg
> > + * 128 bit value
> > + * @param num
> > + * number of bytes to shift left reg by (0-16)
> > + *
> > + * @return
> > + * reg << (num * 8)
> > + */
> > +static inline uint64x2_t
> > +neon_shift_left(uint64x2_t reg, const unsigned int num)
> > +{
> > + uint8x16_t tbl = vld1q_u8(crc_neon_shift_tab + 16 - num);
> > + return vreinterpretq_u64_u8(vqtbl1q_u8(vreinterpretq_u8_u64(reg),
> tbl));
> > +}
> > +
> > /**
> > * @brief Performs one folding round
> > *
> > @@ -46,12 +71,12 @@ crcr32_folding_round(uint64x2_t data_block,
> uint64x2_t precomp,
> > uint64x2_t fold)
> > {
> > uint64x2_t tmp0 = vreinterpretq_u64_p128(vmull_p64(
> > - vgetq_lane_p64(vreinterpretq_p64_u64(fold), 1),
> > + vgetq_lane_p64(vreinterpretq_p64_u64(fold), 0),
> > vgetq_lane_p64(vreinterpretq_p64_u64(precomp),
> 0)));
> >
> > - uint64x2_t tmp1 = vreinterpretq_u64_p128(vmull_p64(
> > - vgetq_lane_p64(vreinterpretq_p64_u64(fold), 0),
> > - vgetq_lane_p64(vreinterpretq_p64_u64(precomp),
> 1)));
> > + uint64x2_t tmp1 = vreinterpretq_u64_p128(vmull_high_p64(
> > + vreinterpretq_p64_u64(fold),
> > + vreinterpretq_p64_u64(precomp)));
> >
> > return veorq_u64(tmp1, veorq_u64(data_block, tmp0));
> > }
> > @@ -98,26 +123,19 @@ static inline uint32_t
> > crcr32_reduce_64_to_32(uint64x2_t data64,
> > uint64x2_t precomp)
> > {
> > - static alignas(16) uint32_t mask1[4] = {
> > - 0xffffffff, 0xffffffff, 0x00000000, 0x00000000
> > - };
> > - static alignas(16) uint32_t mask2[4] = {
> > - 0x00000000, 0xffffffff, 0xffffffff, 0xffffffff
> > - };
> > uint64x2_t tmp0, tmp1, tmp2;
> >
> > - tmp0 = vandq_u64(data64, vld1q_u64((uint64_t *)mask2));
> > + tmp0 = vreinterpretq_u64_u32(
> > + vsetq_lane_u32(0, vreinterpretq_u32_u64(data64), 0));
> >
> > tmp1 = vreinterpretq_u64_p128(vmull_p64(
> > vgetq_lane_p64(vreinterpretq_p64_u64(tmp0), 0),
> > vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 0)));
> > tmp1 = veorq_u64(tmp1, tmp0);
> > - tmp1 = vandq_u64(tmp1, vld1q_u64((uint64_t *)mask1));
> >
> > tmp2 = vreinterpretq_u64_p128(vmull_p64(
> > vgetq_lane_p64(vreinterpretq_p64_u64(tmp1), 0),
> > vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 1)));
> > - tmp2 = veorq_u64(tmp2, tmp1);
> > tmp2 = veorq_u64(tmp2, tmp0);
> >
> > return vgetq_lane_u32(vreinterpretq_u32_u64(tmp2), 2);
> > @@ -177,10 +195,10 @@ crc32_eth_calc_pmull(
> > fold = vld1q_u64((uint64_t *)buffer);
> > fold = veorq_u64(fold, temp);
> > if (unlikely(data_len < 4)) {
> > - fold = vshift_bytes_left(fold, 8 - data_len);
> > + fold = neon_shift_left(fold, 8 - data_len);
> > goto barret_reduction;
> > }
> > - fold = vshift_bytes_left(fold, 16 - data_len);
> > + fold = neon_shift_left(fold, 16 - data_len);
> > goto reduction_128_64;
> > }
> >
> > @@ -201,14 +219,17 @@ crc32_eth_calc_pmull(
> >
> > /** Partial bytes - process last <16 bytes */
> > if (likely(n < data_len)) {
> > - uint64x2_t last16, a, b, mask;
> > + uint8x16_t last16, t1, t2;
> > + uint64x2_t a, b;
> > uint32_t rem = data_len & 15;
> >
> > - last16 = vld1q_u64((const uint64_t *)&data[data_len - 16]);
> > - a = vshift_bytes_left(fold, 16 - rem);
> > - b = vshift_bytes_right(fold, rem);
> > - mask = vshift_bytes_left(vdupq_n_u64(-1), 16 - rem);
> > - b = vorrq_u64(b, vandq_u64(mask, last16));
> > + last16 = vld1q_u8((const uint8_t *)&data[data_len - 16]);
> > + t1 = vld1q_u8(crc_neon_shift_tab + rem);
> > + a =
> vreinterpretq_u64_u8(vqtbl1q_u8(vreinterpretq_u8_u64(fold), t1));
> > + t2 = vmvnq_u8(t1);
> > + t2 = vqtbl1q_u8(vreinterpretq_u8_u64(fold), t2);
> > + t1 = vcgezq_s8(vreinterpretq_s8_u8(t1));
> > + b = vreinterpretq_u64_u8(vbslq_u8(t1, last16, t2));
> >
> > /* k = rk3 & rk4 */
> > fold = crcr32_folding_round(b, k, a);
> > @@ -230,14 +251,14 @@ void
> > rte_net_crc_neon_init(void)
> > {
> > /* Initialize CRC16 data */
> > - uint64_t ccitt_k1_k2[2] = {0x14ff2LLU, 0x19a3cLLU};
> > - uint64_t ccitt_k3_k4[2] = {0x189aeLLU, 0x8e10LLU};
> > + uint64_t ccitt_k1_k2[2] = {0x19a3cLLU, 0x14ff2LLU};
> > + uint64_t ccitt_k3_k4[2] = {0x8e10LLU, 0x189aeLLU};
> > uint64_t ccitt_k5_k6[2] = {0x189aeLLU, 0x114aaLLU};
> > uint64_t ccitt_k7_k8[2] = {0x11c581910LLU, 0x10811LLU};
> >
> > /* Initialize CRC32 data */
> > - uint64_t eth_k1_k2[2] = {0x1c6e41596LLU, 0x154442bd4LLU};
> > - uint64_t eth_k3_k4[2] = {0xccaa009eLLU, 0x1751997d0LLU};
> > + uint64_t eth_k1_k2[2] = {0x154442bd4LLU, 0x1c6e41596LLU};
> > + uint64_t eth_k3_k4[2] = {0x1751997d0LLU, 0xccaa009eLLU};
> > uint64_t eth_k5_k6[2] = {0xccaa009eLLU, 0x163cd6124LLU};
> > uint64_t eth_k7_k8[2] = {0x1f7011640LLU, 0x1db710641LLU};
> >
>
[-- Attachment #2: Type: text/html, Size: 10018 bytes --]
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2026-10-09 3:41 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-10-06 15:18 [PATCH] net/crc: make NEON CRC folding logic similar to x86 SSE Shreesh Adiga
2026-10-09 3:24 ` Wathsala Vithanage
2026-10-09 3:41 ` Shreesh Adiga
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox