RE: [PATCH] eal/x86: optimize memcpy of small 64-byte blocks

["Morten Brørup" <mb@smartsharesystems.com>]

> From: Konstantin Ananyev [mailto:konstantin.ananyev@huawei.com]
> Sent: Thursday, 6 August 2026 09.44
> 
> > > > The implementation for copying 64-byte blocks up to 512 (or 256)
> > > bytes
> > > > does not depend on address alignment with the size of the CPU's
> > > vector
> > > > registers, but is implemented in both unaligned and aligned copy
> > > > functions.
> > > > The main rte_memcpy() function was updated, so
> > > > if the copy size is known at compile time and the other criteria
> > > match,
> > > > the copy is performed without checking alignment requirements.
> > > > This provides two benefits when the optimization comes into play:
> > > > 1. A performance gain, because the address alignment check is
> > > avoided.
> > > > 3. Reduced instruction memory footprint, because the compiler
> only
> > > > generates one instance of the function for copying, instead of
> two
> > > > instances (one in the unaligned copy function, and one in the
> aligned
> > > > copy function).
> > > >
> > > > Furthermore, the temporary alignment mask definition
> (ALIGNMENT_MASK)
> > > > was prefixed by RTE_MEMCPY_ to prevent potential namespace
> collision.
> > > >
> > > > And finally, the superfluous function declaration at the top of
> the
> > > > file was removed, and its description was moved to the function
> > > > definition.
> > > > This improves search results with source code browsers.
> > > >
> > > > Signed-off-by: Morten Brørup <mb@smartsharesystems.com>
> > > > ---
> > > >  lib/eal/x86/include/rte_memcpy.h | 64 ++++++++++++++++++++++----
> ----
> > > --
> > > >  1 file changed, 44 insertions(+), 20 deletions(-)
> > > >
> > > > diff --git a/lib/eal/x86/include/rte_memcpy.h
> > > > b/lib/eal/x86/include/rte_memcpy.h
> > > > index 8ed8c55010..ce5a55c36f 100644
> > > > --- a/lib/eal/x86/include/rte_memcpy.h
> > > > +++ b/lib/eal/x86/include/rte_memcpy.h
> > > > @@ -32,21 +32,6 @@ extern "C" {
> > > >  #define RTE_MEMCPY_AVX
> > > >  #endif
> > > >
> > > > -/**
> > > > - * Copy bytes from one location to another. The locations must
> not
> > > overlap.
> > > > - *
> > > > - * @param dst
> > > > - *   Pointer to the destination of the data.
> > > > - * @param src
> > > > - *   Pointer to the source data.
> > > > - * @param n
> > > > - *   Number of bytes to copy.
> > > > - * @return
> > > > - *   Pointer to the destination data.
> > > > - */
> > > > -static __rte_always_inline void *
> > > > -rte_memcpy(void *__rte_restrict dst, const void *__rte_restrict
> src,
> > > size_t n);
> > > > -
> > > >  /**
> > > >   * Copy bytes from one location to another,
> > > >   * locations must not overlap.
> > > > @@ -187,7 +172,8 @@ rte_mov256(uint8_t *__rte_restrict dst, const
> > > uint8_t
> > > > *__rte_restrict src)
> > > >   * AVX512 implementation below
> > > >   */
> > > >
> > > > -#define ALIGNMENT_MASK 0x3F
> > > > +#define RTE_MEMCPY_ALIGNMENT_MASK 0x3F
> > > > +#define RTE_MEMCPY_BLOCK_64_MAX 512
> > > >
> > > >  /**
> > > >   * Copy 128-byte blocks from one location to another,
> > > > @@ -333,7 +319,8 @@ rte_memcpy_generic_more_than_64(void
> > > *__rte_restrict
> > > > dst, const void *__rte_rest
> > > >   * AVX implementation below
> > > >   */
> > > >
> > > > -#define ALIGNMENT_MASK 0x1F
> > > > +#define RTE_MEMCPY_ALIGNMENT_MASK 0x1F
> > > > +#define RTE_MEMCPY_BLOCK_64_MAX 256
> > >
> > > Wonder why BLOCK_64_MAX is 512 for SSE and AVX512, but 256 for
> AVX2?
> > > Some empirical data or ...?
> >
> > I wonder too.
> > Intel wrote it that way.
> > I considered consolidating on 512 for simplicity.
> > But then I would be changing the algorithm without any data to
> support the
> > change.
> >
> > Now you tickled my curiosity, so I went hunting...
> > It was 512 until DPDK 16.04, but in 16.07 it was changed to 256 with
> this patch:
> >
> https://github.com/DPDK/dpdk/commit/4b42e90ef0e421dc777f2b2e377eb237cd
> > 3675fa
> 
> Interesting, so you just followed the logic in various
> rte_memcpy_generic_more_than_64()
> Implementations, right?

Yes.
And rte_memcpy_aligned_more_than_64().

When copying blocks of 64 bytes up to 512 (or 256) bytes, all code paths (3 generic vector variants and the aligned variant) do exactly the same.
So I consolidated that into one code path, and avoid the address alignment check.
And because I consider it unlikely that rte_memcpy() is called with a size that matches the criteria, I limited it to size known at compile time.

> 
> > >
> > > >
> > > >  /**
> > > >   * Copy 128-byte blocks from one location to another,
> > > > @@ -444,7 +431,8 @@ rte_memcpy_generic_more_than_64(void
> > > *__rte_restrict
> > > > dst, const void *__rte_rest
> > > >   * SSE implementation below
> > > >   */
> > > >
> > > > -#define ALIGNMENT_MASK 0x0F
> > > > +#define RTE_MEMCPY_ALIGNMENT_MASK 0x0F
> > > > +#define RTE_MEMCPY_BLOCK_64_MAX 512
> > > >
> > > >  /**
> > > >   * Macro for copying unaligned block from one location to
> another
> > > with
> > > > constant load offset,
> > > > @@ -673,6 +661,18 @@ rte_memcpy_aligned_more_than_64(void
> > > > *__rte_restrict dst, const void *__rte_rest
> > > >  	return ret;
> > > >  }
> > > >
> > > > +/**
> > > > + * Copy bytes from one location to another. The locations must
> not
> > > overlap.
> > > > + *
> > > > + * @param dst
> > > > + *   Pointer to the destination of the data.
> > > > + * @param src
> > > > + *   Pointer to the source data.
> > > > + * @param n
> > > > + *   Number of bytes to copy.
> > > > + * @return
> > > > + *   Pointer to the destination data.
> > > > + */
> > > >  static __rte_always_inline void *
> > > >  rte_memcpy(void *__rte_restrict dst, const void *__rte_restrict
> src,
> > > size_t n)
> > > >  {
> > > > @@ -707,15 +707,39 @@ rte_memcpy(void *__rte_restrict dst, const
> void
> > > > *__rte_restrict src, size_t n)
> > > >  #endif
> > > >  		return dst;
> > > >  	}
> > > > +	/* Common way for small copy size of 64-byte blocks.
> Unlikely, so
> > > > constant size only */
> > > > +	if (__rte_constant(n) && (n & 63) == 0 && n <=
> > > > RTE_MEMCPY_BLOCK_64_MAX) {
> > >
> > > You probably need a check (static_assert?) that
> RTE_MEMCPY_BLOCK_64_MAX
> > > is
> > > Within allowed values (<= 512+256+128+64).
> > > Otherwise code block below might not be enough to copy everything.
> >
> > Allowed values stop at 512. Note that src and dst are not incremented
> for 512.
> 
> Then it should be 'return' here no?
> Or 'else if ...' after it.

Not needed.
With n=512, the compiler knows that the other conditions (n & 256, 128, 64) are false, and doesn't emit code for it.

> From other side, with current code 0x180 and 0x1c0 look like a
> perfectly valid values for it.
> 
> > I would hope that someone changing the RTE_MEMCPY_BLOCK_64_MAX
> > definition (or introducing a new value if support for a new vector
> register length
> > is added) doesn't do it without considering what it is used for.
> 
> That's exactly the reason why I think it needs either to be documented
> (add comment)
> or self-guarded at compilation time (static_assert, RTE_BUILD_BUG_ON,
> some macro, erc.).
> BTW, curious - what is wrong with static_assert() on it?

Nothing wrong with static_assert(); I just think it is superfluous.

Going back to your question about the need for "return" or "else if", you convinced me now...
I'll add the return for code readability purposes and a static_assert() for safety.

> 
> > A static_assert[]  seems overkill here.
> >
> > >
> > > > +		void *ret = dst;
> > > > +
> > > > +		if (n & 512) {
> > > > +			rte_mov256((uint8_t *)dst + 0 * 256, (const
> uint8_t
> > > *)src
> > > > + 0 * 256);
> > > > +			rte_mov256((uint8_t *)dst + 1 * 256, (const
> uint8_t
> > > *)src
> > > > + 1 * 256);
> > > > +		}
> > > > +		if (n & 256) {
> > > > +			rte_mov256((uint8_t *)dst, (const uint8_t
> *)src);
> > > > +			src = (const uint8_t *)src + 256;
> > > > +			dst = (uint8_t *)dst + 256;
> > > > +		}
> > > > +		if (n & 128) {
> > > > +			rte_mov128((uint8_t *)dst, (const uint8_t
> *)src);
> > > > +			src = (const uint8_t *)src + 128;
> > > > +			dst = (uint8_t *)dst + 128;
> > > > +		}
> > > > +		if (n & 64)
> > > > +			rte_mov64((uint8_t *)dst, (const uint8_t
> *)src);
> > > > +
> > > > +		return ret;
> > > > +	}
> > > >
> > > >  	/* Implementation for size > 64 bytes depends on alignment
> with
> > > vector
> > > > register size. */
> > > > -	if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK))
> > > > +	if (!(((uintptr_t)dst | (uintptr_t)src) &
> > > RTE_MEMCPY_ALIGNMENT_MASK))
> > > >  		return rte_memcpy_aligned_more_than_64(dst, src, n);
> > > >  	else
> > > >  		return rte_memcpy_generic_more_than_64(dst, src, n);
> > > >  }
> > > >
> > > > -#undef ALIGNMENT_MASK
> > > > +#undef RTE_MEMCPY_ALIGNMENT_MASK
> > > > +#undef RTE_MEMCPY_BLOCK_64_MAX
> > > >
> > > >  #ifdef __cplusplus
> > > >  }
> > > > --
> > > > 2.43.0