Re: Add prefetch switch stack hook in scheduler function

Eric Dumazet
2005-07-29T08:30:19Z

[-- Attachment #1: Type: text/plain, Size: 1707 bytes --]

Ingo Molnar a écrit :

> unroll prefetch_range() loops manually.
>
> Signed-off-by: Ingo Molnar <mingo@elte.hu>
>
> include/linux/prefetch.h | 31 +++++++++++++++++++++++++++++--
> 1 files changed, 29 insertions(+), 2 deletions(-)
>
> Index: linux/include/linux/prefetch.h
> ===================================================================
> --- linux.orig/include/linux/prefetch.h
> +++ linux/include/linux/prefetch.h
> @@ -58,11 +58,38 @@ static inline void prefetchw(const void
> static inline void prefetch_range(void *addr, size_t len)
> {
> #ifdef ARCH_HAS_PREFETCH
> - char *cp;
> + char *cp = addr;
> char *end = addr + len;
>
> - for (cp = addr; cp < end; cp += PREFETCH_STRIDE)
> + /*
> + * Unroll agressively:
> + */
> + if (len <= PREFETCH_STRIDE)
> prefetch(cp);
> + else if (len <= 2*PREFETCH_STRIDE) {
> + prefetch(cp);
> + prefetch(cp + PREFETCH_STRIDE);
> + }
> + else if (len <= 3*PREFETCH_STRIDE) {
> + prefetch(cp);
> + prefetch(cp + PREFETCH_STRIDE);
> + prefetch(cp + 2*PREFETCH_STRIDE);
> + }
> + else if (len <= 4*PREFETCH_STRIDE) {
> + prefetch(cp);
> + prefetch(cp + PREFETCH_STRIDE);
> + prefetch(cp + 2*PREFETCH_STRIDE);
> + prefetch(cp + 3*PREFETCH_STRIDE);
> + }
> + else if (len <= 5*PREFETCH_STRIDE) {
> + prefetch(cp);
> + prefetch(cp + PREFETCH_STRIDE);
> + prefetch(cp + 2*PREFETCH_STRIDE);
> + prefetch(cp + 3*PREFETCH_STRIDE);
> + prefetch(cp + 4*PREFETCH_STRIDE);
> + } else
> + for (; cp < end; cp += PREFETCH_STRIDE)
> + prefetch(cp);
> #endif
> }
>
> -

Please test that len is a constant, or else the inlining is too large for the non constant case.

Thank you

[-- Attachment #2: prefetch_range --]
[-- Type: text/plain, Size: 881 bytes --]

static inline void prefetch_range(void *addr, size_t len)
{
char *cp;
char *end = addr + len;

if (__builtin_constant_p(len) && (len <= 5*PREFETCH_STRIDE)) {
if (len <= PREFETCH_STRIDE)
prefetch(cp);
else if (len <= 2*PREFETCH_STRIDE) {
prefetch(cp);
prefetch(cp + PREFETCH_STRIDE);
}
else if (len <= 3*PREFETCH_STRIDE) {
prefetch(cp);
prefetch(cp + PREFETCH_STRIDE);
prefetch(cp + 2*PREFETCH_STRIDE);
}
else if (len <= 4*PREFETCH_STRIDE) {
prefetch(cp);
prefetch(cp + PREFETCH_STRIDE);
prefetch(cp + 2*PREFETCH_STRIDE);
prefetch(cp + 3*PREFETCH_STRIDE);
}
else if (len <= 5*PREFETCH_STRIDE) {
prefetch(cp);
prefetch(cp + PREFETCH_STRIDE);
prefetch(cp + 2*PREFETCH_STRIDE);
prefetch(cp + 3*PREFETCH_STRIDE);
prefetch(cp + 4*PREFETCH_STRIDE);
}
}
else
for (; cp < end; cp += PREFETCH_STRIDE)
prefetch(cp);
}