* [PATCH v2 0/2] util/bufferiszero: Split out hosts, add loongarch64
@ 2024-06-07 0:24 Richard Henderson
2024-06-07 0:24 ` [PATCH v2 1/2] util/bufferiszero: Split out host include files Richard Henderson
2024-06-07 0:24 ` [PATCH v2 2/2] util/bufferiszero: Add loongarch64 vector acceleration Richard Henderson
0 siblings, 2 replies; 4+ messages in thread
From: Richard Henderson @ 2024-06-07 0:24 UTC (permalink / raw)
To: qemu-devel; +Cc: maobibo
Based-on: 20240527211912.14060-1-richard.henderson@linaro.org
("[PATCH 00/18] tcg/loongarch64: Support v64 and v256")
For "util/loongarch64: Detect LASX vector support"
For v2:
* Rename to bufferiszero.c.inc (philmd).
* Add inline assembly for loongarch64.
On cfarm400.cfarm.net (Loongson-3C5000L-LL @ 2.0GHz):
# Start of bufferiszero tests
# buffer_is_zero #0: 1KB 11021 MB/sec
# buffer_is_zero #0: 4KB 32107 MB/sec
# buffer_is_zero #0: 16KB 59118 MB/sec
# buffer_is_zero #0: 64KB 67940 MB/sec
#
# buffer_is_zero #1: 1KB 9540 MB/sec
# buffer_is_zero #1: 4KB 24050 MB/sec
# buffer_is_zero #1: 16KB 38082 MB/sec
# buffer_is_zero #1: 64KB 36399 MB/sec
#
# buffer_is_zero #2: 1KB 8026 MB/sec
# buffer_is_zero #2: 4KB 15493 MB/sec
# buffer_is_zero #2: 16KB 20865 MB/sec
# buffer_is_zero #2: 64KB 19694 MB/sec
r~
Richard Henderson (2):
util/bufferiszero: Split out host include files
util/bufferiszero: Add loongarch64 vector acceleration
util/bufferiszero.c | 191 +-----------------
host/include/aarch64/host/bufferiszero.c.inc | 76 +++++++
host/include/generic/host/bufferiszero.c.inc | 10 +
host/include/i386/host/bufferiszero.c.inc | 124 ++++++++++++
.../loongarch64/host/bufferiszero.c.inc | 143 +++++++++++++
host/include/x86_64/host/bufferiszero.c.inc | 1 +
6 files changed, 355 insertions(+), 190 deletions(-)
create mode 100644 host/include/aarch64/host/bufferiszero.c.inc
create mode 100644 host/include/generic/host/bufferiszero.c.inc
create mode 100644 host/include/i386/host/bufferiszero.c.inc
create mode 100644 host/include/loongarch64/host/bufferiszero.c.inc
create mode 100644 host/include/x86_64/host/bufferiszero.c.inc
--
2.34.1
^ permalink raw reply [flat|nested] 4+ messages in thread* [PATCH v2 1/2] util/bufferiszero: Split out host include files 2024-06-07 0:24 [PATCH v2 0/2] util/bufferiszero: Split out hosts, add loongarch64 Richard Henderson @ 2024-06-07 0:24 ` Richard Henderson 2024-06-07 0:24 ` [PATCH v2 2/2] util/bufferiszero: Add loongarch64 vector acceleration Richard Henderson 1 sibling, 0 replies; 4+ messages in thread From: Richard Henderson @ 2024-06-07 0:24 UTC (permalink / raw) To: qemu-devel; +Cc: maobibo, Philippe Mathieu-Daudé Split out host/bufferiszero.h.inc for x86, aarch64 and generic in order to avoid an overlong ifdef ladder. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- util/bufferiszero.c | 191 +------------------ host/include/aarch64/host/bufferiszero.c.inc | 76 ++++++++ host/include/generic/host/bufferiszero.c.inc | 10 + host/include/i386/host/bufferiszero.c.inc | 124 ++++++++++++ host/include/x86_64/host/bufferiszero.c.inc | 1 + 5 files changed, 212 insertions(+), 190 deletions(-) create mode 100644 host/include/aarch64/host/bufferiszero.c.inc create mode 100644 host/include/generic/host/bufferiszero.c.inc create mode 100644 host/include/i386/host/bufferiszero.c.inc create mode 100644 host/include/x86_64/host/bufferiszero.c.inc diff --git a/util/bufferiszero.c b/util/bufferiszero.c index 11c080e02c..522146dab9 100644 --- a/util/bufferiszero.c +++ b/util/bufferiszero.c @@ -81,196 +81,7 @@ static bool buffer_is_zero_int_ge256(const void *buf, size_t len) return t == 0; } -#if defined(CONFIG_AVX2_OPT) || defined(__SSE2__) -#include <immintrin.h> - -/* Helper for preventing the compiler from reassociating - chains of binary vector operations. */ -#define SSE_REASSOC_BARRIER(vec0, vec1) asm("" : "+x"(vec0), "+x"(vec1)) - -/* Note that these vectorized functions may assume len >= 256. */ - -static bool __attribute__((target("sse2"))) -buffer_zero_sse2(const void *buf, size_t len) -{ - /* Unaligned loads at head/tail. */ - __m128i v = *(__m128i_u *)(buf); - __m128i w = *(__m128i_u *)(buf + len - 16); - /* Align head/tail to 16-byte boundaries. */ - const __m128i *p = QEMU_ALIGN_PTR_DOWN(buf + 16, 16); - const __m128i *e = QEMU_ALIGN_PTR_DOWN(buf + len - 1, 16); - __m128i zero = { 0 }; - - /* Collect a partial block at tail end. */ - v |= e[-1]; w |= e[-2]; - SSE_REASSOC_BARRIER(v, w); - v |= e[-3]; w |= e[-4]; - SSE_REASSOC_BARRIER(v, w); - v |= e[-5]; w |= e[-6]; - SSE_REASSOC_BARRIER(v, w); - v |= e[-7]; v |= w; - - /* - * Loop over complete 128-byte blocks. - * With the head and tail removed, e - p >= 14, so the loop - * must iterate at least once. - */ - do { - v = _mm_cmpeq_epi8(v, zero); - if (unlikely(_mm_movemask_epi8(v) != 0xFFFF)) { - return false; - } - v = p[0]; w = p[1]; - SSE_REASSOC_BARRIER(v, w); - v |= p[2]; w |= p[3]; - SSE_REASSOC_BARRIER(v, w); - v |= p[4]; w |= p[5]; - SSE_REASSOC_BARRIER(v, w); - v |= p[6]; w |= p[7]; - SSE_REASSOC_BARRIER(v, w); - v |= w; - p += 8; - } while (p < e - 7); - - return _mm_movemask_epi8(_mm_cmpeq_epi8(v, zero)) == 0xFFFF; -} - -#ifdef CONFIG_AVX2_OPT -static bool __attribute__((target("avx2"))) -buffer_zero_avx2(const void *buf, size_t len) -{ - /* Unaligned loads at head/tail. */ - __m256i v = *(__m256i_u *)(buf); - __m256i w = *(__m256i_u *)(buf + len - 32); - /* Align head/tail to 32-byte boundaries. */ - const __m256i *p = QEMU_ALIGN_PTR_DOWN(buf + 32, 32); - const __m256i *e = QEMU_ALIGN_PTR_DOWN(buf + len - 1, 32); - __m256i zero = { 0 }; - - /* Collect a partial block at tail end. */ - v |= e[-1]; w |= e[-2]; - SSE_REASSOC_BARRIER(v, w); - v |= e[-3]; w |= e[-4]; - SSE_REASSOC_BARRIER(v, w); - v |= e[-5]; w |= e[-6]; - SSE_REASSOC_BARRIER(v, w); - v |= e[-7]; v |= w; - - /* Loop over complete 256-byte blocks. */ - for (; p < e - 7; p += 8) { - /* PTEST is not profitable here. */ - v = _mm256_cmpeq_epi8(v, zero); - if (unlikely(_mm256_movemask_epi8(v) != 0xFFFFFFFF)) { - return false; - } - v = p[0]; w = p[1]; - SSE_REASSOC_BARRIER(v, w); - v |= p[2]; w |= p[3]; - SSE_REASSOC_BARRIER(v, w); - v |= p[4]; w |= p[5]; - SSE_REASSOC_BARRIER(v, w); - v |= p[6]; w |= p[7]; - SSE_REASSOC_BARRIER(v, w); - v |= w; - } - - return _mm256_movemask_epi8(_mm256_cmpeq_epi8(v, zero)) == 0xFFFFFFFF; -} -#endif /* CONFIG_AVX2_OPT */ - -static biz_accel_fn const accel_table[] = { - buffer_is_zero_int_ge256, - buffer_zero_sse2, -#ifdef CONFIG_AVX2_OPT - buffer_zero_avx2, -#endif -}; - -static unsigned best_accel(void) -{ -#ifdef CONFIG_AVX2_OPT - unsigned info = cpuinfo_init(); - - if (info & CPUINFO_AVX2) { - return 2; - } -#endif - return 1; -} - -#elif defined(__aarch64__) && defined(__ARM_NEON) -#include <arm_neon.h> - -/* - * Helper for preventing the compiler from reassociating - * chains of binary vector operations. - */ -#define REASSOC_BARRIER(vec0, vec1) asm("" : "+w"(vec0), "+w"(vec1)) - -static bool buffer_is_zero_simd(const void *buf, size_t len) -{ - uint32x4_t t0, t1, t2, t3; - - /* Align head/tail to 16-byte boundaries. */ - const uint32x4_t *p = QEMU_ALIGN_PTR_DOWN(buf + 16, 16); - const uint32x4_t *e = QEMU_ALIGN_PTR_DOWN(buf + len - 1, 16); - - /* Unaligned loads at head/tail. */ - t0 = vld1q_u32(buf) | vld1q_u32(buf + len - 16); - - /* Collect a partial block at tail end. */ - t1 = e[-7] | e[-6]; - t2 = e[-5] | e[-4]; - t3 = e[-3] | e[-2]; - t0 |= e[-1]; - REASSOC_BARRIER(t0, t1); - REASSOC_BARRIER(t2, t3); - t0 |= t1; - t2 |= t3; - REASSOC_BARRIER(t0, t2); - t0 |= t2; - - /* - * Loop over complete 128-byte blocks. - * With the head and tail removed, e - p >= 14, so the loop - * must iterate at least once. - */ - do { - /* - * Reduce via UMAXV. Whatever the actual result, - * it will only be zero if all input bytes are zero. - */ - if (unlikely(vmaxvq_u32(t0) != 0)) { - return false; - } - - t0 = p[0] | p[1]; - t1 = p[2] | p[3]; - t2 = p[4] | p[5]; - t3 = p[6] | p[7]; - REASSOC_BARRIER(t0, t1); - REASSOC_BARRIER(t2, t3); - t0 |= t1; - t2 |= t3; - REASSOC_BARRIER(t0, t2); - t0 |= t2; - p += 8; - } while (p < e - 7); - - return vmaxvq_u32(t0) == 0; -} - -#define best_accel() 1 -static biz_accel_fn const accel_table[] = { - buffer_is_zero_int_ge256, - buffer_is_zero_simd, -}; -#else -#define best_accel() 0 -static biz_accel_fn const accel_table[1] = { - buffer_is_zero_int_ge256 -}; -#endif +#include "host/bufferiszero.c.inc" static biz_accel_fn buffer_is_zero_accel; static unsigned accel_index; diff --git a/host/include/aarch64/host/bufferiszero.c.inc b/host/include/aarch64/host/bufferiszero.c.inc new file mode 100644 index 0000000000..947ee7ca1f --- /dev/null +++ b/host/include/aarch64/host/bufferiszero.c.inc @@ -0,0 +1,76 @@ +/* + * SPDX-License-Identifier: GPL-2.0-or-later + * buffer_is_zero acceleration, aarch64 version. + */ + +#ifdef __ARM_NEON +#include <arm_neon.h> + +/* + * Helper for preventing the compiler from reassociating + * chains of binary vector operations. + */ +#define REASSOC_BARRIER(vec0, vec1) asm("" : "+w"(vec0), "+w"(vec1)) + +static bool buffer_is_zero_simd(const void *buf, size_t len) +{ + uint32x4_t t0, t1, t2, t3; + + /* Align head/tail to 16-byte boundaries. */ + const uint32x4_t *p = QEMU_ALIGN_PTR_DOWN(buf + 16, 16); + const uint32x4_t *e = QEMU_ALIGN_PTR_DOWN(buf + len - 1, 16); + + /* Unaligned loads at head/tail. */ + t0 = vld1q_u32(buf) | vld1q_u32(buf + len - 16); + + /* Collect a partial block at tail end. */ + t1 = e[-7] | e[-6]; + t2 = e[-5] | e[-4]; + t3 = e[-3] | e[-2]; + t0 |= e[-1]; + REASSOC_BARRIER(t0, t1); + REASSOC_BARRIER(t2, t3); + t0 |= t1; + t2 |= t3; + REASSOC_BARRIER(t0, t2); + t0 |= t2; + + /* + * Loop over complete 128-byte blocks. + * With the head and tail removed, e - p >= 14, so the loop + * must iterate at least once. + */ + do { + /* + * Reduce via UMAXV. Whatever the actual result, + * it will only be zero if all input bytes are zero. + */ + if (unlikely(vmaxvq_u32(t0) != 0)) { + return false; + } + + t0 = p[0] | p[1]; + t1 = p[2] | p[3]; + t2 = p[4] | p[5]; + t3 = p[6] | p[7]; + REASSOC_BARRIER(t0, t1); + REASSOC_BARRIER(t2, t3); + t0 |= t1; + t2 |= t3; + REASSOC_BARRIER(t0, t2); + t0 |= t2; + p += 8; + } while (p < e - 7); + + return vmaxvq_u32(t0) == 0; +} + +static biz_accel_fn const accel_table[] = { + buffer_is_zero_int_ge256, + buffer_is_zero_simd, +}; + +#define best_accel() 1 +#else +# include "host/include/generic/host/bufferiszero.c.inc" +#endif diff --git a/host/include/generic/host/bufferiszero.c.inc b/host/include/generic/host/bufferiszero.c.inc new file mode 100644 index 0000000000..ea0875c24a --- /dev/null +++ b/host/include/generic/host/bufferiszero.c.inc @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: GPL-2.0-or-later + * buffer_is_zero acceleration, generic version. + */ + +static biz_accel_fn const accel_table[1] = { + buffer_is_zero_int_ge256 +}; + +#define best_accel() 0 diff --git a/host/include/i386/host/bufferiszero.c.inc b/host/include/i386/host/bufferiszero.c.inc new file mode 100644 index 0000000000..3b9605d806 --- /dev/null +++ b/host/include/i386/host/bufferiszero.c.inc @@ -0,0 +1,124 @@ +/* + * SPDX-License-Identifier: GPL-2.0-or-later + * buffer_is_zero acceleration, x86 version. + */ + +#if defined(CONFIG_AVX2_OPT) || defined(__SSE2__) +#include <immintrin.h> + +/* Helper for preventing the compiler from reassociating + chains of binary vector operations. */ +#define SSE_REASSOC_BARRIER(vec0, vec1) asm("" : "+x"(vec0), "+x"(vec1)) + +/* Note that these vectorized functions may assume len >= 256. */ + +static bool __attribute__((target("sse2"))) +buffer_zero_sse2(const void *buf, size_t len) +{ + /* Unaligned loads at head/tail. */ + __m128i v = *(__m128i_u *)(buf); + __m128i w = *(__m128i_u *)(buf + len - 16); + /* Align head/tail to 16-byte boundaries. */ + const __m128i *p = QEMU_ALIGN_PTR_DOWN(buf + 16, 16); + const __m128i *e = QEMU_ALIGN_PTR_DOWN(buf + len - 1, 16); + __m128i zero = { 0 }; + + /* Collect a partial block at tail end. */ + v |= e[-1]; w |= e[-2]; + SSE_REASSOC_BARRIER(v, w); + v |= e[-3]; w |= e[-4]; + SSE_REASSOC_BARRIER(v, w); + v |= e[-5]; w |= e[-6]; + SSE_REASSOC_BARRIER(v, w); + v |= e[-7]; v |= w; + + /* + * Loop over complete 128-byte blocks. + * With the head and tail removed, e - p >= 14, so the loop + * must iterate at least once. + */ + do { + v = _mm_cmpeq_epi8(v, zero); + if (unlikely(_mm_movemask_epi8(v) != 0xFFFF)) { + return false; + } + v = p[0]; w = p[1]; + SSE_REASSOC_BARRIER(v, w); + v |= p[2]; w |= p[3]; + SSE_REASSOC_BARRIER(v, w); + v |= p[4]; w |= p[5]; + SSE_REASSOC_BARRIER(v, w); + v |= p[6]; w |= p[7]; + SSE_REASSOC_BARRIER(v, w); + v |= w; + p += 8; + } while (p < e - 7); + + return _mm_movemask_epi8(_mm_cmpeq_epi8(v, zero)) == 0xFFFF; +} + +#ifdef CONFIG_AVX2_OPT +static bool __attribute__((target("avx2"))) +buffer_zero_avx2(const void *buf, size_t len) +{ + /* Unaligned loads at head/tail. */ + __m256i v = *(__m256i_u *)(buf); + __m256i w = *(__m256i_u *)(buf + len - 32); + /* Align head/tail to 32-byte boundaries. */ + const __m256i *p = QEMU_ALIGN_PTR_DOWN(buf + 32, 32); + const __m256i *e = QEMU_ALIGN_PTR_DOWN(buf + len - 1, 32); + __m256i zero = { 0 }; + + /* Collect a partial block at tail end. */ + v |= e[-1]; w |= e[-2]; + SSE_REASSOC_BARRIER(v, w); + v |= e[-3]; w |= e[-4]; + SSE_REASSOC_BARRIER(v, w); + v |= e[-5]; w |= e[-6]; + SSE_REASSOC_BARRIER(v, w); + v |= e[-7]; v |= w; + + /* Loop over complete 256-byte blocks. */ + for (; p < e - 7; p += 8) { + /* PTEST is not profitable here. */ + v = _mm256_cmpeq_epi8(v, zero); + if (unlikely(_mm256_movemask_epi8(v) != 0xFFFFFFFF)) { + return false; + } + v = p[0]; w = p[1]; + SSE_REASSOC_BARRIER(v, w); + v |= p[2]; w |= p[3]; + SSE_REASSOC_BARRIER(v, w); + v |= p[4]; w |= p[5]; + SSE_REASSOC_BARRIER(v, w); + v |= p[6]; w |= p[7]; + SSE_REASSOC_BARRIER(v, w); + v |= w; + } + + return _mm256_movemask_epi8(_mm256_cmpeq_epi8(v, zero)) == 0xFFFFFFFF; +} +#endif /* CONFIG_AVX2_OPT */ + +static biz_accel_fn const accel_table[] = { + buffer_is_zero_int_ge256, + buffer_zero_sse2, +#ifdef CONFIG_AVX2_OPT + buffer_zero_avx2, +#endif +}; + +static unsigned best_accel(void) +{ +#ifdef CONFIG_AVX2_OPT + unsigned info = cpuinfo_init(); + if (info & CPUINFO_AVX2) { + return 2; + } +#endif + return 1; +} + +#else +# include "host/include/generic/host/bufferiszero.c.inc" +#endif diff --git a/host/include/x86_64/host/bufferiszero.c.inc b/host/include/x86_64/host/bufferiszero.c.inc new file mode 100644 index 0000000000..1d3f1fd6f5 --- /dev/null +++ b/host/include/x86_64/host/bufferiszero.c.inc @@ -0,0 +1 @@ +#include "host/include/i386/host/bufferiszero.c.inc" -- 2.34.1 ^ permalink raw reply related [flat|nested] 4+ messages in thread
* [PATCH v2 2/2] util/bufferiszero: Add loongarch64 vector acceleration 2024-06-07 0:24 [PATCH v2 0/2] util/bufferiszero: Split out hosts, add loongarch64 Richard Henderson 2024-06-07 0:24 ` [PATCH v2 1/2] util/bufferiszero: Split out host include files Richard Henderson @ 2024-06-07 0:24 ` Richard Henderson 2024-06-07 1:52 ` maobibo 1 sibling, 1 reply; 4+ messages in thread From: Richard Henderson @ 2024-06-07 0:24 UTC (permalink / raw) To: qemu-devel; +Cc: maobibo Use inline assembly because no release compiler allows per-function selection of the ISA. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- .../loongarch64/host/bufferiszero.c.inc | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 host/include/loongarch64/host/bufferiszero.c.inc diff --git a/host/include/loongarch64/host/bufferiszero.c.inc b/host/include/loongarch64/host/bufferiszero.c.inc new file mode 100644 index 0000000000..69891eac80 --- /dev/null +++ b/host/include/loongarch64/host/bufferiszero.c.inc @@ -0,0 +1,143 @@ +/* + * SPDX-License-Identifier: GPL-2.0-or-later + * buffer_is_zero acceleration, loongarch64 version. + */ + +/* + * Builtins for LSX and LASX are introduced by gcc 14 and llvm 18, + * but as yet neither has support for attribute target, so neither + * is able to enable the optimization without globally enabling + * vector support. Since we want runtime detection, use assembly. + */ + +static bool buffer_is_zero_lsx(const void *buf, size_t len) +{ + const void *p = QEMU_ALIGN_PTR_DOWN(buf + 16, 16); + const void *e = QEMU_ALIGN_PTR_DOWN(buf + len - 1, 16) - (7 * 16); + const void *l = buf + len; + bool ret; + + asm("vld $vr0,%2,0\n\t" /* first: buf + 0 */ + "vld $vr1,%4,-16\n\t" /* last: buf + len - 16 */ + "vld $vr2,%3,0\n\t" /* e[0] */ + "vld $vr3,%3,16\n\t" /* e[1] */ + "vld $vr4,%3,32\n\t" /* e[2] */ + "vld $vr5,%3,48\n\t" /* e[3] */ + "vld $vr6,%3,64\n\t" /* e[4] */ + "vld $vr7,%3,80\n\t" /* e[5] */ + "vld $vr8,%3,96\n\t" /* e[6] */ + "vor.v $vr0,$vr0,$vr1\n\t" + "vor.v $vr2,$vr2,$vr3\n\t" + "vor.v $vr4,$vr4,$vr5\n\t" + "vor.v $vr6,$vr6,$vr7\n\t" + "vor.v $vr0,$vr0,$vr2\n\t" + "vor.v $vr4,$vr4,$vr6\n\t" + "vor.v $vr0,$vr0,$vr4\n\t" + "vor.v $vr0,$vr0,$vr8\n\t" + "or %0,$r0,$r0\n" /* prepare return false */ + "1:\n\t" + "vsetnez.v $fcc0,$vr0\n\t" + "bcnez $fcc0,2f\n\t" + "vld $vr0,%1,0\n\t" /* p[0] */ + "vld $vr1,%1,16\n\t" /* p[1] */ + "vld $vr2,%1,32\n\t" /* p[2] */ + "vld $vr3,%1,48\n\t" /* p[3] */ + "vld $vr4,%1,64\n\t" /* p[4] */ + "vld $vr5,%1,80\n\t" /* p[5] */ + "vld $vr6,%1,96\n\t" /* p[6] */ + "vld $vr7,%1,112\n\t" /* p[7] */ + "addi.d %1,%1,128\n\t" + "vor.v $vr0,$vr0,$vr1\n\t" + "vor.v $vr2,$vr2,$vr3\n\t" + "vor.v $vr4,$vr4,$vr5\n\t" + "vor.v $vr6,$vr6,$vr7\n\t" + "vor.v $vr0,$vr0,$vr2\n\t" + "vor.v $vr4,$vr4,$vr6\n\t" + "vor.v $vr0,$vr0,$vr4\n\t" + "bltu %1,%3,1b\n\t" + "vsetnez.v $fcc0,$vr0\n\t" + "bcnez $fcc0,2f\n\t" + "ori %0,$r0,1\n" + "2:" + : "=&r"(ret), "+r"(p) + : "r"(buf), "r"(e), "r"(l) + : "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "fcc0"); + + return ret; +} + +static bool buffer_is_zero_lasx(const void *buf, size_t len) +{ + const void *p = QEMU_ALIGN_PTR_DOWN(buf + 32, 32); + const void *e = QEMU_ALIGN_PTR_DOWN(buf + len - 1, 32) - (7 * 32); + const void *l = buf + len; + bool ret; + + asm("xvld $xr0,%2,0\n\t" /* first: buf + 0 */ + "xvld $xr1,%4,-32\n\t" /* last: buf + len - 32 */ + "xvld $xr2,%3,0\n\t" /* e[0] */ + "xvld $xr3,%3,32\n\t" /* e[1] */ + "xvld $xr4,%3,64\n\t" /* e[2] */ + "xvld $xr5,%3,96\n\t" /* e[3] */ + "xvld $xr6,%3,128\n\t" /* e[4] */ + "xvld $xr7,%3,160\n\t" /* e[5] */ + "xvld $xr8,%3,192\n\t" /* e[6] */ + "xvor.v $xr0,$xr0,$xr1\n\t" + "xvor.v $xr2,$xr2,$xr3\n\t" + "xvor.v $xr4,$xr4,$xr5\n\t" + "xvor.v $xr6,$xr6,$xr7\n\t" + "xvor.v $xr0,$xr0,$xr2\n\t" + "xvor.v $xr4,$xr4,$xr6\n\t" + "xvor.v $xr0,$xr0,$xr4\n\t" + "xvor.v $xr0,$xr0,$xr8\n\t" + "or %0,$r0,$r0\n\t" /* prepare return false */ + "bgeu %1,%3,2f\n" + "1:\n\t" + "xvsetnez.v $fcc0,$xr0\n\t" + "bcnez $fcc0,3f\n\t" + "xvld $xr0,%1,0\n\t" /* p[0] */ + "xvld $xr1,%1,32\n\t" /* p[1] */ + "xvld $xr2,%1,64\n\t" /* p[2] */ + "xvld $xr3,%1,96\n\t" /* p[3] */ + "xvld $xr4,%1,128\n\t" /* p[4] */ + "xvld $xr5,%1,160\n\t" /* p[5] */ + "xvld $xr6,%1,192\n\t" /* p[6] */ + "xvld $xr7,%1,224\n\t" /* p[7] */ + "addi.d %1,%1,256\n\t" + "xvor.v $xr0,$xr0,$xr1\n\t" + "xvor.v $xr2,$xr2,$xr3\n\t" + "xvor.v $xr4,$xr4,$xr5\n\t" + "xvor.v $xr6,$xr6,$xr7\n\t" + "xvor.v $xr0,$xr0,$xr2\n\t" + "xvor.v $xr4,$xr4,$xr6\n\t" + "xvor.v $xr0,$xr0,$xr4\n\t" + "bltu %1,%3,1b\n" + "2:\n\t" + "xvsetnez.v $fcc0,$xr0\n\t" + "bcnez $fcc0,3f\n\t" + "ori %0,$r0,1\n" + "3:" + : "=&r"(ret), "+r"(p) + : "r"(buf), "r"(e), "r"(l) + : "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "fcc0"); + + return ret; +} + +static biz_accel_fn const accel_table[] = { + buffer_is_zero_int_ge256, + buffer_is_zero_lsx, + buffer_is_zero_lasx, +}; + +static unsigned best_accel(void) +{ + unsigned info = cpuinfo_init(); + if (info & CPUINFO_LASX) { + return 2; + } + if (info & CPUINFO_LSX) { + return 1; + } + return 0; +} -- 2.34.1 ^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [PATCH v2 2/2] util/bufferiszero: Add loongarch64 vector acceleration 2024-06-07 0:24 ` [PATCH v2 2/2] util/bufferiszero: Add loongarch64 vector acceleration Richard Henderson @ 2024-06-07 1:52 ` maobibo 0 siblings, 0 replies; 4+ messages in thread From: maobibo @ 2024-06-07 1:52 UTC (permalink / raw) To: Richard Henderson, qemu-devel On 2024/6/7 上午8:24, Richard Henderson wrote: > Use inline assembly because no release compiler allows > per-function selection of the ISA. > > Signed-off-by: Richard Henderson <richard.henderson@linaro.org> > --- > .../loongarch64/host/bufferiszero.c.inc | 143 ++++++++++++++++++ > 1 file changed, 143 insertions(+) > create mode 100644 host/include/loongarch64/host/bufferiszero.c.inc > > diff --git a/host/include/loongarch64/host/bufferiszero.c.inc b/host/include/loongarch64/host/bufferiszero.c.inc > new file mode 100644 > index 0000000000..69891eac80 > --- /dev/null > +++ b/host/include/loongarch64/host/bufferiszero.c.inc > @@ -0,0 +1,143 @@ > +/* > + * SPDX-License-Identifier: GPL-2.0-or-later > + * buffer_is_zero acceleration, loongarch64 version. > + */ > + > +/* > + * Builtins for LSX and LASX are introduced by gcc 14 and llvm 18, > + * but as yet neither has support for attribute target, so neither > + * is able to enable the optimization without globally enabling > + * vector support. Since we want runtime detection, use assembly. > + */ > + > +static bool buffer_is_zero_lsx(const void *buf, size_t len) > +{ > + const void *p = QEMU_ALIGN_PTR_DOWN(buf + 16, 16); > + const void *e = QEMU_ALIGN_PTR_DOWN(buf + len - 1, 16) - (7 * 16); > + const void *l = buf + len; > + bool ret; > + > + asm("vld $vr0,%2,0\n\t" /* first: buf + 0 */ > + "vld $vr1,%4,-16\n\t" /* last: buf + len - 16 */ > + "vld $vr2,%3,0\n\t" /* e[0] */ > + "vld $vr3,%3,16\n\t" /* e[1] */ > + "vld $vr4,%3,32\n\t" /* e[2] */ > + "vld $vr5,%3,48\n\t" /* e[3] */ > + "vld $vr6,%3,64\n\t" /* e[4] */ > + "vld $vr7,%3,80\n\t" /* e[5] */ > + "vld $vr8,%3,96\n\t" /* e[6] */ > + "vor.v $vr0,$vr0,$vr1\n\t" > + "vor.v $vr2,$vr2,$vr3\n\t" > + "vor.v $vr4,$vr4,$vr5\n\t" > + "vor.v $vr6,$vr6,$vr7\n\t" > + "vor.v $vr0,$vr0,$vr2\n\t" > + "vor.v $vr4,$vr4,$vr6\n\t" > + "vor.v $vr0,$vr0,$vr4\n\t" > + "vor.v $vr0,$vr0,$vr8\n\t" > + "or %0,$r0,$r0\n" /* prepare return false */ > + "1:\n\t" > + "vsetnez.v $fcc0,$vr0\n\t" > + "bcnez $fcc0,2f\n\t" > + "vld $vr0,%1,0\n\t" /* p[0] */ > + "vld $vr1,%1,16\n\t" /* p[1] */ > + "vld $vr2,%1,32\n\t" /* p[2] */ > + "vld $vr3,%1,48\n\t" /* p[3] */ > + "vld $vr4,%1,64\n\t" /* p[4] */ > + "vld $vr5,%1,80\n\t" /* p[5] */ > + "vld $vr6,%1,96\n\t" /* p[6] */ > + "vld $vr7,%1,112\n\t" /* p[7] */ > + "addi.d %1,%1,128\n\t" > + "vor.v $vr0,$vr0,$vr1\n\t" > + "vor.v $vr2,$vr2,$vr3\n\t" > + "vor.v $vr4,$vr4,$vr5\n\t" > + "vor.v $vr6,$vr6,$vr7\n\t" > + "vor.v $vr0,$vr0,$vr2\n\t" > + "vor.v $vr4,$vr4,$vr6\n\t" > + "vor.v $vr0,$vr0,$vr4\n\t" > + "bltu %1,%3,1b\n\t" > + "vsetnez.v $fcc0,$vr0\n\t" > + "bcnez $fcc0,2f\n\t" > + "ori %0,$r0,1\n" > + "2:" > + : "=&r"(ret), "+r"(p) > + : "r"(buf), "r"(e), "r"(l) > + : "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "fcc0"); > + > + return ret; > +} > + > +static bool buffer_is_zero_lasx(const void *buf, size_t len) > +{ > + const void *p = QEMU_ALIGN_PTR_DOWN(buf + 32, 32); > + const void *e = QEMU_ALIGN_PTR_DOWN(buf + len - 1, 32) - (7 * 32); > + const void *l = buf + len; > + bool ret; > + > + asm("xvld $xr0,%2,0\n\t" /* first: buf + 0 */ > + "xvld $xr1,%4,-32\n\t" /* last: buf + len - 32 */ > + "xvld $xr2,%3,0\n\t" /* e[0] */ > + "xvld $xr3,%3,32\n\t" /* e[1] */ > + "xvld $xr4,%3,64\n\t" /* e[2] */ > + "xvld $xr5,%3,96\n\t" /* e[3] */ > + "xvld $xr6,%3,128\n\t" /* e[4] */ > + "xvld $xr7,%3,160\n\t" /* e[5] */ > + "xvld $xr8,%3,192\n\t" /* e[6] */ > + "xvor.v $xr0,$xr0,$xr1\n\t" > + "xvor.v $xr2,$xr2,$xr3\n\t" > + "xvor.v $xr4,$xr4,$xr5\n\t" > + "xvor.v $xr6,$xr6,$xr7\n\t" > + "xvor.v $xr0,$xr0,$xr2\n\t" > + "xvor.v $xr4,$xr4,$xr6\n\t" > + "xvor.v $xr0,$xr0,$xr4\n\t" > + "xvor.v $xr0,$xr0,$xr8\n\t" > + "or %0,$r0,$r0\n\t" /* prepare return false */ > + "bgeu %1,%3,2f\n" > + "1:\n\t" > + "xvsetnez.v $fcc0,$xr0\n\t" > + "bcnez $fcc0,3f\n\t" > + "xvld $xr0,%1,0\n\t" /* p[0] */ > + "xvld $xr1,%1,32\n\t" /* p[1] */ > + "xvld $xr2,%1,64\n\t" /* p[2] */ > + "xvld $xr3,%1,96\n\t" /* p[3] */ > + "xvld $xr4,%1,128\n\t" /* p[4] */ > + "xvld $xr5,%1,160\n\t" /* p[5] */ > + "xvld $xr6,%1,192\n\t" /* p[6] */ > + "xvld $xr7,%1,224\n\t" /* p[7] */ > + "addi.d %1,%1,256\n\t" > + "xvor.v $xr0,$xr0,$xr1\n\t" > + "xvor.v $xr2,$xr2,$xr3\n\t" > + "xvor.v $xr4,$xr4,$xr5\n\t" > + "xvor.v $xr6,$xr6,$xr7\n\t" > + "xvor.v $xr0,$xr0,$xr2\n\t" > + "xvor.v $xr4,$xr4,$xr6\n\t" > + "xvor.v $xr0,$xr0,$xr4\n\t" > + "bltu %1,%3,1b\n" > + "2:\n\t" > + "xvsetnez.v $fcc0,$xr0\n\t" > + "bcnez $fcc0,3f\n\t" > + "ori %0,$r0,1\n" > + "3:" > + : "=&r"(ret), "+r"(p) > + : "r"(buf), "r"(e), "r"(l) > + : "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "fcc0"); > + > + return ret; > +} > + > +static biz_accel_fn const accel_table[] = { > + buffer_is_zero_int_ge256, > + buffer_is_zero_lsx, > + buffer_is_zero_lasx, > +}; > + > +static unsigned best_accel(void) > +{ > + unsigned info = cpuinfo_init(); > + if (info & CPUINFO_LASX) { > + return 2; > + } > + if (info & CPUINFO_LSX) { > + return 1; > + } > + return 0; > +} > This is better than mine, there is no dependency on compiler option any more. Tested-by: Bibo Mao <maobibo@loongson.cn> ^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2024-06-07 1:53 UTC | newest] Thread overview: 4+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2024-06-07 0:24 [PATCH v2 0/2] util/bufferiszero: Split out hosts, add loongarch64 Richard Henderson 2024-06-07 0:24 ` [PATCH v2 1/2] util/bufferiszero: Split out host include files Richard Henderson 2024-06-07 0:24 ` [PATCH v2 2/2] util/bufferiszero: Add loongarch64 vector acceleration Richard Henderson 2024-06-07 1:52 ` maobibo
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for NNTP newsgroup(s).