* [PATCH v2 1/3] util: Merge cacheflush.c and cacheinfo.c
2022-06-21 1:48 [PATCH v2 0/3] util: Optimize flushing when ppc host has coherent icache Richard Henderson
@ 2022-06-21 1:48 ` Richard Henderson
2022-06-21 15:16 ` Peter Maydell
2022-06-21 1:48 ` [PATCH v2 2/3] util/cacheflush: Merge aarch64 ctr_el0 usage Richard Henderson
2022-06-21 1:48 ` [PATCH v2 3/3] util/cacheflush: Optimize flushing when ppc host has coherent icache Richard Henderson
2 siblings, 1 reply; 7+ messages in thread
From: Richard Henderson @ 2022-06-21 1:48 UTC (permalink / raw)
To: qemu-devel; +Cc: npiggin, qemu-ppc
Combine the two files into cacheflush.c. There's a couple of bits
that would be helpful to share between the two, and combining them
seems better than exporting the bits.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
util/cacheflush.c | 202 +++++++++++++++++++++++++++++++++++++++++++++-
util/cacheinfo.c | 200 ---------------------------------------------
util/meson.build | 2 +-
3 files changed, 202 insertions(+), 202 deletions(-)
delete mode 100644 util/cacheinfo.c
diff --git a/util/cacheflush.c b/util/cacheflush.c
index 4b57186d89..8096afd33c 100644
--- a/util/cacheflush.c
+++ b/util/cacheflush.c
@@ -1,5 +1,5 @@
/*
- * Flush the host cpu caches.
+ * Info about, and flushing the host cpu caches.
*
* This work is licensed under the terms of the GNU GPL, version 2 or later.
* See the COPYING file in the top-level directory.
@@ -9,8 +9,208 @@
#include "qemu/cacheflush.h"
#include "qemu/cacheinfo.h"
#include "qemu/bitops.h"
+#include "qemu/host-utils.h"
+#include "qemu/atomic.h"
+int qemu_icache_linesize = 0;
+int qemu_icache_linesize_log;
+int qemu_dcache_linesize = 0;
+int qemu_dcache_linesize_log;
+
+/*
+ * Operating system specific cache detection mechanisms.
+ */
+
+#if defined(_WIN32)
+
+static void sys_cache_info(int *isize, int *dsize)
+{
+ SYSTEM_LOGICAL_PROCESSOR_INFORMATION *buf;
+ DWORD size = 0;
+ BOOL success;
+ size_t i, n;
+
+ /*
+ * Check for the required buffer size first. Note that if the zero
+ * size we use for the probe results in success, then there is no
+ * data available; fail in that case.
+ */
+ success = GetLogicalProcessorInformation(0, &size);
+ if (success || GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
+ return;
+ }
+
+ n = size / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
+ size = n * sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
+ buf = g_new0(SYSTEM_LOGICAL_PROCESSOR_INFORMATION, n);
+ if (!GetLogicalProcessorInformation(buf, &size)) {
+ goto fail;
+ }
+
+ for (i = 0; i < n; i++) {
+ if (buf[i].Relationship == RelationCache
+ && buf[i].Cache.Level == 1) {
+ switch (buf[i].Cache.Type) {
+ case CacheUnified:
+ *isize = *dsize = buf[i].Cache.LineSize;
+ break;
+ case CacheInstruction:
+ *isize = buf[i].Cache.LineSize;
+ break;
+ case CacheData:
+ *dsize = buf[i].Cache.LineSize;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ fail:
+ g_free(buf);
+}
+
+#elif defined(__APPLE__)
+# include <sys/sysctl.h>
+static void sys_cache_info(int *isize, int *dsize)
+{
+ /* There's only a single sysctl for both I/D cache line sizes. */
+ long size;
+ size_t len = sizeof(size);
+ if (!sysctlbyname("hw.cachelinesize", &size, &len, NULL, 0)) {
+ *isize = *dsize = size;
+ }
+}
+#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
+# include <sys/sysctl.h>
+static void sys_cache_info(int *isize, int *dsize)
+{
+ /* There's only a single sysctl for both I/D cache line sizes. */
+ int size;
+ size_t len = sizeof(size);
+ if (!sysctlbyname("machdep.cacheline_size", &size, &len, NULL, 0)) {
+ *isize = *dsize = size;
+ }
+}
+#else
+/* POSIX */
+
+static void sys_cache_info(int *isize, int *dsize)
+{
+# ifdef _SC_LEVEL1_ICACHE_LINESIZE
+ int tmp_isize = (int) sysconf(_SC_LEVEL1_ICACHE_LINESIZE);
+ if (tmp_isize > 0) {
+ *isize = tmp_isize;
+ }
+# endif
+# ifdef _SC_LEVEL1_DCACHE_LINESIZE
+ int tmp_dsize = (int) sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
+ if (tmp_dsize > 0) {
+ *dsize = tmp_dsize;
+ }
+# endif
+}
+#endif /* sys_cache_info */
+
+
+/*
+ * Architecture (+ OS) specific cache detection mechanisms.
+ */
+
+#if defined(__aarch64__)
+
+static void arch_cache_info(int *isize, int *dsize)
+{
+ if (*isize == 0 || *dsize == 0) {
+ uint64_t ctr;
+
+ /*
+ * The real cache geometry is in CCSIDR_EL1/CLIDR_EL1/CSSELR_EL1,
+ * but (at least under Linux) these are marked protected by the
+ * kernel. However, CTR_EL0 contains the minimum linesize in the
+ * entire hierarchy, and is used by userspace cache flushing.
+ */
+ asm volatile("mrs\t%0, ctr_el0" : "=r"(ctr));
+ if (*isize == 0) {
+ *isize = 4 << (ctr & 0xf);
+ }
+ if (*dsize == 0) {
+ *dsize = 4 << ((ctr >> 16) & 0xf);
+ }
+ }
+}
+
+#elif defined(_ARCH_PPC) && defined(__linux__)
+# include "elf.h"
+
+static void arch_cache_info(int *isize, int *dsize)
+{
+ if (*isize == 0) {
+ *isize = qemu_getauxval(AT_ICACHEBSIZE);
+ }
+ if (*dsize == 0) {
+ *dsize = qemu_getauxval(AT_DCACHEBSIZE);
+ }
+}
+
+#else
+static void arch_cache_info(int *isize, int *dsize) { }
+#endif /* arch_cache_info */
+
+/*
+ * ... and if all else fails ...
+ */
+
+static void fallback_cache_info(int *isize, int *dsize)
+{
+ /* If we can only find one of the two, assume they're the same. */
+ if (*isize) {
+ if (*dsize) {
+ /* Success! */
+ } else {
+ *dsize = *isize;
+ }
+ } else if (*dsize) {
+ *isize = *dsize;
+ } else {
+#if defined(_ARCH_PPC)
+ /*
+ * For PPC, we're going to use the cache sizes computed for
+ * flush_idcache_range. Which means that we must use the
+ * architecture minimum.
+ */
+ *isize = *dsize = 16;
+#else
+ /* Otherwise, 64 bytes is not uncommon. */
+ *isize = *dsize = 64;
+#endif
+ }
+}
+
+static void __attribute__((constructor)) init_cache_info(void)
+{
+ int isize = 0, dsize = 0;
+
+ sys_cache_info(&isize, &dsize);
+ arch_cache_info(&isize, &dsize);
+ fallback_cache_info(&isize, &dsize);
+
+ assert((isize & (isize - 1)) == 0);
+ assert((dsize & (dsize - 1)) == 0);
+
+ qemu_icache_linesize = isize;
+ qemu_icache_linesize_log = ctz32(isize);
+ qemu_dcache_linesize = dsize;
+ qemu_dcache_linesize_log = ctz32(dsize);
+
+ qatomic64_init();
+}
+
+
+/*
+ * Architecture (+ OS) specific cache flushing mechanisms.
+ */
+
#if defined(__i386__) || defined(__x86_64__) || defined(__s390__)
/* Caches are coherent and do not require flushing; symbol inline. */
diff --git a/util/cacheinfo.c b/util/cacheinfo.c
deleted file mode 100644
index ab1644d490..0000000000
--- a/util/cacheinfo.c
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * cacheinfo.c - helpers to query the host about its caches
- *
- * Copyright (C) 2017, Emilio G. Cota <cota@braap.org>
- * License: GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- */
-
-#include "qemu/osdep.h"
-#include "qemu/host-utils.h"
-#include "qemu/atomic.h"
-#include "qemu/cacheinfo.h"
-
-int qemu_icache_linesize = 0;
-int qemu_icache_linesize_log;
-int qemu_dcache_linesize = 0;
-int qemu_dcache_linesize_log;
-
-/*
- * Operating system specific detection mechanisms.
- */
-
-#if defined(_WIN32)
-
-static void sys_cache_info(int *isize, int *dsize)
-{
- SYSTEM_LOGICAL_PROCESSOR_INFORMATION *buf;
- DWORD size = 0;
- BOOL success;
- size_t i, n;
-
- /* Check for the required buffer size first. Note that if the zero
- size we use for the probe results in success, then there is no
- data available; fail in that case. */
- success = GetLogicalProcessorInformation(0, &size);
- if (success || GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
- return;
- }
-
- n = size / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
- size = n * sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
- buf = g_new0(SYSTEM_LOGICAL_PROCESSOR_INFORMATION, n);
- if (!GetLogicalProcessorInformation(buf, &size)) {
- goto fail;
- }
-
- for (i = 0; i < n; i++) {
- if (buf[i].Relationship == RelationCache
- && buf[i].Cache.Level == 1) {
- switch (buf[i].Cache.Type) {
- case CacheUnified:
- *isize = *dsize = buf[i].Cache.LineSize;
- break;
- case CacheInstruction:
- *isize = buf[i].Cache.LineSize;
- break;
- case CacheData:
- *dsize = buf[i].Cache.LineSize;
- break;
- default:
- break;
- }
- }
- }
- fail:
- g_free(buf);
-}
-
-#elif defined(__APPLE__)
-# include <sys/sysctl.h>
-static void sys_cache_info(int *isize, int *dsize)
-{
- /* There's only a single sysctl for both I/D cache line sizes. */
- long size;
- size_t len = sizeof(size);
- if (!sysctlbyname("hw.cachelinesize", &size, &len, NULL, 0)) {
- *isize = *dsize = size;
- }
-}
-#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
-# include <sys/sysctl.h>
-static void sys_cache_info(int *isize, int *dsize)
-{
- /* There's only a single sysctl for both I/D cache line sizes. */
- int size;
- size_t len = sizeof(size);
- if (!sysctlbyname("machdep.cacheline_size", &size, &len, NULL, 0)) {
- *isize = *dsize = size;
- }
-}
-#else
-/* POSIX */
-
-static void sys_cache_info(int *isize, int *dsize)
-{
-# ifdef _SC_LEVEL1_ICACHE_LINESIZE
- int tmp_isize = (int) sysconf(_SC_LEVEL1_ICACHE_LINESIZE);
- if (tmp_isize > 0) {
- *isize = tmp_isize;
- }
-# endif
-# ifdef _SC_LEVEL1_DCACHE_LINESIZE
- int tmp_dsize = (int) sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
- if (tmp_dsize > 0) {
- *dsize = tmp_dsize;
- }
-# endif
-}
-#endif /* sys_cache_info */
-
-/*
- * Architecture (+ OS) specific detection mechanisms.
- */
-
-#if defined(__aarch64__)
-
-static void arch_cache_info(int *isize, int *dsize)
-{
- if (*isize == 0 || *dsize == 0) {
- uint64_t ctr;
-
- /* The real cache geometry is in CCSIDR_EL1/CLIDR_EL1/CSSELR_EL1,
- but (at least under Linux) these are marked protected by the
- kernel. However, CTR_EL0 contains the minimum linesize in the
- entire hierarchy, and is used by userspace cache flushing. */
- asm volatile("mrs\t%0, ctr_el0" : "=r"(ctr));
- if (*isize == 0) {
- *isize = 4 << (ctr & 0xf);
- }
- if (*dsize == 0) {
- *dsize = 4 << ((ctr >> 16) & 0xf);
- }
- }
-}
-
-#elif defined(_ARCH_PPC) && defined(__linux__)
-# include "elf.h"
-
-static void arch_cache_info(int *isize, int *dsize)
-{
- if (*isize == 0) {
- *isize = qemu_getauxval(AT_ICACHEBSIZE);
- }
- if (*dsize == 0) {
- *dsize = qemu_getauxval(AT_DCACHEBSIZE);
- }
-}
-
-#else
-static void arch_cache_info(int *isize, int *dsize) { }
-#endif /* arch_cache_info */
-
-/*
- * ... and if all else fails ...
- */
-
-static void fallback_cache_info(int *isize, int *dsize)
-{
- /* If we can only find one of the two, assume they're the same. */
- if (*isize) {
- if (*dsize) {
- /* Success! */
- } else {
- *dsize = *isize;
- }
- } else if (*dsize) {
- *isize = *dsize;
- } else {
-#if defined(_ARCH_PPC)
- /*
- * For PPC, we're going to use the cache sizes computed for
- * flush_idcache_range. Which means that we must use the
- * architecture minimum.
- */
- *isize = *dsize = 16;
-#else
- /* Otherwise, 64 bytes is not uncommon. */
- *isize = *dsize = 64;
-#endif
- }
-}
-
-static void __attribute__((constructor)) init_cache_info(void)
-{
- int isize = 0, dsize = 0;
-
- sys_cache_info(&isize, &dsize);
- arch_cache_info(&isize, &dsize);
- fallback_cache_info(&isize, &dsize);
-
- assert((isize & (isize - 1)) == 0);
- assert((dsize & (dsize - 1)) == 0);
-
- qemu_icache_linesize = isize;
- qemu_icache_linesize_log = ctz32(isize);
- qemu_dcache_linesize = dsize;
- qemu_dcache_linesize_log = ctz32(dsize);
-
- qatomic64_init();
-}
diff --git a/util/meson.build b/util/meson.build
index 8f16018cd4..4939b0b91c 100644
--- a/util/meson.build
+++ b/util/meson.build
@@ -27,7 +27,7 @@ util_ss.add(files('envlist.c', 'path.c', 'module.c'))
util_ss.add(files('host-utils.c'))
util_ss.add(files('bitmap.c', 'bitops.c'))
util_ss.add(files('fifo8.c'))
-util_ss.add(files('cacheinfo.c', 'cacheflush.c'))
+util_ss.add(files('cacheflush.c'))
util_ss.add(files('error.c', 'error-report.c'))
util_ss.add(files('qemu-print.c'))
util_ss.add(files('id.c'))
--
2.34.1
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH v2 2/3] util/cacheflush: Merge aarch64 ctr_el0 usage
2022-06-21 1:48 [PATCH v2 0/3] util: Optimize flushing when ppc host has coherent icache Richard Henderson
2022-06-21 1:48 ` [PATCH v2 1/3] util: Merge cacheflush.c and cacheinfo.c Richard Henderson
@ 2022-06-21 1:48 ` Richard Henderson
2022-06-21 15:20 ` Peter Maydell
2022-06-21 1:48 ` [PATCH v2 3/3] util/cacheflush: Optimize flushing when ppc host has coherent icache Richard Henderson
2 siblings, 1 reply; 7+ messages in thread
From: Richard Henderson @ 2022-06-21 1:48 UTC (permalink / raw)
To: qemu-devel; +Cc: npiggin, qemu-ppc
Merge init_ctr_el0 into arch_cache_info. In flush_idcache_range,
use the pre-computed line sizes from the global variables.
Use CONFIG_DARWIN in preference to __APPLE__.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
util/cacheflush.c | 44 +++++++++++++++++++-------------------------
1 file changed, 19 insertions(+), 25 deletions(-)
diff --git a/util/cacheflush.c b/util/cacheflush.c
index 8096afd33c..01b6cb7583 100644
--- a/util/cacheflush.c
+++ b/util/cacheflush.c
@@ -70,7 +70,7 @@ static void sys_cache_info(int *isize, int *dsize)
g_free(buf);
}
-#elif defined(__APPLE__)
+#elif defined(CONFIG_DARWIN)
# include <sys/sysctl.h>
static void sys_cache_info(int *isize, int *dsize)
{
@@ -117,20 +117,25 @@ static void sys_cache_info(int *isize, int *dsize)
* Architecture (+ OS) specific cache detection mechanisms.
*/
-#if defined(__aarch64__)
-
+#if defined(__aarch64__) && !defined(CONFIG_DARWIN)
+/* Apple does not expose CTR_EL0, so we must use system interfaces. */
+static uint64_t save_ctr_el0;
static void arch_cache_info(int *isize, int *dsize)
{
- if (*isize == 0 || *dsize == 0) {
- uint64_t ctr;
+ uint64_t ctr;
- /*
- * The real cache geometry is in CCSIDR_EL1/CLIDR_EL1/CSSELR_EL1,
- * but (at least under Linux) these are marked protected by the
- * kernel. However, CTR_EL0 contains the minimum linesize in the
- * entire hierarchy, and is used by userspace cache flushing.
- */
- asm volatile("mrs\t%0, ctr_el0" : "=r"(ctr));
+ /*
+ * The real cache geometry is in CCSIDR_EL1/CLIDR_EL1/CSSELR_EL1,
+ * but (at least under Linux) these are marked protected by the
+ * kernel. However, CTR_EL0 contains the minimum linesize in the
+ * entire hierarchy, and is used by userspace cache flushing.
+ *
+ * We will also use this value in flush_idcache_range.
+ */
+ asm volatile("mrs\t%0, ctr_el0" : "=r"(ctr));
+ save_ctr_el0 = ctr;
+
+ if (*isize == 0 || *dsize == 0) {
if (*isize == 0) {
*isize = 4 << (ctr & 0xf);
}
@@ -228,17 +233,6 @@ void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
}
#else
-/*
- * TODO: unify this with cacheinfo.c.
- * We want to save the whole contents of CTR_EL0, so that we
- * have more than the linesize, but also IDC and DIC.
- */
-static uint64_t save_ctr_el0;
-static void __attribute__((constructor)) init_ctr_el0(void)
-{
- asm volatile("mrs\t%0, ctr_el0" : "=r"(save_ctr_el0));
-}
-
/*
* This is a copy of gcc's __aarch64_sync_cache_range, modified
* to fit this three-operand interface.
@@ -248,8 +242,8 @@ void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
const unsigned CTR_IDC = 1u << 28;
const unsigned CTR_DIC = 1u << 29;
const uint64_t ctr_el0 = save_ctr_el0;
- const uintptr_t icache_lsize = 4 << extract64(ctr_el0, 0, 4);
- const uintptr_t dcache_lsize = 4 << extract64(ctr_el0, 16, 4);
+ const uintptr_t icache_lsize = qemu_icache_linesize;
+ const uintptr_t dcache_lsize = qemu_dcache_linesize;
uintptr_t p;
/*
--
2.34.1
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH v2 3/3] util/cacheflush: Optimize flushing when ppc host has coherent icache
2022-06-21 1:48 [PATCH v2 0/3] util: Optimize flushing when ppc host has coherent icache Richard Henderson
2022-06-21 1:48 ` [PATCH v2 1/3] util: Merge cacheflush.c and cacheinfo.c Richard Henderson
2022-06-21 1:48 ` [PATCH v2 2/3] util/cacheflush: Merge aarch64 ctr_el0 usage Richard Henderson
@ 2022-06-21 1:48 ` Richard Henderson
2022-06-21 15:21 ` Peter Maydell
2 siblings, 1 reply; 7+ messages in thread
From: Richard Henderson @ 2022-06-21 1:48 UTC (permalink / raw)
To: qemu-devel; +Cc: npiggin, qemu-ppc
From: Nicholas Piggin <npiggin@gmail.com>
On linux, the AT_HWCAP bit PPC_FEATURE_ICACHE_SNOOP indicates
that we can use a simplified 3 instruction flush sequence.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Message-Id: <20220519141131.29839-1-npiggin@gmail.com>
[rth: update after merging cacheflush.c and cacheinfo.c]
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
util/cacheflush.c | 25 +++++++++++++++++++++++--
1 file changed, 23 insertions(+), 2 deletions(-)
diff --git a/util/cacheflush.c b/util/cacheflush.c
index 01b6cb7583..2c2c73e085 100644
--- a/util/cacheflush.c
+++ b/util/cacheflush.c
@@ -117,6 +117,10 @@ static void sys_cache_info(int *isize, int *dsize)
* Architecture (+ OS) specific cache detection mechanisms.
*/
+#if defined(__powerpc__)
+static bool have_coherent_icache;
+#endif
+
#if defined(__aarch64__) && !defined(CONFIG_DARWIN)
/* Apple does not expose CTR_EL0, so we must use system interfaces. */
static uint64_t save_ctr_el0;
@@ -156,6 +160,7 @@ static void arch_cache_info(int *isize, int *dsize)
if (*dsize == 0) {
*dsize = qemu_getauxval(AT_DCACHEBSIZE);
}
+ have_coherent_icache = qemu_getauxval(AT_HWCAP) & PPC_FEATURE_ICACHE_SNOOP;
}
#else
@@ -298,8 +303,24 @@ void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
{
uintptr_t p, b, e;
- size_t dsize = qemu_dcache_linesize;
- size_t isize = qemu_icache_linesize;
+ size_t dsize, isize;
+
+ /*
+ * Some processors have coherent caches and support a simplified
+ * flushing procedure. See
+ * POWER9 UM, 4.6.2.2 Instruction Cache Block Invalidate (icbi)
+ * https://ibm.ent.box.com/s/tmklq90ze7aj8f4n32er1mu3sy9u8k3k
+ */
+ if (have_coherent_icache) {
+ asm volatile ("sync\n\t"
+ "icbi 0,%0\n\t"
+ "isync"
+ : : "r"(rx) : "memory");
+ return;
+ }
+
+ dsize = qemu_dcache_linesize;
+ isize = qemu_icache_linesize;
b = rw & ~(dsize - 1);
e = (rw + len + dsize - 1) & ~(dsize - 1);
--
2.34.1
^ permalink raw reply related [flat|nested] 7+ messages in thread