* [kvm-unit-tests PATCH] x86/apic: Gates test_pv_ipi on KVM cpuid, not test device
@ 2023-09-23 10:20 Phil Dennis-Jordan
2023-09-25 15:18 ` Sean Christopherson
0 siblings, 1 reply; 7+ messages in thread
From: Phil Dennis-Jordan @ 2023-09-23 10:20 UTC (permalink / raw)
To: kvm; +Cc: lists, Phil Dennis-Jordan
This changes the test for the KVM IPI hypercall API to be skipped if the
relevant cpuid feature bit is not set or if the KVM cpuid leaf is
missing, rather than the presence of the test device. The latter is an
unreliable inference on non-KVM platforms.
It also adds a skip report when these tests are skipped.
Signed-off-by: Phil Dennis-Jordan <phil@philjordan.eu>
---
lib/x86/processor.h | 19 +++++++++++++++++++
x86/apic.c | 9 ++++++++-
2 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/lib/x86/processor.h b/lib/x86/processor.h
index 44f4fd1e..9a4c0d26 100644
--- a/lib/x86/processor.h
+++ b/lib/x86/processor.h
@@ -284,6 +284,13 @@ static inline bool is_intel(void)
#define X86_FEATURE_VNMI (CPUID(0x8000000A, 0, EDX, 25))
#define X86_FEATURE_AMD_PMU_V2 (CPUID(0x80000022, 0, EAX, 0))
+/*
+ * Hypervisor specific leaves (KVM, ...)
+ * See:
+ * https://kernel.org/doc/html/latest/virt/kvm/x86/cpuid.html
+ */
+#define X86_KVM_FEATURE_PV_SEND_IPI (CPUID(0x40000001, 0, EAX, 11))
+
static inline bool this_cpu_has(u64 feature)
{
u32 input_eax = feature >> 32;
@@ -299,6 +306,18 @@ static inline bool this_cpu_has(u64 feature)
return ((*(tmp + (output_reg % 32))) & (1 << bit));
}
+static inline bool kvm_feature_flags_supported(void)
+{
+ struct cpuid c;
+
+ c = cpuid_indexed(0x40000000, 0);
+ return
+ c.b == 0x4b4d564b
+ && c.c == 0x564b4d56
+ && c.d == 0x4d
+ && (c.a >= 0x40000001 || c.a == 0);
+}
+
struct far_pointer32 {
u32 offset;
u16 selector;
diff --git a/x86/apic.c b/x86/apic.c
index dd7e7834..525e08fd 100644
--- a/x86/apic.c
+++ b/x86/apic.c
@@ -30,6 +30,11 @@ static bool is_xapic_enabled(void)
return (rdmsr(MSR_IA32_APICBASE) & (APIC_EN | APIC_EXTD)) == APIC_EN;
}
+static bool is_kvm_ipi_hypercall_supported(void)
+{
+ return kvm_feature_flags_supported() && this_cpu_has(X86_KVM_FEATURE_PV_SEND_IPI);
+}
+
static void test_lapic_existence(void)
{
u8 version;
@@ -658,8 +663,10 @@ static void test_pv_ipi(void)
int ret;
unsigned long a0 = 0xFFFFFFFF, a1 = 0, a2 = 0xFFFFFFFF, a3 = 0x0;
- if (!test_device_enabled())
+ if (!is_kvm_ipi_hypercall_supported()) {
+ report_skip("PV IPIs testing (No KVM IPI hypercall flag in cpuid)");
return;
+ }
asm volatile("vmcall" : "=a"(ret) :"a"(KVM_HC_SEND_IPI), "b"(a0), "c"(a1), "d"(a2), "S"(a3));
report(!ret, "PV IPIs testing");
--
2.36.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* Re: [kvm-unit-tests PATCH] x86/apic: Gates test_pv_ipi on KVM cpuid, not test device
2023-09-23 10:20 [kvm-unit-tests PATCH] x86/apic: Gates test_pv_ipi on KVM cpuid, not test device Phil Dennis-Jordan
@ 2023-09-25 15:18 ` Sean Christopherson
[not found] ` <CAGCz3vve7RJ+HE8sHOvq1p5-Wc4RpgZwqp0DiCXiSWq0vUpEVw@mail.gmail.com>
0 siblings, 1 reply; 7+ messages in thread
From: Sean Christopherson @ 2023-09-25 15:18 UTC (permalink / raw)
To: Phil Dennis-Jordan; +Cc: kvm, lists
On Sat, Sep 23, 2023, Phil Dennis-Jordan wrote:
> This changes the test for the KVM IPI hypercall API to be skipped if the
> relevant cpuid feature bit is not set or if the KVM cpuid leaf is
> missing, rather than the presence of the test device. The latter is an
> unreliable inference on non-KVM platforms.
>
> It also adds a skip report when these tests are skipped.
>
> Signed-off-by: Phil Dennis-Jordan <phil@philjordan.eu>
> ---
> lib/x86/processor.h | 19 +++++++++++++++++++
> x86/apic.c | 9 ++++++++-
> 2 files changed, 27 insertions(+), 1 deletion(-)
>
> diff --git a/lib/x86/processor.h b/lib/x86/processor.h
> index 44f4fd1e..9a4c0d26 100644
> --- a/lib/x86/processor.h
> +++ b/lib/x86/processor.h
> @@ -284,6 +284,13 @@ static inline bool is_intel(void)
> #define X86_FEATURE_VNMI (CPUID(0x8000000A, 0, EDX, 25))
> #define X86_FEATURE_AMD_PMU_V2 (CPUID(0x80000022, 0, EAX, 0))
>
> +/*
> + * Hypervisor specific leaves (KVM, ...)
> + * See:
> + * https://kernel.org/doc/html/latest/virt/kvm/x86/cpuid.html
> + */
> +#define X86_KVM_FEATURE_PV_SEND_IPI (CPUID(0x40000001, 0, EAX, 11))
We could actually define this using the uapi headers, then there's no need to
reference the kernel docs, e.g.
#define X86_FEATURE_KVM_PV_SEND_IPI (CPUID(KVM_CPUID_FEATURES, 0, EAX, KVM_FEATURE_PV_SEND_IPI)
> +
> static inline bool this_cpu_has(u64 feature)
> {
> u32 input_eax = feature >> 32;
> @@ -299,6 +306,18 @@ static inline bool this_cpu_has(u64 feature)
> return ((*(tmp + (output_reg % 32))) & (1 << bit));
> }
>
> +static inline bool kvm_feature_flags_supported(void)
> +{
> + struct cpuid c;
> +
> + c = cpuid_indexed(0x40000000, 0);
> + return
> + c.b == 0x4b4d564b
> + && c.c == 0x564b4d56
> + && c.d == 0x4d
I would much prefer to provide something similar to the kernel's hypervisor_cpuid_base(),
and then use KVM_SIGNATURE to match the signature. And assert that KVM is placed
at its default base for tests that require KVM paravirt features, i.e. disallow
relocating KVM to e.g. 0x40000100 to make room for Hyper-V.
Something like this (completely untested)
static inline u32 get_hypervisor_cpuid_base(const char *sig)
{
u32 base, signature[3];
if (!this_cpu_has(X86_FEATURE_HYPERVISOR))
return 0;
for (base = 0x40000000; base < 0x40010000; base += 0x100) {
cpuid(base, &eax, &signature[0], &signature[1], &signature[2]);
if (!memcmp(sig, signature, 12))
return base;
}
return 0;
}
static inline bool is_hypervisor_kvm(void)
{
u32 base = get_hypervisor_cpuid_base(KVM_SIGNATURE);
if (!base)
return 0;
/*
* Require that KVM be placed at its default base so that macros can be
* used to query individual KVM feature bits.
*/
TEST_ASSERT(base == KVM_CPUID_SIGNATURE);
return true;
}
> + && (c.a >= 0x40000001 || c.a == 0);
Why allow 0? Though I think we probably forego this check entirely.
> +}
> +
> struct far_pointer32 {
> u32 offset;
> u16 selector;
> diff --git a/x86/apic.c b/x86/apic.c
> index dd7e7834..525e08fd 100644
> --- a/x86/apic.c
> +++ b/x86/apic.c
> @@ -30,6 +30,11 @@ static bool is_xapic_enabled(void)
> return (rdmsr(MSR_IA32_APICBASE) & (APIC_EN | APIC_EXTD)) == APIC_EN;
> }
>
> +static bool is_kvm_ipi_hypercall_supported(void)
> +{
> + return kvm_feature_flags_supported() && this_cpu_has(X86_KVM_FEATURE_PV_SEND_IPI);
> +}
> +
> static void test_lapic_existence(void)
> {
> u8 version;
> @@ -658,8 +663,10 @@ static void test_pv_ipi(void)
> int ret;
> unsigned long a0 = 0xFFFFFFFF, a1 = 0, a2 = 0xFFFFFFFF, a3 = 0x0;
>
> - if (!test_device_enabled())
> + if (!is_kvm_ipi_hypercall_supported()) {
I would rather open code the two independent checks, e.g.
if (!is_hypervisor_kvm() || !this_cpu_has(X86_FEATURE_KVM_PV_SEND_IPI))
Or alternatively, provide a generic helper in processor.h to handle the hypervisor
check, e.g.
static inline this_cpu_has_kvm_feature(...)
Though if we go that route it probably makes sense to play nice with relocating
the base since it would be quite easy to do so.
> + report_skip("PV IPIs testing (No KVM IPI hypercall flag in cpuid)");
> return;
> + }
>
> asm volatile("vmcall" : "=a"(ret) :"a"(KVM_HC_SEND_IPI), "b"(a0), "c"(a1), "d"(a2), "S"(a3));
> report(!ret, "PV IPIs testing");
> --
> 2.36.1
>
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [kvm-unit-tests PATCH] x86/apic: Gates test_pv_ipi on KVM cpuid, not test device
[not found] ` <CAGCz3vve7RJ+HE8sHOvq1p5-Wc4RpgZwqp0DiCXiSWq0vUpEVw@mail.gmail.com>
@ 2023-09-26 16:08 ` Sean Christopherson
2023-09-27 13:14 ` Phil Dennis-Jordan
0 siblings, 1 reply; 7+ messages in thread
From: Sean Christopherson @ 2023-09-26 16:08 UTC (permalink / raw)
To: Phil Dennis-Jordan; +Cc: kvm, Paolo Bonzini
+KVM and Paolo
Please keep the mailing list Cc'd so that other can participate in the conversation,
and so that the mails are archived.
Adding Paolo, who I believe is still not subscribed to kvm@ :-)
On Tue, Sep 26, 2023, Phil Dennis-Jordan wrote:
> Hi Sean,
>
> Thanks for taking a look at this.
>
> On Mon, Sep 25, 2023 at 5:18 PM Sean Christopherson <seanjc@google.com> wrote:
> > On Sat, Sep 23, 2023, Phil Dennis-Jordan wrote:
> > > +#define X86_KVM_FEATURE_PV_SEND_IPI (CPUID(0x40000001, 0, EAX, 11))
> >
> > We could actually define this using the uapi headers, then there's no need to
> > reference the kernel docs, e.g.
> >
> > #define X86_FEATURE_KVM_PV_SEND_IPI (CPUID(KVM_CPUID_FEATURES, 0, EAX, KVM_FEATURE_PV_SEND_IPI)
>
> That sounds sensible, but those symbols are #defined in headers that
> are currently only routinely available on Linux systems. I know this
> is the *KVM* unit test suite, but it's definitely also very useful
> elsewhere.
> (My specific motivation for the patch is that on macOS/HVF, Qemu's
> software APIC implementation is used. The current implementation for
> that turns out to be a massive perf bottleneck and very much
> not-spec-compliant, so I've been improving it. This test suite has
> been very useful for testing/verifying my work, but this IPI test had
> to be disabled.)
Ah, right.
> A few options I can think of:
>
> 1. I notice x86/hyperv.h has a bunch of similar #defines. We could
> collect only the KVM-related x86 constants and helper functions we
> actually need in a corresponding x86/kvm.h. I see there's a kvmclock.h
> for the KVM clock already - that could either be renamed and expanded
> in scope, or left separate.
kvmclock.h is essentially #2, it's a copy of arch/x86/include/asm/pvclock-abi.h
from the kernel.
> 2. Bring a copy of the necessary KVM uapi header file(s) into the
> repo, slightly hacked up to cut down on transitive dependencies. It
> looks like lib/linux/*.h might already be similar instances for other
> Linux bits. Qemu also does this.
This has my vote, though I'd strongly prefer not to strip out anything unless it's
absolutely necessary to get KUT to compile. Grabbing entire files should make it
easier to maintain the copy+pasted code as future updates to re-sync will hopefully
add just the new features.
The attached half-baked patch adds everything except the base "is this KVM?"
check and has only been compile tested on x86, feel free to use it as a starting
point (I wanted to get the basic gist compiling to make sure I wasn't leading you
completely astray)
> > > + && (c.a >= 0x40000001 || c.a == 0);
> >
> > Why allow 0? Though I think we probably forego this check entirely.
>
> "Note also that old hosts set eax value to 0x0. This should be
> interpreted as if the value was 0x40000001. " according to
> https://www.kernel.org/doc/html/v5.7/virt/kvm/cpuid.html
> Though I suppose "old hosts" are probably VERY old and probably don't
> expose the IPI hypercall…
Ha, so your the one that reads the documentation ;-)
> > > @@ -658,8 +663,10 @@ static void test_pv_ipi(void)
> > > int ret;
> > > unsigned long a0 = 0xFFFFFFFF, a1 = 0, a2 = 0xFFFFFFFF, a3 = 0x0;
> > >
> > > - if (!test_device_enabled())
> > > + if (!is_kvm_ipi_hypercall_supported()) {
> >
> > I would rather open code the two independent checks, e.g.
> >
> > if (!is_hypervisor_kvm() || !this_cpu_has(X86_FEATURE_KVM_PV_SEND_IPI))
>
> Makes sense, especially so we don't have to keep rechecking whether
> we're on KVM in case we end up testing for more KVM feature flags at
> some point.
>
> > Or alternatively, provide a generic helper in processor.h to handle the hypervisor
> > check, e.g.
> >
> > static inline this_cpu_has_kvm_feature(...)
> >
> > Though if we go that route it probably makes sense to play nice with relocating
> > the base since it would be quite easy to do so.
>
> Probably overkill as long as we only have this one instance?
Yeah, probably.
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [kvm-unit-tests PATCH] x86/apic: Gates test_pv_ipi on KVM cpuid, not test device
2023-09-26 16:08 ` Sean Christopherson
@ 2023-09-27 13:14 ` Phil Dennis-Jordan
2023-09-27 14:18 ` Sean Christopherson
0 siblings, 1 reply; 7+ messages in thread
From: Phil Dennis-Jordan @ 2023-09-27 13:14 UTC (permalink / raw)
To: Sean Christopherson; +Cc: kvm, Paolo Bonzini
On Tue, Sep 26, 2023 at 6:08 PM Sean Christopherson <seanjc@google.com> wrote:
> Please keep the mailing list Cc'd so that other can participate in the conversation,
> and so that the mails are archived.
Sorry! I've now enabled reply-to-all by default so I can't forget again.
> > 2. Bring a copy of the necessary KVM uapi header file(s) into the
> > repo, slightly hacked up to cut down on transitive dependencies. It
> > looks like lib/linux/*.h might already be similar instances for other
> > Linux bits. Qemu also does this.
>
> This has my vote, though I'd strongly prefer not to strip out anything unless it's
> absolutely necessary to get KUT to compile. Grabbing entire files should make it
> easier to maintain the copy+pasted code as future updates to re-sync will hopefully
> add just the new features.
Yup, makes sense.
> The attached half-baked patch adds everything except the base "is this KVM?"
> check and has only been compile tested on x86, feel free to use it as a starting
> point (I wanted to get the basic gist compiling to make sure I wasn't leading you
> completely astray)
The attachment doesn't seem to have made it, would you mind trying
again? Then I'll put together a v2 of the patch based on that.
Thanks,
Phil
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [kvm-unit-tests PATCH] x86/apic: Gates test_pv_ipi on KVM cpuid, not test device
2023-09-27 13:14 ` Phil Dennis-Jordan
@ 2023-09-27 14:18 ` Sean Christopherson
2023-10-05 20:19 ` Phil Dennis-Jordan
0 siblings, 1 reply; 7+ messages in thread
From: Sean Christopherson @ 2023-09-27 14:18 UTC (permalink / raw)
To: Phil Dennis-Jordan; +Cc: kvm, Paolo Bonzini
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=3Dutf-8, Size: 674 bytes --]
On Wed, Sep 27, 2023, Phil Dennis-Jordan wrote:
> On Tue, Sep 26, 2023 at 6:08=E2=80=AFPM Sean Christopherson <seanjc@googl=
e.com> wrote:
> > The attached half-baked patch adds everything except the base "is this =
KVM?"
> > check and has only been compile tested on x86, feel free to use it as a=
starting
> > point (I wanted to get the basic gist compiling to make sure I wasn't l=
eading you
> > completely astray)
>=20
> The attachment doesn't seem to have made it, would you mind trying
> again? Then I'll put together a v2 of the patch based on that.
Gah, sorry. This is why I usually inline patches, I forget to actually att=
ach
the darn things 50% of the time.
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 3D"0001-tmp.patch" --]
[-- Type: text/x-diff; charset=3Dus-ascii, Size: 7957 bytes --]
From 50e38e262149f8277110f4c965184463bbb7960b Mon Sep 17 00:00:00 2001
From: Sean Christopherson <seanjc@google.com>
Date: Tue, 26 Sep 2023 09:04:35 -0700
Subject: [PATCH] tmp
---
lib/asm-generic/kvm_para.h | 4 +
lib/linux/kvm_para.h | 39 ++++++++++
lib/x86/asm/kvm_para.h | 153 +++++++++++++++++++++++++++++++++++++
lib/x86/processor.h | 4 +
x86/apic.c | 2 +-
5 files changed, 201 insertions(+), 1 deletion(-)
create mode 100644 lib/asm-generic/kvm_para.h
create mode 100644 lib/linux/kvm_para.h
create mode 100644 lib/x86/asm/kvm_para.h
diff --git a/lib/asm-generic/kvm_para.h b/lib/asm-generic/kvm_para.h
new file mode 100644
index 00000000..486f0af7
--- /dev/null
+++ b/lib/asm-generic/kvm_para.h
@@ -0,0 +1,4 @@
+/*
+ * There isn't anything here, but the file must not be empty or patch
+ * will delete it.
+ */
diff --git a/lib/linux/kvm_para.h b/lib/linux/kvm_para.h
new file mode 100644
index 00000000..960c7e93
--- /dev/null
+++ b/lib/linux/kvm_para.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+#ifndef _UAPI__LINUX_KVM_PARA_H
+#define _UAPI__LINUX_KVM_PARA_H
+
+/*
+ * This header file provides a method for making a hypercall to the host
+ * Architectures should define:
+ * - kvm_hypercall0, kvm_hypercall1...
+ * - kvm_arch_para_features
+ * - kvm_para_available
+ */
+
+/* Return values for hypercalls */
+#define KVM_ENOSYS 1000
+#define KVM_EFAULT EFAULT
+#define KVM_EINVAL EINVAL
+#define KVM_E2BIG E2BIG
+#define KVM_EPERM EPERM
+#define KVM_EOPNOTSUPP 95
+
+#define KVM_HC_VAPIC_POLL_IRQ 1
+#define KVM_HC_MMU_OP 2
+#define KVM_HC_FEATURES 3
+#define KVM_HC_PPC_MAP_MAGIC_PAGE 4
+#define KVM_HC_KICK_CPU 5
+#define KVM_HC_MIPS_GET_CLOCK_FREQ 6
+#define KVM_HC_MIPS_EXIT_VM 7
+#define KVM_HC_MIPS_CONSOLE_OUTPUT 8
+#define KVM_HC_CLOCK_PAIRING 9
+#define KVM_HC_SEND_IPI 10
+#define KVM_HC_SCHED_YIELD 11
+#define KVM_HC_MAP_GPA_RANGE 12
+
+/*
+ * hypercalls use architecture specific
+ */
+#include <asm/kvm_para.h>
+
+#endif /* _UAPI__LINUX_KVM_PARA_H */
diff --git a/lib/x86/asm/kvm_para.h b/lib/x86/asm/kvm_para.h
new file mode 100644
index 00000000..6e64b27b
--- /dev/null
+++ b/lib/x86/asm/kvm_para.h
@@ -0,0 +1,153 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+#ifndef _UAPI_ASM_X86_KVM_PARA_H
+#define _UAPI_ASM_X86_KVM_PARA_H
+
+#include <linux/types.h>
+
+/* This CPUID returns the signature 'KVMKVMKVM' in ebx, ecx, and edx. It
+ * should be used to determine that a VM is running under KVM.
+ */
+#define KVM_CPUID_SIGNATURE 0x40000000
+#define KVM_SIGNATURE "KVMKVMKVM\0\0\0"
+
+/* This CPUID returns two feature bitmaps in eax, edx. Before enabling
+ * a particular paravirtualization, the appropriate feature bit should
+ * be checked in eax. The performance hint feature bit should be checked
+ * in edx.
+ */
+#define KVM_CPUID_FEATURES 0x40000001
+#define KVM_FEATURE_CLOCKSOURCE 0
+#define KVM_FEATURE_NOP_IO_DELAY 1
+#define KVM_FEATURE_MMU_OP 2
+/* This indicates that the new set of kvmclock msrs
+ * are available. The use of 0x11 and 0x12 is deprecated
+ */
+#define KVM_FEATURE_CLOCKSOURCE2 3
+#define KVM_FEATURE_ASYNC_PF 4
+#define KVM_FEATURE_STEAL_TIME 5
+#define KVM_FEATURE_PV_EOI 6
+#define KVM_FEATURE_PV_UNHALT 7
+#define KVM_FEATURE_PV_TLB_FLUSH 9
+#define KVM_FEATURE_ASYNC_PF_VMEXIT 10
+#define KVM_FEATURE_PV_SEND_IPI 11
+#define KVM_FEATURE_POLL_CONTROL 12
+#define KVM_FEATURE_PV_SCHED_YIELD 13
+#define KVM_FEATURE_ASYNC_PF_INT 14
+#define KVM_FEATURE_MSI_EXT_DEST_ID 15
+#define KVM_FEATURE_HC_MAP_GPA_RANGE 16
+#define KVM_FEATURE_MIGRATION_CONTROL 17
+
+#define KVM_HINTS_REALTIME 0
+
+/* The last 8 bits are used to indicate how to interpret the flags field
+ * in pvclock structure. If no bits are set, all flags are ignored.
+ */
+#define KVM_FEATURE_CLOCKSOURCE_STABLE_BIT 24
+
+#define MSR_KVM_WALL_CLOCK 0x11
+#define MSR_KVM_SYSTEM_TIME 0x12
+
+#define KVM_MSR_ENABLED 1
+/* Custom MSRs falls in the range 0x4b564d00-0x4b564dff */
+#define MSR_KVM_WALL_CLOCK_NEW 0x4b564d00
+#define MSR_KVM_SYSTEM_TIME_NEW 0x4b564d01
+#define MSR_KVM_ASYNC_PF_EN 0x4b564d02
+#define MSR_KVM_STEAL_TIME 0x4b564d03
+#define MSR_KVM_PV_EOI_EN 0x4b564d04
+#define MSR_KVM_POLL_CONTROL 0x4b564d05
+#define MSR_KVM_ASYNC_PF_INT 0x4b564d06
+#define MSR_KVM_ASYNC_PF_ACK 0x4b564d07
+#define MSR_KVM_MIGRATION_CONTROL 0x4b564d08
+
+struct kvm_steal_time {
+ __u64 steal;
+ __u32 version;
+ __u32 flags;
+ __u8 preempted;
+ __u8 u8_pad[3];
+ __u32 pad[11];
+};
+
+#define KVM_VCPU_PREEMPTED (1 << 0)
+#define KVM_VCPU_FLUSH_TLB (1 << 1)
+
+#define KVM_CLOCK_PAIRING_WALLCLOCK 0
+struct kvm_clock_pairing {
+ __s64 sec;
+ __s64 nsec;
+ __u64 tsc;
+ __u32 flags;
+ __u32 pad[9];
+};
+
+#define KVM_STEAL_ALIGNMENT_BITS 5
+#define KVM_STEAL_VALID_BITS ((-1ULL << (KVM_STEAL_ALIGNMENT_BITS + 1)))
+#define KVM_STEAL_RESERVED_MASK (((1 << KVM_STEAL_ALIGNMENT_BITS) - 1 ) <<=
1)
+
+#define KVM_MAX_MMU_OP_BATCH 32
+
+#define KVM_ASYNC_PF_ENABLED (1 << 0)
+#define KVM_ASYNC_PF_SEND_ALWAYS (1 << 1)
+#define KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT (1 << 2)
+#define KVM_ASYNC_PF_DELIVERY_AS_INT (1 << 3)
+
+/* MSR_KVM_ASYNC_PF_INT */
+#define KVM_ASYNC_PF_VEC_MASK GENMASK(7, 0)
+
+/* MSR_KVM_MIGRATION_CONTROL */
+#define KVM_MIGRATION_READY (1 << 0)
+
+/* KVM_HC_MAP_GPA_RANGE */
+#define KVM_MAP_GPA_RANGE_PAGE_SZ_4K 0
+#define KVM_MAP_GPA_RANGE_PAGE_SZ_2M (1 << 0)
+#define KVM_MAP_GPA_RANGE_PAGE_SZ_1G (1 << 1)
+#define KVM_MAP_GPA_RANGE_ENC_STAT(n) (n << 4)
+#define KVM_MAP_GPA_RANGE_ENCRYPTED KVM_MAP_GPA_RANGE_ENC_STAT(1)
+#define KVM_MAP_GPA_RANGE_DECRYPTED KVM_MAP_GPA_RANGE_ENC_STAT(0)
+
+/* Operations for KVM_HC_MMU_OP */
+#define KVM_MMU_OP_WRITE_PTE 1
+#define KVM_MMU_OP_FLUSH_TLB 2
+#define KVM_MMU_OP_RELEASE_PT 3
+
+/* Payload for KVM_HC_MMU_OP */
+struct kvm_mmu_op_header {
+ __u32 op;
+ __u32 pad;
+};
+
+struct kvm_mmu_op_write_pte {
+ struct kvm_mmu_op_header header;
+ __u64 pte_phys;
+ __u64 pte_val;
+};
+
+struct kvm_mmu_op_flush_tlb {
+ struct kvm_mmu_op_header header;
+};
+
+struct kvm_mmu_op_release_pt {
+ struct kvm_mmu_op_header header;
+ __u64 pt_phys;
+};
+
+#define KVM_PV_REASON_PAGE_NOT_PRESENT 1
+#define KVM_PV_REASON_PAGE_READY 2
+
+struct kvm_vcpu_pv_apf_data {
+ /* Used for 'page not present' events delivered via #PF */
+ __u32 flags;
+
+ /* Used for 'page ready' events delivered via interrupt notification */
+ __u32 token;
+
+ __u8 pad[56];
+ __u32 enabled;
+};
+
+#define KVM_PV_EOI_BIT 0
+#define KVM_PV_EOI_MASK (0x1 << KVM_PV_EOI_BIT)
+#define KVM_PV_EOI_ENABLED KVM_PV_EOI_MASK
+#define KVM_PV_EOI_DISABLED 0x0
+
+#endif /* _UAPI_ASM_X86_KVM_PARA_H */
diff --git a/lib/x86/processor.h b/lib/x86/processor.h
index 44f4fd1e..7a7048f9 100644
--- a/lib/x86/processor.h
+++ b/lib/x86/processor.h
@@ -7,6 +7,8 @@
#include <bitops.h>
#include <stdint.h>
=20
+#include <linux/kvm_para.h>
+
#define NONCANONICAL 0xaaaaaaaaaaaaaaaaull
=20
#ifdef __x86_64__
@@ -284,6 +286,8 @@ static inline bool is_intel(void)
#define X86_FEATURE_VNMI (CPUID(0x8000000A, 0, EDX, 25))
#define X86_FEATURE_AMD_PMU_V2 (CPUID(0x80000022, 0, EAX, 0))
=20
+#define X86_FEATURE_KVM_PV_SEND_IPI (CPUID(KVM_CPUID_FEATURES, 0, EAX, KVM=
_FEATURE_PV_SEND_IPI))
+
static inline bool this_cpu_has(u64 feature)
{
u32 input_eax =3D feature >> 32;
diff --git a/x86/apic.c b/x86/apic.c
index dd7e7834..61334543 100644
--- a/x86/apic.c
+++ b/x86/apic.c
@@ -658,7 +658,7 @@ static void test_pv_ipi(void)
int ret;
unsigned long a0 =3D 0xFFFFFFFF, a1 =3D 0, a2 =3D 0xFFFFFFFF, a3 =3D 0x0;
=20
- if (!test_device_enabled())
+ if (!this_cpu_has(X86_FEATURE_KVM_PV_SEND_IPI))
return;
=20
asm volatile("vmcall" : "=3Da"(ret) :"a"(KVM_HC_SEND_IPI), "b"(a0), "c"(a=
1), "d"(a2), "S"(a3));
base-commit: 038ae991814a8baf220ab2000fa2268d319213ae
--=20
2.42.0.582.g8ccd20d70d-goog
^ permalink raw reply related [flat|nested] 7+ messages in thread
* Re: [kvm-unit-tests PATCH] x86/apic: Gates test_pv_ipi on KVM cpuid, not test device
2023-09-27 14:18 ` Sean Christopherson
@ 2023-10-05 20:19 ` Phil Dennis-Jordan
2023-10-11 12:27 ` Phil Dennis-Jordan
0 siblings, 1 reply; 7+ messages in thread
From: Phil Dennis-Jordan @ 2023-10-05 20:19 UTC (permalink / raw)
To: Sean Christopherson; +Cc: kvm, Paolo Bonzini
[-- Attachment #1: Type: text/plain, Size: 4583 bytes --]
On Wed, Sep 27, 2023 at 4:18 PM Sean Christopherson <seanjc@google.com> wrote:
> Gah, sorry. This is why I usually inline patches, I forget to actually att=
> ach
> the darn things 50% of the time.
Thanks for that, and apologies for only just getting around to taking
a closer look. Out of the box, this doesn't build on macOS at least,
as that's missing <linux/types.h>. I tried going down the rabbit hole
of pulling that header and its various transitive dependencies in from
the Linux tree, but ended up in a horrible mess where those headers
try to define things like bool, which libcflat has already pulled in
from the standard system headers (on Linux I suspect the libc #include
guards match up with the stuff in <linux/*>, so there's no issue).
On macOS, the problem is easy to resolve via a cut-down types.h with a
minimal set of definitions:
#include <libcflat.h>
typedef u8 __u8;
typedef u32 __u32;
typedef u64 __u64;
typedef s64 __s64;
…but I assume that breaks things on Linux. I'm thinking something like
this might work:
#if __LINUX__
#include_next <linux/types.h>
#else
[minimal types.h definitions]
#endif
But I'm unsure if that's really the direction you'd want to go with
this? (And I still need to set myself up with a dev environment on a
physical Linux box that I can test this all on.)
Another option might be a symlinked linux/types.h created by
./configure if not running on Linux?
On the substance of the patch itself:
> unsigned long a0 = 0xFFFFFFFF, a1 = 0, a2 = 0xFFFFFFFF, a3 = 0x0;
> - if (!test_device_enabled())
> + if (!this_cpu_has(X86_FEATURE_KVM_PV_SEND_IPI))
So this check will (erroneously IMO) succeed if we're running on a
non-KVM hypervisor which happens to expose a flag at bit 11 of ecx on
CPUID leaf 0x40000001 page 0, right? With this in mind, your earlier
idea seems better:
if (!is_hypervisor_kvm() ||
!this_cpu_has(X86_FEATURE_KVM_PV_SEND_IPI)) {
So I've gone ahead and made an attempt at fixing up your draft
implementation of is_hypervisor_kvm() below.
The partial struct memcmp in get_hypervisor_cpuid_base is a bit icky;
I'm not sure if that's worth fixing up at the cost of readability.
Thoughts?
(I've attached the full set of WIP changes on top of yours as another
patch. Feel free to squash it all into one if you decide to run with
it.)
Thanks,
Phil
diff --git a/lib/x86/processor.h b/lib/x86/processor.h
index 7a7048f9..3d3930c8 100644
--- a/lib/x86/processor.h
+++ b/lib/x86/processor.h
@@ -240,6 +240,7 @@ static inline bool is_intel(void)
#define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26))
#define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27))
#define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30))
+#define X86_FEATURE_HYPERVISOR (CPUID(0x1, 0, ECX, 31))
#define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7))
#define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9))
#define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19))
@@ -286,7 +287,8 @@ static inline bool is_intel(void)
#define X86_FEATURE_VNMI (CPUID(0x8000000A, 0, EDX, 25))
#define X86_FEATURE_AMD_PMU_V2 (CPUID(0x80000022, 0, EAX, 0))
-#define X86_FEATURE_KVM_PV_SEND_IPI (CPUID(KVM_CPUID_FEATURES, 0,
EAX, KVM_FEATURE_PV_SEND_IPI))
+#define X86_FEATURE_KVM_PV_SEND_IPI \
+ (CPUID(KVM_CPUID_FEATURES, 0, EAX, KVM_FEATURE_PV_SEND_IPI))
static inline bool this_cpu_has(u64 feature)
{
@@ -303,6 +305,40 @@ static inline bool this_cpu_has(u64 feature)
return ((*(tmp + (output_reg % 32))) & (1 << bit));
}
+static inline u32 get_hypervisor_cpuid_base(const char *sig)
+{
+ u32 base;
+ struct cpuid signature;
+
+ if (!this_cpu_has(X86_FEATURE_HYPERVISOR))
+ return 0;
+
+ for (base = 0x40000000; base < 0x40010000; base += 0x100) {
+ signature = cpuid(base);
+
+ if (!memcmp(sig, &signature.b, 12))
+ return base;
+ }
+
+ return 0;
+}
+
+static inline bool is_hypervisor_kvm(void)
+{
+ u32 base = get_hypervisor_cpuid_base(KVM_SIGNATURE);
+
+ if (!base)
+ return false;
+
+ /*
+ * Require that KVM be placed at its default base so that macros can be
+ * used to query individual KVM feature bits.
+ */
+ assert_msg(base == KVM_CPUID_SIGNATURE,
+ "Expect KVM at its default cpuid base (now at: 0x%x)", base);
+ return true;
+}
+
struct far_pointer32 {
u32 offset;
u16 selector;
[-- Attachment #2: 0002-x86-apic-test_pv_ipi-cpuid-check-refinements.patch --]
[-- Type: application/octet-stream, Size: 3787 bytes --]
From 14bf19f3dda2edd2255ba3877af7259974e8a98b Mon Sep 17 00:00:00 2001
From: Phil Dennis-Jordan <phil@philjordan.eu>
Date: Thu, 5 Oct 2023 21:49:48 +0200
Subject: [kvm-unit-tests PATCH 2/4] x86/apic: test_pv_ipi cpuid check
refinements
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Before checking for the KVM IPI hypercall CPUID flag, tests that we’re
actually on KVM, or the flag might mean anything.
Also fixes build on non-Linux machines by adding minimal <linux/types.h>
Signed-off-by: Phil Dennis-Jordan <phil@philjordan.eu>
---
lib/linux/types.h | 15 +++++++++++++++
lib/x86/processor.h | 38 +++++++++++++++++++++++++++++++++++++-
x86/apic.c | 4 +++-
3 files changed, 55 insertions(+), 2 deletions(-)
create mode 100644 lib/linux/types.h
diff --git a/lib/linux/types.h b/lib/linux/types.h
new file mode 100644
index 00000000..f86cd71a
--- /dev/null
+++ b/lib/linux/types.h
@@ -0,0 +1,15 @@
+#ifndef _LIB_LINUX_TYPES_H_
+#ifdef __linux__
+/* On Linux, use the real thing */
+#include_next <linux/types.h>
+
+#else /* !defined(__linux__) */
+/* This is *just* enough for the headers we've pulled in from Linux to compile */
+#include <libcflat.h>
+typedef u8 __u8;
+typedef u32 __u32;
+typedef u64 __u64;
+typedef s64 __s64;
+#endif
+
+#endif
diff --git a/lib/x86/processor.h b/lib/x86/processor.h
index 7a7048f9..3d3930c8 100644
--- a/lib/x86/processor.h
+++ b/lib/x86/processor.h
@@ -240,6 +240,7 @@ static inline bool is_intel(void)
#define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26))
#define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27))
#define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30))
+#define X86_FEATURE_HYPERVISOR (CPUID(0x1, 0, ECX, 31))
#define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7))
#define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9))
#define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19))
@@ -286,7 +287,8 @@ static inline bool is_intel(void)
#define X86_FEATURE_VNMI (CPUID(0x8000000A, 0, EDX, 25))
#define X86_FEATURE_AMD_PMU_V2 (CPUID(0x80000022, 0, EAX, 0))
-#define X86_FEATURE_KVM_PV_SEND_IPI (CPUID(KVM_CPUID_FEATURES, 0, EAX, KVM_FEATURE_PV_SEND_IPI))
+#define X86_FEATURE_KVM_PV_SEND_IPI \
+ (CPUID(KVM_CPUID_FEATURES, 0, EAX, KVM_FEATURE_PV_SEND_IPI))
static inline bool this_cpu_has(u64 feature)
{
@@ -303,6 +305,40 @@ static inline bool this_cpu_has(u64 feature)
return ((*(tmp + (output_reg % 32))) & (1 << bit));
}
+static inline u32 get_hypervisor_cpuid_base(const char *sig)
+{
+ u32 base;
+ struct cpuid signature;
+
+ if (!this_cpu_has(X86_FEATURE_HYPERVISOR))
+ return 0;
+
+ for (base = 0x40000000; base < 0x40010000; base += 0x100) {
+ signature = cpuid(base);
+
+ if (!memcmp(sig, &signature.b, 12))
+ return base;
+ }
+
+ return 0;
+}
+
+static inline bool is_hypervisor_kvm(void)
+{
+ u32 base = get_hypervisor_cpuid_base(KVM_SIGNATURE);
+
+ if (!base)
+ return false;
+
+ /*
+ * Require that KVM be placed at its default base so that macros can be
+ * used to query individual KVM feature bits.
+ */
+ assert_msg(base == KVM_CPUID_SIGNATURE,
+ "Expect KVM at its default cpuid base (now at: 0x%x)", base);
+ return true;
+}
+
struct far_pointer32 {
u32 offset;
u16 selector;
diff --git a/x86/apic.c b/x86/apic.c
index 61334543..783fb740 100644
--- a/x86/apic.c
+++ b/x86/apic.c
@@ -658,8 +658,10 @@ static void test_pv_ipi(void)
int ret;
unsigned long a0 = 0xFFFFFFFF, a1 = 0, a2 = 0xFFFFFFFF, a3 = 0x0;
- if (!this_cpu_has(X86_FEATURE_KVM_PV_SEND_IPI))
+ if (!is_hypervisor_kvm() || !this_cpu_has(X86_FEATURE_KVM_PV_SEND_IPI)) {
+ report_skip("PV IPIs testing");
return;
+ }
asm volatile("vmcall" : "=a"(ret) :"a"(KVM_HC_SEND_IPI), "b"(a0), "c"(a1), "d"(a2), "S"(a3));
report(!ret, "PV IPIs testing");
--
2.36.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* Re: [kvm-unit-tests PATCH] x86/apic: Gates test_pv_ipi on KVM cpuid, not test device
2023-10-05 20:19 ` Phil Dennis-Jordan
@ 2023-10-11 12:27 ` Phil Dennis-Jordan
0 siblings, 0 replies; 7+ messages in thread
From: Phil Dennis-Jordan @ 2023-10-11 12:27 UTC (permalink / raw)
To: Sean Christopherson; +Cc: kvm, Paolo Bonzini
I've now got around to testing this patch on Linux/KVM.
The thing I was worried about, my <linux/types.h> wrapper, appears to
work exactly as intended, everything builds nicely on Linux. So that
aspect of the patch is now a matter of taste.
However, I *have* run into a minor snag with this proposed change: the
X86_FEATURE_KVM_PV_SEND_IPI bit isn't actually enabled by default. It
looks like it needs to be specified explicitly as a +kvm-pv-ipi flag
in the -cpu option on the Qemu command line. KVM itself still handles
the IPI hypercall either way, as there's another flag you'd have to
opt into for only handling advertised hypercalls.
I think the cleanest way to fix this is probably to add +kvm-pv-ipi to
the apic-split, x2apic, and xapic test suites in x86/unittests.cfg and
keep the strict feature flag check in the test code. As I understand
it, Qemu will filter the feature bit if the underlying KVM
implementation doesn't support it, so that would appear to give us the
best compatibility, except perhaps for Qemu versions that predate this
flag, which will presumably fail to run the test suites altogether.
Alternatively, we could simply check whether we're running on KVM and
skip the feature bit check entirely - it certainly wouldn't make any
additional assumptions; as of right now, the master branch already
assumes we're running on KVM *and* the KVM implementation supports the
IPI HC. Dropping the feature bit check doesn't make the patch tangibly
smaller though.
I'll wait a couple of days for any other suggestions or objections,
and in the absence of such I'll roll your draft patch, my
modifications, and the x86/unittests.cfg tweaks into a v2 patch and
re-submit. (And I'll tag it with you, Sean, as Co-authored-by:)
Thanks,
Phil
On Thu, 5 Oct 2023 at 22:19, Phil Dennis-Jordan <lists@philjordan.eu> wrote:
>
> On Wed, Sep 27, 2023 at 4:18 PM Sean Christopherson <seanjc@google.com> wrote:
> > Gah, sorry. This is why I usually inline patches, I forget to actually att=
> > ach
> > the darn things 50% of the time.
>
> Thanks for that, and apologies for only just getting around to taking
> a closer look. Out of the box, this doesn't build on macOS at least,
> as that's missing <linux/types.h>. I tried going down the rabbit hole
> of pulling that header and its various transitive dependencies in from
> the Linux tree, but ended up in a horrible mess where those headers
> try to define things like bool, which libcflat has already pulled in
> from the standard system headers (on Linux I suspect the libc #include
> guards match up with the stuff in <linux/*>, so there's no issue).
> On macOS, the problem is easy to resolve via a cut-down types.h with a
> minimal set of definitions:
>
> #include <libcflat.h>
> typedef u8 __u8;
> typedef u32 __u32;
> typedef u64 __u64;
> typedef s64 __s64;
>
> …but I assume that breaks things on Linux. I'm thinking something like
> this might work:
>
> #if __LINUX__
> #include_next <linux/types.h>
> #else
> [minimal types.h definitions]
> #endif
>
> But I'm unsure if that's really the direction you'd want to go with
> this? (And I still need to set myself up with a dev environment on a
> physical Linux box that I can test this all on.)
>
> Another option might be a symlinked linux/types.h created by
> ./configure if not running on Linux?
>
>
> On the substance of the patch itself:
>
> > unsigned long a0 = 0xFFFFFFFF, a1 = 0, a2 = 0xFFFFFFFF, a3 = 0x0;
> > - if (!test_device_enabled())
> > + if (!this_cpu_has(X86_FEATURE_KVM_PV_SEND_IPI))
>
> So this check will (erroneously IMO) succeed if we're running on a
> non-KVM hypervisor which happens to expose a flag at bit 11 of ecx on
> CPUID leaf 0x40000001 page 0, right? With this in mind, your earlier
> idea seems better:
>
> if (!is_hypervisor_kvm() ||
> !this_cpu_has(X86_FEATURE_KVM_PV_SEND_IPI)) {
>
> So I've gone ahead and made an attempt at fixing up your draft
> implementation of is_hypervisor_kvm() below.
>
> The partial struct memcmp in get_hypervisor_cpuid_base is a bit icky;
> I'm not sure if that's worth fixing up at the cost of readability.
>
>
> Thoughts?
>
> (I've attached the full set of WIP changes on top of yours as another
> patch. Feel free to squash it all into one if you decide to run with
> it.)
>
> Thanks,
> Phil
>
>
> diff --git a/lib/x86/processor.h b/lib/x86/processor.h
> index 7a7048f9..3d3930c8 100644
> --- a/lib/x86/processor.h
> +++ b/lib/x86/processor.h
> @@ -240,6 +240,7 @@ static inline bool is_intel(void)
> #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26))
> #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27))
> #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30))
> +#define X86_FEATURE_HYPERVISOR (CPUID(0x1, 0, ECX, 31))
> #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7))
> #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9))
> #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19))
> @@ -286,7 +287,8 @@ static inline bool is_intel(void)
> #define X86_FEATURE_VNMI (CPUID(0x8000000A, 0, EDX, 25))
> #define X86_FEATURE_AMD_PMU_V2 (CPUID(0x80000022, 0, EAX, 0))
>
> -#define X86_FEATURE_KVM_PV_SEND_IPI (CPUID(KVM_CPUID_FEATURES, 0,
> EAX, KVM_FEATURE_PV_SEND_IPI))
> +#define X86_FEATURE_KVM_PV_SEND_IPI \
> + (CPUID(KVM_CPUID_FEATURES, 0, EAX, KVM_FEATURE_PV_SEND_IPI))
>
> static inline bool this_cpu_has(u64 feature)
> {
> @@ -303,6 +305,40 @@ static inline bool this_cpu_has(u64 feature)
> return ((*(tmp + (output_reg % 32))) & (1 << bit));
> }
>
> +static inline u32 get_hypervisor_cpuid_base(const char *sig)
> +{
> + u32 base;
> + struct cpuid signature;
> +
> + if (!this_cpu_has(X86_FEATURE_HYPERVISOR))
> + return 0;
> +
> + for (base = 0x40000000; base < 0x40010000; base += 0x100) {
> + signature = cpuid(base);
> +
> + if (!memcmp(sig, &signature.b, 12))
> + return base;
> + }
> +
> + return 0;
> +}
> +
> +static inline bool is_hypervisor_kvm(void)
> +{
> + u32 base = get_hypervisor_cpuid_base(KVM_SIGNATURE);
> +
> + if (!base)
> + return false;
> +
> + /*
> + * Require that KVM be placed at its default base so that macros can be
> + * used to query individual KVM feature bits.
> + */
> + assert_msg(base == KVM_CPUID_SIGNATURE,
> + "Expect KVM at its default cpuid base (now at: 0x%x)", base);
> + return true;
> +}
> +
> struct far_pointer32 {
> u32 offset;
> u16 selector;
^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2023-10-11 12:28 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2023-09-23 10:20 [kvm-unit-tests PATCH] x86/apic: Gates test_pv_ipi on KVM cpuid, not test device Phil Dennis-Jordan
2023-09-25 15:18 ` Sean Christopherson
[not found] ` <CAGCz3vve7RJ+HE8sHOvq1p5-Wc4RpgZwqp0DiCXiSWq0vUpEVw@mail.gmail.com>
2023-09-26 16:08 ` Sean Christopherson
2023-09-27 13:14 ` Phil Dennis-Jordan
2023-09-27 14:18 ` Sean Christopherson
2023-10-05 20:19 ` Phil Dennis-Jordan
2023-10-11 12:27 ` Phil Dennis-Jordan
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox