[PATCH kvm-kmod 0/4] Update to 3.19-rc1

[PATCH kvm-kmod 0/4] Update to 3.19-rc1

[Paolo Bonzini]

The largest changes are in the XSAVE support.  Recent kernels added a
separate feature word for XSAVE features, and KVM's CPUID code is relying
on the new definition.  Except for cpu_has_xsaves, it's never accessing
the feature itself: wrap cpu_has_xsaves with kvm_cpu_has_xsaves, and
then there is no problem with out-of-bounds accesses.

Paolo Bonzini (4):
  stubs for xsavec support
  fixes for changes in the iommu and PCI APIs
  fixes for changes in the percpu counter API
  add trace_seq_buffer_ptr

 external-module-compat-comm.h | 32 ++++++++++++++++++++++++++++++++
 external-module-compat.c      | 11 +++++++++++
 linux                         |  2 +-
 sync                          | 14 ++++++++++++--
 x86/external-module-compat.h  | 37 +++++++++++++++++++++++++++++++++++++
 5 files changed, 93 insertions(+), 3 deletions(-)

-- 
1.8.3.1

[PATCH 1/4] stubs for xsavec support

[Paolo Bonzini]

These are needed for KVM changes in 3.18.

Recent kernels added a separate feature word for XSAVE features, and KVM's
CPUID code is relying on the new definition.  Except for cpu_has_xsaves,
it's never accessing the feature itself: wrap cpu_has_xsaves with
kvm_cpu_has_xsaves, and then there is no problem with out-of-bounds
accesses.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 external-module-compat-comm.h |  4 ++++
 external-module-compat.c      | 11 +++++++++++
 sync                          | 14 ++++++++++++--
 x86/external-module-compat.h  | 37 +++++++++++++++++++++++++++++++++++++
 4 files changed, 64 insertions(+), 2 deletions(-)

diff --git a/external-module-compat-comm.h b/external-module-compat-comm.h
index c20b1ed..1717ba3 100644
--- a/external-module-compat-comm.h
+++ b/external-module-compat-comm.h
@@ -1424,3 +1424,7 @@ extern u64 kvm_get_boot_base_ns(struct timekeeper *tk);
 
 #undef is_zero_pfn
 #define is_zero_pfn(pfn) ((pfn) == page_to_pfn(ZERO_PAGE(0)))
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,17,0)
+void *get_xsave_addr(struct xsave_struct *xsave, int feature);
+#endif
diff --git a/external-module-compat.c b/external-module-compat.c
index 38717b6..068ab44 100644
--- a/external-module-compat.c
+++ b/external-module-compat.c
@@ -362,3 +362,14 @@ u64 kvm_get_boot_base_ns(struct timekeeper *tk)
 }
 #endif
 #endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,17,0)
+void *get_xsave_addr(struct xsave_struct *xsave, int feature)
+{
+	int index = fls64(feature) - 1;
+	u32 size, offset, ecx, edx;
+
+	cpuid_count(0xd, index, &size, &offset, &ecx, &edx);
+	return (u8 *)xsave + offset;
+}
+#endif
diff --git a/sync b/sync
index fff85f3..3086b70 100755
--- a/sync
+++ b/sync
@@ -44,8 +44,8 @@ def hack_content(fname, data):
         'set_desc_base set_desc_limit pvclock_vcpu_time_info tboot_enabled '
         'i387_fxsave_struct native_write_msr_safe xsave_struct '
         'fpu_alloc fpu_free fpu_restore_checking fpu_save_init fpu_finit '
-        'load_gdt store_gdt xstate_size cpu_has_xsave __get_user_pages_fast '
-        'set_64bit siginfo_t use_mm '
+        'load_gdt store_gdt xstate_size cpu_has_xsave cpu_has_xsaves '
+        '__get_user_pages_fast set_64bit siginfo_t use_mm '
         'unuse_mm request_threaded_irq init_fpu __this_cpu_read '
         '__this_cpu_write sigset_from_compat '
         'sched_info_on x86_pmu_capability perf_get_x86_pmu_capability '
@@ -299,6 +299,16 @@ def hack_content(fname, data):
         if line == '#endif' and finish_endif:
             w('#endif')
             finish_endif = False
+        if match(r'xcomp_bv'):
+            w('#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0)')
+            w(line)
+            w('#else')
+            # Will be under if (cpu_has_xsaves), which is always 0.  Just
+            # replace with something that compiles, the C statement might
+            # span more than one line.
+            w('WARN(1, "this should never happen"),')
+            w(sub(r'xcomp_bv', 'xstate_bv', line))
+            line = '#endif'
         if match(r'tkr\.'):
             w('#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0)')
             w(line)
diff --git a/x86/external-module-compat.h b/x86/external-module-compat.h
index dec53b6..87cf76a 100644
--- a/x86/external-module-compat.h
+++ b/x86/external-module-compat.h
@@ -428,6 +428,23 @@ static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
 #define X86_FEATURE_MPX		(9*32+14) /* Memory Protection Extension */
 #endif
 
+#if X86_FEATURE_XSAVEOPT < 10 * 32
+#undef X86_FEATURE_XSAVEOPT
+#endif
+#define X86_FEATURE_XSAVEOPT	(10*32+0) /* XSAVEOPT instruction */
+
+#ifndef X86_FEATURE_XSAVEC
+#define X86_FEATURE_XSAVEC	(10*32+1) /* XSAVEC instruction */
+#endif
+
+#ifndef X86_FEATURE_XGETBV1
+#define X86_FEATURE_XGETBV1	(10*32+2) /* "XCR1" register */
+#endif
+
+#ifndef X86_FEATURE_XSAVES
+#define X86_FEATURE_XSAVES	(10*32+3) /* XSAVES instruction */
+#endif
+
 #ifndef MSR_AMD64_PATCH_LOADER
 #define MSR_AMD64_PATCH_LOADER         0xc0010020
 #endif
@@ -586,6 +603,10 @@ static inline void preempt_notifier_sys_exit(void) {}
 #define cpu_has_xsave boot_cpu_has(X86_FEATURE_XSAVE)
 #endif
 
+#ifndef cpu_has_xsaves
+#define cpu_has_xsaves boot_cpu_has(X86_FEATURE_XSAVES)
+#endif
+
 /* EFER_LMA and EFER_LME are missing in pre 2.6.24 i386 kernels */
 #ifndef EFER_LME
 #define _EFER_LME           8  /* Long mode enable */
@@ -1141,6 +1162,16 @@ static inline int kvm_init_fpu(struct task_struct *tsk)
 #define XSTATE_EAGER	(XSTATE_BNDREGS | XSTATE_BNDCSR)
 #endif
 
+#ifndef XSTATE_OPMASK
+#define XSTATE_OPMASK           0x20
+#define XSTATE_ZMM_Hi256        0x40
+#define XSTATE_Hi16_ZMM         0x80
+#endif
+
+#ifndef XSTATE_AVX512
+#define XSTATE_AVX512   (XSTATE_OPMASK | XSTATE_ZMM_Hi256 | XSTATE_Hi16_ZMM)
+#endif
+
 #ifndef XSTATE_EXTEND_MASK
 #define XSTATE_EXTEND_MASK	(~(XSTATE_FPSSE | (1ULL << 63)))
 #endif
@@ -1496,6 +1527,12 @@ static inline int __register_hotcpu_notifier(struct notifier_block *nb)
 }
 #endif
 
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,17,0)
+#define kvm_cpu_has_xsaves	0
+#else /* >= 3.17 */
+#define kvm_cpu_has_xsaves	cpu_has_xsaves
+#endif /* >= 3.17 */
+
 #ifndef MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS
 #define MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS (1ULL << 29)
 #endif
-- 
1.8.3.1

[PATCH 2/4] fixes for changes in the iommu and PCI APIs

[Paolo Bonzini]

These API changes were introduced in 3.18.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 external-module-compat-comm.h | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/external-module-compat-comm.h b/external-module-compat-comm.h
index 1717ba3..6866658 100644
--- a/external-module-compat-comm.h
+++ b/external-module-compat-comm.h
@@ -14,6 +14,7 @@
 #include <linux/kvm_para.h>
 #include <linux/kconfig.h>
 #include <linux/cpu.h>
+#include <linux/pci.h>
 #include <linux/time.h>
 #include <linux/kernel.h>
 #include <asm/processor.h>
@@ -1425,6 +1426,20 @@ extern u64 kvm_get_boot_base_ns(struct timekeeper *tk);
 #undef is_zero_pfn
 #define is_zero_pfn(pfn) ((pfn) == page_to_pfn(ZERO_PAGE(0)))
 
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,18, 0)
+#define iommu_capable(dummy, cap) \
+    iommu_domain_has_cap(kvm->arch.iommu_domain, cap)
+
+static inline void pci_clear_dev_assigned(struct pci_dev *pdev)
+{
+	pdev->dev_flags &= ~PCI_DEV_FLAGS_ASSIGNED;
+}
+
+static inline void pci_set_dev_assigned(struct pci_dev *pdev)
+{
+	pdev->dev_flags |= PCI_DEV_FLAGS_ASSIGNED;
+}
+
 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,17,0)
 void *get_xsave_addr(struct xsave_struct *xsave, int feature);
 #endif
-- 
1.8.3.1

[PATCH 3/4] fixes for changes in the percpu counter API

[Paolo Bonzini]

These API changes were introduced in 3.18.  Update source pointer
since 3.18 now compiles.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 external-module-compat-comm.h | 9 +++++++++
 linux                         | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/external-module-compat-comm.h b/external-module-compat-comm.h
index 6866658..f0e1ce2 100644
--- a/external-module-compat-comm.h
+++ b/external-module-compat-comm.h
@@ -1440,6 +1440,15 @@ static inline void pci_set_dev_assigned(struct pci_dev *pdev)
 	pdev->dev_flags |= PCI_DEV_FLAGS_ASSIGNED;
 }
 
+#undef percpu_counter_init
+#define percpu_counter_init(fbc, value, gfp)                            \
+        ({                                                              \
+                static struct lock_class_key __key;                     \
+                                                                        \
+                __percpu_counter_init(fbc, value, &__key);              \
+        })
+#endif
+
 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,17,0)
 void *get_xsave_addr(struct xsave_struct *xsave, int feature);
 #endif
diff --git a/linux b/linux
index da01e61..b2776bf 160000
--- a/linux
+++ b/linux
@@ -1 +1 @@
-Subproject commit da01e61428aa2b5c424fddc11178498462d8c77f
+Subproject commit b2776bf7149bddd1f4161f14f79520f17fc1d71d
-- 
1.8.3.1

[PATCH 4/4] add trace_seq_buffer_ptr

[Paolo Bonzini]

These API changes were introduced in 3.19-rc1.  Update source pointer
since this is the only required change for 3.19-rc1.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 external-module-compat-comm.h | 4 ++++
 linux                         | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/external-module-compat-comm.h b/external-module-compat-comm.h
index f0e1ce2..66cfde2 100644
--- a/external-module-compat-comm.h
+++ b/external-module-compat-comm.h
@@ -1452,3 +1452,7 @@ static inline void pci_set_dev_assigned(struct pci_dev *pdev)
 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,17,0)
 void *get_xsave_addr(struct xsave_struct *xsave, int feature);
 #endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,19,0)
+#define trace_seq_buffer_ptr(p) ((p)->buffer + (p)->len)
+#endif
diff --git a/linux b/linux
index b2776bf..97bf6af 160000
--- a/linux
+++ b/linux
@@ -1 +1 @@
-Subproject commit b2776bf7149bddd1f4161f14f79520f17fc1d71d
+Subproject commit 97bf6af1f928216fd6c5a66e8a57bfa95a659672
-- 
1.8.3.1

Re: [PATCH 1/4] stubs for xsavec support

[Jan Kiszka]

On 2015-01-08 11:32, Paolo Bonzini wrote:
> These are needed for KVM changes in 3.18.
> 
> Recent kernels added a separate feature word for XSAVE features, and KVM's
> CPUID code is relying on the new definition.  Except for cpu_has_xsaves,
> it's never accessing the feature itself: wrap cpu_has_xsaves with
> kvm_cpu_has_xsaves, and then there is no problem with out-of-bounds
> accesses.
> 
> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
> ---
>  external-module-compat-comm.h |  4 ++++
>  external-module-compat.c      | 11 +++++++++++
>  sync                          | 14 ++++++++++++--
>  x86/external-module-compat.h  | 37 +++++++++++++++++++++++++++++++++++++
>  4 files changed, 64 insertions(+), 2 deletions(-)
> 

...

> diff --git a/x86/external-module-compat.h b/x86/external-module-compat.h
> index dec53b6..87cf76a 100644
> --- a/x86/external-module-compat.h
> +++ b/x86/external-module-compat.h
> @@ -428,6 +428,23 @@ static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
>  #define X86_FEATURE_MPX		(9*32+14) /* Memory Protection Extension */
>  #endif
>  
> +#if X86_FEATURE_XSAVEOPT < 10 * 32
> +#undef X86_FEATURE_XSAVEOPT
> +#endif
> +#define X86_FEATURE_XSAVEOPT	(10*32+0) /* XSAVEOPT instruction */

This causes redefinition warnings if the condition is not met. Was the
plan to put the define before the #endif?

Jan

-- 
Siemens AG, Corporate Technology, CT RTC ITP SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH 1/4] stubs for xsavec support

[Paolo Bonzini]

On 02/02/2015 08:04, Jan Kiszka wrote:
>> > +#if X86_FEATURE_XSAVEOPT < 10 * 32
>> > +#undef X86_FEATURE_XSAVEOPT
>> > +#endif
>> > +#define X86_FEATURE_XSAVEOPT	(10*32+0) /* XSAVEOPT instruction */
> This causes redefinition warnings if the condition is not met. Was the
> plan to put the define before the #endif?

The plan was to match the kernel's definition, which however has a space:

#define X86_FEATURE_XSAVEOPT	(10*32+ 0)

But putting the define before the #endif also works.

Paolo

Re: [PATCH 1/4] stubs for xsavec support

[Jan Kiszka]

On 2015-02-02 10:05, Paolo Bonzini wrote:
> 
> 
> On 02/02/2015 08:04, Jan Kiszka wrote:
>>>> +#if X86_FEATURE_XSAVEOPT < 10 * 32
>>>> +#undef X86_FEATURE_XSAVEOPT
>>>> +#endif
>>>> +#define X86_FEATURE_XSAVEOPT	(10*32+0) /* XSAVEOPT instruction */
>> This causes redefinition warnings if the condition is not met. Was the
>> plan to put the define before the #endif?
> 
> The plan was to match the kernel's definition, which however has a space:
> 
> #define X86_FEATURE_XSAVEOPT	(10*32+ 0)
> 
> But putting the define before the #endif also works.

Finally got my buildbot working again: There are some open issues, maybe
you can have a look at http://buildbot.kiszka.org/kvm-kmod/waterfall,
next branch. At least the 3.17 thing requires fixing but may require
more than a #define.

Thanks,
Jan

-- 
Siemens AG, Corporate Technology, CT RTC ITP SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH 1/4] stubs for xsavec support

[Paolo Bonzini]

On 06/02/2015 20:40, Jan Kiszka wrote:
> On 2015-02-02 10:05, Paolo Bonzini wrote:
>>
>>
>> On 02/02/2015 08:04, Jan Kiszka wrote:
>>>>> +#if X86_FEATURE_XSAVEOPT < 10 * 32
>>>>> +#undef X86_FEATURE_XSAVEOPT
>>>>> +#endif
>>>>> +#define X86_FEATURE_XSAVEOPT	(10*32+0) /* XSAVEOPT instruction */
>>> This causes redefinition warnings if the condition is not met. Was the
>>> plan to put the define before the #endif?
>>
>> The plan was to match the kernel's definition, which however has a space:
>>
>> #define X86_FEATURE_XSAVEOPT	(10*32+ 0)
>>
>> But putting the define before the #endif also works.
> 
> Finally got my buildbot working again: There are some open issues, maybe
> you can have a look at http://buildbot.kiszka.org/kvm-kmod/waterfall,
> next branch. At least the 3.17 thing requires fixing but may require
> more than a #define.

Why is APICv crippled in kvm-kmod even for newer kernels?

If all that's needed is "make it compile with 3.9 and earlier",
you can do something like this:

diff --git a/sync b/sync
index 3086b70..36bed39 100755
--- a/sync
+++ b/sync
@@ -341,13 +341,18 @@ def hack_content(fname, data):
         if line == '\tif (!cpu_has_vmx_apicv())':
             w('#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)')
             w(line)
+            w('#else')
+            w('if (1)')
             line = '#endif'
         if line == '#if IS_ENABLED(CONFIG_KVM)':
             line = '#if 1'
-        if line == '\t\tapic->send_IPI_mask(get_cpu_mask(vcpu->cpu),':
-            line = '\t\t;'
-        if line == '\t\t\t\tPOSTED_INTR_VECTOR);':
-            line = ''
+        if match(r'^\t+apic->send_IPI_mask\(get_cpu_mask\(vcpu->cpu\),$'):
+            w('#if LINUX_VERSION_CODE < KERNEL_VERSION(3,10,0)')
+	    w('\t\t;')
+	    w('#else')
+        if match(r'^\t+POSTED_INTR_VECTOR\);$'):
+            w(line)
+            line = '#endif'
         if line == '\tif (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))':
             w('#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)')
             w(line)

Paolo

Re: [PATCH 1/4] stubs for xsavec support

[Jan Kiszka]

On 2015-02-09 14:17, Paolo Bonzini wrote:
> 
> 
> On 06/02/2015 20:40, Jan Kiszka wrote:
>> On 2015-02-02 10:05, Paolo Bonzini wrote:
>>>
>>>
>>> On 02/02/2015 08:04, Jan Kiszka wrote:
>>>>>> +#if X86_FEATURE_XSAVEOPT < 10 * 32
>>>>>> +#undef X86_FEATURE_XSAVEOPT
>>>>>> +#endif
>>>>>> +#define X86_FEATURE_XSAVEOPT	(10*32+0) /* XSAVEOPT instruction */
>>>> This causes redefinition warnings if the condition is not met. Was the
>>>> plan to put the define before the #endif?
>>>
>>> The plan was to match the kernel's definition, which however has a space:
>>>
>>> #define X86_FEATURE_XSAVEOPT	(10*32+ 0)
>>>
>>> But putting the define before the #endif also works.
>>
>> Finally got my buildbot working again: There are some open issues, maybe
>> you can have a look at http://buildbot.kiszka.org/kvm-kmod/waterfall,
>> next branch. At least the 3.17 thing requires fixing but may require
>> more than a #define.
> 
> Why is APICv crippled in kvm-kmod even for newer kernels?

No idea. Maybe the patcher runs amok? I didn't look into details yet.

> 
> If all that's needed is "make it compile with 3.9 and earlier",
> you can do something like this:
> 
> diff --git a/sync b/sync
> index 3086b70..36bed39 100755
> --- a/sync
> +++ b/sync
> @@ -341,13 +341,18 @@ def hack_content(fname, data):
>          if line == '\tif (!cpu_has_vmx_apicv())':
>              w('#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)')
>              w(line)
> +            w('#else')
> +            w('if (1)')
>              line = '#endif'
>          if line == '#if IS_ENABLED(CONFIG_KVM)':
>              line = '#if 1'
> -        if line == '\t\tapic->send_IPI_mask(get_cpu_mask(vcpu->cpu),':
> -            line = '\t\t;'
> -        if line == '\t\t\t\tPOSTED_INTR_VECTOR);':
> -            line = ''
> +        if match(r'^\t+apic->send_IPI_mask\(get_cpu_mask\(vcpu->cpu\),$'):
> +            w('#if LINUX_VERSION_CODE < KERNEL_VERSION(3,10,0)')
> +	    w('\t\t;')
> +	    w('#else')
> +        if match(r'^\t+POSTED_INTR_VECTOR\);$'):
> +            w(line)
> +            line = '#endif'
>          if line == '\tif (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))':
>              w('#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)')
>              w(line)

Tested already? If so, please provide it as patch. If not: I can run it
later through buildbot here.

Jan

-- 
Siemens AG, Corporate Technology, CT RTC ITP SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH 1/4] stubs for xsavec support

[Paolo Bonzini]

On 09/02/2015 14:44, Jan Kiszka wrote:
> Tested already? If so, please provide it as patch.

Tested but not with APICv, so it's not really worthy except as a compile
check.

Paolo

> If not: I can run it later through buildbot here.