[PATCH v3 0/3] kvm: vmx: enable xsaves for kvm

[PATCH v3 0/3] kvm: vmx: enable xsaves for kvm

[Wanpeng Li]

This patchset is to enable xsaves for kvm part, the patch for 
qemu part will be sent out later. 

The patchset is tested on skylake-client.

v2 -> v3:
 * add kvm_get/set for ia32_xss
 * fix the type XSS_EXIT_BITMAP
 * load host_xss just once in setup_vmcs_config
 * add/clear atuo switch ia32_xss msr in kvm_get/clear
 * add VMX_XSS_EXIT_BITMAP macro
 * add WARN() in handle_xsaves/xrstors
 * export xsaves if related vmcs field is set 

v1 -> v2: 
 * auto switch ia32_xss msr just if this msr is present 

Wanpeng Li (3):
  kvm: x86: Intel XSAVES vmx and msr handle
  kvm: vmx: add kvm_get/set logic to xsaves
  kvm: x86: Enable Intel XSAVES for guest

 arch/x86/include/asm/kvm_host.h |  2 ++
 arch/x86/include/asm/vmx.h      |  3 +++
 arch/x86/include/uapi/asm/vmx.h |  6 ++++-
 arch/x86/kvm/cpuid.c            |  3 ++-
 arch/x86/kvm/vmx.c              | 51 ++++++++++++++++++++++++++++++++++++++++-
 5 files changed, 62 insertions(+), 3 deletions(-)

-- 
1.9.1

[PATCH v3 1/3] kvm: x86: Intel XSAVES vmx and msr handle

[Wanpeng Li]

Intel xsaves vmx and msr handle.

Signed-off-by: Wanpeng Li <wanpeng.li@linux.intel.com>
---
 arch/x86/include/asm/kvm_host.h |  2 ++
 arch/x86/include/asm/vmx.h      |  3 +++
 arch/x86/include/uapi/asm/vmx.h |  6 +++++-
 arch/x86/kvm/vmx.c              | 31 ++++++++++++++++++++++++++++++-
 4 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 2896dbc..0c4c88c 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -362,6 +362,7 @@ struct kvm_vcpu_arch {
 	int mp_state;
 	u64 ia32_misc_enable_msr;
 	bool tpr_access_reporting;
+	u64 ia32_xss;
 
 	/*
 	 * Paging state of the vcpu
@@ -771,6 +772,7 @@ struct kvm_x86_ops {
 			       enum x86_intercept_stage stage);
 	void (*handle_external_intr)(struct kvm_vcpu *vcpu);
 	bool (*mpx_supported)(void);
+	bool (*xsaves_supported)(void);
 
 	int (*check_nested_events)(struct kvm_vcpu *vcpu, bool external_intr);
 
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index bcbfade..45afaee 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -69,6 +69,7 @@
 #define SECONDARY_EXEC_PAUSE_LOOP_EXITING	0x00000400
 #define SECONDARY_EXEC_ENABLE_INVPCID		0x00001000
 #define SECONDARY_EXEC_SHADOW_VMCS              0x00004000
+#define SECONDARY_EXEC_XSAVES			0x00100000
 
 
 #define PIN_BASED_EXT_INTR_MASK                 0x00000001
@@ -159,6 +160,8 @@ enum vmcs_field {
 	EOI_EXIT_BITMAP3_HIGH           = 0x00002023,
 	VMREAD_BITMAP                   = 0x00002026,
 	VMWRITE_BITMAP                  = 0x00002028,
+	XSS_EXIT_BITMAP                 = 0x0000202C,
+	XSS_EXIT_BITMAP_HIGH            = 0x0000202D,
 	GUEST_PHYSICAL_ADDRESS          = 0x00002400,
 	GUEST_PHYSICAL_ADDRESS_HIGH     = 0x00002401,
 	VMCS_LINK_POINTER               = 0x00002800,
diff --git a/arch/x86/include/uapi/asm/vmx.h b/arch/x86/include/uapi/asm/vmx.h
index 990a2fe..b813bf9 100644
--- a/arch/x86/include/uapi/asm/vmx.h
+++ b/arch/x86/include/uapi/asm/vmx.h
@@ -72,6 +72,8 @@
 #define EXIT_REASON_XSETBV              55
 #define EXIT_REASON_APIC_WRITE          56
 #define EXIT_REASON_INVPCID             58
+#define EXIT_REASON_XSAVES              63
+#define EXIT_REASON_XRSTORS             64
 
 #define VMX_EXIT_REASONS \
 	{ EXIT_REASON_EXCEPTION_NMI,         "EXCEPTION_NMI" }, \
@@ -116,6 +118,8 @@
 	{ EXIT_REASON_INVALID_STATE,         "INVALID_STATE" }, \
 	{ EXIT_REASON_INVD,                  "INVD" }, \
 	{ EXIT_REASON_INVVPID,               "INVVPID" }, \
-	{ EXIT_REASON_INVPCID,               "INVPCID" }
+	{ EXIT_REASON_INVPCID,               "INVPCID" }, \
+	{ EXIT_REASON_XSAVES,                "XSAVES" }, \
+	{ EXIT_REASON_XRSTORS,               "XRSTORS" }
 
 #endif /* _UAPIVMX_H */
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 6a951d8..12915f1 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -761,6 +761,7 @@ static u64 construct_eptp(unsigned long root_hpa);
 static void kvm_cpu_vmxon(u64 addr);
 static void kvm_cpu_vmxoff(void);
 static bool vmx_mpx_supported(void);
+static bool vmx_xsaves_supported(void);
 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr);
 static void vmx_set_segment(struct kvm_vcpu *vcpu,
 			    struct kvm_segment *var, int seg);
@@ -2895,7 +2896,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
 			SECONDARY_EXEC_ENABLE_INVPCID |
 			SECONDARY_EXEC_APIC_REGISTER_VIRT |
 			SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
-			SECONDARY_EXEC_SHADOW_VMCS;
+			SECONDARY_EXEC_SHADOW_VMCS |
+			SECONDARY_EXEC_XSAVES;
 		if (adjust_vmx_controls(min2, opt2,
 					MSR_IA32_VMX_PROCBASED_CTLS2,
 					&_cpu_based_2nd_exec_control) < 0)
@@ -4337,6 +4339,7 @@ static void ept_set_mmio_spte_mask(void)
 	kvm_mmu_set_mmio_spte_mask((0x3ull << 62) | 0x6ull);
 }
 
+#define VMX_XSS_EXIT_BITMAP 0
 /*
  * Sets up the vmcs for emulated real mode.
  */
@@ -4446,6 +4449,9 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
 	vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
 	set_cr4_guest_host_mask(vmx);
 
+	if (vmx_xsaves_supported())
+		vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP);
+
 	return 0;
 }
 
@@ -5334,6 +5340,20 @@ static int handle_xsetbv(struct kvm_vcpu *vcpu)
 	return 1;
 }
 
+static int handle_xsaves(struct kvm_vcpu *vcpu)
+{
+	skip_emulated_instruction(vcpu);
+	WARN(1, "this should never happen\n");
+	return 1;
+}
+
+static int handle_xrstors(struct kvm_vcpu *vcpu)
+{
+	skip_emulated_instruction(vcpu);
+	WARN(1, "this should never happen\n");
+	return 1;
+}
+
 static int handle_apic_access(struct kvm_vcpu *vcpu)
 {
 	if (likely(fasteoi)) {
@@ -6951,6 +6971,8 @@ static int (*const kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
 	[EXIT_REASON_MONITOR_INSTRUCTION]     = handle_monitor,
 	[EXIT_REASON_INVEPT]                  = handle_invept,
 	[EXIT_REASON_INVVPID]                 = handle_invvpid,
+	[EXIT_REASON_XSAVES]                  = handle_xsaves,
+	[EXIT_REASON_XRSTORS]                 = handle_xrstors,
 };
 
 static const int kvm_vmx_max_exit_handlers =
@@ -7553,6 +7575,12 @@ static bool vmx_mpx_supported(void)
 		(vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_BNDCFGS);
 }
 
+static bool vmx_xsaves_supported(void)
+{
+	return vmcs_config.cpu_based_2nd_exec_ctrl &
+		SECONDARY_EXEC_XSAVES;
+}
+
 static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
 {
 	u32 exit_intr_info;
@@ -9329,6 +9357,7 @@ static struct kvm_x86_ops vmx_x86_ops = {
 	.check_intercept = vmx_check_intercept,
 	.handle_external_intr = vmx_handle_external_intr,
 	.mpx_supported = vmx_mpx_supported,
+	.xsaves_supported = vmx_xsaves_supported,
 
 	.check_nested_events = vmx_check_nested_events,
 
-- 
1.9.1

[PATCH v3 2/3] kvm: vmx: add kvm_get/set logic to xsaves

[Wanpeng Li]

Add kvm_get/set logic to xsaves. 

Signed-off-by: Wanpeng Li <wanpeng.li@linux.intel.com>
---
 arch/x86/kvm/vmx.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 12915f1..66d1e3d 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -99,6 +99,8 @@ module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
 static bool __read_mostly nested = 0;
 module_param(nested, bool, S_IRUGO);
 
+static u64 __read_mostly host_xss;
+
 #define KVM_GUEST_CR0_MASK (X86_CR0_NW | X86_CR0_CD)
 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST (X86_CR0_WP | X86_CR0_NE)
 #define KVM_VM_CR0_ALWAYS_ON						\
@@ -2570,6 +2572,11 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
 		if (!nested_vmx_allowed(vcpu))
 			return 1;
 		return vmx_get_vmx_msr(vcpu, msr_index, pdata);
+	case MSR_IA32_XSS:
+		if (!vmx_xsaves_supported())
+			return 1;
+		data = vcpu->arch.ia32_xss;
+		break;
 	case MSR_TSC_AUX:
 		if (!to_vmx(vcpu)->rdtscp_enabled)
 			return 1;
@@ -2661,6 +2668,16 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		break;
 	case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
 		return 1; /* they are read-only */
+	case MSR_IA32_XSS:
+		if (!vmx_xsaves_supported())
+			return 1;
+		vcpu->arch.ia32_xss = data;
+		if (vcpu->arch.ia32_xss != host_xss)
+			add_atomic_switch_msr(vmx, MSR_IA32_XSS,
+				vcpu->arch.ia32_xss, host_xss);
+		else
+			clear_atomic_switch_msr(vmx, MSR_IA32_XSS);
+		break;
 	case MSR_TSC_AUX:
 		if (!vmx->rdtscp_enabled)
 			return 1;
@@ -3020,6 +3037,9 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
 		}
 	}
 
+	if (cpu_has_xsaves)
+		rdmsrl(MSR_IA32_XSS, host_xss);
+
 	return 0;
 }
 
-- 
1.9.1

Re: [PATCH v3 0/3] kvm: vmx: enable xsaves for kvm

[Paolo Bonzini]

On 02/12/2014 12:14, Wanpeng Li wrote:
> This patchset is to enable xsaves for kvm part, the patch for 
> qemu part will be sent out later. 
> 
> The patchset is tested on skylake-client.
> 
> v2 -> v3:
>  * add kvm_get/set for ia32_xss
>  * fix the type XSS_EXIT_BITMAP
>  * load host_xss just once in setup_vmcs_config
>  * add/clear atuo switch ia32_xss msr in kvm_get/clear
>  * add VMX_XSS_EXIT_BITMAP macro
>  * add WARN() in handle_xsaves/xrstors
>  * export xsaves if related vmcs field is set 
> 
> v1 -> v2: 
>  * auto switch ia32_xss msr just if this msr is present 
> 
> Wanpeng Li (3):
>   kvm: x86: Intel XSAVES vmx and msr handle
>   kvm: vmx: add kvm_get/set logic to xsaves
>   kvm: x86: Enable Intel XSAVES for guest
> 
>  arch/x86/include/asm/kvm_host.h |  2 ++
>  arch/x86/include/asm/vmx.h      |  3 +++
>  arch/x86/include/uapi/asm/vmx.h |  6 ++++-
>  arch/x86/kvm/cpuid.c            |  3 ++-
>  arch/x86/kvm/vmx.c              | 51 ++++++++++++++++++++++++++++++++++++++++-
>  5 files changed, 62 insertions(+), 3 deletions(-)
> 

We need to return false from an svm_xsaves_supported function too, and 
we need to prevent setting MSR_IA32_XSS to any non-zero value because 
we do not support getting/setting a guest's Trace Packet Configuration 
State.

I will squash this hunk in, and send a new patch to return zero for
CPUID(0xd,i).ECX and CPUID(0xd,i).EDX.

diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 66d1e3d0195e..6e3a4486749c 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -2671,6 +2671,11 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_IA32_XSS:
 		if (!vmx_xsaves_supported())
 			return 1;
+		/* The only supported bit as of Skylake is bit 8, but
+		 * it is not supported on KVM.
+		 */
+		if (data != 0)
+			return 1;
 		vcpu->arch.ia32_xss = data;
 		if (vcpu->arch.ia32_xss != host_xss)
 			add_atomic_switch_msr(vmx, MSR_IA32_XSS,

Paolo

Re: [PATCH v3 0/3] kvm: vmx: enable xsaves for kvm

[Wanpeng Li]

On Tue, Dec 02, 2014 at 02:00:24PM +0100, Paolo Bonzini wrote:
>
>
>On 02/12/2014 12:14, Wanpeng Li wrote:
>> This patchset is to enable xsaves for kvm part, the patch for 
>> qemu part will be sent out later. 
>> 
>> The patchset is tested on skylake-client.
>> 
>> v2 -> v3:
>>  * add kvm_get/set for ia32_xss
>>  * fix the type XSS_EXIT_BITMAP
>>  * load host_xss just once in setup_vmcs_config
>>  * add/clear atuo switch ia32_xss msr in kvm_get/clear
>>  * add VMX_XSS_EXIT_BITMAP macro
>>  * add WARN() in handle_xsaves/xrstors
>>  * export xsaves if related vmcs field is set 
>> 
>> v1 -> v2: 
>>  * auto switch ia32_xss msr just if this msr is present 
>> 
>> Wanpeng Li (3):
>>   kvm: x86: Intel XSAVES vmx and msr handle
>>   kvm: vmx: add kvm_get/set logic to xsaves
>>   kvm: x86: Enable Intel XSAVES for guest
>> 
>>  arch/x86/include/asm/kvm_host.h |  2 ++
>>  arch/x86/include/asm/vmx.h      |  3 +++
>>  arch/x86/include/uapi/asm/vmx.h |  6 ++++-
>>  arch/x86/kvm/cpuid.c            |  3 ++-
>>  arch/x86/kvm/vmx.c              | 51 ++++++++++++++++++++++++++++++++++++++++-
>>  5 files changed, 62 insertions(+), 3 deletions(-)
>> 
>
>We need to return false from an svm_xsaves_supported function too, and 
>we need to prevent setting MSR_IA32_XSS to any non-zero value because 
>we do not support getting/setting a guest's Trace Packet Configuration 
>State.
>

Agreed, I will fix them in next version.

>I will squash this hunk in, and send a new patch to return zero for
>CPUID(0xd,i).ECX and CPUID(0xd,i).EDX.

Cool, I will test it.

Regards,
Wanpeng Li 

>
>diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
>index 66d1e3d0195e..6e3a4486749c 100644
>--- a/arch/x86/kvm/vmx.c
>+++ b/arch/x86/kvm/vmx.c
>@@ -2671,6 +2671,11 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> 	case MSR_IA32_XSS:
> 		if (!vmx_xsaves_supported())
> 			return 1;
>+		/* The only supported bit as of Skylake is bit 8, but
>+		 * it is not supported on KVM.
>+		 */
>+		if (data != 0)
>+			return 1;
> 		vcpu->arch.ia32_xss = data;
> 		if (vcpu->arch.ia32_xss != host_xss)
> 			add_atomic_switch_msr(vmx, MSR_IA32_XSS,
>
>Paolo