Re: [PATCH 2/3] KVM: VMX: Move RESET emulation to vmx_vcpu_reset()

[Vitaly Kuznetsov <vkuznets@redhat.com>]

Sean Christopherson <seanjc@google.com> writes:

> Move vCPU RESET emulation, including initializating of select VMCS state,
> to vmx_vcpu_reset().  Drop the open coded "vCPU load" sequence, as
> ->vcpu_reset() is invoked while the vCPU is properly loaded (which is
> kind of the point of ->vcpu_reset()...).  Hopefully KVM will someday
> expose a dedicated RESET ioctl(), and in the meantime separating "create"
> from "RESET" is a nice cleanup.
>
> Deferring VMCS initialization is effectively a nop as it's impossible to
> safely access the VMCS between the current call site and its new home, as
> both the vCPU and the pCPU are put immediately after init_vmcs(), i.e.
> the VMCS isn't guaranteed to be loaded.
>
> Note, task preemption is not a problem as vmx_sched_in() _can't_ touch
> the VMCS as ->sched_in() is invoked before the vCPU, and thus VMCS, is
> reloaded.  I.e. the preemption path also can't consume VMCS state.
>
> Cc: Reiji Watanabe <reijiw@google.com>
> Signed-off-by: Sean Christopherson <seanjc@google.com>
> ---
>  arch/x86/kvm/vmx/vmx.c | 61 +++++++++++++++++++++---------------------
>  1 file changed, 31 insertions(+), 30 deletions(-)
>
> diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
> index dc274b4c9912..629427cf8f4e 100644
> --- a/arch/x86/kvm/vmx/vmx.c
> +++ b/arch/x86/kvm/vmx/vmx.c
> @@ -4327,10 +4327,6 @@ static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx)
>  
>  #define VMX_XSS_EXIT_BITMAP 0
>  
> -/*
> - * Noting that the initialization of Guest-state Area of VMCS is in
> - * vmx_vcpu_reset().
> - */
>  static void init_vmcs(struct vcpu_vmx *vmx)
>  {
>  	if (nested)
> @@ -4435,10 +4431,39 @@ static void init_vmcs(struct vcpu_vmx *vmx)
>  	vmx_setup_uret_msrs(vmx);
>  }
>  
> +static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu)
> +{
> +	struct vcpu_vmx *vmx = to_vmx(vcpu);
> +
> +	init_vmcs(vmx);
> +
> +	if (nested)
> +		memcpy(&vmx->nested.msrs, &vmcs_config.nested, sizeof(vmx->nested.msrs));
> +
> +	vcpu_setup_sgx_lepubkeyhash(vcpu);
> +
> +	vmx->nested.posted_intr_nv = -1;
> +	vmx->nested.current_vmptr = -1ull;
> +	vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID;

What would happen in (hypothetical) case when enlightened VMCS is
currently in use? If we zap 'hv_evmcs_vmptr' here, the consequent
nested_release_evmcs() (called from
nested_vmx_handle_enlightened_vmptrld(), for example) will not do 
kvm_vcpu_unmap() while it should.

This, however, got me thinking: should we free all-things-nested with
free_nested()/nested_vmx_free_vcpu() upon vcpu reset? I can't seem to
find us doing that... (I do remember that INIT is blocked in VMX-root
mode and nobody else besides kvm_arch_vcpu_create()/
kvm_apic_accept_events() seems to call kvm_vcpu_reset()) but maybe we
should at least add a WARN_ON() guardian here?

Side topic: we don't seem to reset Hyper-V specific MSRs either,
probably relying on userspace VMM doing the right thing but this is also
not ideal I believe.

> +
> +	vcpu->arch.microcode_version = 0x100000000ULL;
> +	vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED;
> +
> +	/*
> +	 * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR
> +	 * or POSTED_INTR_WAKEUP_VECTOR.
> +	 */
> +	vmx->pi_desc.nv = POSTED_INTR_VECTOR;
> +	vmx->pi_desc.sn = 1;
> +}
> +
>  static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
>  {
>  	struct vcpu_vmx *vmx = to_vmx(vcpu);
>  
> +	if (!init_event)
> +		__vmx_vcpu_reset(vcpu);
> +
>  	vmx->rmode.vm86_active = 0;
>  	vmx->spec_ctrl = 0;
>  
> @@ -6797,7 +6822,7 @@ static int vmx_create_vcpu(struct kvm_vcpu *vcpu)
>  {
>  	struct vmx_uret_msr *tsx_ctrl;
>  	struct vcpu_vmx *vmx;
> -	int i, cpu, err;
> +	int i, err;
>  
>  	BUILD_BUG_ON(offsetof(struct vcpu_vmx, vcpu) != 0);
>  	vmx = to_vmx(vcpu);
> @@ -6856,12 +6881,7 @@ static int vmx_create_vcpu(struct kvm_vcpu *vcpu)
>  	}
>  
>  	vmx->loaded_vmcs = &vmx->vmcs01;
> -	cpu = get_cpu();
> -	vmx_vcpu_load(vcpu, cpu);
> -	vcpu->cpu = cpu;
> -	init_vmcs(vmx);
> -	vmx_vcpu_put(vcpu);
> -	put_cpu();
> +
>  	if (cpu_need_virtualize_apic_accesses(vcpu)) {
>  		err = alloc_apic_access_page(vcpu->kvm);
>  		if (err)
> @@ -6874,25 +6894,6 @@ static int vmx_create_vcpu(struct kvm_vcpu *vcpu)
>  			goto free_vmcs;
>  	}
>  
> -	if (nested)
> -		memcpy(&vmx->nested.msrs, &vmcs_config.nested, sizeof(vmx->nested.msrs));
> -
> -	vcpu_setup_sgx_lepubkeyhash(vcpu);
> -
> -	vmx->nested.posted_intr_nv = -1;
> -	vmx->nested.current_vmptr = -1ull;
> -	vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID;
> -
> -	vcpu->arch.microcode_version = 0x100000000ULL;
> -	vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED;
> -
> -	/*
> -	 * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR
> -	 * or POSTED_INTR_WAKEUP_VECTOR.
> -	 */
> -	vmx->pi_desc.nv = POSTED_INTR_VECTOR;
> -	vmx->pi_desc.sn = 1;
> -
>  	return 0;
>  
>  free_vmcs:

-- 
Vitaly