[PATCH 0/6] x86/kvm: add boot parameters for max vcpu configs

[PATCH 0/6] x86/kvm: add boot parameters for max vcpu configs

[Juergen Gross]

In order to be able to have a single kernel for supporting even huge
numbers of vcpus per guest some arrays should be sized dynamically.

The easiest way to do that is to add boot parameters for the maximum
number of vcpus and the highest supported vcpu-id overwriting the
normal default.

This patch series is doing that for x86. The same scheme can be easily
adapted to other architectures, but I don't want to do that in the
first iteration.

In the long term I'd suggest to have a per-guest setting of the two
parameters allowing to spare some memory for smaller guests. OTOH this
would require new ioctl()s and respective qemu modifications, so I let
those away for now.

I've tested the series not to break normal guest operation and the new
parameters to be effective on x86. For Arm64 I did a compile test only.

Juergen Gross (6):
  x86/kvm: fix vcpu-id indexed array sizes
  x86/kvm: remove non-x86 stuff from arch/x86/kvm/ioapic.h
  x86/kvm: add boot parameter for maximum vcpu-id
  x86/kvm: introduce per cpu vcpu masks
  kvm: allocate vcpu pointer array separately
  x86/kvm: add boot parameter for setting max number of vcpus per guest

 .../admin-guide/kernel-parameters.txt         | 18 +++++++
 arch/arm64/kvm/arm.c                          | 28 +++++++++--
 arch/x86/include/asm/kvm_host.h               | 22 ++++++---
 arch/x86/kvm/hyperv.c                         | 25 +++++++---
 arch/x86/kvm/ioapic.c                         | 14 +++++-
 arch/x86/kvm/ioapic.h                         |  8 +--
 arch/x86/kvm/irq_comm.c                       |  9 +++-
 arch/x86/kvm/x86.c                            | 49 ++++++++++++++++++-
 include/linux/kvm_host.h                      | 17 ++++++-
 9 files changed, 160 insertions(+), 30 deletions(-)

-- 
2.26.2


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH 5/6] kvm: allocate vcpu pointer array separately

[Juergen Gross]

Prepare support of very large vcpu numbers per guest by moving the
vcpu pointer array out of struct kvm.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 arch/arm64/kvm/arm.c            | 28 ++++++++++++++++++++++++----
 arch/x86/include/asm/kvm_host.h |  5 +----
 arch/x86/kvm/x86.c              | 19 +++++++++++++++++++
 include/linux/kvm_host.h        | 17 +++++++++++++++--
 4 files changed, 59 insertions(+), 10 deletions(-)

diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index e720148232a0..4f055408fe9f 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -280,18 +280,38 @@ long kvm_arch_dev_ioctl(struct file *filp,
 
 struct kvm *kvm_arch_alloc_vm(void)
 {
+	struct kvm *kvm;
+
 	if (!has_vhe())
-		return kzalloc(sizeof(struct kvm), GFP_KERNEL);
+		kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
+	else
+		kvm = vzalloc(sizeof(struct kvm));
 
-	return vzalloc(sizeof(struct kvm));
+	if (!kvm)
+		return NULL;
+
+	if (!has_vhe())
+		kvm->vcpus = kcalloc(KVM_MAX_VCPUS, sizeof(void *), GFP_KERNEL);
+	else
+		kvm->vcpus = vzalloc(KVM_MAX_VCPUS * sizeof(void *));
+
+	if (!kvm->vcpus) {
+		kvm_arch_free_vm(kvm);
+		kvm = NULL;
+	}
+
+	return kvm;
 }
 
 void kvm_arch_free_vm(struct kvm *kvm)
 {
-	if (!has_vhe())
+	if (!has_vhe()) {
+		kfree(kvm->vcpus);
 		kfree(kvm);
-	else
+	} else {
+		vfree(kvm->vcpus);
 		vfree(kvm);
+	}
 }
 
 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 79138c91f83d..39cbc4b6bffb 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1440,10 +1440,7 @@ static inline void kvm_ops_static_call_update(void)
 }
 
 #define __KVM_HAVE_ARCH_VM_ALLOC
-static inline struct kvm *kvm_arch_alloc_vm(void)
-{
-	return __vmalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
-}
+struct kvm *kvm_arch_alloc_vm(void);
 void kvm_arch_free_vm(struct kvm *kvm);
 
 #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 3af398ef1fc9..a9b0bb2221ea 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -10741,9 +10741,28 @@ void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
 	static_call(kvm_x86_sched_in)(vcpu, cpu);
 }
 
+struct kvm *kvm_arch_alloc_vm(void)
+{
+	struct kvm *kvm;
+
+	kvm = __vmalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+	if (!kvm)
+		return NULL;
+
+	kvm->vcpus = __vmalloc(KVM_MAX_VCPUS * sizeof(void *),
+			       GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+	if (!kvm->vcpus) {
+		vfree(kvm);
+		kvm = NULL;
+	}
+
+	return kvm;
+}
+
 void kvm_arch_free_vm(struct kvm *kvm)
 {
 	kfree(to_kvm_hv(kvm)->hv_pa_pg);
+	vfree(kvm->vcpus);
 	vfree(kvm);
 }
 
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 8583ed3ff344..e424ef1078a1 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -525,7 +525,7 @@ struct kvm {
 	struct mutex slots_lock;
 	struct mm_struct *mm; /* userspace tied to this vm */
 	struct kvm_memslots __rcu *memslots[KVM_ADDRESS_SPACE_NUM];
-	struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
+	struct kvm_vcpu **vcpus;
 
 	/*
 	 * created_vcpus is protected by kvm->lock, and is incremented
@@ -1022,11 +1022,24 @@ void kvm_arch_pre_destroy_vm(struct kvm *kvm);
  */
 static inline struct kvm *kvm_arch_alloc_vm(void)
 {
-	return kzalloc(sizeof(struct kvm), GFP_KERNEL);
+	struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
+
+	if (!kvm)
+		return NULL;
+
+	kvm->vcpus = kcalloc(KVM_MAX_VCPUS, sizeof(void *), GFP_KERNEL);
+	if (!kvm->vcpus) {
+		kfree(kvm);
+		kvm = NULL;
+	}
+
+	return kvm;
 }
 
 static inline void kvm_arch_free_vm(struct kvm *kvm)
 {
+	if (kvm)
+		kfree(kvm->vcpus);
 	kfree(kvm);
 }
 #endif
-- 
2.26.2


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH 5/6] kvm: allocate vcpu pointer array separately

[Paolo Bonzini]

On 01/07/21 17:41, Juergen Gross wrote:
>   {
> -	if (!has_vhe())
> +	if (!has_vhe()) {
> +		kfree(kvm->vcpus);
>   		kfree(kvm);
> -	else
> +	} else {
> +		vfree(kvm->vcpus);
>   		vfree(kvm);
> +	}
>   }
>   
>   int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
> diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
> index 79138c91f83d..39cbc4b6bffb 100644
> --- a/arch/x86/include/asm/kvm_host.h
> +++ b/arch/x86/include/asm/kvm_host.h
> @@ -1440,10 +1440,7 @@ static inline void kvm_ops_static_call_update(void)
>   }
>   
>   #define __KVM_HAVE_ARCH_VM_ALLOC
> -static inline struct kvm *kvm_arch_alloc_vm(void)
> -{
> -	return __vmalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
> -}
> +struct kvm *kvm_arch_alloc_vm(void);
>   void kvm_arch_free_vm(struct kvm *kvm);
>   
>   #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index 3af398ef1fc9..a9b0bb2221ea 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -10741,9 +10741,28 @@ void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
>   	static_call(kvm_x86_sched_in)(vcpu, cpu);
>   }
>   
> +struct kvm *kvm_arch_alloc_vm(void)
> +{
> +	struct kvm *kvm;
> +
> +	kvm = __vmalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
> +	if (!kvm)
> +		return NULL;
> +
> +	kvm->vcpus = __vmalloc(KVM_MAX_VCPUS * sizeof(void *),
> +			       GFP_KERNEL_ACCOUNT | __GFP_ZERO);
> +	if (!kvm->vcpus) {
> +		vfree(kvm);
> +		kvm = NULL;
> +	}
> +

Let's keep this cleaner:

1) use kvfree in the common version of kvm_arch_free_vm

2) split __KVM_HAVE_ARCH_VM_ALLOC and __KVM_HAVE_ARCH_VM_FREE (ARM does 
not need it once kvfree is used)

3) define a __kvm_arch_free_vm version that is defined even if 
!__KVM_HAVE_ARCH_VM_FREE, and which can be used on x86.

Paolo


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH 0/6] x86/kvm: add boot parameters for max vcpu configs

[Paolo Bonzini]

On 01/07/21 17:40, Juergen Gross wrote:
> In order to be able to have a single kernel for supporting even huge
> numbers of vcpus per guest some arrays should be sized dynamically.
> 
> The easiest way to do that is to add boot parameters for the maximum
> number of vcpus and the highest supported vcpu-id overwriting the
> normal default.
> 
> This patch series is doing that for x86. The same scheme can be easily
> adapted to other architectures, but I don't want to do that in the
> first iteration.
> 
> In the long term I'd suggest to have a per-guest setting of the two
> parameters allowing to spare some memory for smaller guests. OTOH this
> would require new ioctl()s and respective qemu modifications, so I let
> those away for now.
> 
> I've tested the series not to break normal guest operation and the new
> parameters to be effective on x86. For Arm64 I did a compile test only.
> 
> Juergen Gross (6):
>    x86/kvm: fix vcpu-id indexed array sizes
>    x86/kvm: remove non-x86 stuff from arch/x86/kvm/ioapic.h
>    x86/kvm: add boot parameter for maximum vcpu-id
>    x86/kvm: introduce per cpu vcpu masks
>    kvm: allocate vcpu pointer array separately
>    x86/kvm: add boot parameter for setting max number of vcpus per guest
> 
>   .../admin-guide/kernel-parameters.txt         | 18 +++++++
>   arch/arm64/kvm/arm.c                          | 28 +++++++++--
>   arch/x86/include/asm/kvm_host.h               | 22 ++++++---
>   arch/x86/kvm/hyperv.c                         | 25 +++++++---
>   arch/x86/kvm/ioapic.c                         | 14 +++++-
>   arch/x86/kvm/ioapic.h                         |  8 +--
>   arch/x86/kvm/irq_comm.c                       |  9 +++-
>   arch/x86/kvm/x86.c                            | 49 ++++++++++++++++++-
>   include/linux/kvm_host.h                      | 17 ++++++-
>   9 files changed, 160 insertions(+), 30 deletions(-)
> 

Queued patches 1-2, thanks (1 for stable too).

Paolo


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH 5/6] kvm: allocate vcpu pointer array separately

[Juergen Gross]

[-- Attachment #1.1.1.1: Type: text/plain, Size: 2252 bytes --]

On 26.07.21 15:40, Paolo Bonzini wrote:
> On 01/07/21 17:41, Juergen Gross wrote:
>>   {
>> -    if (!has_vhe())
>> +    if (!has_vhe()) {
>> +        kfree(kvm->vcpus);
>>           kfree(kvm);
>> -    else
>> +    } else {
>> +        vfree(kvm->vcpus);
>>           vfree(kvm);
>> +    }
>>   }
>>   int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
>> diff --git a/arch/x86/include/asm/kvm_host.h 
>> b/arch/x86/include/asm/kvm_host.h
>> index 79138c91f83d..39cbc4b6bffb 100644
>> --- a/arch/x86/include/asm/kvm_host.h
>> +++ b/arch/x86/include/asm/kvm_host.h
>> @@ -1440,10 +1440,7 @@ static inline void 
>> kvm_ops_static_call_update(void)
>>   }
>>   #define __KVM_HAVE_ARCH_VM_ALLOC
>> -static inline struct kvm *kvm_arch_alloc_vm(void)
>> -{
>> -    return __vmalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT | 
>> __GFP_ZERO);
>> -}
>> +struct kvm *kvm_arch_alloc_vm(void);
>>   void kvm_arch_free_vm(struct kvm *kvm);
>>   #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB
>> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
>> index 3af398ef1fc9..a9b0bb2221ea 100644
>> --- a/arch/x86/kvm/x86.c
>> +++ b/arch/x86/kvm/x86.c
>> @@ -10741,9 +10741,28 @@ void kvm_arch_sched_in(struct kvm_vcpu *vcpu, 
>> int cpu)
>>       static_call(kvm_x86_sched_in)(vcpu, cpu);
>>   }
>> +struct kvm *kvm_arch_alloc_vm(void)
>> +{
>> +    struct kvm *kvm;
>> +
>> +    kvm = __vmalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT | 
>> __GFP_ZERO);
>> +    if (!kvm)
>> +        return NULL;
>> +
>> +    kvm->vcpus = __vmalloc(KVM_MAX_VCPUS * sizeof(void *),
>> +                   GFP_KERNEL_ACCOUNT | __GFP_ZERO);
>> +    if (!kvm->vcpus) {
>> +        vfree(kvm);
>> +        kvm = NULL;
>> +    }
>> +
> 
> Let's keep this cleaner:
> 
> 1) use kvfree in the common version of kvm_arch_free_vm
> 
> 2) split __KVM_HAVE_ARCH_VM_ALLOC and __KVM_HAVE_ARCH_VM_FREE (ARM does 
> not need it once kvfree is used)
> 
> 3) define a __kvm_arch_free_vm version that is defined even if 
> !__KVM_HAVE_ARCH_VM_FREE, and which can be used on x86.

Okay, will do so.


Juergen

[-- Attachment #1.1.1.2: OpenPGP public key --]
[-- Type: application/pgp-keys, Size: 3135 bytes --]

[-- Attachment #1.2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 495 bytes --]

[-- Attachment #2: Type: text/plain, Size: 176 bytes --]

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH 5/6] kvm: allocate vcpu pointer array separately

[Marc Zyngier]

On 2021-07-26 14:46, Juergen Gross wrote:
> On 26.07.21 15:40, Paolo Bonzini wrote:
>> On 01/07/21 17:41, Juergen Gross wrote:
>>>   {
>>> -    if (!has_vhe())
>>> +    if (!has_vhe()) {
>>> +        kfree(kvm->vcpus);
>>>           kfree(kvm);
>>> -    else
>>> +    } else {
>>> +        vfree(kvm->vcpus);
>>>           vfree(kvm);
>>> +    }
>>>   }
>>>   int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
>>> diff --git a/arch/x86/include/asm/kvm_host.h 
>>> b/arch/x86/include/asm/kvm_host.h
>>> index 79138c91f83d..39cbc4b6bffb 100644
>>> --- a/arch/x86/include/asm/kvm_host.h
>>> +++ b/arch/x86/include/asm/kvm_host.h
>>> @@ -1440,10 +1440,7 @@ static inline void 
>>> kvm_ops_static_call_update(void)
>>>   }
>>>   #define __KVM_HAVE_ARCH_VM_ALLOC
>>> -static inline struct kvm *kvm_arch_alloc_vm(void)
>>> -{
>>> -    return __vmalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT | 
>>> __GFP_ZERO);
>>> -}
>>> +struct kvm *kvm_arch_alloc_vm(void);
>>>   void kvm_arch_free_vm(struct kvm *kvm);
>>>   #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB
>>> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
>>> index 3af398ef1fc9..a9b0bb2221ea 100644
>>> --- a/arch/x86/kvm/x86.c
>>> +++ b/arch/x86/kvm/x86.c
>>> @@ -10741,9 +10741,28 @@ void kvm_arch_sched_in(struct kvm_vcpu 
>>> *vcpu, int cpu)
>>>       static_call(kvm_x86_sched_in)(vcpu, cpu);
>>>   }
>>> +struct kvm *kvm_arch_alloc_vm(void)
>>> +{
>>> +    struct kvm *kvm;
>>> +
>>> +    kvm = __vmalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT | 
>>> __GFP_ZERO);
>>> +    if (!kvm)
>>> +        return NULL;
>>> +
>>> +    kvm->vcpus = __vmalloc(KVM_MAX_VCPUS * sizeof(void *),
>>> +                   GFP_KERNEL_ACCOUNT | __GFP_ZERO);
>>> +    if (!kvm->vcpus) {
>>> +        vfree(kvm);
>>> +        kvm = NULL;
>>> +    }
>>> +
>> 
>> Let's keep this cleaner:
>> 
>> 1) use kvfree in the common version of kvm_arch_free_vm
>> 
>> 2) split __KVM_HAVE_ARCH_VM_ALLOC and __KVM_HAVE_ARCH_VM_FREE (ARM 
>> does not need it once kvfree is used)
>> 
>> 3) define a __kvm_arch_free_vm version that is defined even if 
>> !__KVM_HAVE_ARCH_VM_FREE, and which can be used on x86.
> 
> Okay, will do so.

I'd appreciate if you could Cc me on the whole series, and
not just the single arm64 patch.

Thanks,

         M.
-- 
Jazz is not dead. It just smells funny...

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel