[PATCH] pc: q35: Bump max_cpus to 4096

[PATCH] pc: q35: Bump max_cpus to 4096

[Ani Sinha]

Since commit f10a570b093e6 ("KVM: x86: Add CONFIG_KVM_MAX_NR_VCPUS to allow up to 4096 vCPUs")
Linux kernel can support upto a maximum number of 4096 vCPUS when MAXSMP is
enabled in the kernel. So bump up the max_cpus value for q35 machines versions
8.3 and newer to 4096. Older q35 machines versions 8.2 and older continue to
support 1024 maximum vcpus as before.

If KVM is not able to support the specified number of vcpus, QEMU would
return the following error messages:

$ ./qemu-system-x86_64 -cpu host -accel kvm -machine q35 -smp 4096
qemu-system-x86_64: -accel kvm: warning: Number of SMP cpus requested (4096) exceeds the recommended cpus supported by KVM (12)
Number of SMP cpus requested (4096) exceeds the maximum cpus supported by KVM (1024)

Cc: Daniel P. Berrangé <berrange@redhat.com>
Cc: Igor Mammedov <imammedo@redhat.com>
Cc: Michael S. Tsirkin <mst@redhat.com>
Cc: Julia Suvorova <jusual@redhat.com>
Signed-off-by: Ani Sinha <anisinha@redhat.com>
---
 hw/i386/pc_q35.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
index 4f3e5412f6..2ed57814e1 100644
--- a/hw/i386/pc_q35.c
+++ b/hw/i386/pc_q35.c
@@ -375,7 +375,7 @@ static void pc_q35_machine_options(MachineClass *m)
     m->default_nic = "e1000e";
     m->default_kernel_irqchip_split = false;
     m->no_floppy = 1;
-    m->max_cpus = 1024;
+    m->max_cpus = 4096;
     m->no_parallel = !module_object_class_by_name(TYPE_ISA_PARALLEL);
     machine_class_allow_dynamic_sysbus_dev(m, TYPE_AMD_IOMMU_DEVICE);
     machine_class_allow_dynamic_sysbus_dev(m, TYPE_INTEL_IOMMU_DEVICE);
@@ -383,12 +383,22 @@ static void pc_q35_machine_options(MachineClass *m)
     machine_class_allow_dynamic_sysbus_dev(m, TYPE_VMBUS_BRIDGE);
 }
 
-static void pc_q35_8_2_machine_options(MachineClass *m)
+static void pc_q35_8_3_machine_options(MachineClass *m)
 {
     pc_q35_machine_options(m);
     m->alias = "q35";
 }
 
+DEFINE_Q35_MACHINE(v8_3, "pc-q35-8.3", NULL,
+                   pc_q35_8_3_machine_options);
+
+static void pc_q35_8_2_machine_options(MachineClass *m)
+{
+    pc_q35_8_3_machine_options(m);
+    m->alias = NULL;
+    m->max_cpus = 1024;
+}
+
 DEFINE_Q35_MACHINE(v8_2, "pc-q35-8.2", NULL,
                    pc_q35_8_2_machine_options);
 
@@ -396,7 +406,6 @@ static void pc_q35_8_1_machine_options(MachineClass *m)
 {
     PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
     pc_q35_8_2_machine_options(m);
-    m->alias = NULL;
     pcmc->broken_32bit_mem_addr_check = true;
     compat_props_add(m->compat_props, hw_compat_8_1, hw_compat_8_1_len);
     compat_props_add(m->compat_props, pc_compat_8_1, pc_compat_8_1_len);
-- 
2.42.0

Re: [PATCH] pc: q35: Bump max_cpus to 4096

[Daniel P. Berrangé]

On Fri, Dec 08, 2023 at 05:56:11PM +0530, Ani Sinha wrote:
> Since commit f10a570b093e6 ("KVM: x86: Add CONFIG_KVM_MAX_NR_VCPUS to allow up to 4096 vCPUs")
> Linux kernel can support upto a maximum number of 4096 vCPUS when MAXSMP is
> enabled in the kernel. So bump up the max_cpus value for q35 machines versions
> 8.3 and newer to 4096. Older q35 machines versions 8.2 and older continue to
> support 1024 maximum vcpus as before.
> 
> If KVM is not able to support the specified number of vcpus, QEMU would
> return the following error messages:
> 
> $ ./qemu-system-x86_64 -cpu host -accel kvm -machine q35 -smp 4096
> qemu-system-x86_64: -accel kvm: warning: Number of SMP cpus requested (4096) exceeds the recommended cpus supported by KVM (12)
> Number of SMP cpus requested (4096) exceeds the maximum cpus supported by KVM (1024)
> 
> Cc: Daniel P. Berrangé <berrange@redhat.com>
> Cc: Igor Mammedov <imammedo@redhat.com>
> Cc: Michael S. Tsirkin <mst@redhat.com>
> Cc: Julia Suvorova <jusual@redhat.com>
> Signed-off-by: Ani Sinha <anisinha@redhat.com>
> ---
>  hw/i386/pc_q35.c | 15 ++++++++++++---
>  1 file changed, 12 insertions(+), 3 deletions(-)

What testing has been done to confirm if QEMU is actually capable of
booting a guest with this CPU count, either UEFI or SeaBIOS or both ?

Historically every time we wanted to raise max cpus we've seen limits
or scalability problems that needed fixing first. The previous bump to
1024 had been implicitly proven via downstream testing we had done in
RHEL, and had required the switch to SMBIOS v3 entrypoint.

> 
> diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
> index 4f3e5412f6..2ed57814e1 100644
> --- a/hw/i386/pc_q35.c
> +++ b/hw/i386/pc_q35.c
> @@ -375,7 +375,7 @@ static void pc_q35_machine_options(MachineClass *m)
>      m->default_nic = "e1000e";
>      m->default_kernel_irqchip_split = false;
>      m->no_floppy = 1;
> -    m->max_cpus = 1024;
> +    m->max_cpus = 4096;
>      m->no_parallel = !module_object_class_by_name(TYPE_ISA_PARALLEL);
>      machine_class_allow_dynamic_sysbus_dev(m, TYPE_AMD_IOMMU_DEVICE);
>      machine_class_allow_dynamic_sysbus_dev(m, TYPE_INTEL_IOMMU_DEVICE);
> @@ -383,12 +383,22 @@ static void pc_q35_machine_options(MachineClass *m)
>      machine_class_allow_dynamic_sysbus_dev(m, TYPE_VMBUS_BRIDGE);
>  }
>  
> -static void pc_q35_8_2_machine_options(MachineClass *m)
> +static void pc_q35_8_3_machine_options(MachineClass *m)
>  {
>      pc_q35_machine_options(m);
>      m->alias = "q35";
>  }
>  
> +DEFINE_Q35_MACHINE(v8_3, "pc-q35-8.3", NULL,
> +                   pc_q35_8_3_machine_options);
> +
> +static void pc_q35_8_2_machine_options(MachineClass *m)
> +{
> +    pc_q35_8_3_machine_options(m);
> +    m->alias = NULL;
> +    m->max_cpus = 1024;
> +}
> +
>  DEFINE_Q35_MACHINE(v8_2, "pc-q35-8.2", NULL,
>                     pc_q35_8_2_machine_options);
>  
> @@ -396,7 +406,6 @@ static void pc_q35_8_1_machine_options(MachineClass *m)
>  {
>      PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
>      pc_q35_8_2_machine_options(m);
> -    m->alias = NULL;
>      pcmc->broken_32bit_mem_addr_check = true;
>      compat_props_add(m->compat_props, hw_compat_8_1, hw_compat_8_1_len);
>      compat_props_add(m->compat_props, pc_compat_8_1, pc_compat_8_1_len);
> -- 
> 2.42.0
> 

With regards,
Daniel
-- 
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|

Re: [PATCH] pc: q35: Bump max_cpus to 4096

[Ani Sinha]

> On 08-Dec-2023, at 6:27 PM, Daniel P. Berrangé <berrange@redhat.com> wrote:
> 
> On Fri, Dec 08, 2023 at 05:56:11PM +0530, Ani Sinha wrote:
>> Since commit f10a570b093e6 ("KVM: x86: Add CONFIG_KVM_MAX_NR_VCPUS to allow up to 4096 vCPUs")
>> Linux kernel can support upto a maximum number of 4096 vCPUS when MAXSMP is
>> enabled in the kernel. So bump up the max_cpus value for q35 machines versions
>> 8.3 and newer to 4096. Older q35 machines versions 8.2 and older continue to
>> support 1024 maximum vcpus as before.
>> 
>> If KVM is not able to support the specified number of vcpus, QEMU would
>> return the following error messages:
>> 
>> $ ./qemu-system-x86_64 -cpu host -accel kvm -machine q35 -smp 4096
>> qemu-system-x86_64: -accel kvm: warning: Number of SMP cpus requested (4096) exceeds the recommended cpus supported by KVM (12)
>> Number of SMP cpus requested (4096) exceeds the maximum cpus supported by KVM (1024)
>> 
>> Cc: Daniel P. Berrangé <berrange@redhat.com>
>> Cc: Igor Mammedov <imammedo@redhat.com>
>> Cc: Michael S. Tsirkin <mst@redhat.com>
>> Cc: Julia Suvorova <jusual@redhat.com>
>> Signed-off-by: Ani Sinha <anisinha@redhat.com>
>> ---
>> hw/i386/pc_q35.c | 15 ++++++++++++---
>> 1 file changed, 12 insertions(+), 3 deletions(-)
> 
> What testing has been done to confirm if QEMU is actually capable of
> booting a guest with this CPU count, either UEFI or SeaBIOS or both ?

I admit we did not test this with 4096 cpus.

It was tested downstream with edk2 with modified kernel and increased QEMU limit for 
https://bugzilla.redhat.com/show_bug.cgi?id=1983086

> We validated a ~48TB, 1728 cores, and 32 socket vm using legacy
> bios from smbios 3.0, the latest qemu modified with higher vcpu limits, a=
nd
> modified kernel limits.

I am trying to get some more clarity on the testing front and checking what max values for max_cpu we can test with.

> 
> Historically every time we wanted to raise max cpus we've seen limits
> or scalability problems that needed fixing first. The previous bump to
> 1024 had been implicitly proven via downstream testing we had done in
> RHEL, and had required the switch to SMBIOS v3 entrypoint.
> 
>> 
>> diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
>> index 4f3e5412f6..2ed57814e1 100644
>> --- a/hw/i386/pc_q35.c
>> +++ b/hw/i386/pc_q35.c
>> @@ -375,7 +375,7 @@ static void pc_q35_machine_options(MachineClass *m)
>>     m->default_nic = "e1000e";
>>     m->default_kernel_irqchip_split = false;
>>     m->no_floppy = 1;
>> -    m->max_cpus = 1024;
>> +    m->max_cpus = 4096;
>>     m->no_parallel = !module_object_class_by_name(TYPE_ISA_PARALLEL);
>>     machine_class_allow_dynamic_sysbus_dev(m, TYPE_AMD_IOMMU_DEVICE);
>>     machine_class_allow_dynamic_sysbus_dev(m, TYPE_INTEL_IOMMU_DEVICE);
>> @@ -383,12 +383,22 @@ static void pc_q35_machine_options(MachineClass *m)
>>     machine_class_allow_dynamic_sysbus_dev(m, TYPE_VMBUS_BRIDGE);
>> }
>> 
>> -static void pc_q35_8_2_machine_options(MachineClass *m)
>> +static void pc_q35_8_3_machine_options(MachineClass *m)
>> {
>>     pc_q35_machine_options(m);
>>     m->alias = "q35";
>> }
>> 
>> +DEFINE_Q35_MACHINE(v8_3, "pc-q35-8.3", NULL,
>> +                   pc_q35_8_3_machine_options);
>> +
>> +static void pc_q35_8_2_machine_options(MachineClass *m)
>> +{
>> +    pc_q35_8_3_machine_options(m);
>> +    m->alias = NULL;
>> +    m->max_cpus = 1024;
>> +}
>> +
>> DEFINE_Q35_MACHINE(v8_2, "pc-q35-8.2", NULL,
>>                    pc_q35_8_2_machine_options);
>> 
>> @@ -396,7 +406,6 @@ static void pc_q35_8_1_machine_options(MachineClass *m)
>> {
>>     PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
>>     pc_q35_8_2_machine_options(m);
>> -    m->alias = NULL;
>>     pcmc->broken_32bit_mem_addr_check = true;
>>     compat_props_add(m->compat_props, hw_compat_8_1, hw_compat_8_1_len);
>>     compat_props_add(m->compat_props, pc_compat_8_1, pc_compat_8_1_len);
>> -- 
>> 2.42.0
>> 
> 
> With regards,
> Daniel
> -- 
> |: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org         -o-            https://fstop138.berrange.com :|
> |: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|

Re: [PATCH] pc: q35: Bump max_cpus to 4096

[Thomas Huth]

On 08/12/2023 13.26, Ani Sinha wrote:
> Since commit f10a570b093e6 ("KVM: x86: Add CONFIG_KVM_MAX_NR_VCPUS to allow up to 4096 vCPUs")
> Linux kernel can support upto a maximum number of 4096 vCPUS when MAXSMP is
> enabled in the kernel. So bump up the max_cpus value for q35 machines versions
> 8.3 and newer to 4096. Older q35 machines versions 8.2 and older continue to
> support 1024 maximum vcpus as before.
> 
> If KVM is not able to support the specified number of vcpus, QEMU would
> return the following error messages:
> 
> $ ./qemu-system-x86_64 -cpu host -accel kvm -machine q35 -smp 4096
> qemu-system-x86_64: -accel kvm: warning: Number of SMP cpus requested (4096) exceeds the recommended cpus supported by KVM (12)
> Number of SMP cpus requested (4096) exceeds the maximum cpus supported by KVM (1024)
> 
> Cc: Daniel P. Berrangé <berrange@redhat.com>
> Cc: Igor Mammedov <imammedo@redhat.com>
> Cc: Michael S. Tsirkin <mst@redhat.com>
> Cc: Julia Suvorova <jusual@redhat.com>
> Signed-off-by: Ani Sinha <anisinha@redhat.com>
> ---
>   hw/i386/pc_q35.c | 15 ++++++++++++---
>   1 file changed, 12 insertions(+), 3 deletions(-)
> 
> diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
> index 4f3e5412f6..2ed57814e1 100644
> --- a/hw/i386/pc_q35.c
> +++ b/hw/i386/pc_q35.c
> @@ -375,7 +375,7 @@ static void pc_q35_machine_options(MachineClass *m)
>       m->default_nic = "e1000e";
>       m->default_kernel_irqchip_split = false;
>       m->no_floppy = 1;
> -    m->max_cpus = 1024;
> +    m->max_cpus = 4096;
>       m->no_parallel = !module_object_class_by_name(TYPE_ISA_PARALLEL);
>       machine_class_allow_dynamic_sysbus_dev(m, TYPE_AMD_IOMMU_DEVICE);
>       machine_class_allow_dynamic_sysbus_dev(m, TYPE_INTEL_IOMMU_DEVICE);
> @@ -383,12 +383,22 @@ static void pc_q35_machine_options(MachineClass *m)
>       machine_class_allow_dynamic_sysbus_dev(m, TYPE_VMBUS_BRIDGE);
>   }
>   
> -static void pc_q35_8_2_machine_options(MachineClass *m)
> +static void pc_q35_8_3_machine_options(MachineClass *m)

  Hi Ani!

The next QEMU version after 8.2 is 9.0. Maybe best if you rebase your patch 
on top of:

  https://lore.kernel.org/qemu-devel/20231120094259.1191804-1-cohuck@redhat.com/

   Thomas

Re: [PATCH] pc: q35: Bump max_cpus to 4096

[Ani Sinha]

> On 11-Dec-2023, at 11:27 AM, Thomas Huth <thuth@redhat.com> wrote:
> 
> On 08/12/2023 13.26, Ani Sinha wrote:
>> Since commit f10a570b093e6 ("KVM: x86: Add CONFIG_KVM_MAX_NR_VCPUS to allow up to 4096 vCPUs")
>> Linux kernel can support upto a maximum number of 4096 vCPUS when MAXSMP is
>> enabled in the kernel. So bump up the max_cpus value for q35 machines versions
>> 8.3 and newer to 4096. Older q35 machines versions 8.2 and older continue to
>> support 1024 maximum vcpus as before.
>> If KVM is not able to support the specified number of vcpus, QEMU would
>> return the following error messages:
>> $ ./qemu-system-x86_64 -cpu host -accel kvm -machine q35 -smp 4096
>> qemu-system-x86_64: -accel kvm: warning: Number of SMP cpus requested (4096) exceeds the recommended cpus supported by KVM (12)
>> Number of SMP cpus requested (4096) exceeds the maximum cpus supported by KVM (1024)
>> Cc: Daniel P. Berrangé <berrange@redhat.com>
>> Cc: Igor Mammedov <imammedo@redhat.com>
>> Cc: Michael S. Tsirkin <mst@redhat.com>
>> Cc: Julia Suvorova <jusual@redhat.com>
>> Signed-off-by: Ani Sinha <anisinha@redhat.com>
>> ---
>>  hw/i386/pc_q35.c | 15 ++++++++++++---
>>  1 file changed, 12 insertions(+), 3 deletions(-)
>> diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
>> index 4f3e5412f6..2ed57814e1 100644
>> --- a/hw/i386/pc_q35.c
>> +++ b/hw/i386/pc_q35.c
>> @@ -375,7 +375,7 @@ static void pc_q35_machine_options(MachineClass *m)
>>      m->default_nic = "e1000e";
>>      m->default_kernel_irqchip_split = false;
>>      m->no_floppy = 1;
>> -    m->max_cpus = 1024;
>> +    m->max_cpus = 4096;
>>      m->no_parallel = !module_object_class_by_name(TYPE_ISA_PARALLEL);
>>      machine_class_allow_dynamic_sysbus_dev(m, TYPE_AMD_IOMMU_DEVICE);
>>      machine_class_allow_dynamic_sysbus_dev(m, TYPE_INTEL_IOMMU_DEVICE);
>> @@ -383,12 +383,22 @@ static void pc_q35_machine_options(MachineClass *m)
>>      machine_class_allow_dynamic_sysbus_dev(m, TYPE_VMBUS_BRIDGE);
>>  }
>>  -static void pc_q35_8_2_machine_options(MachineClass *m)
>> +static void pc_q35_8_3_machine_options(MachineClass *m)
> 
> Hi Ani!
> 
> The next QEMU version after 8.2 is 9.0.

Ah yes I completely forgot about our versioning scheme.


> Maybe best if you rebase your patch on top of:
> 
> https://lore.kernel.org/qemu-devel/20231120094259.1191804-1-cohuck@redhat.com/

Will do. I am trying to get some data with testing on edk2. It seems on legacy bios, we have done some testing downstream with 4096 vcpus with qemu modifications and patched host kernel [1] and the linux guest boots fine. With edk2 it seems we have tested with 1880 vcpus and it works fine with the patched edk2 [2]. I am working downstream with HPE folks to get some testing done with 4096 vcpus.

[1] patched with https://github.com/kvm-x86/linux/commit/f10a570b093e60c6bd3f210ae909f014f421352a .
[2] https://github.com/tianocore/edk2/pull/4181/commits/03ca8224f995eda75e2769ed9a2fc437e321db8e