* [Qemu-devel] [RFC PATCH 1/7] x86: add x86_cpu_unrealizefn() for cpu apic remove
2014-07-11 9:59 [Qemu-devel] [RFC PATCH 0/7] i386: add cpu hot remove support Gu Zheng
@ 2014-07-11 9:59 ` Gu Zheng
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 2/7] i386: add cpu device_del support Gu Zheng
` (7 subsequent siblings)
8 siblings, 0 replies; 16+ messages in thread
From: Gu Zheng @ 2014-07-11 9:59 UTC (permalink / raw)
To: qemu-devel
Cc: tangchen, Gu Zheng, isimatu.yasuaki, imammedo, Chen Fan,
anshul.makkar, afaerber
From: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
From: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
Implement x86_cpu_unrealizefn() for corresponding x86_cpu_realizefn(),
which is mostly used to clean the apic related allocation and vmstates
at here.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
---
hw/i386/kvm/apic.c | 8 +++++++
hw/intc/apic.c | 10 ++++++++
hw/intc/apic_common.c | 23 +++++++++++++++++++-
include/hw/cpu/icc_bus.h | 1 +
include/hw/i386/apic_internal.h | 1 +
target-i386/cpu-qom.h | 1 +
target-i386/cpu.c | 44 +++++++++++++++++++++++++++++++++++++++
7 files changed, 87 insertions(+), 1 deletions(-)
diff --git a/hw/i386/kvm/apic.c b/hw/i386/kvm/apic.c
index e873b50..593ca19 100644
--- a/hw/i386/kvm/apic.c
+++ b/hw/i386/kvm/apic.c
@@ -183,11 +183,19 @@ static void kvm_apic_realize(DeviceState *dev, Error **errp)
}
}
+static void kvm_apic_unrealize(DeviceState *dev, Error **errp)
+{
+ APICCommonState *s = APIC_COMMON(dev);
+
+ memory_region_destroy(&s->io_memory);
+}
+
static void kvm_apic_class_init(ObjectClass *klass, void *data)
{
APICCommonClass *k = APIC_COMMON_CLASS(klass);
k->realize = kvm_apic_realize;
+ k->unrealize = kvm_apic_unrealize;
k->set_base = kvm_apic_set_base;
k->set_tpr = kvm_apic_set_tpr;
k->get_tpr = kvm_apic_get_tpr;
diff --git a/hw/intc/apic.c b/hw/intc/apic.c
index ef19e55..efdbf5b 100644
--- a/hw/intc/apic.c
+++ b/hw/intc/apic.c
@@ -885,11 +885,21 @@ static void apic_realize(DeviceState *dev, Error **errp)
msi_supported = true;
}
+static void apic_unrealize(DeviceState *dev, Error **errp)
+{
+ APICCommonState *s = APIC_COMMON(dev);
+
+ memory_region_destroy(&s->io_memory);
+ timer_free(s->timer);
+ local_apics[s->idx] = NULL;
+}
+
static void apic_class_init(ObjectClass *klass, void *data)
{
APICCommonClass *k = APIC_COMMON_CLASS(klass);
k->realize = apic_realize;
+ k->unrealize = apic_unrealize;
k->set_base = apic_set_base;
k->set_tpr = apic_set_tpr;
k->get_tpr = apic_get_tpr;
diff --git a/hw/intc/apic_common.c b/hw/intc/apic_common.c
index 029f67d..8d17be1 100644
--- a/hw/intc/apic_common.c
+++ b/hw/intc/apic_common.c
@@ -289,12 +289,13 @@ static int apic_load_old(QEMUFile *f, void *opaque, int version_id)
return 0;
}
+static int apic_no;
+
static void apic_common_realize(DeviceState *dev, Error **errp)
{
APICCommonState *s = APIC_COMMON(dev);
APICCommonClass *info;
static DeviceState *vapic;
- static int apic_no;
static bool mmio_registered;
if (apic_no >= MAX_APICS) {
@@ -324,6 +325,25 @@ static void apic_common_realize(DeviceState *dev, Error **errp)
}
+static void apic_common_unrealize(DeviceState *dev, Error **errp)
+{
+ APICCommonState *s = APIC_COMMON(dev);
+ APICCommonClass *info = APIC_COMMON_GET_CLASS(s);
+
+ if (apic_no <= 0) {
+ error_setg(errp, "%s exit failed.",
+ object_get_typename(OBJECT(dev)));
+ return;
+ }
+ apic_no--;
+
+ info->unrealize(dev, errp);
+
+ if (apic_report_tpr_access && info->enable_tpr_reporting) {
+ info->enable_tpr_reporting(s, false);
+ }
+}
+
static void apic_dispatch_pre_save(void *opaque)
{
APICCommonState *s = APIC_COMMON(opaque);
@@ -394,6 +414,7 @@ static void apic_common_class_init(ObjectClass *klass, void *data)
dc->reset = apic_reset_common;
dc->props = apic_properties_common;
idc->realize = apic_common_realize;
+ idc->unrealize = apic_common_unrealize;
/*
* Reason: APIC and CPU need to be wired up by
* x86_cpu_apic_create()
diff --git a/include/hw/cpu/icc_bus.h b/include/hw/cpu/icc_bus.h
index 98a979f..75ed309 100644
--- a/include/hw/cpu/icc_bus.h
+++ b/include/hw/cpu/icc_bus.h
@@ -67,6 +67,7 @@ typedef struct ICCDeviceClass {
/*< public >*/
DeviceRealize realize;
+ DeviceUnrealize unrealize;
} ICCDeviceClass;
#define TYPE_ICC_DEVICE "icc-device"
diff --git a/include/hw/i386/apic_internal.h b/include/hw/i386/apic_internal.h
index 8a645cf..32fb354 100644
--- a/include/hw/i386/apic_internal.h
+++ b/include/hw/i386/apic_internal.h
@@ -82,6 +82,7 @@ typedef struct APICCommonClass
ICCDeviceClass parent_class;
DeviceRealize realize;
+ DeviceUnrealize unrealize;
void (*set_base)(APICCommonState *s, uint64_t val);
void (*set_tpr)(APICCommonState *s, uint8_t val);
uint8_t (*get_tpr)(APICCommonState *s);
diff --git a/target-i386/cpu-qom.h b/target-i386/cpu-qom.h
index 71a1b97..2239105 100644
--- a/target-i386/cpu-qom.h
+++ b/target-i386/cpu-qom.h
@@ -65,6 +65,7 @@ typedef struct X86CPUClass {
bool kvm_required;
DeviceRealize parent_realize;
+ DeviceUnrealize parent_unrealize;
void (*parent_reset)(CPUState *cpu);
} X86CPUClass;
diff --git a/target-i386/cpu.c b/target-i386/cpu.c
index 881ff25..56043fa 100644
--- a/target-i386/cpu.c
+++ b/target-i386/cpu.c
@@ -2715,10 +2715,31 @@ static void x86_cpu_apic_realize(X86CPU *cpu, Error **errp)
return;
}
}
+
+static void x86_cpu_apic_unrealize(X86CPU *cpu, Error **errp)
+{
+ Error *local_err = NULL;
+
+ if (cpu->apic_state == NULL) {
+ return;
+ }
+
+ object_property_set_bool(OBJECT(cpu->apic_state),
+ false, "realized", &local_err);
+ if (local_err != NULL) {
+ error_propagate(errp, local_err);
+ return;
+ }
+
+ object_unparent(OBJECT(cpu->apic_state));
+}
#else
static void x86_cpu_apic_realize(X86CPU *cpu, Error **errp)
{
}
+static void x86_cpu_apic_unrealize(X86CPU *cpu, Error **errp)
+{
+}
#endif
static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
@@ -2781,6 +2802,27 @@ out:
}
}
+static void x86_cpu_unrealizefn(DeviceState *dev, Error **errp)
+{
+ X86CPU *cpu = X86_CPU(dev);
+ CPUClass *cc = CPU_GET_CLASS(dev);
+ Error *local_err = NULL;
+
+ if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
+ vmstate_unregister(NULL, &vmstate_cpu_common, cpu);
+ }
+
+ if (cc->vmsd != NULL) {
+ vmstate_unregister(NULL, cc->vmsd, cpu);
+ }
+
+ x86_cpu_apic_unrealize(cpu, &local_err);
+ if (local_err != NULL) {
+ error_propagate(errp, local_err);
+ return;
+ }
+}
+
/* Enables contiguous-apic-ID mode, for compatibility */
static bool compat_apic_id_mode;
@@ -2960,7 +3002,9 @@ static void x86_cpu_common_class_init(ObjectClass *oc, void *data)
DeviceClass *dc = DEVICE_CLASS(oc);
xcc->parent_realize = dc->realize;
+ xcc->parent_unrealize = dc->unrealize;
dc->realize = x86_cpu_realizefn;
+ dc->unrealize = x86_cpu_unrealizefn;
dc->bus_type = TYPE_ICC_BUS;
dc->props = x86_cpu_properties;
--
1.7.7
^ permalink raw reply related [flat|nested] 16+ messages in thread* [Qemu-devel] [RFC PATCH 2/7] i386: add cpu device_del support
2014-07-11 9:59 [Qemu-devel] [RFC PATCH 0/7] i386: add cpu hot remove support Gu Zheng
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 1/7] x86: add x86_cpu_unrealizefn() for cpu apic remove Gu Zheng
@ 2014-07-11 9:59 ` Gu Zheng
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 3/7] qom cpu: rename variable 'cpu_added_notifier' to 'cpu_hotplug_notifier' Gu Zheng
` (6 subsequent siblings)
8 siblings, 0 replies; 16+ messages in thread
From: Gu Zheng @ 2014-07-11 9:59 UTC (permalink / raw)
To: qemu-devel
Cc: tangchen, Chen Fan, isimatu.yasuaki, imammedo, Gu Zheng,
anshul.makkar, afaerber
In order to add cpu(i386) device_del support, introduce the cpu hot
unplug hanlde(x86_cpu_unplug) which will trigger the unrealize routine,
and just register it to the cpu class unplug handle.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
---
target-i386/cpu.c | 22 ++++++++++++++++++++++
1 files changed, 22 insertions(+), 0 deletions(-)
diff --git a/target-i386/cpu.c b/target-i386/cpu.c
index 56043fa..06e969b 100644
--- a/target-i386/cpu.c
+++ b/target-i386/cpu.c
@@ -2995,6 +2995,27 @@ static Property x86_cpu_properties[] = {
DEFINE_PROP_END_OF_LIST()
};
+static int x86_cpu_unplug(DeviceState *dev)
+{
+ CPUState *cpu = CPU(dev);
+ X86CPUClass *xcc;
+ Error *err = NULL;
+
+ if (cpu == first_cpu) {
+ error_report("Can not remove the first cpu!\n");
+ return -1;
+ }
+
+ xcc = X86_CPU_GET_CLASS(DEVICE(cpu));
+ xcc->parent_unrealize(DEVICE(cpu), &err);
+ if (err != NULL) {
+ error_report("%s", error_get_pretty(err));
+ error_free(err);
+ return -1;
+ }
+ return 0;
+}
+
static void x86_cpu_common_class_init(ObjectClass *oc, void *data)
{
X86CPUClass *xcc = X86_CPU_CLASS(oc);
@@ -3007,6 +3028,7 @@ static void x86_cpu_common_class_init(ObjectClass *oc, void *data)
dc->unrealize = x86_cpu_unrealizefn;
dc->bus_type = TYPE_ICC_BUS;
dc->props = x86_cpu_properties;
+ dc->unplug = x86_cpu_unplug;
xcc->parent_reset = cc->reset;
cc->reset = x86_cpu_reset;
--
1.7.7
^ permalink raw reply related [flat|nested] 16+ messages in thread* [Qemu-devel] [RFC PATCH 3/7] qom cpu: rename variable 'cpu_added_notifier' to 'cpu_hotplug_notifier'
2014-07-11 9:59 [Qemu-devel] [RFC PATCH 0/7] i386: add cpu hot remove support Gu Zheng
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 1/7] x86: add x86_cpu_unrealizefn() for cpu apic remove Gu Zheng
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 2/7] i386: add cpu device_del support Gu Zheng
@ 2014-07-11 9:59 ` Gu Zheng
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 4/7] qom cpu: add UNPLUG cpu notify support Gu Zheng
` (5 subsequent siblings)
8 siblings, 0 replies; 16+ messages in thread
From: Gu Zheng @ 2014-07-11 9:59 UTC (permalink / raw)
To: qemu-devel
Cc: tangchen, Gu Zheng, isimatu.yasuaki, imammedo, Chen Fan,
anshul.makkar, afaerber
From: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
Rename variable 'cpu_added_notifier' to 'cpu_hotplug_notifier' for
adding remove vcpu notifier support.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
---
hw/acpi/ich9.c | 8 ++++----
hw/acpi/piix4.c | 10 +++++-----
hw/i386/pc.c | 2 +-
include/hw/acpi/ich9.h | 2 +-
include/sysemu/sysemu.h | 2 +-
qom/cpu.c | 10 +++++-----
6 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/hw/acpi/ich9.c b/hw/acpi/ich9.c
index e7d6c77..fa4150a 100644
--- a/hw/acpi/ich9.c
+++ b/hw/acpi/ich9.c
@@ -209,9 +209,9 @@ static void pm_powerdown_req(Notifier *n, void *opaque)
acpi_pm1_evt_power_down(&pm->acpi_regs);
}
-static void ich9_cpu_added_req(Notifier *n, void *opaque)
+static void ich9_cpu_hotplug_req(Notifier *n, void *opaque)
{
- ICH9LPCPMRegs *pm = container_of(n, ICH9LPCPMRegs, cpu_added_notifier);
+ ICH9LPCPMRegs *pm = container_of(n, ICH9LPCPMRegs, cpu_hotplug_notifier);
assert(pm != NULL);
AcpiCpuHotplug_add(&pm->acpi_regs.gpe, &pm->gpe_cpu, CPU(opaque));
@@ -246,8 +246,8 @@ void ich9_pm_init(PCIDevice *lpc_pci, ICH9LPCPMRegs *pm,
AcpiCpuHotplug_init(pci_address_space_io(lpc_pci), OBJECT(lpc_pci),
&pm->gpe_cpu, ICH9_CPU_HOTPLUG_IO_BASE);
- pm->cpu_added_notifier.notify = ich9_cpu_added_req;
- qemu_register_cpu_added_notifier(&pm->cpu_added_notifier);
+ pm->cpu_hotplug_notifier.notify = ich9_cpu_hotplug_req;
+ qemu_register_cpu_hotplug_notifier(&pm->cpu_hotplug_notifier);
if (pm->acpi_memory_hotplug.is_enabled) {
acpi_memory_hotplug_init(pci_address_space_io(lpc_pci), OBJECT(lpc_pci),
diff --git a/hw/acpi/piix4.c b/hw/acpi/piix4.c
index b72b34e..5cd6300 100644
--- a/hw/acpi/piix4.c
+++ b/hw/acpi/piix4.c
@@ -83,7 +83,7 @@ typedef struct PIIX4PMState {
uint8_t s4_val;
AcpiCpuHotplug gpe_cpu;
- Notifier cpu_added_notifier;
+ Notifier cpu_hotplug_notifier;
MemHotplugState acpi_memory_hotplug;
} PIIX4PMState;
@@ -544,9 +544,9 @@ static const MemoryRegionOps piix4_gpe_ops = {
.endianness = DEVICE_LITTLE_ENDIAN,
};
-static void piix4_cpu_added_req(Notifier *n, void *opaque)
+static void piix4_cpu_hotplug(Notifier *n, void *opaque)
{
- PIIX4PMState *s = container_of(n, PIIX4PMState, cpu_added_notifier);
+ PIIX4PMState *s = container_of(n, PIIX4PMState, cpu_hotplug_notifier);
assert(s != NULL);
AcpiCpuHotplug_add(&s->ar.gpe, &s->gpe_cpu, CPU(opaque));
@@ -565,8 +565,8 @@ static void piix4_acpi_system_hot_add_init(MemoryRegion *parent,
AcpiCpuHotplug_init(parent, OBJECT(s), &s->gpe_cpu,
PIIX4_CPU_HOTPLUG_IO_BASE);
- s->cpu_added_notifier.notify = piix4_cpu_added_req;
- qemu_register_cpu_added_notifier(&s->cpu_added_notifier);
+ s->cpu_hotplug_notifier.notify = piix4_cpu_hotplug;
+ qemu_register_cpu_hotplug_notifier(&s->cpu_hotplug_notifier);
if (s->acpi_memory_hotplug.is_enabled) {
acpi_memory_hotplug_init(parent, OBJECT(s), &s->acpi_memory_hotplug);
diff --git a/hw/i386/pc.c b/hw/i386/pc.c
index 2cf22b1..bb474a8 100644
--- a/hw/i386/pc.c
+++ b/hw/i386/pc.c
@@ -413,7 +413,7 @@ void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
/* init CPU hotplug notifier */
cpu_hotplug_cb.rtc_state = s;
cpu_hotplug_cb.cpu_added_notifier.notify = rtc_notify_cpu_added;
- qemu_register_cpu_added_notifier(&cpu_hotplug_cb.cpu_added_notifier);
+ qemu_register_cpu_hotplug_notifier(&cpu_hotplug_cb.cpu_added_notifier);
if (set_boot_dev(s, boot_device)) {
exit(1);
diff --git a/include/hw/acpi/ich9.h b/include/hw/acpi/ich9.h
index 7e42448..bbb5d2c 100644
--- a/include/hw/acpi/ich9.h
+++ b/include/hw/acpi/ich9.h
@@ -47,7 +47,7 @@ typedef struct ICH9LPCPMRegs {
Notifier powerdown_notifier;
AcpiCpuHotplug gpe_cpu;
- Notifier cpu_added_notifier;
+ Notifier cpu_hotplug_notifier;
MemHotplugState acpi_memory_hotplug;
} ICH9LPCPMRegs;
diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index d8539fd..6810f70 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -184,7 +184,7 @@ void do_pci_device_hot_remove(Monitor *mon, const QDict *qdict);
void drive_hot_add(Monitor *mon, const QDict *qdict);
/* CPU hotplug */
-void qemu_register_cpu_added_notifier(Notifier *notifier);
+void qemu_register_cpu_hotplug_notifier(Notifier *notifier);
/* pcie aer error injection */
void pcie_aer_inject_error_print(Monitor *mon, const QObject *data);
diff --git a/qom/cpu.c b/qom/cpu.c
index bf16590..add92b1 100644
--- a/qom/cpu.c
+++ b/qom/cpu.c
@@ -108,12 +108,12 @@ static void cpu_common_get_memory_mapping(CPUState *cpu,
}
/* CPU hot-plug notifiers */
-static NotifierList cpu_added_notifiers =
- NOTIFIER_LIST_INITIALIZER(cpu_add_notifiers);
+static NotifierList cpu_hotplug_notifiers =
+ NOTIFIER_LIST_INITIALIZER(cpu_hotplug_notifiers);
-void qemu_register_cpu_added_notifier(Notifier *notifier)
+void qemu_register_cpu_hotplug_notifier(Notifier *notifier)
{
- notifier_list_add(&cpu_added_notifiers, notifier);
+ notifier_list_add(&cpu_hotplug_notifiers, notifier);
}
void cpu_reset_interrupt(CPUState *cpu, int mask)
@@ -305,7 +305,7 @@ static void cpu_common_realizefn(DeviceState *dev, Error **errp)
if (dev->hotplugged) {
cpu_synchronize_post_init(cpu);
- notifier_list_notify(&cpu_added_notifiers, dev);
+ notifier_list_notify(&cpu_hotplug_notifiers, dev);
cpu_resume(cpu);
}
}
--
1.7.7
^ permalink raw reply related [flat|nested] 16+ messages in thread* [Qemu-devel] [RFC PATCH 4/7] qom cpu: add UNPLUG cpu notify support
2014-07-11 9:59 [Qemu-devel] [RFC PATCH 0/7] i386: add cpu hot remove support Gu Zheng
` (2 preceding siblings ...)
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 3/7] qom cpu: rename variable 'cpu_added_notifier' to 'cpu_hotplug_notifier' Gu Zheng
@ 2014-07-11 9:59 ` Gu Zheng
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 5/7] i386: implement pc interface cpu_common_unrealizefn() in qom/cpu.c Gu Zheng
` (4 subsequent siblings)
8 siblings, 0 replies; 16+ messages in thread
From: Gu Zheng @ 2014-07-11 9:59 UTC (permalink / raw)
To: qemu-devel
Cc: tangchen, Chen Fan, isimatu.yasuaki, imammedo, Gu Zheng,
anshul.makkar, afaerber
Introduce a common cpu hotplug notifier(CPUNotifier)
to support UNPLUG cpu notify.
Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/acpi/cpu_hotplug.c | 15 +++++++++++----
hw/acpi/ich9.c | 5 ++++-
hw/acpi/piix4.c | 11 +++++++----
include/hw/acpi/cpu_hotplug.h | 13 ++++++++++++-
qom/cpu.c | 7 ++++++-
5 files changed, 40 insertions(+), 11 deletions(-)
diff --git a/hw/acpi/cpu_hotplug.c b/hw/acpi/cpu_hotplug.c
index 2ad83a0..56cb316 100644
--- a/hw/acpi/cpu_hotplug.c
+++ b/hw/acpi/cpu_hotplug.c
@@ -36,15 +36,22 @@ static const MemoryRegionOps AcpiCpuHotplug_ops = {
},
};
-void AcpiCpuHotplug_add(ACPIGPE *gpe, AcpiCpuHotplug *g, CPUState *cpu)
+void AcpiCpuHotplug_handle(ACPIGPE *gpe, AcpiCpuHotplug *g,
+ CPUNotifier *notify)
{
- CPUClass *k = CPU_GET_CLASS(cpu);
+ CPUClass *k = CPU_GET_CLASS(notify->dev);
+ HotplugEventType type = notify->type;
int64_t cpu_id;
*gpe->sts = *gpe->sts | ACPI_CPU_HOTPLUG_STATUS;
- cpu_id = k->get_arch_id(CPU(cpu));
+ cpu_id = k->get_arch_id(CPU(notify->dev));
g_assert((cpu_id / 8) < ACPI_GPE_PROC_LEN);
- g->sts[cpu_id / 8] |= (1 << (cpu_id % 8));
+
+ if (type == PLUG) {
+ g->sts[cpu_id / 8] |= (1 << (cpu_id % 8));
+ } else {
+ g->sts[cpu_id / 8] &= ~(1 << (cpu_id % 8));
+ }
}
void AcpiCpuHotplug_init(MemoryRegion *parent, Object *owner,
diff --git a/hw/acpi/ich9.c b/hw/acpi/ich9.c
index fa4150a..4d9e4b2 100644
--- a/hw/acpi/ich9.c
+++ b/hw/acpi/ich9.c
@@ -212,9 +212,12 @@ static void pm_powerdown_req(Notifier *n, void *opaque)
static void ich9_cpu_hotplug_req(Notifier *n, void *opaque)
{
ICH9LPCPMRegs *pm = container_of(n, ICH9LPCPMRegs, cpu_hotplug_notifier);
+ CPUNotifier *notifier = (CPUNotifier *)opaque;
assert(pm != NULL);
- AcpiCpuHotplug_add(&pm->acpi_regs.gpe, &pm->gpe_cpu, CPU(opaque));
+
+ AcpiCpuHotplug_handle(&pm->acpi_regs.gpe, &pm->gpe_cpu, notifier);
+
acpi_update_sci(&pm->acpi_regs, pm->irq);
}
diff --git a/hw/acpi/piix4.c b/hw/acpi/piix4.c
index 5cd6300..cc15b60 100644
--- a/hw/acpi/piix4.c
+++ b/hw/acpi/piix4.c
@@ -93,7 +93,7 @@ typedef struct PIIX4PMState {
#define PIIX4_PM(obj) \
OBJECT_CHECK(PIIX4PMState, (obj), TYPE_PIIX4_PM)
-static void piix4_acpi_system_hot_add_init(MemoryRegion *parent,
+static void piix4_acpi_system_hotplug_init(MemoryRegion *parent,
PCIBus *bus, PIIX4PMState *s);
#define ACPI_ENABLE 0xf1
@@ -465,7 +465,7 @@ static int piix4_pm_initfn(PCIDevice *dev)
qemu_add_machine_init_done_notifier(&s->machine_ready);
qemu_register_reset(piix4_reset, s);
- piix4_acpi_system_hot_add_init(pci_address_space_io(dev), dev->bus, s);
+ piix4_acpi_system_hotplug_init(pci_address_space_io(dev), dev->bus, s);
piix4_pm_add_propeties(s);
return 0;
@@ -547,13 +547,16 @@ static const MemoryRegionOps piix4_gpe_ops = {
static void piix4_cpu_hotplug(Notifier *n, void *opaque)
{
PIIX4PMState *s = container_of(n, PIIX4PMState, cpu_hotplug_notifier);
+ CPUNotifier *notifier = (CPUNotifier *)opaque;
assert(s != NULL);
- AcpiCpuHotplug_add(&s->ar.gpe, &s->gpe_cpu, CPU(opaque));
+
+ AcpiCpuHotplug_handle(&s->ar.gpe, &s->gpe_cpu, notifier);
+
acpi_update_sci(&s->ar, s->irq);
}
-static void piix4_acpi_system_hot_add_init(MemoryRegion *parent,
+static void piix4_acpi_system_hotplug_init(MemoryRegion *parent,
PCIBus *bus, PIIX4PMState *s)
{
memory_region_init_io(&s->io_gpe, OBJECT(s), &piix4_gpe_ops, s,
diff --git a/include/hw/acpi/cpu_hotplug.h b/include/hw/acpi/cpu_hotplug.h
index 9e5d30c..4fe0066 100644
--- a/include/hw/acpi/cpu_hotplug.h
+++ b/include/hw/acpi/cpu_hotplug.h
@@ -15,12 +15,23 @@
#include "hw/acpi/acpi.h"
#include "hw/acpi/pc-hotplug.h"
+typedef enum {
+ PLUG,
+ UNPLUG,
+} HotplugEventType;
+
+typedef struct CPUNotifier {
+ DeviceState *dev;
+ HotplugEventType type;
+} CPUNotifier;
+
typedef struct AcpiCpuHotplug {
MemoryRegion io;
uint8_t sts[ACPI_GPE_PROC_LEN];
} AcpiCpuHotplug;
-void AcpiCpuHotplug_add(ACPIGPE *gpe, AcpiCpuHotplug *g, CPUState *cpu);
+void AcpiCpuHotplug_handle(ACPIGPE *gpe, AcpiCpuHotplug *g,
+ CPUNotifier *notify);
void AcpiCpuHotplug_init(MemoryRegion *parent, Object *owner,
AcpiCpuHotplug *gpe_cpu, uint16_t base);
diff --git a/qom/cpu.c b/qom/cpu.c
index add92b1..f921282 100644
--- a/qom/cpu.c
+++ b/qom/cpu.c
@@ -25,6 +25,7 @@
#include "qemu/log.h"
#include "qemu/error-report.h"
#include "sysemu/sysemu.h"
+#include "hw/acpi/cpu_hotplug.h"
bool cpu_exists(int64_t id)
{
@@ -304,8 +305,12 @@ static void cpu_common_realizefn(DeviceState *dev, Error **errp)
cpu_vmstate_register(cpu);
if (dev->hotplugged) {
+ CPUNotifier notifier;
+
+ notifier.dev = dev;
+ notifier.type = PLUG;
cpu_synchronize_post_init(cpu);
- notifier_list_notify(&cpu_hotplug_notifiers, dev);
+ notifier_list_notify(&cpu_hotplug_notifiers, ¬ifier);
cpu_resume(cpu);
}
}
--
1.7.7
^ permalink raw reply related [flat|nested] 16+ messages in thread* [Qemu-devel] [RFC PATCH 5/7] i386: implement pc interface cpu_common_unrealizefn() in qom/cpu.c
2014-07-11 9:59 [Qemu-devel] [RFC PATCH 0/7] i386: add cpu hot remove support Gu Zheng
` (3 preceding siblings ...)
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 4/7] qom cpu: add UNPLUG cpu notify support Gu Zheng
@ 2014-07-11 9:59 ` Gu Zheng
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 6/7] cpu hotplug: implement function cpu_status_write() for vcpu ejection Gu Zheng
` (3 subsequent siblings)
8 siblings, 0 replies; 16+ messages in thread
From: Gu Zheng @ 2014-07-11 9:59 UTC (permalink / raw)
To: qemu-devel
Cc: tangchen, Gu Zheng, isimatu.yasuaki, imammedo, Chen Fan,
anshul.makkar, afaerber
From: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
add interface cpu_common_unrealizefn() for emiting vcpu unplug
notifier to ACPI, then ACPI could send sci interrupt
to OS for hot-remove vcpu.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
---
qom/cpu.c | 12 ++++++++++++
1 files changed, 12 insertions(+), 0 deletions(-)
diff --git a/qom/cpu.c b/qom/cpu.c
index f921282..8bb2b4f 100644
--- a/qom/cpu.c
+++ b/qom/cpu.c
@@ -315,6 +315,17 @@ static void cpu_common_realizefn(DeviceState *dev, Error **errp)
}
}
+static void cpu_common_unrealizefn(DeviceState *dev, Error **errp)
+{
+ CPUNotifier notifier;
+
+ notifier.dev = dev;
+ notifier.type = UNPLUG;
+
+ notifier_list_notify(&cpu_hotplug_notifiers, ¬ifier);
+}
+
+
static void cpu_common_initfn(Object *obj)
{
CPUState *cpu = CPU(obj);
@@ -348,6 +359,7 @@ static void cpu_class_init(ObjectClass *klass, void *data)
k->gdb_write_register = cpu_common_gdb_write_register;
k->virtio_is_big_endian = cpu_common_virtio_is_big_endian;
dc->realize = cpu_common_realizefn;
+ dc->unrealize = cpu_common_unrealizefn;
/*
* Reason: CPUs still need special care by board code: wiring up
* IRQs, adding reset handlers, halting non-first CPUs, ...
--
1.7.7
^ permalink raw reply related [flat|nested] 16+ messages in thread* [Qemu-devel] [RFC PATCH 6/7] cpu hotplug: implement function cpu_status_write() for vcpu ejection
2014-07-11 9:59 [Qemu-devel] [RFC PATCH 0/7] i386: add cpu hot remove support Gu Zheng
` (4 preceding siblings ...)
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 5/7] i386: implement pc interface cpu_common_unrealizefn() in qom/cpu.c Gu Zheng
@ 2014-07-11 9:59 ` Gu Zheng
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 7/7] cpus: reclaim allocated vCPU objects Gu Zheng
` (2 subsequent siblings)
8 siblings, 0 replies; 16+ messages in thread
From: Gu Zheng @ 2014-07-11 9:59 UTC (permalink / raw)
To: qemu-devel
Cc: tangchen, Gu Zheng, isimatu.yasuaki, imammedo, Chen Fan,
anshul.makkar, afaerber
From: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
When OS ejected a vcpu (like: echo 1 > /sys/bus/acpi/devices/LNXCPUXX/eject),
it would call acpi EJ0 method, the firmware need to write the new cpumap, QEMU
would know which vcpu need to be ejected.
TODO:
-confirm the hotplug result via OST if guest support it.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
---
cpus.c | 7 ++++++
hw/acpi/cpu_hotplug.c | 40 ++++++++++++++++++++++++++++++++++++-
hw/i386/acpi-dsdt-cpu-hotplug.dsl | 6 ++++-
include/hw/acpi/cpu_hotplug.h | 1 +
include/qom/cpu.h | 9 ++++++++
5 files changed, 61 insertions(+), 2 deletions(-)
diff --git a/cpus.c b/cpus.c
index 5e7f2cf..4dfb889 100644
--- a/cpus.c
+++ b/cpus.c
@@ -1117,6 +1117,13 @@ void resume_all_vcpus(void)
}
}
+void cpu_remove(CPUState *cpu)
+{
+ cpu->stop = true;
+ cpu->exit = true;
+ qemu_cpu_kick(cpu);
+}
+
/* For temporary buffers for forming a name */
#define VCPU_THREAD_NAME_SIZE 16
diff --git a/hw/acpi/cpu_hotplug.c b/hw/acpi/cpu_hotplug.c
index 56cb316..7cbce69 100644
--- a/hw/acpi/cpu_hotplug.c
+++ b/hw/acpi/cpu_hotplug.c
@@ -20,10 +20,46 @@ static uint64_t cpu_status_read(void *opaque, hwaddr addr, unsigned int size)
return val;
}
+static void acpi_eject_vcpu(AcpiCpuHotplug *cpus_status, int64_t cpu_id)
+{
+ CPUState *cpu;
+
+ CPU_FOREACH(cpu) {
+ CPUClass *cc = CPU_GET_CLASS(cpu);
+ int64_t id = cc->get_arch_id(cpu);
+
+ if (cpu_id == id) {
+ cpus_status->old_sts[cpu_id / 8] &= ~(1 << (cpu_id % 8));
+ cpus_status->sts[cpu_id / 8] &= ~(1 << (cpu_id % 8));
+ cpu_remove(cpu);
+ break;
+ }
+ }
+}
+
static void cpu_status_write(void *opaque, hwaddr addr, uint64_t data,
unsigned int size)
{
- /* TODO: implement VCPU removal on guest signal that CPU can be removed */
+ AcpiCpuHotplug *cpus = opaque;
+ uint8_t val;
+ int i;
+ int64_t cpu_id = -1;
+
+ val = cpus->old_sts[addr] ^ data;
+
+ if (val == 0) {
+ return;
+ }
+
+ for (i = 0; i < 8; i++) {
+ if (val & 1 << i) {
+ cpu_id = 8 * addr + i;
+ }
+ }
+
+ if (cpu_id != -1) {
+ acpi_eject_vcpu(cpus, cpu_id);
+ }
}
static const MemoryRegionOps AcpiCpuHotplug_ops = {
@@ -49,6 +85,7 @@ void AcpiCpuHotplug_handle(ACPIGPE *gpe, AcpiCpuHotplug *g,
if (type == PLUG) {
g->sts[cpu_id / 8] |= (1 << (cpu_id % 8));
+ g->old_sts[cpu_id / 8] |= (1 << (cpu_id % 8));
} else {
g->sts[cpu_id / 8] &= ~(1 << (cpu_id % 8));
}
@@ -65,6 +102,7 @@ void AcpiCpuHotplug_init(MemoryRegion *parent, Object *owner,
g_assert((id / 8) < ACPI_GPE_PROC_LEN);
gpe_cpu->sts[id / 8] |= (1 << (id % 8));
+ gpe_cpu->old_sts[id / 8] |= (1 << (id % 8));
}
memory_region_init_io(&gpe_cpu->io, owner, &AcpiCpuHotplug_ops,
gpe_cpu, "acpi-cpu-hotplug", ACPI_GPE_PROC_LEN);
diff --git a/hw/i386/acpi-dsdt-cpu-hotplug.dsl b/hw/i386/acpi-dsdt-cpu-hotplug.dsl
index 34aab5a..9485f12 100644
--- a/hw/i386/acpi-dsdt-cpu-hotplug.dsl
+++ b/hw/i386/acpi-dsdt-cpu-hotplug.dsl
@@ -50,7 +50,11 @@ Scope(\_SB) {
}
Method(CPEJ, 2, NotSerialized) {
// _EJ0 method - eject callback
- Sleep(200)
+ Store(Zero, Index(CPON, ToInteger(Arg0)))
+ Store(One, Local0)
+ ShiftLeft(Local0, Arg0, Local0)
+ Not(Local0, Local0)
+ And(PRS, Local0, PRS)
}
#define CPU_STATUS_LEN ACPI_GPE_PROC_LEN
diff --git a/include/hw/acpi/cpu_hotplug.h b/include/hw/acpi/cpu_hotplug.h
index 4fe0066..3bffc69 100644
--- a/include/hw/acpi/cpu_hotplug.h
+++ b/include/hw/acpi/cpu_hotplug.h
@@ -28,6 +28,7 @@ typedef struct CPUNotifier {
typedef struct AcpiCpuHotplug {
MemoryRegion io;
uint8_t sts[ACPI_GPE_PROC_LEN];
+ uint8_t old_sts[ACPI_GPE_PROC_LEN];
} AcpiCpuHotplug;
void AcpiCpuHotplug_handle(ACPIGPE *gpe, AcpiCpuHotplug *g,
diff --git a/include/qom/cpu.h b/include/qom/cpu.h
index 2fc00ef..9108dc6 100644
--- a/include/qom/cpu.h
+++ b/include/qom/cpu.h
@@ -236,6 +236,7 @@ struct CPUState {
bool created;
bool stop;
bool stopped;
+ bool exit;
volatile sig_atomic_t exit_request;
uint32_t interrupt_request;
int singlestep_enabled;
@@ -600,6 +601,14 @@ void cpu_exit(CPUState *cpu);
void cpu_resume(CPUState *cpu);
/**
+ * cpu_remove:
+ * @cpu: The vCPU to remove.
+ *
+ * Requests the CPU @cpu to be removed.
+ */
+void cpu_remove(CPUState *cpu);
+
+/**
* qemu_init_vcpu:
* @cpu: The vCPU to initialize.
*
--
1.7.7
^ permalink raw reply related [flat|nested] 16+ messages in thread* [Qemu-devel] [RFC PATCH 7/7] cpus: reclaim allocated vCPU objects
2014-07-11 9:59 [Qemu-devel] [RFC PATCH 0/7] i386: add cpu hot remove support Gu Zheng
` (5 preceding siblings ...)
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 6/7] cpu hotplug: implement function cpu_status_write() for vcpu ejection Gu Zheng
@ 2014-07-11 9:59 ` Gu Zheng
2014-07-17 16:24 ` Anshul Makkar
2014-07-15 1:16 ` [Qemu-devel] [RFC PATCH 0/7] i386: add cpu hot remove support Gu Zheng
2014-07-22 2:24 ` Gu Zheng
8 siblings, 1 reply; 16+ messages in thread
From: Gu Zheng @ 2014-07-11 9:59 UTC (permalink / raw)
To: qemu-devel
Cc: tangchen, Chen Fan, isimatu.yasuaki, imammedo, Gu Zheng,
anshul.makkar, afaerber
After ACPI get a signal to eject a vCPU, the vCPU must be
removed from CPU list，before the vCPU really removed, then
release the all related vCPU objects.
But we do not close KVM vcpu fd, just record it into a list, in
order to reuse it.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
---
cpus.c | 37 ++++++++++++++++++++++++++++++++
include/sysemu/kvm.h | 1 +
kvm-all.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++-
3 files changed, 94 insertions(+), 1 deletions(-)
diff --git a/cpus.c b/cpus.c
index 4dfb889..9a73407 100644
--- a/cpus.c
+++ b/cpus.c
@@ -786,6 +786,24 @@ void async_run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
qemu_cpu_kick(cpu);
}
+static void qemu_kvm_destroy_vcpu(CPUState *cpu)
+{
+ CPU_REMOVE(cpu);
+
+ if (kvm_destroy_vcpu(cpu) < 0) {
+ fprintf(stderr, "kvm_destroy_vcpu failed.\n");
+ exit(1);
+ }
+
+ object_unparent(OBJECT(cpu));
+}
+
+static void qemu_tcg_destroy_vcpu(CPUState *cpu)
+{
+ CPU_REMOVE(cpu);
+ object_unparent(OBJECT(cpu));
+}
+
static void flush_queued_work(CPUState *cpu)
{
struct qemu_work_item *wi;
@@ -877,6 +895,11 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
}
}
qemu_kvm_wait_io_event(cpu);
+ if (cpu->exit && !cpu_can_run(cpu)) {
+ qemu_kvm_destroy_vcpu(cpu);
+ qemu_mutex_unlock(&qemu_global_mutex);
+ return NULL;
+ }
}
return NULL;
@@ -929,6 +952,7 @@ static void tcg_exec_all(void);
static void *qemu_tcg_cpu_thread_fn(void *arg)
{
CPUState *cpu = arg;
+ CPUState *remove_cpu = NULL;
qemu_tcg_init_cpu_signals();
qemu_thread_get_self(cpu->thread);
@@ -961,6 +985,16 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
}
}
qemu_tcg_wait_io_event();
+ CPU_FOREACH(cpu) {
+ if (cpu->exit && !cpu_can_run(cpu)) {
+ remove_cpu = cpu;
+ break;
+ }
+ }
+ if (remove_cpu) {
+ qemu_tcg_destroy_vcpu(remove_cpu);
+ remove_cpu = NULL;
+ }
}
return NULL;
@@ -1316,6 +1350,9 @@ static void tcg_exec_all(void)
break;
}
} else if (cpu->stop || cpu->stopped) {
+ if (cpu->exit) {
+ next_cpu = CPU_NEXT(cpu);
+ }
break;
}
}
diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
index 174ea36..88e2403 100644
--- a/include/sysemu/kvm.h
+++ b/include/sysemu/kvm.h
@@ -178,6 +178,7 @@ int kvm_has_intx_set_mask(void);
int kvm_init_vcpu(CPUState *cpu);
int kvm_cpu_exec(CPUState *cpu);
+int kvm_destroy_vcpu(CPUState *cpu);
#ifdef NEED_CPU_H
diff --git a/kvm-all.c b/kvm-all.c
index 3ae30ee..25e2a43 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -74,6 +74,12 @@ typedef struct KVMSlot
typedef struct kvm_dirty_log KVMDirtyLog;
+struct KVMParkedVcpu {
+ unsigned long vcpu_id;
+ int kvm_fd;
+ QLIST_ENTRY(KVMParkedVcpu) node;
+};
+
struct KVMState
{
KVMSlot *slots;
@@ -108,6 +114,7 @@ struct KVMState
QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
bool direct_msi;
#endif
+ QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus;
};
KVMState *kvm_state;
@@ -226,6 +233,53 @@ static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
}
+int kvm_destroy_vcpu(CPUState *cpu)
+{
+ KVMState *s = kvm_state;
+ long mmap_size;
+ struct KVMParkedVcpu *vcpu = NULL;
+ int ret = 0;
+
+ DPRINTF("kvm_destroy_vcpu\n");
+
+ mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
+ if (mmap_size < 0) {
+ ret = mmap_size;
+ DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
+ goto err;
+ }
+
+ ret = munmap(cpu->kvm_run, mmap_size);
+ if (ret < 0) {
+ goto err;
+ }
+
+ vcpu = g_malloc0(sizeof(*vcpu));
+ vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
+ vcpu->kvm_fd = cpu->kvm_fd;
+ QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
+err:
+ return ret;
+}
+
+static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id)
+{
+ struct KVMParkedVcpu *cpu;
+
+ QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
+ if (cpu->vcpu_id == vcpu_id) {
+ int kvm_fd;
+
+ QLIST_REMOVE(cpu, node);
+ kvm_fd = cpu->kvm_fd;
+ g_free(cpu);
+ return kvm_fd;
+ }
+ }
+
+ return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id);
+}
+
int kvm_init_vcpu(CPUState *cpu)
{
KVMState *s = kvm_state;
@@ -234,7 +288,7 @@ int kvm_init_vcpu(CPUState *cpu)
DPRINTF("kvm_init_vcpu\n");
- ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
+ ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu));
if (ret < 0) {
DPRINTF("kvm_create_vcpu failed\n");
goto err;
@@ -1404,6 +1458,7 @@ int kvm_init(MachineClass *mc)
#ifdef KVM_CAP_SET_GUEST_DEBUG
QTAILQ_INIT(&s->kvm_sw_breakpoints);
#endif
+ QLIST_INIT(&s->kvm_parked_vcpus);
s->vmfd = -1;
s->fd = qemu_open("/dev/kvm", O_RDWR);
if (s->fd == -1) {
--
1.7.7
^ permalink raw reply related [flat|nested] 16+ messages in thread* Re: [Qemu-devel] [RFC PATCH 7/7] cpus: reclaim allocated vCPU objects
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 7/7] cpus: reclaim allocated vCPU objects Gu Zheng
@ 2014-07-17 16:24 ` Anshul Makkar
2014-07-18 2:09 ` Gu Zheng
0 siblings, 1 reply; 16+ messages in thread
From: Anshul Makkar @ 2014-07-17 16:24 UTC (permalink / raw)
To: Gu Zheng
Cc: qemu-devel, tangchen, isimatu.yasuaki, Chen Fan, Igor Mammedov,
afaerber
Are we not going to introduce new command cpu_del for deleting the cpu ?
I couldn't find any patch for addition of cpu_del command. Is this
intentional and we intend to use device_del (and similarly device_add)
for cpu hot(un)plug or just skipped to be added later. I have the
patch for the same which I can release, if the intent is to add this
command.
Thanks
Anshul Makkar
On Fri, Jul 11, 2014 at 11:59 AM, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
> After ACPI get a signal to eject a vCPU, the vCPU must be
> removed from CPU list，before the vCPU really removed, then
> release the all related vCPU objects.
> But we do not close KVM vcpu fd, just record it into a list, in
> order to reuse it.
>
> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
> Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
> ---
> cpus.c | 37 ++++++++++++++++++++++++++++++++
> include/sysemu/kvm.h | 1 +
> kvm-all.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++-
> 3 files changed, 94 insertions(+), 1 deletions(-)
>
> diff --git a/cpus.c b/cpus.c
> index 4dfb889..9a73407 100644
> --- a/cpus.c
> +++ b/cpus.c
> @@ -786,6 +786,24 @@ void async_run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
> qemu_cpu_kick(cpu);
> }
>
> +static void qemu_kvm_destroy_vcpu(CPUState *cpu)
> +{
> + CPU_REMOVE(cpu);
> +
> + if (kvm_destroy_vcpu(cpu) < 0) {
> + fprintf(stderr, "kvm_destroy_vcpu failed.\n");
> + exit(1);
> + }
> +
> + object_unparent(OBJECT(cpu));
> +}
> +
> +static void qemu_tcg_destroy_vcpu(CPUState *cpu)
> +{
> + CPU_REMOVE(cpu);
> + object_unparent(OBJECT(cpu));
> +}
> +
> static void flush_queued_work(CPUState *cpu)
> {
> struct qemu_work_item *wi;
> @@ -877,6 +895,11 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
> }
> }
> qemu_kvm_wait_io_event(cpu);
> + if (cpu->exit && !cpu_can_run(cpu)) {
> + qemu_kvm_destroy_vcpu(cpu);
> + qemu_mutex_unlock(&qemu_global_mutex);
> + return NULL;
> + }
> }
>
> return NULL;
> @@ -929,6 +952,7 @@ static void tcg_exec_all(void);
> static void *qemu_tcg_cpu_thread_fn(void *arg)
> {
> CPUState *cpu = arg;
> + CPUState *remove_cpu = NULL;
>
> qemu_tcg_init_cpu_signals();
> qemu_thread_get_self(cpu->thread);
> @@ -961,6 +985,16 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
> }
> }
> qemu_tcg_wait_io_event();
> + CPU_FOREACH(cpu) {
> + if (cpu->exit && !cpu_can_run(cpu)) {
> + remove_cpu = cpu;
> + break;
> + }
> + }
> + if (remove_cpu) {
> + qemu_tcg_destroy_vcpu(remove_cpu);
> + remove_cpu = NULL;
> + }
> }
>
> return NULL;
> @@ -1316,6 +1350,9 @@ static void tcg_exec_all(void)
> break;
> }
> } else if (cpu->stop || cpu->stopped) {
> + if (cpu->exit) {
> + next_cpu = CPU_NEXT(cpu);
> + }
> break;
> }
> }
> diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
> index 174ea36..88e2403 100644
> --- a/include/sysemu/kvm.h
> +++ b/include/sysemu/kvm.h
> @@ -178,6 +178,7 @@ int kvm_has_intx_set_mask(void);
>
> int kvm_init_vcpu(CPUState *cpu);
> int kvm_cpu_exec(CPUState *cpu);
> +int kvm_destroy_vcpu(CPUState *cpu);
>
> #ifdef NEED_CPU_H
>
> diff --git a/kvm-all.c b/kvm-all.c
> index 3ae30ee..25e2a43 100644
> --- a/kvm-all.c
> +++ b/kvm-all.c
> @@ -74,6 +74,12 @@ typedef struct KVMSlot
>
> typedef struct kvm_dirty_log KVMDirtyLog;
>
> +struct KVMParkedVcpu {
> + unsigned long vcpu_id;
> + int kvm_fd;
> + QLIST_ENTRY(KVMParkedVcpu) node;
> +};
> +
> struct KVMState
> {
> KVMSlot *slots;
> @@ -108,6 +114,7 @@ struct KVMState
> QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
> bool direct_msi;
> #endif
> + QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus;
> };
>
> KVMState *kvm_state;
> @@ -226,6 +233,53 @@ static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
> return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
> }
>
> +int kvm_destroy_vcpu(CPUState *cpu)
> +{
> + KVMState *s = kvm_state;
> + long mmap_size;
> + struct KVMParkedVcpu *vcpu = NULL;
> + int ret = 0;
> +
> + DPRINTF("kvm_destroy_vcpu\n");
> +
> + mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
> + if (mmap_size < 0) {
> + ret = mmap_size;
> + DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
> + goto err;
> + }
> +
> + ret = munmap(cpu->kvm_run, mmap_size);
> + if (ret < 0) {
> + goto err;
> + }
> +
> + vcpu = g_malloc0(sizeof(*vcpu));
> + vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
> + vcpu->kvm_fd = cpu->kvm_fd;
> + QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
> +err:
> + return ret;
> +}
> +
> +static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id)
> +{
> + struct KVMParkedVcpu *cpu;
> +
> + QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
> + if (cpu->vcpu_id == vcpu_id) {
> + int kvm_fd;
> +
> + QLIST_REMOVE(cpu, node);
> + kvm_fd = cpu->kvm_fd;
> + g_free(cpu);
> + return kvm_fd;
> + }
> + }
> +
> + return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id);
> +}
> +
> int kvm_init_vcpu(CPUState *cpu)
> {
> KVMState *s = kvm_state;
> @@ -234,7 +288,7 @@ int kvm_init_vcpu(CPUState *cpu)
>
> DPRINTF("kvm_init_vcpu\n");
>
> - ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
> + ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu));
> if (ret < 0) {
> DPRINTF("kvm_create_vcpu failed\n");
> goto err;
> @@ -1404,6 +1458,7 @@ int kvm_init(MachineClass *mc)
> #ifdef KVM_CAP_SET_GUEST_DEBUG
> QTAILQ_INIT(&s->kvm_sw_breakpoints);
> #endif
> + QLIST_INIT(&s->kvm_parked_vcpus);
> s->vmfd = -1;
> s->fd = qemu_open("/dev/kvm", O_RDWR);
> if (s->fd == -1) {
> --
> 1.7.7
>
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [Qemu-devel] [RFC PATCH 7/7] cpus: reclaim allocated vCPU objects
2014-07-17 16:24 ` Anshul Makkar
@ 2014-07-18 2:09 ` Gu Zheng
2014-07-30 14:31 ` Anshul Makkar
0 siblings, 1 reply; 16+ messages in thread
From: Gu Zheng @ 2014-07-18 2:09 UTC (permalink / raw)
To: Anshul Makkar
Cc: qemu-devel, tangchen, isimatu.yasuaki, Chen Fan, Igor Mammedov,
afaerber
Hi Anshul,
On 07/18/2014 12:24 AM, Anshul Makkar wrote:
> Are we not going to introduce new command cpu_del for deleting the cpu ?
>
> I couldn't find any patch for addition of cpu_del command. Is this
> intentional and we intend to use device_del (and similarly device_add)
> for cpu hot(un)plug or just skipped to be added later. I have the
> patch for the same which I can release, if the intent is to add this
> command.
The "device_add/device_del" interface is the approved way to support add/del cpu,
which is also more common and elegant than "cpu_add/del".
<http://wiki.qemu.org/Features/CPUHotplug>
so we intend to use device_del rather than the cpu_del.
And IMO, the cpu_add will be replaced by "device_add" sooner or later.
Thanks,
Gu
>
> Thanks
> Anshul Makkar
>
> On Fri, Jul 11, 2014 at 11:59 AM, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
>> After ACPI get a signal to eject a vCPU, the vCPU must be
>> removed from CPU list，before the vCPU really removed, then
>> release the all related vCPU objects.
>> But we do not close KVM vcpu fd, just record it into a list, in
>> order to reuse it.
>>
>> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
>> Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
>> ---
>> cpus.c | 37 ++++++++++++++++++++++++++++++++
>> include/sysemu/kvm.h | 1 +
>> kvm-all.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++-
>> 3 files changed, 94 insertions(+), 1 deletions(-)
>>
>> diff --git a/cpus.c b/cpus.c
>> index 4dfb889..9a73407 100644
>> --- a/cpus.c
>> +++ b/cpus.c
>> @@ -786,6 +786,24 @@ void async_run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
>> qemu_cpu_kick(cpu);
>> }
>>
>> +static void qemu_kvm_destroy_vcpu(CPUState *cpu)
>> +{
>> + CPU_REMOVE(cpu);
>> +
>> + if (kvm_destroy_vcpu(cpu) < 0) {
>> + fprintf(stderr, "kvm_destroy_vcpu failed.\n");
>> + exit(1);
>> + }
>> +
>> + object_unparent(OBJECT(cpu));
>> +}
>> +
>> +static void qemu_tcg_destroy_vcpu(CPUState *cpu)
>> +{
>> + CPU_REMOVE(cpu);
>> + object_unparent(OBJECT(cpu));
>> +}
>> +
>> static void flush_queued_work(CPUState *cpu)
>> {
>> struct qemu_work_item *wi;
>> @@ -877,6 +895,11 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
>> }
>> }
>> qemu_kvm_wait_io_event(cpu);
>> + if (cpu->exit && !cpu_can_run(cpu)) {
>> + qemu_kvm_destroy_vcpu(cpu);
>> + qemu_mutex_unlock(&qemu_global_mutex);
>> + return NULL;
>> + }
>> }
>>
>> return NULL;
>> @@ -929,6 +952,7 @@ static void tcg_exec_all(void);
>> static void *qemu_tcg_cpu_thread_fn(void *arg)
>> {
>> CPUState *cpu = arg;
>> + CPUState *remove_cpu = NULL;
>>
>> qemu_tcg_init_cpu_signals();
>> qemu_thread_get_self(cpu->thread);
>> @@ -961,6 +985,16 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
>> }
>> }
>> qemu_tcg_wait_io_event();
>> + CPU_FOREACH(cpu) {
>> + if (cpu->exit && !cpu_can_run(cpu)) {
>> + remove_cpu = cpu;
>> + break;
>> + }
>> + }
>> + if (remove_cpu) {
>> + qemu_tcg_destroy_vcpu(remove_cpu);
>> + remove_cpu = NULL;
>> + }
>> }
>>
>> return NULL;
>> @@ -1316,6 +1350,9 @@ static void tcg_exec_all(void)
>> break;
>> }
>> } else if (cpu->stop || cpu->stopped) {
>> + if (cpu->exit) {
>> + next_cpu = CPU_NEXT(cpu);
>> + }
>> break;
>> }
>> }
>> diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
>> index 174ea36..88e2403 100644
>> --- a/include/sysemu/kvm.h
>> +++ b/include/sysemu/kvm.h
>> @@ -178,6 +178,7 @@ int kvm_has_intx_set_mask(void);
>>
>> int kvm_init_vcpu(CPUState *cpu);
>> int kvm_cpu_exec(CPUState *cpu);
>> +int kvm_destroy_vcpu(CPUState *cpu);
>>
>> #ifdef NEED_CPU_H
>>
>> diff --git a/kvm-all.c b/kvm-all.c
>> index 3ae30ee..25e2a43 100644
>> --- a/kvm-all.c
>> +++ b/kvm-all.c
>> @@ -74,6 +74,12 @@ typedef struct KVMSlot
>>
>> typedef struct kvm_dirty_log KVMDirtyLog;
>>
>> +struct KVMParkedVcpu {
>> + unsigned long vcpu_id;
>> + int kvm_fd;
>> + QLIST_ENTRY(KVMParkedVcpu) node;
>> +};
>> +
>> struct KVMState
>> {
>> KVMSlot *slots;
>> @@ -108,6 +114,7 @@ struct KVMState
>> QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
>> bool direct_msi;
>> #endif
>> + QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus;
>> };
>>
>> KVMState *kvm_state;
>> @@ -226,6 +233,53 @@ static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
>> return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
>> }
>>
>> +int kvm_destroy_vcpu(CPUState *cpu)
>> +{
>> + KVMState *s = kvm_state;
>> + long mmap_size;
>> + struct KVMParkedVcpu *vcpu = NULL;
>> + int ret = 0;
>> +
>> + DPRINTF("kvm_destroy_vcpu\n");
>> +
>> + mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
>> + if (mmap_size < 0) {
>> + ret = mmap_size;
>> + DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
>> + goto err;
>> + }
>> +
>> + ret = munmap(cpu->kvm_run, mmap_size);
>> + if (ret < 0) {
>> + goto err;
>> + }
>> +
>> + vcpu = g_malloc0(sizeof(*vcpu));
>> + vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
>> + vcpu->kvm_fd = cpu->kvm_fd;
>> + QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
>> +err:
>> + return ret;
>> +}
>> +
>> +static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id)
>> +{
>> + struct KVMParkedVcpu *cpu;
>> +
>> + QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
>> + if (cpu->vcpu_id == vcpu_id) {
>> + int kvm_fd;
>> +
>> + QLIST_REMOVE(cpu, node);
>> + kvm_fd = cpu->kvm_fd;
>> + g_free(cpu);
>> + return kvm_fd;
>> + }
>> + }
>> +
>> + return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id);
>> +}
>> +
>> int kvm_init_vcpu(CPUState *cpu)
>> {
>> KVMState *s = kvm_state;
>> @@ -234,7 +288,7 @@ int kvm_init_vcpu(CPUState *cpu)
>>
>> DPRINTF("kvm_init_vcpu\n");
>>
>> - ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
>> + ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu));
>> if (ret < 0) {
>> DPRINTF("kvm_create_vcpu failed\n");
>> goto err;
>> @@ -1404,6 +1458,7 @@ int kvm_init(MachineClass *mc)
>> #ifdef KVM_CAP_SET_GUEST_DEBUG
>> QTAILQ_INIT(&s->kvm_sw_breakpoints);
>> #endif
>> + QLIST_INIT(&s->kvm_parked_vcpus);
>> s->vmfd = -1;
>> s->fd = qemu_open("/dev/kvm", O_RDWR);
>> if (s->fd == -1) {
>> --
>> 1.7.7
>>
> .
>
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [Qemu-devel] [RFC PATCH 7/7] cpus: reclaim allocated vCPU objects
2014-07-18 2:09 ` Gu Zheng
@ 2014-07-30 14:31 ` Anshul Makkar
2014-08-01 4:42 ` Gu Zheng
0 siblings, 1 reply; 16+ messages in thread
From: Anshul Makkar @ 2014-07-30 14:31 UTC (permalink / raw)
To: Gu Zheng
Cc: qemu-devel, tangchen, isimatu.yasuaki, Chen Fan, Igor Mammedov,
afaerber
Hi,
I am testing the cpu-hotunplug patches. I observed that after the
deletion of the cpu with id = x, if I cpu-add the same cpu again id =
x, then qemu exits with the error that file descriptor already exists.
On debugging I found that if I give cpu-add <apic-id = x>, then
qemu_kvm_cpu_thread_fn->kvm_init_vcpu is called which sends an IOCTL
(KVM_CREATE_VCPU) to kvm to create a new fd. As the fd already exists
in KVM as we never delete the fd from the kernel and just park it in
separate list, it returns false and QEMU exits. In the above code
flow, no where its being checked if we have the cpu with cpuid = x
available in the parked list and we can reuse it.
Am I missing something or this bit is yet to be implmented.
Thanks
Anshul Makkar
On Fri, Jul 18, 2014 at 4:09 AM, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
> Hi Anshul,
> On 07/18/2014 12:24 AM, Anshul Makkar wrote:
>
>> Are we not going to introduce new command cpu_del for deleting the cpu ?
>>
>> I couldn't find any patch for addition of cpu_del command. Is this
>> intentional and we intend to use device_del (and similarly device_add)
>> for cpu hot(un)plug or just skipped to be added later. I have the
>> patch for the same which I can release, if the intent is to add this
>> command.
>
> The "device_add/device_del" interface is the approved way to support add/del cpu,
> which is also more common and elegant than "cpu_add/del".
> <http://wiki.qemu.org/Features/CPUHotplug>
> so we intend to use device_del rather than the cpu_del.
> And IMO, the cpu_add will be replaced by "device_add" sooner or later.
>
> Thanks,
> Gu
>
>>
>> Thanks
>> Anshul Makkar
>>
>> On Fri, Jul 11, 2014 at 11:59 AM, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
>>> After ACPI get a signal to eject a vCPU, the vCPU must be
>>> removed from CPU list，before the vCPU really removed, then
>>> release the all related vCPU objects.
>>> But we do not close KVM vcpu fd, just record it into a list, in
>>> order to reuse it.
>>>
>>> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
>>> Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
>>> ---
>>> cpus.c | 37 ++++++++++++++++++++++++++++++++
>>> include/sysemu/kvm.h | 1 +
>>> kvm-all.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++-
>>> 3 files changed, 94 insertions(+), 1 deletions(-)
>>>
>>> diff --git a/cpus.c b/cpus.c
>>> index 4dfb889..9a73407 100644
>>> --- a/cpus.c
>>> +++ b/cpus.c
>>> @@ -786,6 +786,24 @@ void async_run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
>>> qemu_cpu_kick(cpu);
>>> }
>>>
>>> +static void qemu_kvm_destroy_vcpu(CPUState *cpu)
>>> +{
>>> + CPU_REMOVE(cpu);
>>> +
>>> + if (kvm_destroy_vcpu(cpu) < 0) {
>>> + fprintf(stderr, "kvm_destroy_vcpu failed.\n");
>>> + exit(1);
>>> + }
>>> +
>>> + object_unparent(OBJECT(cpu));
>>> +}
>>> +
>>> +static void qemu_tcg_destroy_vcpu(CPUState *cpu)
>>> +{
>>> + CPU_REMOVE(cpu);
>>> + object_unparent(OBJECT(cpu));
>>> +}
>>> +
>>> static void flush_queued_work(CPUState *cpu)
>>> {
>>> struct qemu_work_item *wi;
>>> @@ -877,6 +895,11 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
>>> }
>>> }
>>> qemu_kvm_wait_io_event(cpu);
>>> + if (cpu->exit && !cpu_can_run(cpu)) {
>>> + qemu_kvm_destroy_vcpu(cpu);
>>> + qemu_mutex_unlock(&qemu_global_mutex);
>>> + return NULL;
>>> + }
>>> }
>>>
>>> return NULL;
>>> @@ -929,6 +952,7 @@ static void tcg_exec_all(void);
>>> static void *qemu_tcg_cpu_thread_fn(void *arg)
>>> {
>>> CPUState *cpu = arg;
>>> + CPUState *remove_cpu = NULL;
>>>
>>> qemu_tcg_init_cpu_signals();
>>> qemu_thread_get_self(cpu->thread);
>>> @@ -961,6 +985,16 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
>>> }
>>> }
>>> qemu_tcg_wait_io_event();
>>> + CPU_FOREACH(cpu) {
>>> + if (cpu->exit && !cpu_can_run(cpu)) {
>>> + remove_cpu = cpu;
>>> + break;
>>> + }
>>> + }
>>> + if (remove_cpu) {
>>> + qemu_tcg_destroy_vcpu(remove_cpu);
>>> + remove_cpu = NULL;
>>> + }
>>> }
>>>
>>> return NULL;
>>> @@ -1316,6 +1350,9 @@ static void tcg_exec_all(void)
>>> break;
>>> }
>>> } else if (cpu->stop || cpu->stopped) {
>>> + if (cpu->exit) {
>>> + next_cpu = CPU_NEXT(cpu);
>>> + }
>>> break;
>>> }
>>> }
>>> diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
>>> index 174ea36..88e2403 100644
>>> --- a/include/sysemu/kvm.h
>>> +++ b/include/sysemu/kvm.h
>>> @@ -178,6 +178,7 @@ int kvm_has_intx_set_mask(void);
>>>
>>> int kvm_init_vcpu(CPUState *cpu);
>>> int kvm_cpu_exec(CPUState *cpu);
>>> +int kvm_destroy_vcpu(CPUState *cpu);
>>>
>>> #ifdef NEED_CPU_H
>>>
>>> diff --git a/kvm-all.c b/kvm-all.c
>>> index 3ae30ee..25e2a43 100644
>>> --- a/kvm-all.c
>>> +++ b/kvm-all.c
>>> @@ -74,6 +74,12 @@ typedef struct KVMSlot
>>>
>>> typedef struct kvm_dirty_log KVMDirtyLog;
>>>
>>> +struct KVMParkedVcpu {
>>> + unsigned long vcpu_id;
>>> + int kvm_fd;
>>> + QLIST_ENTRY(KVMParkedVcpu) node;
>>> +};
>>> +
>>> struct KVMState
>>> {
>>> KVMSlot *slots;
>>> @@ -108,6 +114,7 @@ struct KVMState
>>> QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
>>> bool direct_msi;
>>> #endif
>>> + QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus;
>>> };
>>>
>>> KVMState *kvm_state;
>>> @@ -226,6 +233,53 @@ static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
>>> return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
>>> }
>>>
>>> +int kvm_destroy_vcpu(CPUState *cpu)
>>> +{
>>> + KVMState *s = kvm_state;
>>> + long mmap_size;
>>> + struct KVMParkedVcpu *vcpu = NULL;
>>> + int ret = 0;
>>> +
>>> + DPRINTF("kvm_destroy_vcpu\n");
>>> +
>>> + mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
>>> + if (mmap_size < 0) {
>>> + ret = mmap_size;
>>> + DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
>>> + goto err;
>>> + }
>>> +
>>> + ret = munmap(cpu->kvm_run, mmap_size);
>>> + if (ret < 0) {
>>> + goto err;
>>> + }
>>> +
>>> + vcpu = g_malloc0(sizeof(*vcpu));
>>> + vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
>>> + vcpu->kvm_fd = cpu->kvm_fd;
>>> + QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
>>> +err:
>>> + return ret;
>>> +}
>>> +
>>> +static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id)
>>> +{
>>> + struct KVMParkedVcpu *cpu;
>>> +
>>> + QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
>>> + if (cpu->vcpu_id == vcpu_id) {
>>> + int kvm_fd;
>>> +
>>> + QLIST_REMOVE(cpu, node);
>>> + kvm_fd = cpu->kvm_fd;
>>> + g_free(cpu);
>>> + return kvm_fd;
>>> + }
>>> + }
>>> +
>>> + return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id);
>>> +}
>>> +
>>> int kvm_init_vcpu(CPUState *cpu)
>>> {
>>> KVMState *s = kvm_state;
>>> @@ -234,7 +288,7 @@ int kvm_init_vcpu(CPUState *cpu)
>>>
>>> DPRINTF("kvm_init_vcpu\n");
>>>
>>> - ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
>>> + ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu));
>>> if (ret < 0) {
>>> DPRINTF("kvm_create_vcpu failed\n");
>>> goto err;
>>> @@ -1404,6 +1458,7 @@ int kvm_init(MachineClass *mc)
>>> #ifdef KVM_CAP_SET_GUEST_DEBUG
>>> QTAILQ_INIT(&s->kvm_sw_breakpoints);
>>> #endif
>>> + QLIST_INIT(&s->kvm_parked_vcpus);
>>> s->vmfd = -1;
>>> s->fd = qemu_open("/dev/kvm", O_RDWR);
>>> if (s->fd == -1) {
>>> --
>>> 1.7.7
>>>
>> .
>>
>
>
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [Qemu-devel] [RFC PATCH 7/7] cpus: reclaim allocated vCPU objects
2014-07-30 14:31 ` Anshul Makkar
@ 2014-08-01 4:42 ` Gu Zheng
2014-08-01 15:34 ` Anshul Makkar
0 siblings, 1 reply; 16+ messages in thread
From: Gu Zheng @ 2014-08-01 4:42 UTC (permalink / raw)
To: Anshul Makkar
Cc: qemu-devel, tangchen, isimatu.yasuaki, Chen Fan, Igor Mammedov,
afaerber
Hi Anshul,
Thanks for your test.
On 07/30/2014 10:31 PM, Anshul Makkar wrote:
> Hi,
>
> I am testing the cpu-hotunplug patches. I observed that after the
> deletion of the cpu with id = x, if I cpu-add the same cpu again id =
> x, then qemu exits with the error that file descriptor already exists.
Could you please offer the whole reproduce routine? In my test box, we
can add a removed cpu with the id.
>
> On debugging I found that if I give cpu-add <apic-id = x>, then
> qemu_kvm_cpu_thread_fn->kvm_init_vcpu is called which sends an IOCTL
> (KVM_CREATE_VCPU) to kvm to create a new fd. As the fd already exists
> in KVM as we never delete the fd from the kernel and just park it in
> separate list, it returns false and QEMU exits. In the above code
> flow, no where its being checked if we have the cpu with cpuid = x
> available in the parked list and we can reuse it.
>
> Am I missing something or this bit is yet to be implmented.
Yes, it is implemented, in the same way as you mention above, please refer
to function kvm_get_vcpu().
Thanks,
Gu
>
> Thanks
> Anshul Makkar
>
> On Fri, Jul 18, 2014 at 4:09 AM, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
>> Hi Anshul,
>> On 07/18/2014 12:24 AM, Anshul Makkar wrote:
>>
>>> Are we not going to introduce new command cpu_del for deleting the cpu ?
>>>
>>> I couldn't find any patch for addition of cpu_del command. Is this
>>> intentional and we intend to use device_del (and similarly device_add)
>>> for cpu hot(un)plug or just skipped to be added later. I have the
>>> patch for the same which I can release, if the intent is to add this
>>> command.
>>
>> The "device_add/device_del" interface is the approved way to support add/del cpu,
>> which is also more common and elegant than "cpu_add/del".
>> <http://wiki.qemu.org/Features/CPUHotplug>
>> so we intend to use device_del rather than the cpu_del.
>> And IMO, the cpu_add will be replaced by "device_add" sooner or later.
>>
>> Thanks,
>> Gu
>>
>>>
>>> Thanks
>>> Anshul Makkar
>>>
>>> On Fri, Jul 11, 2014 at 11:59 AM, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
>>>> After ACPI get a signal to eject a vCPU, the vCPU must be
>>>> removed from CPU list，before the vCPU really removed, then
>>>> release the all related vCPU objects.
>>>> But we do not close KVM vcpu fd, just record it into a list, in
>>>> order to reuse it.
>>>>
>>>> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
>>>> Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
>>>> ---
>>>> cpus.c | 37 ++++++++++++++++++++++++++++++++
>>>> include/sysemu/kvm.h | 1 +
>>>> kvm-all.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++-
>>>> 3 files changed, 94 insertions(+), 1 deletions(-)
>>>>
>>>> diff --git a/cpus.c b/cpus.c
>>>> index 4dfb889..9a73407 100644
>>>> --- a/cpus.c
>>>> +++ b/cpus.c
>>>> @@ -786,6 +786,24 @@ void async_run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
>>>> qemu_cpu_kick(cpu);
>>>> }
>>>>
>>>> +static void qemu_kvm_destroy_vcpu(CPUState *cpu)
>>>> +{
>>>> + CPU_REMOVE(cpu);
>>>> +
>>>> + if (kvm_destroy_vcpu(cpu) < 0) {
>>>> + fprintf(stderr, "kvm_destroy_vcpu failed.\n");
>>>> + exit(1);
>>>> + }
>>>> +
>>>> + object_unparent(OBJECT(cpu));
>>>> +}
>>>> +
>>>> +static void qemu_tcg_destroy_vcpu(CPUState *cpu)
>>>> +{
>>>> + CPU_REMOVE(cpu);
>>>> + object_unparent(OBJECT(cpu));
>>>> +}
>>>> +
>>>> static void flush_queued_work(CPUState *cpu)
>>>> {
>>>> struct qemu_work_item *wi;
>>>> @@ -877,6 +895,11 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
>>>> }
>>>> }
>>>> qemu_kvm_wait_io_event(cpu);
>>>> + if (cpu->exit && !cpu_can_run(cpu)) {
>>>> + qemu_kvm_destroy_vcpu(cpu);
>>>> + qemu_mutex_unlock(&qemu_global_mutex);
>>>> + return NULL;
>>>> + }
>>>> }
>>>>
>>>> return NULL;
>>>> @@ -929,6 +952,7 @@ static void tcg_exec_all(void);
>>>> static void *qemu_tcg_cpu_thread_fn(void *arg)
>>>> {
>>>> CPUState *cpu = arg;
>>>> + CPUState *remove_cpu = NULL;
>>>>
>>>> qemu_tcg_init_cpu_signals();
>>>> qemu_thread_get_self(cpu->thread);
>>>> @@ -961,6 +985,16 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
>>>> }
>>>> }
>>>> qemu_tcg_wait_io_event();
>>>> + CPU_FOREACH(cpu) {
>>>> + if (cpu->exit && !cpu_can_run(cpu)) {
>>>> + remove_cpu = cpu;
>>>> + break;
>>>> + }
>>>> + }
>>>> + if (remove_cpu) {
>>>> + qemu_tcg_destroy_vcpu(remove_cpu);
>>>> + remove_cpu = NULL;
>>>> + }
>>>> }
>>>>
>>>> return NULL;
>>>> @@ -1316,6 +1350,9 @@ static void tcg_exec_all(void)
>>>> break;
>>>> }
>>>> } else if (cpu->stop || cpu->stopped) {
>>>> + if (cpu->exit) {
>>>> + next_cpu = CPU_NEXT(cpu);
>>>> + }
>>>> break;
>>>> }
>>>> }
>>>> diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
>>>> index 174ea36..88e2403 100644
>>>> --- a/include/sysemu/kvm.h
>>>> +++ b/include/sysemu/kvm.h
>>>> @@ -178,6 +178,7 @@ int kvm_has_intx_set_mask(void);
>>>>
>>>> int kvm_init_vcpu(CPUState *cpu);
>>>> int kvm_cpu_exec(CPUState *cpu);
>>>> +int kvm_destroy_vcpu(CPUState *cpu);
>>>>
>>>> #ifdef NEED_CPU_H
>>>>
>>>> diff --git a/kvm-all.c b/kvm-all.c
>>>> index 3ae30ee..25e2a43 100644
>>>> --- a/kvm-all.c
>>>> +++ b/kvm-all.c
>>>> @@ -74,6 +74,12 @@ typedef struct KVMSlot
>>>>
>>>> typedef struct kvm_dirty_log KVMDirtyLog;
>>>>
>>>> +struct KVMParkedVcpu {
>>>> + unsigned long vcpu_id;
>>>> + int kvm_fd;
>>>> + QLIST_ENTRY(KVMParkedVcpu) node;
>>>> +};
>>>> +
>>>> struct KVMState
>>>> {
>>>> KVMSlot *slots;
>>>> @@ -108,6 +114,7 @@ struct KVMState
>>>> QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
>>>> bool direct_msi;
>>>> #endif
>>>> + QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus;
>>>> };
>>>>
>>>> KVMState *kvm_state;
>>>> @@ -226,6 +233,53 @@ static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
>>>> return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
>>>> }
>>>>
>>>> +int kvm_destroy_vcpu(CPUState *cpu)
>>>> +{
>>>> + KVMState *s = kvm_state;
>>>> + long mmap_size;
>>>> + struct KVMParkedVcpu *vcpu = NULL;
>>>> + int ret = 0;
>>>> +
>>>> + DPRINTF("kvm_destroy_vcpu\n");
>>>> +
>>>> + mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
>>>> + if (mmap_size < 0) {
>>>> + ret = mmap_size;
>>>> + DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
>>>> + goto err;
>>>> + }
>>>> +
>>>> + ret = munmap(cpu->kvm_run, mmap_size);
>>>> + if (ret < 0) {
>>>> + goto err;
>>>> + }
>>>> +
>>>> + vcpu = g_malloc0(sizeof(*vcpu));
>>>> + vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
>>>> + vcpu->kvm_fd = cpu->kvm_fd;
>>>> + QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
>>>> +err:
>>>> + return ret;
>>>> +}
>>>> +
>>>> +static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id)
>>>> +{
>>>> + struct KVMParkedVcpu *cpu;
>>>> +
>>>> + QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
>>>> + if (cpu->vcpu_id == vcpu_id) {
>>>> + int kvm_fd;
>>>> +
>>>> + QLIST_REMOVE(cpu, node);
>>>> + kvm_fd = cpu->kvm_fd;
>>>> + g_free(cpu);
>>>> + return kvm_fd;
>>>> + }
>>>> + }
>>>> +
>>>> + return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id);
>>>> +}
>>>> +
>>>> int kvm_init_vcpu(CPUState *cpu)
>>>> {
>>>> KVMState *s = kvm_state;
>>>> @@ -234,7 +288,7 @@ int kvm_init_vcpu(CPUState *cpu)
>>>>
>>>> DPRINTF("kvm_init_vcpu\n");
>>>>
>>>> - ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
>>>> + ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu));
>>>> if (ret < 0) {
>>>> DPRINTF("kvm_create_vcpu failed\n");
>>>> goto err;
>>>> @@ -1404,6 +1458,7 @@ int kvm_init(MachineClass *mc)
>>>> #ifdef KVM_CAP_SET_GUEST_DEBUG
>>>> QTAILQ_INIT(&s->kvm_sw_breakpoints);
>>>> #endif
>>>> + QLIST_INIT(&s->kvm_parked_vcpus);
>>>> s->vmfd = -1;
>>>> s->fd = qemu_open("/dev/kvm", O_RDWR);
>>>> if (s->fd == -1) {
>>>> --
>>>> 1.7.7
>>>>
>>> .
>>>
>>
>>
> .
>
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [Qemu-devel] [RFC PATCH 7/7] cpus: reclaim allocated vCPU objects
2014-08-01 4:42 ` Gu Zheng
@ 2014-08-01 15:34 ` Anshul Makkar
2014-08-07 4:53 ` Gu Zheng
0 siblings, 1 reply; 16+ messages in thread
From: Anshul Makkar @ 2014-08-01 15:34 UTC (permalink / raw)
To: Gu Zheng
Cc: qemu-devel, tangchen, isimatu.yasuaki, Chen Fan, Igor Mammedov,
afaerber
Hi Gu,
Thanks for clarifying.
Ah I missed that bit of the patch. Sorry about that and for making noise.
Yes, now cpu-hotplug and unplug works fine. Next week I plan to run a
series of automated and stress test. Will keep the group posted about
the results.
Thanks
Anshul Makkar
On Fri, Aug 1, 2014 at 6:42 AM, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
> Hi Anshul,
> Thanks for your test.
> On 07/30/2014 10:31 PM, Anshul Makkar wrote:
>
>> Hi,
>>
>> I am testing the cpu-hotunplug patches. I observed that after the
>> deletion of the cpu with id = x, if I cpu-add the same cpu again id =
>> x, then qemu exits with the error that file descriptor already exists.
>
> Could you please offer the whole reproduce routine? In my test box, we
> can add a removed cpu with the id.
>
>>
>> On debugging I found that if I give cpu-add <apic-id = x>, then
>> qemu_kvm_cpu_thread_fn->kvm_init_vcpu is called which sends an IOCTL
>> (KVM_CREATE_VCPU) to kvm to create a new fd. As the fd already exists
>> in KVM as we never delete the fd from the kernel and just park it in
>> separate list, it returns false and QEMU exits. In the above code
>> flow, no where its being checked if we have the cpu with cpuid = x
>> available in the parked list and we can reuse it.
>>
>> Am I missing something or this bit is yet to be implmented.
>
> Yes, it is implemented, in the same way as you mention above, please refer
> to function kvm_get_vcpu().
>
> Thanks,
> Gu
>
>>
>> Thanks
>> Anshul Makkar
>>
>> On Fri, Jul 18, 2014 at 4:09 AM, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
>>> Hi Anshul,
>>> On 07/18/2014 12:24 AM, Anshul Makkar wrote:
>>>
>>>> Are we not going to introduce new command cpu_del for deleting the cpu ?
>>>>
>>>> I couldn't find any patch for addition of cpu_del command. Is this
>>>> intentional and we intend to use device_del (and similarly device_add)
>>>> for cpu hot(un)plug or just skipped to be added later. I have the
>>>> patch for the same which I can release, if the intent is to add this
>>>> command.
>>>
>>> The "device_add/device_del" interface is the approved way to support add/del cpu,
>>> which is also more common and elegant than "cpu_add/del".
>>> <http://wiki.qemu.org/Features/CPUHotplug>
>>> so we intend to use device_del rather than the cpu_del.
>>> And IMO, the cpu_add will be replaced by "device_add" sooner or later.
>>>
>>> Thanks,
>>> Gu
>>>
>>>>
>>>> Thanks
>>>> Anshul Makkar
>>>>
>>>> On Fri, Jul 11, 2014 at 11:59 AM, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
>>>>> After ACPI get a signal to eject a vCPU, the vCPU must be
>>>>> removed from CPU list，before the vCPU really removed, then
>>>>> release the all related vCPU objects.
>>>>> But we do not close KVM vcpu fd, just record it into a list, in
>>>>> order to reuse it.
>>>>>
>>>>> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
>>>>> Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
>>>>> ---
>>>>> cpus.c | 37 ++++++++++++++++++++++++++++++++
>>>>> include/sysemu/kvm.h | 1 +
>>>>> kvm-all.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++-
>>>>> 3 files changed, 94 insertions(+), 1 deletions(-)
>>>>>
>>>>> diff --git a/cpus.c b/cpus.c
>>>>> index 4dfb889..9a73407 100644
>>>>> --- a/cpus.c
>>>>> +++ b/cpus.c
>>>>> @@ -786,6 +786,24 @@ void async_run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
>>>>> qemu_cpu_kick(cpu);
>>>>> }
>>>>>
>>>>> +static void qemu_kvm_destroy_vcpu(CPUState *cpu)
>>>>> +{
>>>>> + CPU_REMOVE(cpu);
>>>>> +
>>>>> + if (kvm_destroy_vcpu(cpu) < 0) {
>>>>> + fprintf(stderr, "kvm_destroy_vcpu failed.\n");
>>>>> + exit(1);
>>>>> + }
>>>>> +
>>>>> + object_unparent(OBJECT(cpu));
>>>>> +}
>>>>> +
>>>>> +static void qemu_tcg_destroy_vcpu(CPUState *cpu)
>>>>> +{
>>>>> + CPU_REMOVE(cpu);
>>>>> + object_unparent(OBJECT(cpu));
>>>>> +}
>>>>> +
>>>>> static void flush_queued_work(CPUState *cpu)
>>>>> {
>>>>> struct qemu_work_item *wi;
>>>>> @@ -877,6 +895,11 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
>>>>> }
>>>>> }
>>>>> qemu_kvm_wait_io_event(cpu);
>>>>> + if (cpu->exit && !cpu_can_run(cpu)) {
>>>>> + qemu_kvm_destroy_vcpu(cpu);
>>>>> + qemu_mutex_unlock(&qemu_global_mutex);
>>>>> + return NULL;
>>>>> + }
>>>>> }
>>>>>
>>>>> return NULL;
>>>>> @@ -929,6 +952,7 @@ static void tcg_exec_all(void);
>>>>> static void *qemu_tcg_cpu_thread_fn(void *arg)
>>>>> {
>>>>> CPUState *cpu = arg;
>>>>> + CPUState *remove_cpu = NULL;
>>>>>
>>>>> qemu_tcg_init_cpu_signals();
>>>>> qemu_thread_get_self(cpu->thread);
>>>>> @@ -961,6 +985,16 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
>>>>> }
>>>>> }
>>>>> qemu_tcg_wait_io_event();
>>>>> + CPU_FOREACH(cpu) {
>>>>> + if (cpu->exit && !cpu_can_run(cpu)) {
>>>>> + remove_cpu = cpu;
>>>>> + break;
>>>>> + }
>>>>> + }
>>>>> + if (remove_cpu) {
>>>>> + qemu_tcg_destroy_vcpu(remove_cpu);
>>>>> + remove_cpu = NULL;
>>>>> + }
>>>>> }
>>>>>
>>>>> return NULL;
>>>>> @@ -1316,6 +1350,9 @@ static void tcg_exec_all(void)
>>>>> break;
>>>>> }
>>>>> } else if (cpu->stop || cpu->stopped) {
>>>>> + if (cpu->exit) {
>>>>> + next_cpu = CPU_NEXT(cpu);
>>>>> + }
>>>>> break;
>>>>> }
>>>>> }
>>>>> diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
>>>>> index 174ea36..88e2403 100644
>>>>> --- a/include/sysemu/kvm.h
>>>>> +++ b/include/sysemu/kvm.h
>>>>> @@ -178,6 +178,7 @@ int kvm_has_intx_set_mask(void);
>>>>>
>>>>> int kvm_init_vcpu(CPUState *cpu);
>>>>> int kvm_cpu_exec(CPUState *cpu);
>>>>> +int kvm_destroy_vcpu(CPUState *cpu);
>>>>>
>>>>> #ifdef NEED_CPU_H
>>>>>
>>>>> diff --git a/kvm-all.c b/kvm-all.c
>>>>> index 3ae30ee..25e2a43 100644
>>>>> --- a/kvm-all.c
>>>>> +++ b/kvm-all.c
>>>>> @@ -74,6 +74,12 @@ typedef struct KVMSlot
>>>>>
>>>>> typedef struct kvm_dirty_log KVMDirtyLog;
>>>>>
>>>>> +struct KVMParkedVcpu {
>>>>> + unsigned long vcpu_id;
>>>>> + int kvm_fd;
>>>>> + QLIST_ENTRY(KVMParkedVcpu) node;
>>>>> +};
>>>>> +
>>>>> struct KVMState
>>>>> {
>>>>> KVMSlot *slots;
>>>>> @@ -108,6 +114,7 @@ struct KVMState
>>>>> QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
>>>>> bool direct_msi;
>>>>> #endif
>>>>> + QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus;
>>>>> };
>>>>>
>>>>> KVMState *kvm_state;
>>>>> @@ -226,6 +233,53 @@ static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
>>>>> return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
>>>>> }
>>>>>
>>>>> +int kvm_destroy_vcpu(CPUState *cpu)
>>>>> +{
>>>>> + KVMState *s = kvm_state;
>>>>> + long mmap_size;
>>>>> + struct KVMParkedVcpu *vcpu = NULL;
>>>>> + int ret = 0;
>>>>> +
>>>>> + DPRINTF("kvm_destroy_vcpu\n");
>>>>> +
>>>>> + mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
>>>>> + if (mmap_size < 0) {
>>>>> + ret = mmap_size;
>>>>> + DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
>>>>> + goto err;
>>>>> + }
>>>>> +
>>>>> + ret = munmap(cpu->kvm_run, mmap_size);
>>>>> + if (ret < 0) {
>>>>> + goto err;
>>>>> + }
>>>>> +
>>>>> + vcpu = g_malloc0(sizeof(*vcpu));
>>>>> + vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
>>>>> + vcpu->kvm_fd = cpu->kvm_fd;
>>>>> + QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
>>>>> +err:
>>>>> + return ret;
>>>>> +}
>>>>> +
>>>>> +static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id)
>>>>> +{
>>>>> + struct KVMParkedVcpu *cpu;
>>>>> +
>>>>> + QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
>>>>> + if (cpu->vcpu_id == vcpu_id) {
>>>>> + int kvm_fd;
>>>>> +
>>>>> + QLIST_REMOVE(cpu, node);
>>>>> + kvm_fd = cpu->kvm_fd;
>>>>> + g_free(cpu);
>>>>> + return kvm_fd;
>>>>> + }
>>>>> + }
>>>>> +
>>>>> + return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id);
>>>>> +}
>>>>> +
>>>>> int kvm_init_vcpu(CPUState *cpu)
>>>>> {
>>>>> KVMState *s = kvm_state;
>>>>> @@ -234,7 +288,7 @@ int kvm_init_vcpu(CPUState *cpu)
>>>>>
>>>>> DPRINTF("kvm_init_vcpu\n");
>>>>>
>>>>> - ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
>>>>> + ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu));
>>>>> if (ret < 0) {
>>>>> DPRINTF("kvm_create_vcpu failed\n");
>>>>> goto err;
>>>>> @@ -1404,6 +1458,7 @@ int kvm_init(MachineClass *mc)
>>>>> #ifdef KVM_CAP_SET_GUEST_DEBUG
>>>>> QTAILQ_INIT(&s->kvm_sw_breakpoints);
>>>>> #endif
>>>>> + QLIST_INIT(&s->kvm_parked_vcpus);
>>>>> s->vmfd = -1;
>>>>> s->fd = qemu_open("/dev/kvm", O_RDWR);
>>>>> if (s->fd == -1) {
>>>>> --
>>>>> 1.7.7
>>>>>
>>>> .
>>>>
>>>
>>>
>> .
>>
>
>
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [Qemu-devel] [RFC PATCH 7/7] cpus: reclaim allocated vCPU objects
2014-08-01 15:34 ` Anshul Makkar
@ 2014-08-07 4:53 ` Gu Zheng
0 siblings, 0 replies; 16+ messages in thread
From: Gu Zheng @ 2014-08-07 4:53 UTC (permalink / raw)
To: Anshul Makkar
Cc: qemu-devel, tangchen, isimatu.yasuaki, Chen Fan, Igor Mammedov,
afaerber
Hi Anshul,
I rebased these two parts on latest QEMU tree (no functional change), and
will send out later.
So if you like, please refer to the new one, it will be easy for you to
review and test.
Thanks,
Gu
On 08/01/2014 11:34 PM, Anshul Makkar wrote:
> Hi Gu,
>
> Thanks for clarifying.
>
> Ah I missed that bit of the patch. Sorry about that and for making noise.
>
> Yes, now cpu-hotplug and unplug works fine. Next week I plan to run a
> series of automated and stress test. Will keep the group posted about
> the results.
>
> Thanks
> Anshul Makkar
>
> On Fri, Aug 1, 2014 at 6:42 AM, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
>> Hi Anshul,
>> Thanks for your test.
>> On 07/30/2014 10:31 PM, Anshul Makkar wrote:
>>
>>> Hi,
>>>
>>> I am testing the cpu-hotunplug patches. I observed that after the
>>> deletion of the cpu with id = x, if I cpu-add the same cpu again id =
>>> x, then qemu exits with the error that file descriptor already exists.
>>
>> Could you please offer the whole reproduce routine? In my test box, we
>> can add a removed cpu with the id.
>>
>>>
>>> On debugging I found that if I give cpu-add <apic-id = x>, then
>>> qemu_kvm_cpu_thread_fn->kvm_init_vcpu is called which sends an IOCTL
>>> (KVM_CREATE_VCPU) to kvm to create a new fd. As the fd already exists
>>> in KVM as we never delete the fd from the kernel and just park it in
>>> separate list, it returns false and QEMU exits. In the above code
>>> flow, no where its being checked if we have the cpu with cpuid = x
>>> available in the parked list and we can reuse it.
>>>
>>> Am I missing something or this bit is yet to be implmented.
>>
>> Yes, it is implemented, in the same way as you mention above, please refer
>> to function kvm_get_vcpu().
>>
>> Thanks,
>> Gu
>>
>>>
>>> Thanks
>>> Anshul Makkar
>>>
>>> On Fri, Jul 18, 2014 at 4:09 AM, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
>>>> Hi Anshul,
>>>> On 07/18/2014 12:24 AM, Anshul Makkar wrote:
>>>>
>>>>> Are we not going to introduce new command cpu_del for deleting the cpu ?
>>>>>
>>>>> I couldn't find any patch for addition of cpu_del command. Is this
>>>>> intentional and we intend to use device_del (and similarly device_add)
>>>>> for cpu hot(un)plug or just skipped to be added later. I have the
>>>>> patch for the same which I can release, if the intent is to add this
>>>>> command.
>>>>
>>>> The "device_add/device_del" interface is the approved way to support add/del cpu,
>>>> which is also more common and elegant than "cpu_add/del".
>>>> <http://wiki.qemu.org/Features/CPUHotplug>
>>>> so we intend to use device_del rather than the cpu_del.
>>>> And IMO, the cpu_add will be replaced by "device_add" sooner or later.
>>>>
>>>> Thanks,
>>>> Gu
>>>>
>>>>>
>>>>> Thanks
>>>>> Anshul Makkar
>>>>>
>>>>> On Fri, Jul 11, 2014 at 11:59 AM, Gu Zheng <guz.fnst@cn.fujitsu.com> wrote:
>>>>>> After ACPI get a signal to eject a vCPU, the vCPU must be
>>>>>> removed from CPU list，before the vCPU really removed, then
>>>>>> release the all related vCPU objects.
>>>>>> But we do not close KVM vcpu fd, just record it into a list, in
>>>>>> order to reuse it.
>>>>>>
>>>>>> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
>>>>>> Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
>>>>>> ---
>>>>>> cpus.c | 37 ++++++++++++++++++++++++++++++++
>>>>>> include/sysemu/kvm.h | 1 +
>>>>>> kvm-all.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++-
>>>>>> 3 files changed, 94 insertions(+), 1 deletions(-)
>>>>>>
>>>>>> diff --git a/cpus.c b/cpus.c
>>>>>> index 4dfb889..9a73407 100644
>>>>>> --- a/cpus.c
>>>>>> +++ b/cpus.c
>>>>>> @@ -786,6 +786,24 @@ void async_run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
>>>>>> qemu_cpu_kick(cpu);
>>>>>> }
>>>>>>
>>>>>> +static void qemu_kvm_destroy_vcpu(CPUState *cpu)
>>>>>> +{
>>>>>> + CPU_REMOVE(cpu);
>>>>>> +
>>>>>> + if (kvm_destroy_vcpu(cpu) < 0) {
>>>>>> + fprintf(stderr, "kvm_destroy_vcpu failed.\n");
>>>>>> + exit(1);
>>>>>> + }
>>>>>> +
>>>>>> + object_unparent(OBJECT(cpu));
>>>>>> +}
>>>>>> +
>>>>>> +static void qemu_tcg_destroy_vcpu(CPUState *cpu)
>>>>>> +{
>>>>>> + CPU_REMOVE(cpu);
>>>>>> + object_unparent(OBJECT(cpu));
>>>>>> +}
>>>>>> +
>>>>>> static void flush_queued_work(CPUState *cpu)
>>>>>> {
>>>>>> struct qemu_work_item *wi;
>>>>>> @@ -877,6 +895,11 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
>>>>>> }
>>>>>> }
>>>>>> qemu_kvm_wait_io_event(cpu);
>>>>>> + if (cpu->exit && !cpu_can_run(cpu)) {
>>>>>> + qemu_kvm_destroy_vcpu(cpu);
>>>>>> + qemu_mutex_unlock(&qemu_global_mutex);
>>>>>> + return NULL;
>>>>>> + }
>>>>>> }
>>>>>>
>>>>>> return NULL;
>>>>>> @@ -929,6 +952,7 @@ static void tcg_exec_all(void);
>>>>>> static void *qemu_tcg_cpu_thread_fn(void *arg)
>>>>>> {
>>>>>> CPUState *cpu = arg;
>>>>>> + CPUState *remove_cpu = NULL;
>>>>>>
>>>>>> qemu_tcg_init_cpu_signals();
>>>>>> qemu_thread_get_self(cpu->thread);
>>>>>> @@ -961,6 +985,16 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
>>>>>> }
>>>>>> }
>>>>>> qemu_tcg_wait_io_event();
>>>>>> + CPU_FOREACH(cpu) {
>>>>>> + if (cpu->exit && !cpu_can_run(cpu)) {
>>>>>> + remove_cpu = cpu;
>>>>>> + break;
>>>>>> + }
>>>>>> + }
>>>>>> + if (remove_cpu) {
>>>>>> + qemu_tcg_destroy_vcpu(remove_cpu);
>>>>>> + remove_cpu = NULL;
>>>>>> + }
>>>>>> }
>>>>>>
>>>>>> return NULL;
>>>>>> @@ -1316,6 +1350,9 @@ static void tcg_exec_all(void)
>>>>>> break;
>>>>>> }
>>>>>> } else if (cpu->stop || cpu->stopped) {
>>>>>> + if (cpu->exit) {
>>>>>> + next_cpu = CPU_NEXT(cpu);
>>>>>> + }
>>>>>> break;
>>>>>> }
>>>>>> }
>>>>>> diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
>>>>>> index 174ea36..88e2403 100644
>>>>>> --- a/include/sysemu/kvm.h
>>>>>> +++ b/include/sysemu/kvm.h
>>>>>> @@ -178,6 +178,7 @@ int kvm_has_intx_set_mask(void);
>>>>>>
>>>>>> int kvm_init_vcpu(CPUState *cpu);
>>>>>> int kvm_cpu_exec(CPUState *cpu);
>>>>>> +int kvm_destroy_vcpu(CPUState *cpu);
>>>>>>
>>>>>> #ifdef NEED_CPU_H
>>>>>>
>>>>>> diff --git a/kvm-all.c b/kvm-all.c
>>>>>> index 3ae30ee..25e2a43 100644
>>>>>> --- a/kvm-all.c
>>>>>> +++ b/kvm-all.c
>>>>>> @@ -74,6 +74,12 @@ typedef struct KVMSlot
>>>>>>
>>>>>> typedef struct kvm_dirty_log KVMDirtyLog;
>>>>>>
>>>>>> +struct KVMParkedVcpu {
>>>>>> + unsigned long vcpu_id;
>>>>>> + int kvm_fd;
>>>>>> + QLIST_ENTRY(KVMParkedVcpu) node;
>>>>>> +};
>>>>>> +
>>>>>> struct KVMState
>>>>>> {
>>>>>> KVMSlot *slots;
>>>>>> @@ -108,6 +114,7 @@ struct KVMState
>>>>>> QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
>>>>>> bool direct_msi;
>>>>>> #endif
>>>>>> + QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus;
>>>>>> };
>>>>>>
>>>>>> KVMState *kvm_state;
>>>>>> @@ -226,6 +233,53 @@ static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
>>>>>> return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
>>>>>> }
>>>>>>
>>>>>> +int kvm_destroy_vcpu(CPUState *cpu)
>>>>>> +{
>>>>>> + KVMState *s = kvm_state;
>>>>>> + long mmap_size;
>>>>>> + struct KVMParkedVcpu *vcpu = NULL;
>>>>>> + int ret = 0;
>>>>>> +
>>>>>> + DPRINTF("kvm_destroy_vcpu\n");
>>>>>> +
>>>>>> + mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
>>>>>> + if (mmap_size < 0) {
>>>>>> + ret = mmap_size;
>>>>>> + DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
>>>>>> + goto err;
>>>>>> + }
>>>>>> +
>>>>>> + ret = munmap(cpu->kvm_run, mmap_size);
>>>>>> + if (ret < 0) {
>>>>>> + goto err;
>>>>>> + }
>>>>>> +
>>>>>> + vcpu = g_malloc0(sizeof(*vcpu));
>>>>>> + vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
>>>>>> + vcpu->kvm_fd = cpu->kvm_fd;
>>>>>> + QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
>>>>>> +err:
>>>>>> + return ret;
>>>>>> +}
>>>>>> +
>>>>>> +static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id)
>>>>>> +{
>>>>>> + struct KVMParkedVcpu *cpu;
>>>>>> +
>>>>>> + QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
>>>>>> + if (cpu->vcpu_id == vcpu_id) {
>>>>>> + int kvm_fd;
>>>>>> +
>>>>>> + QLIST_REMOVE(cpu, node);
>>>>>> + kvm_fd = cpu->kvm_fd;
>>>>>> + g_free(cpu);
>>>>>> + return kvm_fd;
>>>>>> + }
>>>>>> + }
>>>>>> +
>>>>>> + return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id);
>>>>>> +}
>>>>>> +
>>>>>> int kvm_init_vcpu(CPUState *cpu)
>>>>>> {
>>>>>> KVMState *s = kvm_state;
>>>>>> @@ -234,7 +288,7 @@ int kvm_init_vcpu(CPUState *cpu)
>>>>>>
>>>>>> DPRINTF("kvm_init_vcpu\n");
>>>>>>
>>>>>> - ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
>>>>>> + ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu));
>>>>>> if (ret < 0) {
>>>>>> DPRINTF("kvm_create_vcpu failed\n");
>>>>>> goto err;
>>>>>> @@ -1404,6 +1458,7 @@ int kvm_init(MachineClass *mc)
>>>>>> #ifdef KVM_CAP_SET_GUEST_DEBUG
>>>>>> QTAILQ_INIT(&s->kvm_sw_breakpoints);
>>>>>> #endif
>>>>>> + QLIST_INIT(&s->kvm_parked_vcpus);
>>>>>> s->vmfd = -1;
>>>>>> s->fd = qemu_open("/dev/kvm", O_RDWR);
>>>>>> if (s->fd == -1) {
>>>>>> --
>>>>>> 1.7.7
>>>>>>
>>>>> .
>>>>>
>>>>
>>>>
>>> .
>>>
>>
>>
> .
>
^ permalink raw reply [flat|nested] 16+ messages in thread
* Re: [Qemu-devel] [RFC PATCH 0/7] i386: add cpu hot remove support
2014-07-11 9:59 [Qemu-devel] [RFC PATCH 0/7] i386: add cpu hot remove support Gu Zheng
` (6 preceding siblings ...)
2014-07-11 9:59 ` [Qemu-devel] [RFC PATCH 7/7] cpus: reclaim allocated vCPU objects Gu Zheng
@ 2014-07-15 1:16 ` Gu Zheng
2014-07-22 2:24 ` Gu Zheng
8 siblings, 0 replies; 16+ messages in thread
From: Gu Zheng @ 2014-07-15 1:16 UTC (permalink / raw)
To: Gu Zheng
Cc: qemu-devel, tangchen, isimatu.yasuaki, imammedo, anshul.makkar,
afaerber
ping...
On 07/11/2014 05:59 PM, Gu Zheng wrote:
> Via implementing ACPI standard methods _EJ0 in ACPI table, after Guest OS remove
> one vCPU online, the fireware will store removed bitmap to QEMU, then QEMU could
> know to notify the assigned vCPU of exiting. meanwhile, intruduce the QOM command
> 'device_del' to remove vCPU from QEMU itself.
>
> This job is based on Chen Fan's previous "cpu-del" patchset:
> https://lists.nongnu.org/archive/html/qemu-devel/2013-12/msg04266.html
>
> And it depends on the previous "device_add foo-x86_64-cpu" patchset:
> https://lists.nongnu.org/archive/html/qemu-devel/2014-06/msg06690.html
>
> Chen Fan (4):
> x86: add x86_cpu_unrealizefn() for cpu apic remove
> qom cpu: rename variable 'cpu_added_notifier' to
> 'cpu_hotplug_notifier'
> i386: implement pc interface cpu_common_unrealizefn() in qom/cpu.c
> cpu hotplug: implement function cpu_status_write() for vcpu ejection
>
> Gu Zheng (3):
> i386: add cpu device_del support
> qom cpu: add UNPLUG cpu notify support
> cpus: reclaim allocated vCPU objects
>
> cpus.c | 44 ++++++++++++++++++++++++
> hw/acpi/cpu_hotplug.c | 55 ++++++++++++++++++++++++++++---
> hw/acpi/ich9.c | 13 ++++---
> hw/acpi/piix4.c | 21 +++++++-----
> hw/i386/acpi-dsdt-cpu-hotplug.dsl | 6 +++-
> hw/i386/kvm/apic.c | 8 ++++
> hw/i386/pc.c | 2 +-
> hw/intc/apic.c | 10 +++++
> hw/intc/apic_common.c | 23 ++++++++++++-
> include/hw/acpi/cpu_hotplug.h | 14 +++++++-
> include/hw/acpi/ich9.h | 2 +-
> include/hw/cpu/icc_bus.h | 1 +
> include/hw/i386/apic_internal.h | 1 +
> include/qom/cpu.h | 9 +++++
> include/sysemu/kvm.h | 1 +
> include/sysemu/sysemu.h | 2 +-
> kvm-all.c | 57 +++++++++++++++++++++++++++++++-
> qom/cpu.c | 27 ++++++++++++---
> target-i386/cpu-qom.h | 1 +
> target-i386/cpu.c | 66 +++++++++++++++++++++++++++++++++++++
> 20 files changed, 332 insertions(+), 31 deletions(-)
>
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [Qemu-devel] [RFC PATCH 0/7] i386: add cpu hot remove support
2014-07-11 9:59 [Qemu-devel] [RFC PATCH 0/7] i386: add cpu hot remove support Gu Zheng
` (7 preceding siblings ...)
2014-07-15 1:16 ` [Qemu-devel] [RFC PATCH 0/7] i386: add cpu hot remove support Gu Zheng
@ 2014-07-22 2:24 ` Gu Zheng
8 siblings, 0 replies; 16+ messages in thread
From: Gu Zheng @ 2014-07-22 2:24 UTC (permalink / raw)
To: Gu Zheng
Cc: qemu-devel, tangchen, isimatu.yasuaki, imammedo, anshul.makkar,
afaerber
Hi Igor, Andreas,
How about this patchset? Could you please help to review it?
Any comment is welcome.
Regards,
Gu
On 07/11/2014 05:59 PM, Gu Zheng wrote:
> Via implementing ACPI standard methods _EJ0 in ACPI table, after Guest OS remove
> one vCPU online, the fireware will store removed bitmap to QEMU, then QEMU could
> know to notify the assigned vCPU of exiting. meanwhile, intruduce the QOM command
> 'device_del' to remove vCPU from QEMU itself.
>
> This job is based on Chen Fan's previous "cpu-del" patchset:
> https://lists.nongnu.org/archive/html/qemu-devel/2013-12/msg04266.html
>
> And it depends on the previous "device_add foo-x86_64-cpu" patchset:
> https://lists.nongnu.org/archive/html/qemu-devel/2014-06/msg06690.html
>
> Chen Fan (4):
> x86: add x86_cpu_unrealizefn() for cpu apic remove
> qom cpu: rename variable 'cpu_added_notifier' to
> 'cpu_hotplug_notifier'
> i386: implement pc interface cpu_common_unrealizefn() in qom/cpu.c
> cpu hotplug: implement function cpu_status_write() for vcpu ejection
>
> Gu Zheng (3):
> i386: add cpu device_del support
> qom cpu: add UNPLUG cpu notify support
> cpus: reclaim allocated vCPU objects
>
> cpus.c | 44 ++++++++++++++++++++++++
> hw/acpi/cpu_hotplug.c | 55 ++++++++++++++++++++++++++++---
> hw/acpi/ich9.c | 13 ++++---
> hw/acpi/piix4.c | 21 +++++++-----
> hw/i386/acpi-dsdt-cpu-hotplug.dsl | 6 +++-
> hw/i386/kvm/apic.c | 8 ++++
> hw/i386/pc.c | 2 +-
> hw/intc/apic.c | 10 +++++
> hw/intc/apic_common.c | 23 ++++++++++++-
> include/hw/acpi/cpu_hotplug.h | 14 +++++++-
> include/hw/acpi/ich9.h | 2 +-
> include/hw/cpu/icc_bus.h | 1 +
> include/hw/i386/apic_internal.h | 1 +
> include/qom/cpu.h | 9 +++++
> include/sysemu/kvm.h | 1 +
> include/sysemu/sysemu.h | 2 +-
> kvm-all.c | 57 +++++++++++++++++++++++++++++++-
> qom/cpu.c | 27 ++++++++++++---
> target-i386/cpu-qom.h | 1 +
> target-i386/cpu.c | 66 +++++++++++++++++++++++++++++++++++++
> 20 files changed, 332 insertions(+), 31 deletions(-)
>
^ permalink raw reply [flat|nested] 16+ messages in thread