[Qemu-devel] [RFC PATCH v2 19/23] spapr: CPU hot unplug support

[Bharata B Rao <bharata@linux.vnet.ibm.com>]

Support hot removal of CPU for sPAPR guests by sending the hot
unplug notification to the guest via EPOW interrupt.

Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
---
 hw/ppc/spapr.c            | 78 ++++++++++++++++++++++++++++++++++++++++++++++-
 linux-headers/linux/kvm.h |  1 +
 target-ppc/kvm.c          |  7 +++++
 target-ppc/kvm_ppc.h      |  6 ++++
 4 files changed, 91 insertions(+), 1 deletion(-)

diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index b48994b..7b8784d 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -1468,6 +1468,12 @@ static void spapr_cpu_init(PowerPCCPU *cpu)
     qemu_register_reset(spapr_cpu_reset, cpu);
 }
 
+static void spapr_cpu_destroy(PowerPCCPU *cpu)
+{
+    xics_cpu_destroy(spapr->icp, cpu);
+    qemu_unregister_reset(spapr_cpu_reset, cpu);
+}
+
 /* pSeries LPAR / sPAPR hardware init */
 static void ppc_spapr_init(MachineState *machine)
 {
@@ -1880,6 +1886,18 @@ static void spapr_cpu_hotplug_add(DeviceState *dev, CPUState *cs, Error **errp)
     }
 }
 
+static void spapr_cpu_hotplug_remove(DeviceState *dev, CPUState *cs,
+                                     Error **errp)
+{
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    int id = ppc_get_vcpu_dt_id(cpu);
+    sPAPRDRConnector *drc =
+        spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id);
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+
+    drck->detach(drc, dev, NULL, NULL, errp);
+}
+
 static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
                             Error **errp)
 {
@@ -1911,6 +1929,51 @@ static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
     return;
 }
 
+static int spapr_cpu_unplug(Object *obj, void *opaque)
+{
+    Error **errp = opaque;
+    DeviceState *dev = DEVICE(obj);
+    CPUState *cs = CPU(dev);
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    int id = ppc_get_vcpu_dt_id(cpu);
+    int smt = kvmppc_smt_threads();
+    sPAPRDRConnector *drc =
+        spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id);
+
+    spapr_cpu_destroy(cpu);
+
+    /*
+     * SMT threads return from here, only main thread (core) will
+     * continue and signal hot unplug event to the guest.
+     */
+    if ((id % smt) != 0) {
+        return 0;
+    }
+    g_assert(drc);
+
+    spapr_cpu_hotplug_remove(dev, cs, errp);
+    if (*errp) {
+        return -1;
+    }
+    spapr_hotplug_req_remove_event(drc);
+
+    return 0;
+}
+
+static int spapr_cpu_core_unplug(Object *obj, void *opaque)
+{
+    Error **errp = opaque;
+
+    object_child_foreach(obj, spapr_cpu_unplug, errp);
+    return 0;
+}
+
+static void spapr_cpu_socket_unplug(HotplugHandler *hotplug_dev,
+                            DeviceState *dev, Error **errp)
+{
+    object_child_foreach(OBJECT(dev), spapr_cpu_core_unplug, errp);
+}
+
 static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
                                       DeviceState *dev, Error **errp)
 {
@@ -1926,10 +1989,21 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
     }
 }
 
+static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev,
+                                      DeviceState *dev, Error **errp)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) {
+        if (dev->hotplugged && spapr->dr_cpu_enabled) {
+            spapr_cpu_socket_unplug(hotplug_dev, dev, errp);
+        }
+    }
+}
+
 static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine,
                                              DeviceState *dev)
 {
-    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
+    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU) ||
+        object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) {
         return HOTPLUG_HANDLER(machine);
     }
     return NULL;
@@ -1953,6 +2027,8 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data)
     mc->has_dynamic_sysbus = true;
     mc->get_hotplug_handler = spapr_get_hotpug_handler;
     hc->plug = spapr_machine_device_plug;
+    hc->unplug = spapr_machine_device_unplug;
+
     smc->dr_phb_enabled = false;
     smc->dr_cpu_enabled = false;
     smc->dr_lmb_enabled = false;
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 12045a1..0c1be07 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -761,6 +761,7 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_PPC_FIXUP_HCALL 103
 #define KVM_CAP_PPC_ENABLE_HCALL 104
 #define KVM_CAP_CHECK_EXTENSION_VM 105
+#define KVM_CAP_SPAPR_REUSE_VCPU 107
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c
index 1edf2b5..ee23bf6 100644
--- a/target-ppc/kvm.c
+++ b/target-ppc/kvm.c
@@ -72,6 +72,7 @@ static int cap_ppc_watchdog;
 static int cap_papr;
 static int cap_htab_fd;
 static int cap_fixup_hcalls;
+static int cap_spapr_reuse_vcpu;
 
 static uint32_t debug_inst_opcode;
 
@@ -114,6 +115,7 @@ int kvm_arch_init(KVMState *s)
      * only activated after this by kvmppc_set_papr() */
     cap_htab_fd = kvm_check_extension(s, KVM_CAP_PPC_HTAB_FD);
     cap_fixup_hcalls = kvm_check_extension(s, KVM_CAP_PPC_FIXUP_HCALL);
+    cap_spapr_reuse_vcpu = kvm_check_extension(s, KVM_CAP_SPAPR_REUSE_VCPU);
 
     if (!cap_interrupt_level) {
         fprintf(stderr, "KVM: Couldn't find level irq capability. Expect the "
@@ -2408,3 +2410,8 @@ int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
 {
     return 0;
 }
+
+bool kvmppc_spapr_reuse_vcpu(void)
+{
+    return cap_spapr_reuse_vcpu;
+}
diff --git a/target-ppc/kvm_ppc.h b/target-ppc/kvm_ppc.h
index 2e0224c..c831229 100644
--- a/target-ppc/kvm_ppc.h
+++ b/target-ppc/kvm_ppc.h
@@ -40,6 +40,7 @@ void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd,
 int kvmppc_remove_spapr_tce(void *table, int pfd, uint32_t window_size);
 int kvmppc_reset_htab(int shift_hint);
 uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift);
+bool kvmppc_spapr_reuse_vcpu(void);
 #endif /* !CONFIG_USER_ONLY */
 bool kvmppc_has_cap_epr(void);
 int kvmppc_define_rtas_kernel_token(uint32_t token, const char *function);
@@ -185,6 +186,11 @@ static inline int kvmppc_update_sdr1(CPUPPCState *env)
     return 0;
 }
 
+static inline bool kvmppc_spapr_reuse_vcpu(void)
+{
+    return false;
+}
+
 #endif /* !CONFIG_USER_ONLY */
 
 static inline bool kvmppc_has_cap_epr(void)
-- 
2.1.0