Re: [Qemu-devel] [Qemu-ppc] [RFC PATCH v2 19/23] spapr: CPU hot unplug support

[Bharata B Rao <bharata@linux.vnet.ibm.com>]

On Tue, Apr 07, 2015 at 04:45:17PM +1000, Alexey Kardashevskiy wrote:
> On 03/24/2015 12:36 AM, Bharata B Rao wrote:
> >Support hot removal of CPU for sPAPR guests by sending the hot
> >unplug notification to the guest via EPOW interrupt.
> >
> >Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
> >---
> >  hw/ppc/spapr.c            | 78 ++++++++++++++++++++++++++++++++++++++++++++++-
> >  linux-headers/linux/kvm.h |  1 +
> >  target-ppc/kvm.c          |  7 +++++
> >  target-ppc/kvm_ppc.h      |  6 ++++
> >  4 files changed, 91 insertions(+), 1 deletion(-)
> >
> >diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> >index b48994b..7b8784d 100644
> >--- a/hw/ppc/spapr.c
> >+++ b/hw/ppc/spapr.c
> >@@ -1468,6 +1468,12 @@ static void spapr_cpu_init(PowerPCCPU *cpu)
> >      qemu_register_reset(spapr_cpu_reset, cpu);
> >  }
> >
> >+static void spapr_cpu_destroy(PowerPCCPU *cpu)
> >+{
> >+    xics_cpu_destroy(spapr->icp, cpu);
> >+    qemu_unregister_reset(spapr_cpu_reset, cpu);
> >+}
> >+
> >  /* pSeries LPAR / sPAPR hardware init */
> >  static void ppc_spapr_init(MachineState *machine)
> >  {
> >@@ -1880,6 +1886,18 @@ static void spapr_cpu_hotplug_add(DeviceState *dev, CPUState *cs, Error **errp)
> >      }
> >  }
> >
> >+static void spapr_cpu_hotplug_remove(DeviceState *dev, CPUState *cs,
> >+                                     Error **errp)
> >+{
> >+    PowerPCCPU *cpu = POWERPC_CPU(cs);
> >+    int id = ppc_get_vcpu_dt_id(cpu);
> >+    sPAPRDRConnector *drc =
> >+        spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id);
> >+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
> >+
> >+    drck->detach(drc, dev, NULL, NULL, errp);
> >+}
> >+
> >  static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
> >                              Error **errp)
> >  {
> >@@ -1911,6 +1929,51 @@ static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
> >      return;
> >  }
> >
> >+static int spapr_cpu_unplug(Object *obj, void *opaque)
> >+{
> >+    Error **errp = opaque;
> >+    DeviceState *dev = DEVICE(obj);
> >+    CPUState *cs = CPU(dev);
> >+    PowerPCCPU *cpu = POWERPC_CPU(cs);
> >+    int id = ppc_get_vcpu_dt_id(cpu);
> >+    int smt = kvmppc_smt_threads();
> >+    sPAPRDRConnector *drc =
> >+        spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id);
> >+
> >+    spapr_cpu_destroy(cpu);
> >+
> >+    /*
> >+     * SMT threads return from here, only main thread (core) will
> >+     * continue and signal hot unplug event to the guest.
> >+     */
> >+    if ((id % smt) != 0) {
> >+        return 0;
> >+    }
> >+    g_assert(drc);
> >+
> >+    spapr_cpu_hotplug_remove(dev, cs, errp);
> >+    if (*errp) {
> >+        return -1;
> >+    }
> >+    spapr_hotplug_req_remove_event(drc);
> >+
> >+    return 0;
> >+}
> >+
> >+static int spapr_cpu_core_unplug(Object *obj, void *opaque)
> >+{
> >+    Error **errp = opaque;
> >+
> >+    object_child_foreach(obj, spapr_cpu_unplug, errp);
> >+    return 0;
> >+}
> >+
> >+static void spapr_cpu_socket_unplug(HotplugHandler *hotplug_dev,
> >+                            DeviceState *dev, Error **errp)
> >+{
> >+    object_child_foreach(OBJECT(dev), spapr_cpu_core_unplug, errp);
> >+}
> >+
> >  static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
> >                                        DeviceState *dev, Error **errp)
> >  {
> >@@ -1926,10 +1989,21 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
> >      }
> >  }
> >
> >+static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev,
> >+                                      DeviceState *dev, Error **errp)
> >+{
> >+    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) {
> >+        if (dev->hotplugged && spapr->dr_cpu_enabled) {
> >+            spapr_cpu_socket_unplug(hotplug_dev, dev, errp);
> >+        }
> >+    }
> >+}
> >+
> >  static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine,
> >                                               DeviceState *dev)
> >  {
> >-    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
> >+    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU) ||
> >+        object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) {
> 
> 
> What is this change for? I mean why is not it always socket-only? Commit log
> would not hurt here...

In the hot add case (do_device_add), the CPU socket device is realized first
which will realize the CPU core devices. Core devices will realize the CPU
thread devices. So the ->plug() operation happens as part of CPU thread devices
and hence hotplug_handler is returned only for TYPE_CPU.

However in case of hot remove, qdev_unplug() directly does ->unplug() and
hence I need to return the hotplug_handler for TYPE_CPU_SOCKET also.
This ensures that ->unplug() gets called for socket object where I take
care of recursively walking down the core and thread objects and unplugging
the CPU thread object eventually.

Regards,
Bharata.