Re: [Qemu-devel] [RFC PATCH v2 9/9] spapr: CPU hot unplug support

[David Gibson <david@gibson.dropbear.id.au>]

[-- Attachment #1: Type: text/plain, Size: 7348 bytes --]

On Fri, Mar 11, 2016 at 10:24:38AM +0530, Bharata B Rao wrote:
> Remove the CPU core device by removing the underlying CPU thread devices.
> Hot removal of CPU for sPAPR guests is achieved by sending the hot unplug
> notification to the guest. Release the vCPU object after CPU hot unplug so
> that vCPU fd can be parked and reused.
> 
> Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
> ---
>  hw/ppc/spapr.c                  | 21 ++++++++++
>  hw/ppc/spapr_cpu_core.c         | 86 +++++++++++++++++++++++++++++++++++++++++
>  include/hw/ppc/spapr.h          |  1 +
>  include/hw/ppc/spapr_cpu_core.h | 12 ++++++
>  4 files changed, 120 insertions(+)
> 
> diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> index 822c87d..b1e9ba2 100644
> --- a/hw/ppc/spapr.c
> +++ b/hw/ppc/spapr.c
> @@ -2345,7 +2345,12 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
>  
>          spapr_memory_plug(hotplug_dev, dev, node, errp);
>      } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
> +        /*
> +         * TODO: Move this check to pre_plug handler at which point
> +         * spapr_core_release() won't be necessary.
> +         */
>          if (!smc->dr_cpu_enabled && dev->hotplugged) {
> +            spapr_core_release(dev);
>              error_setg(errp, "CPU hotplug not supported for this machine");
>              return;
>          }

This hunk doesn't look like its related to unplug.  Did it belong in
another patch?

> @@ -2353,11 +2358,27 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
>      }
>  }
>  
> +void spapr_cpu_destroy(PowerPCCPU *cpu)
> +{
> +    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
> +
> +    xics_cpu_destroy(spapr->icp, cpu);
> +    qemu_unregister_reset(spapr_cpu_reset, cpu);
> +}
> +
>  static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev,
>                                        DeviceState *dev, Error **errp)
>  {
> +    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine());
> +
>      if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
>          error_setg(errp, "Memory hot unplug not supported by sPAPR");
> +    } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
> +        if (!smc->dr_cpu_enabled) {
> +            error_setg(errp, "CPU hot unplug not supported on this machine");
> +            return;
> +        }
> +        spapr_core_unplug(hotplug_dev, dev, errp);
>      }
>  }
>  
> diff --git a/hw/ppc/spapr_cpu_core.c b/hw/ppc/spapr_cpu_core.c
> index db8de32..dd391bd 100644
> --- a/hw/ppc/spapr_cpu_core.c
> +++ b/hw/ppc/spapr_cpu_core.c
> @@ -84,6 +84,92 @@ void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
>      }
>  }
>  
> +static void spapr_cpu_core_cleanup(struct sPAPRCPUUnplugList *unplug_list)
> +{
> +    sPAPRCPUUnplug *unplug, *next;
> +    Object *cpu;
> +
> +    QLIST_FOREACH_SAFE(unplug, unplug_list, node, next) {
> +        cpu = unplug->cpu;
> +        object_unparent(cpu);
> +        QLIST_REMOVE(unplug, node);
> +        g_free(unplug);
> +    }
> +}
> +
> +static void spapr_add_cpu_to_unplug_list(Object *cpu,
> +                                         struct sPAPRCPUUnplugList *unplug_list)
> +{
> +    sPAPRCPUUnplug *unplug = g_malloc(sizeof(*unplug));
> +
> +    unplug->cpu = cpu;
> +    QLIST_INSERT_HEAD(unplug_list, unplug, node);
> +}
> +
> +static int spapr_cpu_release(Object *obj, void *opaque)
> +{
> +    DeviceState *dev = DEVICE(obj);
> +    CPUState *cs = CPU(dev);
> +    PowerPCCPU *cpu = POWERPC_CPU(cs);
> +    struct sPAPRCPUUnplugList *unplug_list = opaque;
> +
> +    spapr_cpu_destroy(cpu);
> +    cpu_remove_sync(cs);
> +
> +    /*
> +     * We are still walking the core object's children list, and
> +     * hence can't cleanup this CPU thread object just yet. Put
> +     * it on a list for later removal.
> +     */
> +    spapr_add_cpu_to_unplug_list(obj, unplug_list);
> +    return 0;
> +}
> +
> +void spapr_core_release(DeviceState *dev)
> +{
> +    struct sPAPRCPUUnplugList unplug_list;
> +    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
> +    sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev));
> +    int core_dt_id = object_property_get_int(OBJECT(dev), "core", NULL);
> +    int smt = kvmppc_smt_threads();
> +
> +    QLIST_INIT(&unplug_list);
> +    object_child_foreach(OBJECT(dev), spapr_cpu_release, &unplug_list);
> +    spapr_cpu_core_cleanup(&unplug_list);
> +    spapr->cores[core_dt_id / smt] = NULL;
> +
> +    g_free(core->threads);
> +}
> +
> +static void spapr_core_release_unparent(DeviceState *dev, void *opaque)
> +{
> +    spapr_core_release(dev);
> +    object_unparent(OBJECT(dev));
> +}
> +
> +void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
> +                       Error **errp)
> +{
> +    sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev));
> +    PowerPCCPU *cpu = &core->threads[0];
> +    int id = ppc_get_vcpu_dt_id(cpu);
> +    sPAPRDRConnector *drc =
> +        spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id);
> +    sPAPRDRConnectorClass *drck;
> +    Error *local_err = NULL;
> +
> +    g_assert(drc);
> +
> +    drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
> +    drck->detach(drc, dev, spapr_core_release_unparent, NULL, &local_err);
> +    if (local_err) {
> +        error_propagate(errp, local_err);
> +        return;
> +    }
> +
> +    spapr_hotplug_req_remove_by_index(drc);
> +}
> +
>  static void spapr_cpu_core_create_threads(sPAPRCPUCore *core, int threads,
>                                            Error **errp)
>  {
> diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h
> index 8957072..41d0928 100644
> --- a/include/hw/ppc/spapr.h
> +++ b/include/hw/ppc/spapr.h
> @@ -591,6 +591,7 @@ void spapr_cpu_init(sPAPRMachineState *spapr, PowerPCCPU *cpu, Error **errp);
>  void spapr_cpu_reset(void *opaque);
>  void *spapr_populate_hotplug_cpu_dt(DeviceState *dev, CPUState *cs,
>                                      int *fdt_offset, sPAPRMachineState *spapr);
> +void spapr_cpu_destroy(PowerPCCPU *cpu);
>  
>  /* rtas-configure-connector state */
>  struct sPAPRConfigureConnectorState {
> diff --git a/include/hw/ppc/spapr_cpu_core.h b/include/hw/ppc/spapr_cpu_core.h
> index 980d8ae..75c6c58 100644
> --- a/include/hw/ppc/spapr_cpu_core.h
> +++ b/include/hw/ppc/spapr_cpu_core.h
> @@ -27,4 +27,16 @@ typedef struct sPAPRCPUCore {
>  
>  void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
>                       Error **errp);
> +void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
> +                       Error **errp);
> +void spapr_core_release(DeviceState *dev);
> +
> +/* List to store unplugged CPU objects for cleanup during unplug */
> +typedef struct sPAPRCPUUnplug {
> +    Object *cpu;
> +    QLIST_ENTRY(sPAPRCPUUnplug) node;
> +} sPAPRCPUUnplug;
> +
> +QLIST_HEAD(sPAPRCPUUnplugList, sPAPRCPUUnplug);
> +
>  #endif

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]