Re: [Qemu-devel] [Qemu-ppc] [RFC PATCH v2 19/23] spapr: CPU hot unplug support

[Alexey Kardashevskiy <aik@ozlabs.ru>]

On 03/24/2015 12:36 AM, Bharata B Rao wrote:
> Support hot removal of CPU for sPAPR guests by sending the hot
> unplug notification to the guest via EPOW interrupt.
>
> Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
> ---
>   hw/ppc/spapr.c            | 78 ++++++++++++++++++++++++++++++++++++++++++++++-
>   linux-headers/linux/kvm.h |  1 +
>   target-ppc/kvm.c          |  7 +++++
>   target-ppc/kvm_ppc.h      |  6 ++++
>   4 files changed, 91 insertions(+), 1 deletion(-)
>
> diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> index b48994b..7b8784d 100644
> --- a/hw/ppc/spapr.c
> +++ b/hw/ppc/spapr.c
> @@ -1468,6 +1468,12 @@ static void spapr_cpu_init(PowerPCCPU *cpu)
>       qemu_register_reset(spapr_cpu_reset, cpu);
>   }
>
> +static void spapr_cpu_destroy(PowerPCCPU *cpu)
> +{
> +    xics_cpu_destroy(spapr->icp, cpu);
> +    qemu_unregister_reset(spapr_cpu_reset, cpu);
> +}
> +
>   /* pSeries LPAR / sPAPR hardware init */
>   static void ppc_spapr_init(MachineState *machine)
>   {
> @@ -1880,6 +1886,18 @@ static void spapr_cpu_hotplug_add(DeviceState *dev, CPUState *cs, Error **errp)
>       }
>   }
>
> +static void spapr_cpu_hotplug_remove(DeviceState *dev, CPUState *cs,
> +                                     Error **errp)
> +{
> +    PowerPCCPU *cpu = POWERPC_CPU(cs);
> +    int id = ppc_get_vcpu_dt_id(cpu);
> +    sPAPRDRConnector *drc =
> +        spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id);
> +    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
> +
> +    drck->detach(drc, dev, NULL, NULL, errp);
> +}
> +
>   static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
>                               Error **errp)
>   {
> @@ -1911,6 +1929,51 @@ static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
>       return;
>   }
>
> +static int spapr_cpu_unplug(Object *obj, void *opaque)
> +{
> +    Error **errp = opaque;
> +    DeviceState *dev = DEVICE(obj);
> +    CPUState *cs = CPU(dev);
> +    PowerPCCPU *cpu = POWERPC_CPU(cs);
> +    int id = ppc_get_vcpu_dt_id(cpu);
> +    int smt = kvmppc_smt_threads();
> +    sPAPRDRConnector *drc =
> +        spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id);
> +
> +    spapr_cpu_destroy(cpu);
> +
> +    /*
> +     * SMT threads return from here, only main thread (core) will
> +     * continue and signal hot unplug event to the guest.
> +     */
> +    if ((id % smt) != 0) {
> +        return 0;
> +    }
> +    g_assert(drc);
> +
> +    spapr_cpu_hotplug_remove(dev, cs, errp);
> +    if (*errp) {
> +        return -1;
> +    }
> +    spapr_hotplug_req_remove_event(drc);
> +
> +    return 0;
> +}
> +
> +static int spapr_cpu_core_unplug(Object *obj, void *opaque)
> +{
> +    Error **errp = opaque;
> +
> +    object_child_foreach(obj, spapr_cpu_unplug, errp);
> +    return 0;
> +}
> +
> +static void spapr_cpu_socket_unplug(HotplugHandler *hotplug_dev,
> +                            DeviceState *dev, Error **errp)
> +{
> +    object_child_foreach(OBJECT(dev), spapr_cpu_core_unplug, errp);
> +}
> +
>   static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
>                                         DeviceState *dev, Error **errp)
>   {
> @@ -1926,10 +1989,21 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
>       }
>   }
>
> +static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev,
> +                                      DeviceState *dev, Error **errp)
> +{
> +    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) {
> +        if (dev->hotplugged && spapr->dr_cpu_enabled) {
> +            spapr_cpu_socket_unplug(hotplug_dev, dev, errp);
> +        }
> +    }
> +}
> +
>   static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine,
>                                                DeviceState *dev)
>   {
> -    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
> +    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU) ||
> +        object_dynamic_cast(OBJECT(dev), TYPE_CPU_SOCKET)) {


What is this change for? I mean why is not it always socket-only? Commit 
log would not hurt here...


>           return HOTPLUG_HANDLER(machine);
>       }
>       return NULL;
> @@ -1953,6 +2027,8 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data)
>       mc->has_dynamic_sysbus = true;
>       mc->get_hotplug_handler = spapr_get_hotpug_handler;
>       hc->plug = spapr_machine_device_plug;
> +    hc->unplug = spapr_machine_device_unplug;
> +
>       smc->dr_phb_enabled = false;
>       smc->dr_cpu_enabled = false;
>       smc->dr_lmb_enabled = false;
> diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
> index 12045a1..0c1be07 100644
> --- a/linux-headers/linux/kvm.h
> +++ b/linux-headers/linux/kvm.h
> @@ -761,6 +761,7 @@ struct kvm_ppc_smmu_info {
>   #define KVM_CAP_PPC_FIXUP_HCALL 103
>   #define KVM_CAP_PPC_ENABLE_HCALL 104
>   #define KVM_CAP_CHECK_EXTENSION_VM 105
> +#define KVM_CAP_SPAPR_REUSE_VCPU 107
>
>   #ifdef KVM_CAP_IRQ_ROUTING
>
> diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c
> index 1edf2b5..ee23bf6 100644
> --- a/target-ppc/kvm.c
> +++ b/target-ppc/kvm.c
> @@ -72,6 +72,7 @@ static int cap_ppc_watchdog;
>   static int cap_papr;
>   static int cap_htab_fd;
>   static int cap_fixup_hcalls;
> +static int cap_spapr_reuse_vcpu;
>
>   static uint32_t debug_inst_opcode;
>
> @@ -114,6 +115,7 @@ int kvm_arch_init(KVMState *s)
>        * only activated after this by kvmppc_set_papr() */
>       cap_htab_fd = kvm_check_extension(s, KVM_CAP_PPC_HTAB_FD);
>       cap_fixup_hcalls = kvm_check_extension(s, KVM_CAP_PPC_FIXUP_HCALL);
> +    cap_spapr_reuse_vcpu = kvm_check_extension(s, KVM_CAP_SPAPR_REUSE_VCPU);
>
>       if (!cap_interrupt_level) {
>           fprintf(stderr, "KVM: Couldn't find level irq capability. Expect the "
> @@ -2408,3 +2410,8 @@ int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
>   {
>       return 0;
>   }
> +
> +bool kvmppc_spapr_reuse_vcpu(void)
> +{
> +    return cap_spapr_reuse_vcpu;
> +}
> diff --git a/target-ppc/kvm_ppc.h b/target-ppc/kvm_ppc.h
> index 2e0224c..c831229 100644
> --- a/target-ppc/kvm_ppc.h
> +++ b/target-ppc/kvm_ppc.h
> @@ -40,6 +40,7 @@ void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd,
>   int kvmppc_remove_spapr_tce(void *table, int pfd, uint32_t window_size);
>   int kvmppc_reset_htab(int shift_hint);
>   uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift);
> +bool kvmppc_spapr_reuse_vcpu(void);
>   #endif /* !CONFIG_USER_ONLY */
>   bool kvmppc_has_cap_epr(void);
>   int kvmppc_define_rtas_kernel_token(uint32_t token, const char *function);
> @@ -185,6 +186,11 @@ static inline int kvmppc_update_sdr1(CPUPPCState *env)
>       return 0;
>   }
>
> +static inline bool kvmppc_spapr_reuse_vcpu(void)
> +{
> +    return false;
> +}
> +
>   #endif /* !CONFIG_USER_ONLY */
>
>   static inline bool kvmppc_has_cap_epr(void)
>


-- 
Alexey