From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([2001:4830:134:3::10]:39740) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1YpXXR-0001cd-H1 for qemu-devel@nongnu.org; Tue, 05 May 2015 03:45:11 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1YpXXO-0002z1-Jf for qemu-devel@nongnu.org; Tue, 05 May 2015 03:45:09 -0400 Date: Tue, 5 May 2015 16:59:51 +1000 From: David Gibson Message-ID: <20150505065951.GJ14090@voom.redhat.com> References: <1429858066-12088-1-git-send-email-bharata@linux.vnet.ibm.com> <1429858066-12088-13-git-send-email-bharata@linux.vnet.ibm.com> MIME-Version: 1.0 Content-Type: multipart/signed; micalg=pgp-sha1; protocol="application/pgp-signature"; boundary="HSQ3hISbU3Um6hch" Content-Disposition: inline In-Reply-To: <1429858066-12088-13-git-send-email-bharata@linux.vnet.ibm.com> Subject: Re: [Qemu-devel] [RFC PATCH v3 12/24] spapr: CPU hotplug support List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Bharata B Rao Cc: mdroth@linux.vnet.ibm.com, aik@ozlabs.ru, agraf@suse.de, qemu-devel@nongnu.org, qemu-ppc@nongnu.org, tyreld@linux.vnet.ibm.com, nfont@linux.vnet.ibm.com, imammedo@redhat.com, afaerber@suse.de --HSQ3hISbU3Um6hch Content-Type: text/plain; charset=us-ascii Content-Disposition: inline Content-Transfer-Encoding: quoted-printable On Fri, Apr 24, 2015 at 12:17:34PM +0530, Bharata B Rao wrote: > Support CPU hotplug via device-add command. Set up device tree > entries for the hotplugged CPU core and use the exising EPOW event > infrastructure to send CPU hotplug notification to the guest. >=20 > Also support cold plugged CPUs that are specified by -device option > on cmdline. >=20 > Signed-off-by: Bharata B Rao > --- > hw/ppc/spapr.c | 129 ++++++++++++++++++++++++++++++++++++++++++++= ++++++ > hw/ppc/spapr_events.c | 8 ++-- > hw/ppc/spapr_rtas.c | 11 +++++ > 3 files changed, 145 insertions(+), 3 deletions(-) >=20 > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c > index b526b7d..9b0701c 100644 > --- a/hw/ppc/spapr.c > +++ b/hw/ppc/spapr.c > @@ -33,6 +33,7 @@ > #include "sysemu/block-backend.h" > #include "sysemu/cpus.h" > #include "sysemu/kvm.h" > +#include "sysemu/device_tree.h" > #include "kvm_ppc.h" > #include "mmu-hash64.h" > #include "qom/cpu.h" > @@ -662,6 +663,17 @@ static void spapr_populate_cpu_dt(CPUState *cs, void= *fdt, int offset) > unsigned sockets =3D opts ? qemu_opt_get_number(opts, "sockets", 0) = : 0; > uint32_t cpus_per_socket =3D sockets ? (smp_cpus / sockets) : 1; > uint32_t pft_size_prop[] =3D {0, cpu_to_be32(spapr->htab_shift)}; > + sPAPRDRConnector *drc; > + sPAPRDRConnectorClass *drck; > + int drc_index; > + > + if (spapr->dr_cpu_enabled) { > + drc =3D spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, in= dex); > + g_assert(drc); > + drck =3D SPAPR_DR_CONNECTOR_GET_CLASS(drc); > + drc_index =3D drck->get_index(drc); > + _FDT((fdt_setprop_cell(fdt, offset, "ibm,my-drc-index", drc_inde= x))); > + } > =20 > _FDT((fdt_setprop_cell(fdt, offset, "reg", index))); > _FDT((fdt_setprop_string(fdt, offset, "device_type", "cpu"))); > @@ -1850,6 +1862,114 @@ static void spapr_nmi(NMIState *n, int cpu_index,= Error **errp) > } > } > =20 > +static void *spapr_populate_hotplug_cpu_dt(DeviceState *dev, CPUState *c= s, > + int *fdt_offset) > +{ > + PowerPCCPU *cpu =3D POWERPC_CPU(cs); > + DeviceClass *dc =3D DEVICE_GET_CLASS(cs); > + int id =3D ppc_get_vcpu_dt_id(cpu); > + void *fdt; > + int offset, fdt_size; > + char *nodename; > + > + fdt =3D create_device_tree(&fdt_size); > + nodename =3D g_strdup_printf("%s@%x", dc->fw_name, id); > + offset =3D fdt_add_subnode(fdt, 0, nodename); > + > + spapr_populate_cpu_dt(cs, fdt, offset); > + g_free(nodename); > + > + *fdt_offset =3D offset; > + return fdt; > +} > + > +static void spapr_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > + Error **errp) > +{ > + CPUState *cs =3D CPU(dev); > + PowerPCCPU *cpu =3D POWERPC_CPU(cs); > + int id =3D ppc_get_vcpu_dt_id(cpu); > + sPAPRDRConnector *drc =3D > + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); > + sPAPRDRConnectorClass *drck; > + int smt =3D kvmppc_smt_threads(); > + Error *local_err =3D NULL; > + void *fdt =3D NULL; > + int i, fdt_offset =3D 0; > + > + /* Set NUMA node for the added CPUs */ > + for (i =3D 0; i < nb_numa_nodes; i++) { > + if (test_bit(cs->cpu_index, numa_info[i].node_cpu)) { > + cs->numa_node =3D i; > + break; > + } > + } > + > + /* > + * SMT threads return from here, only main thread (core) will > + * continue and signal hotplug event to the guest. > + */ > + if ((id % smt) !=3D 0) { > + return; > + } Couldn't you avoid this by attaching this call to the core device, rather than the individual vcpu thread objects? > + if (!spapr->dr_cpu_enabled) { > + /* > + * This is a cold plugged CPU but the machine doesn't support > + * DR. So skip the hotplug path ensuring that the CPU is brought > + * up online with out an associated DR connector. > + */ > + return; > + } > + > + g_assert(drc); > + > + /* > + * Setup CPU DT entries only for hotplugged CPUs. For boot time or > + * coldplugged CPUs DT entries are setup in spapr_finalize_fdt(). > + */ > + if (dev->hotplugged) { > + fdt =3D spapr_populate_hotplug_cpu_dt(dev, cs, &fdt_offset); > + } > + > + drck =3D SPAPR_DR_CONNECTOR_GET_CLASS(drc); > + drck->attach(drc, dev, fdt, fdt_offset, !dev->hotplugged, &local_err= ); > + if (local_err) { > + g_free(fdt); > + error_propagate(errp, local_err); > + return; > + } > + > + /* > + * We send hotplug notification interrupt to the guest only in case > + * of hotplugged CPUs. > + */ > + if (dev->hotplugged) { > + spapr_hotplug_req_add_event(drc); > + } else { > + /* > + * HACK to support removal of hotplugged CPU after VM migration: > + * > + * Since we want to be able to hot-remove those coldplugged CPUs > + * started at boot time using -device option at the target VM, w= e set > + * the right allocation_state and isolation_state for them, whic= h for > + * the hotplugged CPUs would be set via RTAS calls done from the > + * guest during hotplug. > + * > + * This allows the coldplugged CPUs started using -device option= to > + * have the right isolation and allocation states as expected by= the > + * CPU hot removal code. > + * > + * This hack will be removed once we have DRC states migrated as= part > + * of VM migration. > + */ > + drck->set_allocation_state(drc, SPAPR_DR_ALLOCATION_STATE_USABLE= ); > + drck->set_isolation_state(drc, SPAPR_DR_ISOLATION_STATE_UNISOLAT= ED); > + } > + > + return; > +} > + > static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > DeviceState *dev, Error **errp) > { > @@ -1858,6 +1978,15 @@ static void spapr_machine_device_plug(HotplugHandl= er *hotplug_dev, > PowerPCCPU *cpu =3D POWERPC_CPU(cs); > =20 > spapr_cpu_init(cpu); > + spapr_cpu_reset(cpu); I'm a little surprised these get called here, rather than in the creation / realize path of the core qdev. > + /* > + * Fail hotplug on machines where CPU DR isn't enabled. > + */ > + if (!spapr->dr_cpu_enabled && dev->hotplugged) { > + return; > + } > + spapr_cpu_plug(hotplug_dev, dev, errp); > } > } > =20 > diff --git a/hw/ppc/spapr_events.c b/hw/ppc/spapr_events.c > index be82815..4ae818a 100644 > --- a/hw/ppc/spapr_events.c > +++ b/hw/ppc/spapr_events.c > @@ -421,14 +421,16 @@ static void spapr_hotplug_req_event(sPAPRDRConnecto= r *drc, uint8_t hp_action) > hp->hdr.section_length =3D cpu_to_be16(sizeof(*hp)); > hp->hdr.section_version =3D 1; /* includes extended modifier */ > hp->hotplug_action =3D hp_action; > - > + hp->drc.index =3D cpu_to_be32(drck->get_index(drc)); > + hp->hotplug_identifier =3D RTAS_LOG_V6_HP_ID_DRC_INDEX; > =20 > switch (drc_type) { > case SPAPR_DR_CONNECTOR_TYPE_PCI: > - hp->drc.index =3D cpu_to_be32(drck->get_index(drc)); > - hp->hotplug_identifier =3D RTAS_LOG_V6_HP_ID_DRC_INDEX; > hp->hotplug_type =3D RTAS_LOG_V6_HP_TYPE_PCI; > break; > + case SPAPR_DR_CONNECTOR_TYPE_CPU: > + hp->hotplug_type =3D RTAS_LOG_V6_HP_TYPE_CPU; > + break; > default: > /* we shouldn't be signaling hotplug events for resources > * that don't support them > diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c > index 57ec97a..48aeb86 100644 > --- a/hw/ppc/spapr_rtas.c > +++ b/hw/ppc/spapr_rtas.c > @@ -121,6 +121,16 @@ static void rtas_query_cpu_stopped_state(PowerPCCPU = *cpu_, > rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR); > } > =20 > +static void spapr_cpu_set_endianness(PowerPCCPU *cpu) > +{ > + PowerPCCPU *fcpu =3D POWERPC_CPU(first_cpu); > + PowerPCCPUClass *pcc =3D POWERPC_CPU_GET_CLASS(fcpu); > + > + if (!(*pcc->interrupts_big_endian)(fcpu)) { > + cpu->env.spr[SPR_LPCR] |=3D LPCR_ILE; > + } > +} > + > static void rtas_start_cpu(PowerPCCPU *cpu_, sPAPREnvironment *spapr, > uint32_t token, uint32_t nargs, > target_ulong args, > @@ -157,6 +167,7 @@ static void rtas_start_cpu(PowerPCCPU *cpu_, sPAPREnv= ironment *spapr, > env->nip =3D start; > env->gpr[3] =3D r3; > cs->halted =3D 0; > + spapr_cpu_set_endianness(cpu); > =20 > qemu_cpu_kick(cs); > =20 --=20 David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson --HSQ3hISbU3Um6hch Content-Type: application/pgp-signature -----BEGIN PGP SIGNATURE----- Version: GnuPG v1 iQIcBAEBAgAGBQJVSGpnAAoJEGw4ysog2bOStTcQAN3QMuqNGz9TDDyw+6Z8wh1B seUxGhyQlblx09sevr6q/wEplK4AA2J5oc7S644/TQI3ptu83/WC/ZGudPRqtAFb jMf8J7sJOv+Z1uRBLztDsktw2HILK8CNeXCHMROg4otvml49BuwLSjUn+y9H69Jh 5IOGymeW+k0tcY3wmdyJ6dj/H559VyrEyc44Jm6bRmt0QomkUzTcV9Ajnqbx9Ppv p/CQg5qWlbFY5pLc8BDA8/kVQAwBsqxd4CYNrYOZnb8maie0H/uNwMTTt5qSyqoP yaCmzu4//pduN9xI5dB2ZI35jcs7a0BgfIercksgocr8o7QGxlsL9d/IfhlvlfXB sBfcQTHhkQ0NmaEEdLW0P4cqKBi/riES4yJmD2JDs+m5MtPN8UCN1D4/ArP4+mRC klRM1+3OdkpPTx3WdpfE26Dt7+LYaNbNrg27t3fXSvXia0Qi0V3XdQEqVmcmq3oj a41cW7vP967STM2AIFAZqE2Mo4r5GC5BLdvZwYeaC+pd5zL/AQYToMVgwOM+t0cg w7xU1AE30WkneiXEaCxXn4GJ2pmnGD+jVY5O2bMGstGzRBxMTHXTDuP80oxKvxMm t8hDE8x5A9YWgeCFup7aNX4HTTJIxaoxJZosKrmMeAVIdxRQm2Sg7TTf/pd1twPM yw24ReCsq9u1R7ezd5Ss =Bbou -----END PGP SIGNATURE----- --HSQ3hISbU3Um6hch--