From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([2001:4830:134:3::10]:56154) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1acmEf-00070C-PS for qemu-devel@nongnu.org; Sun, 06 Mar 2016 22:53:35 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1acmEc-00005z-G3 for qemu-devel@nongnu.org; Sun, 06 Mar 2016 22:53:33 -0500 Date: Mon, 7 Mar 2016 14:49:06 +1100 From: David Gibson Message-ID: <20160307034906.GF22546@voom.fritz.box> References: <1457074461-14285-1-git-send-email-bharata@linux.vnet.ibm.com> <1457074461-14285-9-git-send-email-bharata@linux.vnet.ibm.com> MIME-Version: 1.0 Content-Type: multipart/signed; micalg=pgp-sha1; protocol="application/pgp-signature"; boundary="9ADF8FXzFeE7X4jE" Content-Disposition: inline In-Reply-To: <1457074461-14285-9-git-send-email-bharata@linux.vnet.ibm.com> Subject: Re: [Qemu-devel] [RFC PATCH v1 08/10] spapr: CPU hotplug support List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Bharata B Rao Cc: mjrosato@linux.vnet.ibm.com, agraf@suse.de, thuth@redhat.com, pkrempa@redhat.com, ehabkost@redhat.com, aik@ozlabs.ru, qemu-devel@nongnu.org, armbru@redhat.com, borntraeger@de.ibm.com, qemu-ppc@nongnu.org, pbonzini@redhat.com, imammedo@redhat.com, afaerber@suse.de, mdroth@linux.vnet.ibm.com --9ADF8FXzFeE7X4jE Content-Type: text/plain; charset=us-ascii Content-Disposition: inline Content-Transfer-Encoding: quoted-printable On Fri, Mar 04, 2016 at 12:24:19PM +0530, Bharata B Rao wrote: > Set up device tree entries for the hotplugged CPU core and use the > exising EPOW event infrastructure to send CPU hotplug notification to > the guest. >=20 > Signed-off-by: Bharata B Rao > --- > hw/ppc/spapr.c | 73 +++++++++++++++++++++++++++++++++++= +++++- > hw/ppc/spapr_cpu_core.c | 60 +++++++++++++++++++++++++++++++++ > hw/ppc/spapr_events.c | 3 ++ > hw/ppc/spapr_rtas.c | 24 ++++++++++++++ > include/hw/ppc/spapr.h | 4 +++ > include/hw/ppc/spapr_cpu_core.h | 2 ++ > 6 files changed, 165 insertions(+), 1 deletion(-) >=20 > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c > index 5acb612..6c4ac50 100644 > --- a/hw/ppc/spapr.c > +++ b/hw/ppc/spapr.c > @@ -603,6 +603,18 @@ static void spapr_populate_cpu_dt(CPUState *cs, void= *fdt, int offset, > size_t page_sizes_prop_size; > uint32_t vcpus_per_socket =3D smp_threads * smp_cores; > uint32_t pft_size_prop[] =3D {0, cpu_to_be32(spapr->htab_shift)}; > + sPAPRMachineClass *smc =3D SPAPR_MACHINE_GET_CLASS(qdev_get_machine(= )); > + sPAPRDRConnector *drc; > + sPAPRDRConnectorClass *drck; > + int drc_index; > + > + if (smc->dr_cpu_enabled) { > + drc =3D spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, in= dex); > + g_assert(drc); > + drck =3D SPAPR_DR_CONNECTOR_GET_CLASS(drc); > + drc_index =3D drck->get_index(drc); > + _FDT((fdt_setprop_cell(fdt, offset, "ibm,my-drc-index", drc_inde= x))); > + } > =20 > /* Note: we keep CI large pages off for now because a 64K capable gu= est > * provisioned with large pages might otherwise try to map a qemu > @@ -987,6 +999,16 @@ static void spapr_finalize_fdt(sPAPRMachineState *sp= apr, > _FDT(spapr_drc_populate_dt(fdt, 0, NULL, SPAPR_DR_CONNECTOR_TYPE= _LMB)); > } > =20 > + if (smc->dr_cpu_enabled) { > + int offset =3D fdt_path_offset(fdt, "/cpus"); > + ret =3D spapr_drc_populate_dt(fdt, offset, NULL, > + SPAPR_DR_CONNECTOR_TYPE_CPU); > + if (ret < 0) { > + error_report("Couldn't set up CPU DR device tree properties"= ); > + exit(1); > + } > + } > + > _FDT((fdt_pack(fdt))); > =20 > if (fdt_totalsize(fdt) > FDT_MAX_SIZE) { > @@ -1181,7 +1203,7 @@ static void ppc_spapr_reset(void) > =20 > } > =20 > -static void spapr_cpu_reset(void *opaque) > +void spapr_cpu_reset(void *opaque) > { > sPAPRMachineState *spapr =3D SPAPR_MACHINE(qdev_get_machine()); > PowerPCCPU *cpu =3D opaque; > @@ -1622,6 +1644,8 @@ static void spapr_boot_set(void *opaque, const char= *boot_device, > void spapr_cpu_init(sPAPRMachineState *spapr, PowerPCCPU *cpu, Error **e= rrp) > { > CPUPPCState *env =3D &cpu->env; > + CPUState *cs =3D CPU(cpu); > + int i; > =20 > /* Set time-base frequency to 512 MHz */ > cpu_ppc_tb_init(env, TIMEBASE_FREQ); > @@ -1646,6 +1670,14 @@ void spapr_cpu_init(sPAPRMachineState *spapr, Powe= rPCCPU *cpu, Error **errp) > } > } > =20 > + /* Set NUMA node for the added CPUs */ > + for (i =3D 0; i < nb_numa_nodes; i++) { > + if (test_bit(cs->cpu_index, numa_info[i].node_cpu)) { > + cs->numa_node =3D i; > + break; > + } > + } > + This hunk seems like it belongs in a different patch. > xics_cpu_setup(spapr->icp, cpu); > qemu_register_reset(spapr_cpu_reset, cpu); > } > @@ -1768,6 +1800,7 @@ static void ppc_spapr_init(MachineState *machine) > char *filename; > int spapr_cores =3D smp_cpus / smp_threads; > int spapr_max_cores =3D max_cpus / smp_threads; > + int smt =3D kvmppc_smt_threads(); > =20 > if (smp_cpus % smp_threads) { > error_report("smp_cpus (%u) must be multiple of threads (%u)", > @@ -1834,6 +1867,15 @@ static void ppc_spapr_init(MachineState *machine) > spapr_validate_node_memory(machine, &error_fatal); > } > =20 > + if (smc->dr_cpu_enabled) { > + for (i =3D 0; i < spapr_max_cores; i++) { > + sPAPRDRConnector *drc =3D > + spapr_dr_connector_new(OBJECT(spapr), > + SPAPR_DR_CONNECTOR_TYPE_CPU, i * = smt); > + qemu_register_reset(spapr_drc_reset, drc); > + } > + } > + Nit: would this be cleaner to include in the same loop that constructs the (empty) links and boot-time cpu cores? > /* init CPUs */ > if (machine->cpu_model =3D=3D NULL) { > machine->cpu_model =3D kvm_enabled() ? "host" : "POWER7"; > @@ -2267,6 +2309,27 @@ out: > error_propagate(errp, local_err); > } > =20 > +void *spapr_populate_hotplug_cpu_dt(DeviceState *dev, CPUState *cs, > + int *fdt_offset, sPAPRMachineState *= spapr) > +{ > + PowerPCCPU *cpu =3D POWERPC_CPU(cs); > + DeviceClass *dc =3D DEVICE_GET_CLASS(cs); > + int id =3D ppc_get_vcpu_dt_id(cpu); > + void *fdt; > + int offset, fdt_size; > + char *nodename; > + > + fdt =3D create_device_tree(&fdt_size); > + nodename =3D g_strdup_printf("%s@%x", dc->fw_name, id); > + offset =3D fdt_add_subnode(fdt, 0, nodename); > + > + spapr_populate_cpu_dt(cs, fdt, offset, spapr); > + g_free(nodename); > + > + *fdt_offset =3D offset; > + return fdt; > +} > + > static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > DeviceState *dev, Error **errp) > { > @@ -2307,6 +2370,12 @@ static void spapr_machine_device_plug(HotplugHandl= er *hotplug_dev, > } > =20 > spapr_memory_plug(hotplug_dev, dev, node, errp); > + } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) { > + if (!smc->dr_cpu_enabled && dev->hotplugged) { > + error_setg(errp, "CPU hotplug not supported for this machine= "); > + return; > + } > + spapr_core_plug(hotplug_dev, dev, errp); > } > } > =20 > @@ -2366,6 +2435,7 @@ static void spapr_machine_class_init(ObjectClass *o= c, void *data) > mc->cpu_index_to_socket_id =3D spapr_cpu_index_to_socket_id; > =20 > smc->dr_lmb_enabled =3D true; > + smc->dr_cpu_enabled =3D true; > fwc->get_dev_path =3D spapr_get_fw_dev_path; > nc->nmi_monitor_handler =3D spapr_nmi; > } > @@ -2445,6 +2515,7 @@ static void spapr_machine_2_5_class_options(Machine= Class *mc) > =20 > spapr_machine_2_6_class_options(mc); > smc->use_ohci_by_default =3D true; > + smc->dr_cpu_enabled =3D false; > SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_5); > } > =20 > diff --git a/hw/ppc/spapr_cpu_core.c b/hw/ppc/spapr_cpu_core.c > index 9ddf3ce..4c233d7 100644 > --- a/hw/ppc/spapr_cpu_core.c > +++ b/hw/ppc/spapr_cpu_core.c > @@ -14,6 +14,65 @@ > #include "qapi/visitor.h" > #include > =20 > +void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > + Error **errp) > +{ > + sPAPRMachineClass *smc =3D SPAPR_MACHINE_GET_CLASS(qdev_get_machine(= )); > + sPAPRMachineState *ms =3D SPAPR_MACHINE(qdev_get_machine()); > + sPAPRCPUCore *core =3D SPAPR_CPU_CORE(OBJECT(dev)); > + PowerPCCPU *cpu =3D &core->threads[0]; > + CPUState *cs =3D CPU(cpu); > + int id =3D ppc_get_vcpu_dt_id(cpu); > + sPAPRDRConnector *drc =3D > + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); > + sPAPRDRConnectorClass *drck; > + Error *local_err =3D NULL; > + void *fdt =3D NULL; > + int fdt_offset =3D 0; > + > + if (!smc->dr_cpu_enabled) { > + /* > + * This is a cold plugged CPU core but the machine doesn't suppo= rt > + * DR. So skip the hotplug path ensuring that the core is brought > + * up online with out an associated DR connector. > + */ > + return; > + } > + > + g_assert(drc); > + > + /* > + * Setup CPU DT entries only for hotplugged CPUs. For boot time or > + * coldplugged CPUs DT entries are setup in spapr_finalize_fdt(). > + */ > + if (dev->hotplugged) { > + fdt =3D spapr_populate_hotplug_cpu_dt(dev, cs, &fdt_offset, ms); > + dev->hotplugged =3D true; > + } > + > + drck =3D SPAPR_DR_CONNECTOR_GET_CLASS(drc); > + drck->attach(drc, dev, fdt, fdt_offset, !dev->hotplugged, &local_err= ); > + if (local_err) { > + g_free(fdt); > + error_propagate(errp, local_err); > + return; > + } > + > + if (dev->hotplugged) { > + /* > + * Send hotplug notification interrupt to the guest only in case > + * of hotplugged CPUs. > + */ > + spapr_hotplug_req_add_by_index(drc); > + } else { > + /* > + * Set the right DRC states for cold plugged CPU. > + */ > + drck->set_allocation_state(drc, SPAPR_DR_ALLOCATION_STATE_USABLE= ); > + drck->set_isolation_state(drc, SPAPR_DR_ISOLATION_STATE_UNISOLAT= ED); > + } > +} > + > static int spapr_cpu_core_realize_child(Object *child, void *opaque) > { > Error **errp =3D opaque; > @@ -30,6 +89,7 @@ static int spapr_cpu_core_realize_child(Object *child, = void *opaque) > if (*errp) { > return 1; > } > + spapr_cpu_reset(cpu); This also looks like it belongs in a different patch. > return 0; > } > diff --git a/hw/ppc/spapr_events.c b/hw/ppc/spapr_events.c > index 39f4682..10340e1 100644 > --- a/hw/ppc/spapr_events.c > +++ b/hw/ppc/spapr_events.c > @@ -437,6 +437,9 @@ static void spapr_hotplug_req_event(uint8_t hp_id, ui= nt8_t hp_action, > case SPAPR_DR_CONNECTOR_TYPE_LMB: > hp->hotplug_type =3D RTAS_LOG_V6_HP_TYPE_MEMORY; > break; > + case SPAPR_DR_CONNECTOR_TYPE_CPU: > + hp->hotplug_type =3D RTAS_LOG_V6_HP_TYPE_CPU; > + break; > default: > /* we shouldn't be signaling hotplug events for resources > * that don't support them > diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c > index b7c5ebd..cc0369e 100644 > --- a/hw/ppc/spapr_rtas.c > +++ b/hw/ppc/spapr_rtas.c > @@ -34,6 +34,7 @@ > =20 > #include "hw/ppc/spapr.h" > #include "hw/ppc/spapr_vio.h" > +#include "hw/ppc/ppc.h" > #include "qapi-event.h" > #include "hw/boards.h" > =20 > @@ -161,6 +162,27 @@ static void rtas_query_cpu_stopped_state(PowerPCCPU = *cpu_, > rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR); > } > =20 > +/* > + * Set the timebase offset of the CPU to that of first CPU. > + * This helps hotplugged CPU to have the correct timebase offset. > + */ > +static void spapr_cpu_update_tb_offset(PowerPCCPU *cpu) > +{ > + PowerPCCPU *fcpu =3D POWERPC_CPU(first_cpu); > + > + cpu->env.tb_env->tb_offset =3D fcpu->env.tb_env->tb_offset; > +} > + > +static void spapr_cpu_set_endianness(PowerPCCPU *cpu) > +{ > + PowerPCCPU *fcpu =3D POWERPC_CPU(first_cpu); > + PowerPCCPUClass *pcc =3D POWERPC_CPU_GET_CLASS(fcpu); > + > + if (!pcc->interrupts_big_endian(fcpu)) { > + cpu->env.spr[SPR_LPCR] |=3D LPCR_ILE; > + } > +} > + > static void rtas_start_cpu(PowerPCCPU *cpu_, sPAPRMachineState *spapr, > uint32_t token, uint32_t nargs, > target_ulong args, > @@ -197,6 +219,8 @@ static void rtas_start_cpu(PowerPCCPU *cpu_, sPAPRMac= hineState *spapr, > env->nip =3D start; > env->gpr[3] =3D r3; > cs->halted =3D 0; > + spapr_cpu_set_endianness(cpu); > + spapr_cpu_update_tb_offset(cpu); > =20 > qemu_cpu_kick(cs); > =20 > diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h > index 06e6cec..d1a0af8 100644 > --- a/include/hw/ppc/spapr.h > +++ b/include/hw/ppc/spapr.h > @@ -36,6 +36,7 @@ struct sPAPRMachineClass { > =20 > /*< public >*/ > bool dr_lmb_enabled; /* enable dynamic-reconfig/hotplug of LMB= s */ > + bool dr_cpu_enabled; /* enable dynamic-reconfig/hotplug of CPU= s */ > bool use_ohci_by_default; /* use USB-OHCI instead of XHCI */ > }; > =20 > @@ -589,6 +590,9 @@ void spapr_hotplug_req_add_by_count(sPAPRDRConnectorT= ype drc_type, > void spapr_hotplug_req_remove_by_count(sPAPRDRConnectorType drc_type, > uint32_t count); > void spapr_cpu_init(sPAPRMachineState *spapr, PowerPCCPU *cpu, Error **e= rrp); > +void spapr_cpu_reset(void *opaque); > +void *spapr_populate_hotplug_cpu_dt(DeviceState *dev, CPUState *cs, > + int *fdt_offset, sPAPRMachineState *= spapr); > =20 > /* rtas-configure-connector state */ > struct sPAPRConfigureConnectorState { > diff --git a/include/hw/ppc/spapr_cpu_core.h b/include/hw/ppc/spapr_cpu_c= ore.h > index ddaff6f..9879398 100644 > --- a/include/hw/ppc/spapr_cpu_core.h > +++ b/include/hw/ppc/spapr_cpu_core.h > @@ -26,4 +26,6 @@ typedef struct sPAPRCPUCore { > PowerPCCPU *threads; > } sPAPRCPUCore; > =20 > +void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > + Error **errp); > #endif --=20 David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson --9ADF8FXzFeE7X4jE Content-Type: application/pgp-signature; name="signature.asc" -----BEGIN PGP SIGNATURE----- Version: GnuPG v1 iQIcBAEBAgAGBQJW3PoyAAoJEGw4ysog2bOSwSYQAMaYdnoDmwtVdWHXcvDqoNMy yBGxYKIGL3glpMfZZRI2jBj1F0G6P1sl2W5DPvJtBdqMLQYr2OGj+5hvLTrAfbZb gevQyo1Co7NFu7JYW5z6eX/yxXjcmzY00UsHgihumJE7P/HfzTAPznxxuG83sIzS F3Tj1OMezOy748tkgNC3vw6JjSIsFaq5BvQvkRvoQA6jWVBT8AgxZaq9lDrU2U0G aCzsPZkfEEiPAxPssLqwy9QLz9odF0OeA45q3Sgf5UK4GGTBLk3shyKyomBZGaWQ yiz9s9iePzom8GEVRYlnbpYJshD5p38Ns5ToIjKoJJZNJaeP3CtgrTdQD/Ip+m3G ieOwRT7qzVQL+PCAZjSyAfvPIYac1hw6PjlufeCOueNf7WZYJjN7AyawJuoAcaON XdxUSu6TzYx4Ltw6iqzdMXTsLXH98Yj6Jki+f8RtSUV7pwlDMc7d519FaJ8vl6Au bXy/ivvU11q6u3/cEMZsujGi35zz0PCiqKh2Z5z2OqwIZfNBV/4F9WAg2N182sGy ySjhu831OgJc746SM9/Ga1IO+lpbzXBcRhgu70wd/lWrWdgKSNvMdRMJki42lEMi 0/XU7ZYmfNLDhDNjUZtu+SvY0jw5zUjZt6wZyhXv0dcmYnfpJBodop9Kk4w8UuTr 7juPLG+LrT7hJIBfaH1t =30Ht -----END PGP SIGNATURE----- --9ADF8FXzFeE7X4jE--