[PATCH] powerpc/powernv: implement NMI IPI with OPAL_SIGNAL_SYSTEM_RESET

[PATCH] powerpc/powernv: implement NMI IPI with OPAL_SIGNAL_SYSTEM_RESET

[Nicholas Piggin]

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
Hi,

This goes with the previous NMI IPI series, and a new version of
Alistair's opal API I posted to the skiboot list.

Only RFC until the firmware API is agreed on.

Thanks,
Nick

 arch/powerpc/include/asm/opal-api.h            | 21 ++++++++++++++++++++-
 arch/powerpc/include/asm/opal.h                |  1 +
 arch/powerpc/platforms/powernv/opal-wrappers.S |  1 +
 arch/powerpc/platforms/powernv/powernv.h       |  1 +
 arch/powerpc/platforms/powernv/setup.c         |  3 +++
 arch/powerpc/platforms/powernv/smp.c           | 24 ++++++++++++++++++++++++
 6 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/arch/powerpc/include/asm/opal-api.h b/arch/powerpc/include/asm/opal-api.h
index 0e2e57bcab50..53f1c09cab5d 100644
--- a/arch/powerpc/include/asm/opal-api.h
+++ b/arch/powerpc/include/asm/opal-api.h
@@ -167,7 +167,26 @@
 #define OPAL_INT_EOI				124
 #define OPAL_INT_SET_MFRR			125
 #define OPAL_PCI_TCE_KILL			126
-#define OPAL_LAST				126
+#define OPAL_NMMU_SET_PTCR			127
+#define OPAL_XIVE_RESET				128
+#define OPAL_XIVE_GET_IRQ_INFO			129
+#define OPAL_XIVE_GET_IRQ_CONFIG		130
+#define OPAL_XIVE_SET_IRQ_CONFIG		131
+#define OPAL_XIVE_GET_QUEUE_INFO		132
+#define OPAL_XIVE_SET_QUEUE_INFO		133
+#define OPAL_XIVE_DONATE_PAGE			134
+#define OPAL_XIVE_ALLOCATE_VP_BLOCK		135
+#define OPAL_XIVE_FREE_VP_BLOCK			136
+#define OPAL_XIVE_GET_VP_INFO			137
+#define OPAL_XIVE_SET_VP_INFO			138
+#define OPAL_XIVE_ALLOCATE_IRQ			139
+#define OPAL_XIVE_FREE_IRQ			140
+#define OPAL_XIVE_RESERVED1			141
+#define OPAL_XIVE_RESERVED2			142
+#define OPAL_XIVE_RESERVED3			143
+#define OPAL_XIVE_RESERVED4			144
+#define OPAL_SIGNAL_SYSTEM_RESET		145
+#define OPAL_LAST				145
 
 /* Device tree flags */
 
diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h
index 5c7db0f1a708..893d422febf6 100644
--- a/arch/powerpc/include/asm/opal.h
+++ b/arch/powerpc/include/asm/opal.h
@@ -232,6 +232,7 @@ int64_t opal_pci_tce_kill(uint64_t phb_id, uint32_t kill_type,
 int64_t opal_rm_pci_tce_kill(uint64_t phb_id, uint32_t kill_type,
 			     uint32_t pe_num, uint32_t tce_size,
 			     uint64_t dma_addr, uint32_t npages);
+int64_t opal_signal_system_reset(int32_t cpu);
 
 /* Internal functions */
 extern int early_init_dt_scan_opal(unsigned long node, const char *uname,
diff --git a/arch/powerpc/platforms/powernv/opal-wrappers.S b/arch/powerpc/platforms/powernv/opal-wrappers.S
index 3aa40f1b20f5..e1e9ef11f100 100644
--- a/arch/powerpc/platforms/powernv/opal-wrappers.S
+++ b/arch/powerpc/platforms/powernv/opal-wrappers.S
@@ -312,3 +312,4 @@ OPAL_CALL(opal_int_set_mfrr,			OPAL_INT_SET_MFRR);
 OPAL_CALL_REAL(opal_rm_int_set_mfrr,		OPAL_INT_SET_MFRR);
 OPAL_CALL(opal_pci_tce_kill,			OPAL_PCI_TCE_KILL);
 OPAL_CALL_REAL(opal_rm_pci_tce_kill,		OPAL_PCI_TCE_KILL);
+OPAL_CALL(opal_signal_system_reset,		OPAL_SIGNAL_SYSTEM_RESET);
diff --git a/arch/powerpc/platforms/powernv/powernv.h b/arch/powerpc/platforms/powernv/powernv.h
index da7c843ac7f1..6b17bf7190e7 100644
--- a/arch/powerpc/platforms/powernv/powernv.h
+++ b/arch/powerpc/platforms/powernv/powernv.h
@@ -3,6 +3,7 @@
 
 #ifdef CONFIG_SMP
 extern void pnv_smp_init(void);
+extern int pnv_system_reset_exception(struct pt_regs *regs);
 #else
 static inline void pnv_smp_init(void) { }
 #endif
diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c
index d50c7d99baaf..2ebb429e272e 100644
--- a/arch/powerpc/platforms/powernv/setup.c
+++ b/arch/powerpc/platforms/powernv/setup.c
@@ -260,6 +260,9 @@ static void __init pnv_setup_machdep_opal(void)
 	ppc_md.restart = pnv_restart;
 	pm_power_off = pnv_power_off;
 	ppc_md.halt = pnv_halt;
+#ifdef CONFIG_SMP
+	ppc_md.system_reset_exception = pnv_system_reset_exception;
+#endif
 	ppc_md.machine_check_exception = opal_machine_check;
 	ppc_md.mce_check_early_recovery = opal_mce_check_early_recovery;
 	ppc_md.hmi_exception_early = opal_hmi_exception_early;
diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c
index 092ec1f7b58d..f90555f75723 100644
--- a/arch/powerpc/platforms/powernv/smp.c
+++ b/arch/powerpc/platforms/powernv/smp.c
@@ -241,6 +241,28 @@ static int pnv_cpu_bootable(unsigned int nr)
 	return smp_generic_cpu_bootable(nr);
 }
 
+int pnv_system_reset_exception(struct pt_regs *regs)
+{
+	if (smp_handle_nmi_ipi(regs))
+		return 1;
+	return 0;
+}
+
+static int pnv_cause_nmi_ipi(int cpu)
+{
+	int64_t rc;
+
+	rc = opal_signal_system_reset(cpu);
+	if (rc == OPAL_SUCCESS)
+		return 1;
+
+	/*
+	 * Don't cope with OPAL_PARTIAL yet (just punt to regular IPI)
+	 */
+
+	return 0;
+}
+
 static struct smp_ops_t pnv_smp_ops = {
 	.message_pass	= smp_muxed_ipi_message_pass,
 	.cause_ipi	= NULL,	/* Filled at runtime by xics_smp_probe() */
@@ -258,6 +280,8 @@ static struct smp_ops_t pnv_smp_ops = {
 /* This is called very early during platform setup_arch */
 void __init pnv_smp_init(void)
 {
+	if (opal_check_token(OPAL_SIGNAL_SYSTEM_RESET))
+		pnv_smp_ops.cause_nmi_ipi = pnv_cause_nmi_ipi;
 	smp_ops = &pnv_smp_ops;
 
 #ifdef CONFIG_HOTPLUG_CPU
-- 
2.11.0

Re: [PATCH] powerpc/powernv: implement NMI IPI with OPAL_SIGNAL_SYSTEM_RESET

[Nicholas Piggin]

On Fri,  3 Feb 2017 00:25:11 +1000
Nicholas Piggin <npiggin@gmail.com> wrote:

> This goes with the previous NMI IPI series, and a new version of
> Alistair's opal API I posted to the skiboot list.

And here is the incremental bit that is required for Alistair's
hardware implementation to work.

If the opal broacast call fails with OPAL_PARTIAL, then we designate
a bouncer CPU on another core to send NMI IPIs back to our sibling
threads.

Probably needs more discussion and testing about how to detect and
handle failure cases and future compatibility for different types of
restrictions, but at least it works in mambo.

Of course the other option rather than doing this in Linux is call
into opal in the system reset handler and have it do the bouncing.
Something to consider before we finalise the API.

Thanks,
Nick

---
 arch/powerpc/platforms/powernv/smp.c | 88 +++++++++++++++++++++++++++++++++---
 1 file changed, 82 insertions(+), 6 deletions(-)

diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c
index f90555f75723..4241fda9df9e 100644
--- a/arch/powerpc/platforms/powernv/smp.c
+++ b/arch/powerpc/platforms/powernv/smp.c
@@ -241,8 +241,32 @@ static int pnv_cpu_bootable(unsigned int nr)
 	return smp_generic_cpu_bootable(nr);
 }
 
+static int nmi_ipi_bounce_cpu;
+static int nmi_ipi_bounce_cpu_done;
+static int nmi_ipi_bounce_target_core;
+static int nmi_ipi_bounce_target_exclude;
+
 int pnv_system_reset_exception(struct pt_regs *regs)
 {
+	smp_rmb();
+	if (nmi_ipi_bounce_cpu == smp_processor_id()) {
+		int64_t rc;
+		int c;
+
+		for_each_online_cpu(c) {
+			if (!cpumask_test_cpu(c, cpu_sibling_mask(nmi_ipi_bounce_target_core)))
+				continue;
+			if (c == nmi_ipi_bounce_target_exclude)
+				continue;
+			rc = opal_signal_system_reset(c);
+			if (rc != OPAL_SUCCESS) {
+				nmi_ipi_bounce_cpu_done = -1;
+				return 1;
+			}
+		}
+		nmi_ipi_bounce_cpu_done = 1;
+	}
+
 	if (smp_handle_nmi_ipi(regs))
 		return 1;
 	return 0;
@@ -252,13 +276,65 @@ static int pnv_cause_nmi_ipi(int cpu)
 {
 	int64_t rc;
 
-	rc = opal_signal_system_reset(cpu);
-	if (rc == OPAL_SUCCESS)
-		return 1;
+	if (cpu >= 0) {
+		rc = opal_signal_system_reset(cpu);
+		if (rc == OPAL_SUCCESS)
+			return 1;
+	} else {
+		/*
+		 * Test bounce behavior with broadcast IPI.
+		 */
+		rc = OPAL_PARTIAL;
+	}
+	if (rc == OPAL_PARTIAL) {
+		int c;
 
-	/*
-	 * Don't cope with OPAL_PARTIAL yet (just punt to regular IPI)
-	 */
+		/*
+		 * Some platforms can not send NMI to sibling threads in
+		 * the same core. We can designate one inter-core target
+		 * to bounce NMIs back to our sibling threads.
+		 */
+
+		if (cpu >= 0) {
+			/*
+			 * Don't support bouncing unicast NMIs yet (because
+			 * that would have to raise an NMI on an unrelated
+			 * CPU. Revisit this if callers start using unicast.
+			 */
+			return 0;
+		}
+
+		nmi_ipi_bounce_cpu = -1;
+		nmi_ipi_bounce_cpu_done = 0;
+		nmi_ipi_bounce_target_core = -1;
+		nmi_ipi_bounce_target_exclude = -1;
+		smp_wmb();
+
+		for_each_online_cpu(c) {
+			if (cpumask_test_cpu(c, cpu_sibling_mask(smp_processor_id())))
+				continue;
+
+			if (nmi_ipi_bounce_cpu == -1) {
+				nmi_ipi_bounce_cpu = c;
+				nmi_ipi_bounce_target_core = smp_processor_id();
+				if (cpu == NMI_IPI_ALL_OTHERS)
+					nmi_ipi_bounce_target_exclude = smp_processor_id();
+			}
+
+			rc = opal_signal_system_reset(c);
+			if (rc != OPAL_SUCCESS)
+				return 0;
+		}
+
+		if (nmi_ipi_bounce_cpu == -1)
+			return 0; /* could not find a bouncer */
+
+		while (!nmi_ipi_bounce_cpu_done)
+			cpu_relax();
+
+		if (nmi_ipi_bounce_cpu_done == 1)
+			return 1; /* bounce worked */
+	}
 
 	return 0;
 }
-- 
2.11.0

Re: [PATCH] powerpc/powernv: implement NMI IPI with OPAL_SIGNAL_SYSTEM_RESET

[Alistair Popple]

On Fri, 3 Feb 2017 03:20:48 PM Nicholas Piggin wrote:
> On Fri,  3 Feb 2017 00:25:11 +1000
> Nicholas Piggin <npiggin@gmail.com> wrote:
>
> > This goes with the previous NMI IPI series, and a new version of
> > Alistair's opal API I posted to the skiboot list.
>
> And here is the incremental bit that is required for Alistair's
> hardware implementation to work.
>
> If the opal broacast call fails with OPAL_PARTIAL, then we designate
> a bouncer CPU on another core to send NMI IPIs back to our sibling
> threads.
>
> Probably needs more discussion and testing about how to detect and
> handle failure cases and future compatibility for different types of
> restrictions, but at least it works in mambo.

Looking at the Mambo implementation you recently posted I couldn't see
where OPAL_PARTIAL was returned. So I guess you have other Skiboot
patches to test this path using Mambo? I'm wondering because they
could be useful for testing the skiboot HW side as well...

>
> Of course the other option rather than doing this in Linux is call
> into opal in the system reset handler and have it do the bouncing.
> Something to consider before we finalise the API.

That might not be such a bad idea. OPAL could queue up the threads it
couldn't reset and then wait until opal_sreset_complete() is called
from an eligible thread to reset the ones it couldn't do in the
original call.

I might try prototyping something like this when I get some time. One
issue would be if there is only a single core in the system, but
that's unlikely and I think that's probably something we can't support
in any case as cores can't sreset threads on the same core, at least
on P8.

- Alistair

> Thanks,
> Nick
>
> ---
>  arch/powerpc/platforms/powernv/smp.c | 88 +++++++++++++++++++++++++++++++++---
>  1 file changed, 82 insertions(+), 6 deletions(-)
>
> diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c
> index f90555f75723..4241fda9df9e 100644
> --- a/arch/powerpc/platforms/powernv/smp.c
> +++ b/arch/powerpc/platforms/powernv/smp.c
> @@ -241,8 +241,32 @@ static int pnv_cpu_bootable(unsigned int nr)
>  	return smp_generic_cpu_bootable(nr);
>  }
>
> +static int nmi_ipi_bounce_cpu;
> +static int nmi_ipi_bounce_cpu_done;
> +static int nmi_ipi_bounce_target_core;
> +static int nmi_ipi_bounce_target_exclude;
> +
>  int pnv_system_reset_exception(struct pt_regs *regs)
>  {
> +	smp_rmb();
> +	if (nmi_ipi_bounce_cpu == smp_processor_id()) {
> +		int64_t rc;
> +		int c;
> +
> +		for_each_online_cpu(c) {
> +			if (!cpumask_test_cpu(c, cpu_sibling_mask(nmi_ipi_bounce_target_core)))
> +				continue;
> +			if (c == nmi_ipi_bounce_target_exclude)
> +				continue;
> +			rc = opal_signal_system_reset(c);
> +			if (rc != OPAL_SUCCESS) {
> +				nmi_ipi_bounce_cpu_done = -1;
> +				return 1;
> +			}
> +		}
> +		nmi_ipi_bounce_cpu_done = 1;
> +	}
> +
>  	if (smp_handle_nmi_ipi(regs))
>  		return 1;
>  	return 0;
> @@ -252,13 +276,65 @@ static int pnv_cause_nmi_ipi(int cpu)
>  {
>  	int64_t rc;
>
> -	rc = opal_signal_system_reset(cpu);
> -	if (rc == OPAL_SUCCESS)
> -		return 1;
> +	if (cpu >= 0) {
> +		rc = opal_signal_system_reset(cpu);
> +		if (rc == OPAL_SUCCESS)
> +			return 1;
> +	} else {
> +		/*
> +		 * Test bounce behavior with broadcast IPI.
> +		 */
> +		rc = OPAL_PARTIAL;
> +	}
> +	if (rc == OPAL_PARTIAL) {
> +		int c;
>
> -	/*
> -	 * Don't cope with OPAL_PARTIAL yet (just punt to regular IPI)
> -	 */
> +		/*
> +		 * Some platforms can not send NMI to sibling threads in
> +		 * the same core. We can designate one inter-core target
> +		 * to bounce NMIs back to our sibling threads.
> +		 */
> +
> +		if (cpu >= 0) {
> +			/*
> +			 * Don't support bouncing unicast NMIs yet (because
> +			 * that would have to raise an NMI on an unrelated
> +			 * CPU. Revisit this if callers start using unicast.
> +			 */
> +			return 0;
> +		}
> +
> +		nmi_ipi_bounce_cpu = -1;
> +		nmi_ipi_bounce_cpu_done = 0;
> +		nmi_ipi_bounce_target_core = -1;
> +		nmi_ipi_bounce_target_exclude = -1;
> +		smp_wmb();
> +
> +		for_each_online_cpu(c) {
> +			if (cpumask_test_cpu(c, cpu_sibling_mask(smp_processor_id())))
> +				continue;
> +
> +			if (nmi_ipi_bounce_cpu == -1) {
> +				nmi_ipi_bounce_cpu = c;
> +				nmi_ipi_bounce_target_core = smp_processor_id();
> +				if (cpu == NMI_IPI_ALL_OTHERS)
> +					nmi_ipi_bounce_target_exclude = smp_processor_id();
> +			}
> +
> +			rc = opal_signal_system_reset(c);
> +			if (rc != OPAL_SUCCESS)
> +				return 0;
> +		}
> +
> +		if (nmi_ipi_bounce_cpu == -1)
> +			return 0; /* could not find a bouncer */
> +
> +		while (!nmi_ipi_bounce_cpu_done)
> +			cpu_relax();
> +
> +		if (nmi_ipi_bounce_cpu_done == 1)
> +			return 1; /* bounce worked */
> +	}
>
>  	return 0;
>  }
>

Re: [PATCH] powerpc/powernv: implement NMI IPI with OPAL_SIGNAL_SYSTEM_RESET

[Nicholas Piggin]

On Fri, 03 Feb 2017 17:00:19 +1100
Alistair Popple <alistair@popple.id.au> wrote:

> On Fri, 3 Feb 2017 03:20:48 PM Nicholas Piggin wrote:
> > On Fri,  3 Feb 2017 00:25:11 +1000
> > Nicholas Piggin <npiggin@gmail.com> wrote:
> >  
> > > This goes with the previous NMI IPI series, and a new version of
> > > Alistair's opal API I posted to the skiboot list.  
> >
> > And here is the incremental bit that is required for Alistair's
> > hardware implementation to work.
> >
> > If the opal broacast call fails with OPAL_PARTIAL, then we designate
> > a bouncer CPU on another core to send NMI IPIs back to our sibling
> > threads.
> >
> > Probably needs more discussion and testing about how to detect and
> > handle failure cases and future compatibility for different types of
> > restrictions, but at least it works in mambo.  
> 
> Looking at the Mambo implementation you recently posted I couldn't see
> where OPAL_PARTIAL was returned. So I guess you have other Skiboot
> patches to test this path using Mambo? I'm wondering because they
> could be useful for testing the skiboot HW side as well...

Oh I just did it all on the Linux side, just forced it to always
do bounce rather than try broadcast first, I was lazy. But we'll
make mambo match the hardware implementation as far as possible
once you've got something.

> > Of course the other option rather than doing this in Linux is call
> > into opal in the system reset handler and have it do the bouncing.
> > Something to consider before we finalise the API.  
> 
> That might not be such a bad idea. OPAL could queue up the threads it
> couldn't reset and then wait until opal_sreset_complete() is called
> from an eligible thread to reset the ones it couldn't do in the
> original call.

Yeah, IIRC we thought it might have been harder to do in firmware,
but that may not be the case.

It may not be a bad idea in general for opal to have a token
available to hook system resets too.

> I might try prototyping something like this when I get some time. One
> issue would be if there is only a single core in the system, but
> that's unlikely and I think that's probably something we can't support
> in any case as cores can't sreset threads on the same core, at least
> on P8.

Yes on powernv it would be unusual, except maybe mambo where we
can make it work.

Thanks,
Nick