[PATCH 1/2] powerpc: add Book E support to 64-bit hibernation

[PATCH 1/2] powerpc: add Book E support to 64-bit hibernation

[Wang Dongsheng]

Update the 64-bit hibernation code to support Book E CPUs.
Some registers and instructions are not defined for Book3e
(SDR reg, tlbia instruction).

SDR: Storage Description Register. Book3S and Book3E have different
address translation mode, we do not need HTABORG & HTABSIZE to
translate virtual address to real address.

More registers are saved in BookE-64bit.(TCR, SPRGx, ...)

Signed-off-by: Wang Dongsheng <dongsheng.wang@freescale.com>
---
* History:
* Wood Scott(A): Please investigate the issue of whether we are loading
*                kernel module code in this step
* R: Kernel will allocate the memory for module code segment and data
*    segment. First allocate a memory, and copy the umod to hdr members
*    of the struct load_info. Due to the temporary assigned module belongs
*    to the kernel space, so it belongs to the kernel data.
*
*    The kernel of all data will be saved when hibernation suspend. So
*    the module which has already been inserted will be saved.

 arch/powerpc/kernel/swsusp_asm64.S | 64 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 62 insertions(+), 2 deletions(-)

diff --git a/arch/powerpc/kernel/swsusp_asm64.S b/arch/powerpc/kernel/swsusp_asm64.S
index 86ac1d9..608e4ceb 100644
--- a/arch/powerpc/kernel/swsusp_asm64.S
+++ b/arch/powerpc/kernel/swsusp_asm64.S
@@ -46,10 +46,29 @@
 #define SL_r29		0xe8
 #define SL_r30		0xf0
 #define SL_r31		0xf8
-#define SL_SIZE		SL_r31+8
+#define SL_SPRG0	0x100
+#define SL_SPRG1	0x108
+#define SL_SPRG2	0x110
+#define SL_SPRG3	0x118
+#define SL_SPRG4	0x120
+#define SL_SPRG5	0x128
+#define SL_SPRG6	0x130
+#define SL_SPRG7	0x138
+#define SL_TCR		0x140
+#define SL_SIZE		SL_TCR+8
 
 /* these macros rely on the save area being
  * pointed to by r11 */
+
+#define SAVE_SPR(register)		\
+	mfspr	r0,SPRN_##register	;\
+	std	r0,SL_##register(r11)
+#define RESTORE_SPR(register)		\
+	ld	r0,SL_##register(r11)	;\
+	mtspr	SPRN_##register,r0
+#define RESTORE_SPRG(n)			\
+	ld	r0,SL_SPRG##n(r11)	;\
+	mtsprg	n,r0
 #define SAVE_SPECIAL(special)		\
 	mf##special	r0		;\
 	std	r0, SL_##special(r11)
@@ -103,8 +122,21 @@ _GLOBAL(swsusp_arch_suspend)
 	SAVE_REGISTER(r30)
 	SAVE_REGISTER(r31)
 	SAVE_SPECIAL(MSR)
-	SAVE_SPECIAL(SDR1)
 	SAVE_SPECIAL(XER)
+#ifdef CONFIG_PPC_BOOK3S_64
+	SAVE_SPECIAL(SDR1)
+#else
+	SAVE_SPR(TCR)
+	/* Save SPRGs */
+	SAVE_SPR(SPRG0)
+	SAVE_SPR(SPRG1)
+	SAVE_SPR(SPRG2)
+	SAVE_SPR(SPRG3)
+	SAVE_SPR(SPRG4)
+	SAVE_SPR(SPRG5)
+	SAVE_SPR(SPRG6)
+	SAVE_SPR(SPRG7)
+#endif
 
 	/* we push the stack up 128 bytes but don't store the
 	 * stack pointer on the stack like a real stackframe */
@@ -151,6 +183,7 @@ copy_page_loop:
 	bne+	copyloop
 nothing_to_copy:
 
+#ifdef CONFIG_PPC_BOOK3S_64
 	/* flush caches */
 	lis	r3, 0x10
 	mtctr	r3
@@ -167,6 +200,7 @@ nothing_to_copy:
 	sync
 
 	tlbia
+#endif
 
 	ld	r11,swsusp_save_area_ptr@toc(r2)
 
@@ -208,16 +242,42 @@ nothing_to_copy:
 	RESTORE_REGISTER(r29)
 	RESTORE_REGISTER(r30)
 	RESTORE_REGISTER(r31)
+
+#ifdef CONFIG_PPC_BOOK3S_64
 	/* can't use RESTORE_SPECIAL(MSR) */
 	ld	r0, SL_MSR(r11)
 	mtmsrd	r0, 0
 	RESTORE_SPECIAL(SDR1)
+#else
+	/* Save SPRGs */
+	RESTORE_SPRG(0)
+	RESTORE_SPRG(1)
+	RESTORE_SPRG(2)
+	RESTORE_SPRG(3)
+	RESTORE_SPRG(4)
+	RESTORE_SPRG(5)
+	RESTORE_SPRG(6)
+	RESTORE_SPRG(7)
+
+	RESTORE_SPECIAL(MSR)
+
+	/* Restore TCR and clear any pending bits in TSR. */
+	RESTORE_SPR(TCR)
+	lis	r0, (TSR_ENW | TSR_WIS | TSR_DIS | TSR_FIS)@h
+	mtspr	SPRN_TSR,r0
+
+	/* Kick decrementer */
+	li	r0,1
+	mtdec	r0
+#endif
 	RESTORE_SPECIAL(XER)
 
 	sync
 
 	addi	r1,r1,-128
+#ifdef CONFIG_PPC_BOOK3S_64
 	bl	slb_flush_and_rebolt
+#endif
 	bl	do_after_copyback
 	addi	r1,r1,128
 
-- 
1.8.0

[PATCH 2/2] powerpc/hibernate: add restore mmu context after resume

[Wang Dongsheng]

add restore_mmu_context to replace switch_mmu_context in
restore_processor_state, because the switch_mmu_context will do
a whole pile of stuff that are probably completely unnecessary.

There just need to restore the existing process context, and
invalidate TLB for boot core.

Signed-off-by: Wang Dongsheng <dongsheng.wang@freescale.com>
---
 arch/powerpc/include/asm/tlbflush.h |  2 ++
 arch/powerpc/kernel/swsusp.c        |  4 +---
 arch/powerpc/mm/tlb_nohash.c        | 12 ++++++++++++
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/arch/powerpc/include/asm/tlbflush.h b/arch/powerpc/include/asm/tlbflush.h
index 61a5927..c401fe3 100644
--- a/arch/powerpc/include/asm/tlbflush.h
+++ b/arch/powerpc/include/asm/tlbflush.h
@@ -44,6 +44,8 @@ extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmadd
 extern void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
 				   int tsize, int ind);
 
+extern void restore_mmu_context(void);
+
 #ifdef CONFIG_SMP
 extern void flush_tlb_mm(struct mm_struct *mm);
 extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
diff --git a/arch/powerpc/kernel/swsusp.c b/arch/powerpc/kernel/swsusp.c
index eae33e1..0b104d7 100644
--- a/arch/powerpc/kernel/swsusp.c
+++ b/arch/powerpc/kernel/swsusp.c
@@ -32,7 +32,5 @@ void save_processor_state(void)
 
 void restore_processor_state(void)
 {
-#ifdef CONFIG_PPC32
-	switch_mmu_context(current->active_mm, current->active_mm);
-#endif
+	restore_mmu_context();
 }
diff --git a/arch/powerpc/mm/tlb_nohash.c b/arch/powerpc/mm/tlb_nohash.c
index df32a83..a5a0708 100644
--- a/arch/powerpc/mm/tlb_nohash.c
+++ b/arch/powerpc/mm/tlb_nohash.c
@@ -39,10 +39,12 @@
 #include <linux/of_fdt.h>
 #include <linux/hugetlb.h>
 
+#include <asm/current.h>
 #include <asm/tlbflush.h>
 #include <asm/tlb.h>
 #include <asm/code-patching.h>
 #include <asm/hugetlb.h>
+#include <asm/mmu_context.h>
 
 #include "mmu_decl.h"
 
@@ -193,6 +195,16 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 }
 EXPORT_SYMBOL(local_flush_tlb_page);
 
+void restore_mmu_context(void)
+{
+	struct mm_struct *mm = current->active_mm;
+
+	set_context(mm->context.id, mm->pgd);
+
+	_tlbil_all();
+}
+EXPORT_SYMBOL(restore_mmu_context);
+
 /*
  * And here are the SMP non-local implementations
  */
-- 
1.8.0

Re: [PATCH 1/2] powerpc: add Book E support to 64-bit hibernation

[Scott Wood]

On 06/09/2013 05:37:39 AM, Wang Dongsheng wrote:
>  /* these macros rely on the save area being
>   * pointed to by r11 */
> +
> +#define SAVE_SPR(register)		\
> +	mfspr	r0,SPRN_##register	;\
> +	std	r0,SL_##register(r11)
> +#define RESTORE_SPR(register)		\
> +	ld	r0,SL_##register(r11)	;\
> +	mtspr	SPRN_##register,r0
> +#define RESTORE_SPRG(n)			\
> +	ld	r0,SL_SPRG##n(r11)	;\
> +	mtsprg	n,r0
>  #define SAVE_SPECIAL(special)		\
>  	mf##special	r0		;\
>  	std	r0, SL_##special(r11)

Is there a particular SPR that you're trying to save, for which =20
SAVE_SPECIAL doesn't work?

> +#else
> +	/* Save SPRGs */
> +	RESTORE_SPRG(0)
> +	RESTORE_SPRG(1)
> +	RESTORE_SPRG(2)
> +	RESTORE_SPRG(3)
> +	RESTORE_SPRG(4)
> +	RESTORE_SPRG(5)
> +	RESTORE_SPRG(6)
> +	RESTORE_SPRG(7)

Why do we need this on book3e and not on book3s?

> +
> +	RESTORE_SPECIAL(MSR)
> +
> +	/* Restore TCR and clear any pending bits in TSR. */
> +	RESTORE_SPR(TCR)
> +	lis	r0, (TSR_ENW | TSR_WIS | TSR_DIS | TSR_FIS)@h
> +	mtspr	SPRN_TSR,r0

Please be internally consistent with whitespace after commas, even if =20
the rest of the file is already inconsistent. :-P

> +
> +	/* Kick decrementer */
> +	li	r0,1
> +	mtdec	r0

Why doesn't book3s need to kick the decrementer?

-Scott=

RE: [PATCH 1/2] powerpc: add Book E support to 64-bit hibernation

[Wang Dongsheng-B40534]

> -----Original Message-----
> From: Wood Scott-B07421
> Sent: Thursday, June 13, 2013 6:04 AM
> To: Wang Dongsheng-B40534
> Cc: benh@kernel.crashing.org; johannes@sipsolutions.net; anton@enomsg.org=
;
> galak@kernel.crashing.org; linuxppc-dev@lists.ozlabs.org; Wang Dongsheng-
> B40534
> Subject: Re: [PATCH 1/2] powerpc: add Book E support to 64-bit
> hibernation
>=20
> On 06/09/2013 05:37:39 AM, Wang Dongsheng wrote:
> >  /* these macros rely on the save area being
> >   * pointed to by r11 */
> > +
> > +#define SAVE_SPR(register)		\
> > +	mfspr	r0,SPRN_##register	;\
> > +	std	r0,SL_##register(r11)
> > +#define RESTORE_SPR(register)		\
> > +	ld	r0,SL_##register(r11)	;\
> > +	mtspr	SPRN_##register,r0
> > +#define RESTORE_SPRG(n)			\
> > +	ld	r0,SL_SPRG##n(r11)	;\
> > +	mtsprg	n,r0
> >  #define SAVE_SPECIAL(special)		\
> >  	mf##special	r0		;\
> >  	std	r0, SL_##special(r11)
>=20
> Is there a particular SPR that you're trying to save, for which
> SAVE_SPECIAL doesn't work?
>=20
Yes, like pid, tcr.

> > +#else
> > +	/* Save SPRGs */
> > +	RESTORE_SPRG(0)
> > +	RESTORE_SPRG(1)
> > +	RESTORE_SPRG(2)
> > +	RESTORE_SPRG(3)
> > +	RESTORE_SPRG(4)
> > +	RESTORE_SPRG(5)
> > +	RESTORE_SPRG(6)
> > +	RESTORE_SPRG(7)
>=20
> Why do we need this on book3e and not on book3s?
>=20
Book3e: SPRG1 used save paca, SPRG2 be defined SPRN_SPRG_TLB_EXFRAME,...
I think those register should be save, even now some SPRG register not be u=
se.

Book3s: Sorry, I not clear why book3s not do this. I think Anton or Ben cou=
ld know the reason.

> > +
> > +	RESTORE_SPECIAL(MSR)
> > +
> > +	/* Restore TCR and clear any pending bits in TSR. */
> > +	RESTORE_SPR(TCR)
> > +	lis	r0, (TSR_ENW | TSR_WIS | TSR_DIS | TSR_FIS)@h
> > +	mtspr	SPRN_TSR,r0
>=20
> Please be internally consistent with whitespace after commas, even if the
> rest of the file is already inconsistent. :-P
>=20
Thanks.

> > +
> > +	/* Kick decrementer */
> > +	li	r0,1
> > +	mtdec	r0
>=20
> Why doesn't book3s need to kick the decrementer?
>=20
Sorry, I not clear why book3s not do this. I think Anton or Ben could know =
the reason.

> -Scott

Re: [PATCH 1/2] powerpc: add Book E support to 64-bit hibernation

[Scott Wood]

On 06/13/2013 04:55:43 AM, Wang Dongsheng-B40534 wrote:
> > > +#else
> > > +	/* Save SPRGs */
> > > +	RESTORE_SPRG(0)
> > > +	RESTORE_SPRG(1)
> > > +	RESTORE_SPRG(2)
> > > +	RESTORE_SPRG(3)
> > > +	RESTORE_SPRG(4)
> > > +	RESTORE_SPRG(5)
> > > +	RESTORE_SPRG(6)
> > > +	RESTORE_SPRG(7)
> >
> > Why do we need this on book3e and not on book3s?
> >
> Book3e: SPRG1 used save paca, SPRG2 be defined =20
> SPRN_SPRG_TLB_EXFRAME,...
> I think those register should be save, even now some SPRG register =20
> not be use.

Are those expected/allowed to change as a result of the restore?

-Scott=

RE: [PATCH 1/2] powerpc: add Book E support to 64-bit hibernation

[Wang Dongsheng-B40534]

> -----Original Message-----
> From: Wood Scott-B07421
> Sent: Friday, June 14, 2013 12:51 AM
> To: Wang Dongsheng-B40534
> Cc: Wood Scott-B07421; benh@kernel.crashing.org;
> johannes@sipsolutions.net; anton@enomsg.org; galak@kernel.crashing.org;
> linuxppc-dev@lists.ozlabs.org
> Subject: Re: [PATCH 1/2] powerpc: add Book E support to 64-bit
> hibernation
>=20
> On 06/13/2013 04:55:43 AM, Wang Dongsheng-B40534 wrote:
> > > > +#else
> > > > +	/* Save SPRGs */
> > > > +	RESTORE_SPRG(0)
> > > > +	RESTORE_SPRG(1)
> > > > +	RESTORE_SPRG(2)
> > > > +	RESTORE_SPRG(3)
> > > > +	RESTORE_SPRG(4)
> > > > +	RESTORE_SPRG(5)
> > > > +	RESTORE_SPRG(6)
> > > > +	RESTORE_SPRG(7)
> > >
> > > Why do we need this on book3e and not on book3s?
> > >
> > Book3e: SPRG1 used save paca, SPRG2 be defined
> > SPRN_SPRG_TLB_EXFRAME,...
> > I think those register should be save, even now some SPRG register not
> > be use.
>=20
> Are those expected/allowed to change as a result of the restore?
>=20
Those registers are used by software, some allowed to change.
Exception handling is used in some registers, see exception-64e.h
These registers can be modified and saved.

> -Scott

Re: [PATCH 1/2] powerpc: add Book E support to 64-bit hibernation

[Scott Wood]

On 06/17/2013 12:54:32 AM, Wang Dongsheng-B40534 wrote:
>=20
>=20
> > -----Original Message-----
> > From: Wood Scott-B07421
> > Sent: Friday, June 14, 2013 12:51 AM
> > To: Wang Dongsheng-B40534
> > Cc: Wood Scott-B07421; benh@kernel.crashing.org;
> > johannes@sipsolutions.net; anton@enomsg.org; =20
> galak@kernel.crashing.org;
> > linuxppc-dev@lists.ozlabs.org
> > Subject: Re: [PATCH 1/2] powerpc: add Book E support to 64-bit
> > hibernation
> >
> > On 06/13/2013 04:55:43 AM, Wang Dongsheng-B40534 wrote:
> > > > > +#else
> > > > > +	/* Save SPRGs */
> > > > > +	RESTORE_SPRG(0)
> > > > > +	RESTORE_SPRG(1)
> > > > > +	RESTORE_SPRG(2)
> > > > > +	RESTORE_SPRG(3)
> > > > > +	RESTORE_SPRG(4)
> > > > > +	RESTORE_SPRG(5)
> > > > > +	RESTORE_SPRG(6)
> > > > > +	RESTORE_SPRG(7)
> > > >
> > > > Why do we need this on book3e and not on book3s?
> > > >
> > > Book3e: SPRG1 used save paca, SPRG2 be defined
> > > SPRN_SPRG_TLB_EXFRAME,...
> > > I think those register should be save, even now some SPRG =20
> register not
> > > be use.
> >
> > Are those expected/allowed to change as a result of the restore?
> >
> Those registers are used by software, some allowed to change.
> Exception handling is used in some registers, see exception-64e.h
> These registers can be modified and saved.

I really doubt the exception scratch registers need to be saved -- =20
we're not trying to restore into the middle of an exception =20
prolog/epilog.

book3s has the PACA as well and they don't save it.  Don't we rely on =20
things like boot-time memory allocations happening in the same place =20
when we resume?  extlb is part of the PACA, so the same applies.

Granted, this isn't performance critical so it may seem better to =20
save/restore just in case, but there's value in not unnecessarily =20
deviating from what book3s does.

-Scott=

Re: [PATCH 1/2] powerpc: add Book E support to 64-bit hibernation

[Benjamin Herrenschmidt]

On Mon, 2013-06-17 at 19:01 -0500, Scott Wood wrote:
> I really doubt the exception scratch registers need to be saved --  
> we're not trying to restore into the middle of an exception  
> prolog/epilog.
> 
> book3s has the PACA as well and they don't save it.  Don't we rely on  
> things like boot-time memory allocations happening in the same place  
> when we resume?  extlb is part of the PACA, so the same applies.

I doubt we seriously tested hibernation :-) The PACA SPR should
definitely be saved/restored.

> Granted, this isn't performance critical so it may seem better to  
> save/restore just in case, but there's value in not unnecessarily  
> deviating from what book3s does.

Cheers,
Ben.

Re: [PATCH 1/2] powerpc: add Book E support to 64-bit hibernation

[Scott Wood]

On 06/17/2013 07:17:30 PM, Benjamin Herrenschmidt wrote:
> On Mon, 2013-06-17 at 19:01 -0500, Scott Wood wrote:
> > I really doubt the exception scratch registers need to be saved --
> > we're not trying to restore into the middle of an exception
> > prolog/epilog.
> >
> > book3s has the PACA as well and they don't save it.  Don't we rely =20
> on
> > things like boot-time memory allocations happening in the same place
> > when we resume?  extlb is part of the PACA, so the same applies.
>=20
> I doubt we seriously tested hibernation :-) The PACA SPR should
> definitely be saved/restored.

OK.  It's not obvious to me how much the entire mechanism depends on =20
things like boot time allocations being the same each time -- if we do =20
depend on that in general, then the PACA shouldn't change on a =20
particular CPU, right?

Is it possible to restore on a different CPU than we saved on?  If so, =20
could restoring the PACA leave us pointing to a different CPU's PACA?

-Scott=

Re: [PATCH 1/2] powerpc: add Book E support to 64-bit hibernation

[Benjamin Herrenschmidt]

On Mon, 2013-06-17 at 19:22 -0500, Scott Wood wrote:
> On 06/17/2013 07:17:30 PM, Benjamin Herrenschmidt wrote:
> > On Mon, 2013-06-17 at 19:01 -0500, Scott Wood wrote:
> > > I really doubt the exception scratch registers need to be saved --
> > > we're not trying to restore into the middle of an exception
> > > prolog/epilog.
> > >
> > > book3s has the PACA as well and they don't save it.  Don't we rely  
> > on
> > > things like boot-time memory allocations happening in the same place
> > > when we resume?  extlb is part of the PACA, so the same applies.
> > 
> > I doubt we seriously tested hibernation :-) The PACA SPR should
> > definitely be saved/restored.
> 
> OK.  It's not obvious to me how much the entire mechanism depends on  
> things like boot time allocations being the same each time -- if we do  
> depend on that in general, then the PACA shouldn't change on a  
> particular CPU, right?

No we shouldn't be depending on that stuff.

> Is it possible to restore on a different CPU than we saved on?  If so,  
> could restoring the PACA leave us pointing to a different CPU's PACA?

Today on your code no, but of course this needs to be handled, the PACA
should be restored along with other thing, but for the right
processor :-)

Cheers,
Ben.

RE: [PATCH 1/2] powerpc: add Book E support to 64-bit hibernation

[Wang Dongsheng-B40534]

Hi scott,

Could you apply this patch?

Thanks.

-dongsheng

> -----Original Message-----
> From: Wang Dongsheng-B40534
> Sent: Sunday, June 09, 2013 6:38 PM
> To: benh@kernel.crashing.org
> Cc: johannes@sipsolutions.net; anton@enomsg.org; Wood Scott-B07421;
> galak@kernel.crashing.org; linuxppc-dev@lists.ozlabs.org; Wang Dongsheng-
> B40534
> Subject: [PATCH 1/2] powerpc: add Book E support to 64-bit hibernation
>=20
> Update the 64-bit hibernation code to support Book E CPUs.
> Some registers and instructions are not defined for Book3e
> (SDR reg, tlbia instruction).
>=20
> SDR: Storage Description Register. Book3S and Book3E have different
> address translation mode, we do not need HTABORG & HTABSIZE to
> translate virtual address to real address.
>=20
> More registers are saved in BookE-64bit.(TCR, SPRGx, ...)
>=20
> Signed-off-by: Wang Dongsheng <dongsheng.wang@freescale.com>
> ---
> * History:
> * Wood Scott(A): Please investigate the issue of whether we are loading
> *                kernel module code in this step
> * R: Kernel will allocate the memory for module code segment and data
> *    segment. First allocate a memory, and copy the umod to hdr members
> *    of the struct load_info. Due to the temporary assigned module
> belongs
> *    to the kernel space, so it belongs to the kernel data.
> *
> *    The kernel of all data will be saved when hibernation suspend. So
> *    the module which has already been inserted will be saved.
>=20
>  arch/powerpc/kernel/swsusp_asm64.S | 64
> ++++++++++++++++++++++++++++++++++++--
>  1 file changed, 62 insertions(+), 2 deletions(-)
>=20
> diff --git a/arch/powerpc/kernel/swsusp_asm64.S
> b/arch/powerpc/kernel/swsusp_asm64.S
> index 86ac1d9..608e4ceb 100644
> --- a/arch/powerpc/kernel/swsusp_asm64.S
> +++ b/arch/powerpc/kernel/swsusp_asm64.S
> @@ -46,10 +46,29 @@
>  #define SL_r29		0xe8
>  #define SL_r30		0xf0
>  #define SL_r31		0xf8
> -#define SL_SIZE		SL_r31+8
> +#define SL_SPRG0	0x100
> +#define SL_SPRG1	0x108
> +#define SL_SPRG2	0x110
> +#define SL_SPRG3	0x118
> +#define SL_SPRG4	0x120
> +#define SL_SPRG5	0x128
> +#define SL_SPRG6	0x130
> +#define SL_SPRG7	0x138
> +#define SL_TCR		0x140
> +#define SL_SIZE		SL_TCR+8
>=20
>  /* these macros rely on the save area being
>   * pointed to by r11 */
> +
> +#define SAVE_SPR(register)		\
> +	mfspr	r0,SPRN_##register	;\
> +	std	r0,SL_##register(r11)
> +#define RESTORE_SPR(register)		\
> +	ld	r0,SL_##register(r11)	;\
> +	mtspr	SPRN_##register,r0
> +#define RESTORE_SPRG(n)			\
> +	ld	r0,SL_SPRG##n(r11)	;\
> +	mtsprg	n,r0
>  #define SAVE_SPECIAL(special)		\
>  	mf##special	r0		;\
>  	std	r0, SL_##special(r11)
> @@ -103,8 +122,21 @@ _GLOBAL(swsusp_arch_suspend)
>  	SAVE_REGISTER(r30)
>  	SAVE_REGISTER(r31)
>  	SAVE_SPECIAL(MSR)
> -	SAVE_SPECIAL(SDR1)
>  	SAVE_SPECIAL(XER)
> +#ifdef CONFIG_PPC_BOOK3S_64
> +	SAVE_SPECIAL(SDR1)
> +#else
> +	SAVE_SPR(TCR)
> +	/* Save SPRGs */
> +	SAVE_SPR(SPRG0)
> +	SAVE_SPR(SPRG1)
> +	SAVE_SPR(SPRG2)
> +	SAVE_SPR(SPRG3)
> +	SAVE_SPR(SPRG4)
> +	SAVE_SPR(SPRG5)
> +	SAVE_SPR(SPRG6)
> +	SAVE_SPR(SPRG7)
> +#endif
>=20
>  	/* we push the stack up 128 bytes but don't store the
>  	 * stack pointer on the stack like a real stackframe */
> @@ -151,6 +183,7 @@ copy_page_loop:
>  	bne+	copyloop
>  nothing_to_copy:
>=20
> +#ifdef CONFIG_PPC_BOOK3S_64
>  	/* flush caches */
>  	lis	r3, 0x10
>  	mtctr	r3
> @@ -167,6 +200,7 @@ nothing_to_copy:
>  	sync
>=20
>  	tlbia
> +#endif
>=20
>  	ld	r11,swsusp_save_area_ptr@toc(r2)
>=20
> @@ -208,16 +242,42 @@ nothing_to_copy:
>  	RESTORE_REGISTER(r29)
>  	RESTORE_REGISTER(r30)
>  	RESTORE_REGISTER(r31)
> +
> +#ifdef CONFIG_PPC_BOOK3S_64
>  	/* can't use RESTORE_SPECIAL(MSR) */
>  	ld	r0, SL_MSR(r11)
>  	mtmsrd	r0, 0
>  	RESTORE_SPECIAL(SDR1)
> +#else
> +	/* Save SPRGs */
> +	RESTORE_SPRG(0)
> +	RESTORE_SPRG(1)
> +	RESTORE_SPRG(2)
> +	RESTORE_SPRG(3)
> +	RESTORE_SPRG(4)
> +	RESTORE_SPRG(5)
> +	RESTORE_SPRG(6)
> +	RESTORE_SPRG(7)
> +
> +	RESTORE_SPECIAL(MSR)
> +
> +	/* Restore TCR and clear any pending bits in TSR. */
> +	RESTORE_SPR(TCR)
> +	lis	r0, (TSR_ENW | TSR_WIS | TSR_DIS | TSR_FIS)@h
> +	mtspr	SPRN_TSR,r0
> +
> +	/* Kick decrementer */
> +	li	r0,1
> +	mtdec	r0
> +#endif
>  	RESTORE_SPECIAL(XER)
>=20
>  	sync
>=20
>  	addi	r1,r1,-128
> +#ifdef CONFIG_PPC_BOOK3S_64
>  	bl	slb_flush_and_rebolt
> +#endif
>  	bl	do_after_copyback
>  	addi	r1,r1,128
>=20
> --
> 1.8.0

Re: [PATCH 1/2] powerpc: add Book E support to 64-bit hibernation

[Benjamin Herrenschmidt]

On Wed, 2013-07-10 at 09:41 +0000, Wang Dongsheng-B40534 wrote:
> Hi scott,
> 
> Could you apply this patch?

There were a numbre of comments, were there addressed ? Do you need to
save all the SPRGs for example ?

Ben.

> Thanks.
> 
> -dongsheng
> 
> > -----Original Message-----
> > From: Wang Dongsheng-B40534
> > Sent: Sunday, June 09, 2013 6:38 PM
> > To: benh@kernel.crashing.org
> > Cc: johannes@sipsolutions.net; anton@enomsg.org; Wood Scott-B07421;
> > galak@kernel.crashing.org; linuxppc-dev@lists.ozlabs.org; Wang Dongsheng-
> > B40534
> > Subject: [PATCH 1/2] powerpc: add Book E support to 64-bit hibernation
> > 
> > Update the 64-bit hibernation code to support Book E CPUs.
> > Some registers and instructions are not defined for Book3e
> > (SDR reg, tlbia instruction).
> > 
> > SDR: Storage Description Register. Book3S and Book3E have different
> > address translation mode, we do not need HTABORG & HTABSIZE to
> > translate virtual address to real address.
> > 
> > More registers are saved in BookE-64bit.(TCR, SPRGx, ...)
> > 
> > Signed-off-by: Wang Dongsheng <dongsheng.wang@freescale.com>
> > ---
> > * History:
> > * Wood Scott(A): Please investigate the issue of whether we are loading
> > *                kernel module code in this step
> > * R: Kernel will allocate the memory for module code segment and data
> > *    segment. First allocate a memory, and copy the umod to hdr members
> > *    of the struct load_info. Due to the temporary assigned module
> > belongs
> > *    to the kernel space, so it belongs to the kernel data.
> > *
> > *    The kernel of all data will be saved when hibernation suspend. So
> > *    the module which has already been inserted will be saved.
> > 
> >  arch/powerpc/kernel/swsusp_asm64.S | 64
> > ++++++++++++++++++++++++++++++++++++--
> >  1 file changed, 62 insertions(+), 2 deletions(-)
> > 
> > diff --git a/arch/powerpc/kernel/swsusp_asm64.S
> > b/arch/powerpc/kernel/swsusp_asm64.S
> > index 86ac1d9..608e4ceb 100644
> > --- a/arch/powerpc/kernel/swsusp_asm64.S
> > +++ b/arch/powerpc/kernel/swsusp_asm64.S
> > @@ -46,10 +46,29 @@
> >  #define SL_r29		0xe8
> >  #define SL_r30		0xf0
> >  #define SL_r31		0xf8
> > -#define SL_SIZE		SL_r31+8
> > +#define SL_SPRG0	0x100
> > +#define SL_SPRG1	0x108
> > +#define SL_SPRG2	0x110
> > +#define SL_SPRG3	0x118
> > +#define SL_SPRG4	0x120
> > +#define SL_SPRG5	0x128
> > +#define SL_SPRG6	0x130
> > +#define SL_SPRG7	0x138
> > +#define SL_TCR		0x140
> > +#define SL_SIZE		SL_TCR+8
> > 
> >  /* these macros rely on the save area being
> >   * pointed to by r11 */
> > +
> > +#define SAVE_SPR(register)		\
> > +	mfspr	r0,SPRN_##register	;\
> > +	std	r0,SL_##register(r11)
> > +#define RESTORE_SPR(register)		\
> > +	ld	r0,SL_##register(r11)	;\
> > +	mtspr	SPRN_##register,r0
> > +#define RESTORE_SPRG(n)			\
> > +	ld	r0,SL_SPRG##n(r11)	;\
> > +	mtsprg	n,r0
> >  #define SAVE_SPECIAL(special)		\
> >  	mf##special	r0		;\
> >  	std	r0, SL_##special(r11)
> > @@ -103,8 +122,21 @@ _GLOBAL(swsusp_arch_suspend)
> >  	SAVE_REGISTER(r30)
> >  	SAVE_REGISTER(r31)
> >  	SAVE_SPECIAL(MSR)
> > -	SAVE_SPECIAL(SDR1)
> >  	SAVE_SPECIAL(XER)
> > +#ifdef CONFIG_PPC_BOOK3S_64
> > +	SAVE_SPECIAL(SDR1)
> > +#else
> > +	SAVE_SPR(TCR)
> > +	/* Save SPRGs */
> > +	SAVE_SPR(SPRG0)
> > +	SAVE_SPR(SPRG1)
> > +	SAVE_SPR(SPRG2)
> > +	SAVE_SPR(SPRG3)
> > +	SAVE_SPR(SPRG4)
> > +	SAVE_SPR(SPRG5)
> > +	SAVE_SPR(SPRG6)
> > +	SAVE_SPR(SPRG7)
> > +#endif
> > 
> >  	/* we push the stack up 128 bytes but don't store the
> >  	 * stack pointer on the stack like a real stackframe */
> > @@ -151,6 +183,7 @@ copy_page_loop:
> >  	bne+	copyloop
> >  nothing_to_copy:
> > 
> > +#ifdef CONFIG_PPC_BOOK3S_64
> >  	/* flush caches */
> >  	lis	r3, 0x10
> >  	mtctr	r3
> > @@ -167,6 +200,7 @@ nothing_to_copy:
> >  	sync
> > 
> >  	tlbia
> > +#endif
> > 
> >  	ld	r11,swsusp_save_area_ptr@toc(r2)
> > 
> > @@ -208,16 +242,42 @@ nothing_to_copy:
> >  	RESTORE_REGISTER(r29)
> >  	RESTORE_REGISTER(r30)
> >  	RESTORE_REGISTER(r31)
> > +
> > +#ifdef CONFIG_PPC_BOOK3S_64
> >  	/* can't use RESTORE_SPECIAL(MSR) */
> >  	ld	r0, SL_MSR(r11)
> >  	mtmsrd	r0, 0
> >  	RESTORE_SPECIAL(SDR1)
> > +#else
> > +	/* Save SPRGs */
> > +	RESTORE_SPRG(0)
> > +	RESTORE_SPRG(1)
> > +	RESTORE_SPRG(2)
> > +	RESTORE_SPRG(3)
> > +	RESTORE_SPRG(4)
> > +	RESTORE_SPRG(5)
> > +	RESTORE_SPRG(6)
> > +	RESTORE_SPRG(7)
> > +
> > +	RESTORE_SPECIAL(MSR)
> > +
> > +	/* Restore TCR and clear any pending bits in TSR. */
> > +	RESTORE_SPR(TCR)
> > +	lis	r0, (TSR_ENW | TSR_WIS | TSR_DIS | TSR_FIS)@h
> > +	mtspr	SPRN_TSR,r0
> > +
> > +	/* Kick decrementer */
> > +	li	r0,1
> > +	mtdec	r0
> > +#endif
> >  	RESTORE_SPECIAL(XER)
> > 
> >  	sync
> > 
> >  	addi	r1,r1,-128
> > +#ifdef CONFIG_PPC_BOOK3S_64
> >  	bl	slb_flush_and_rebolt
> > +#endif
> >  	bl	do_after_copyback
> >  	addi	r1,r1,128
> > 
> > --
> > 1.8.0
> 

RE: [PATCH 1/2] powerpc: add Book E support to 64-bit hibernation

[Wang Dongsheng-B40534]

DQoNCj4gLS0tLS1PcmlnaW5hbCBNZXNzYWdlLS0tLS0NCj4gRnJvbTogQmVuamFtaW4gSGVycmVu
c2NobWlkdCBbbWFpbHRvOmJlbmhAa2VybmVsLmNyYXNoaW5nLm9yZ10NCj4gU2VudDogV2VkbmVz
ZGF5LCBKdWx5IDEwLCAyMDEzIDU6NTIgUE0NCj4gVG86IFdhbmcgRG9uZ3NoZW5nLUI0MDUzNA0K
PiBDYzogV29vZCBTY290dC1CMDc0MjE7IGpvaGFubmVzQHNpcHNvbHV0aW9ucy5uZXQ7IGFudG9u
QGVub21zZy5vcmc7DQo+IGdhbGFrQGtlcm5lbC5jcmFzaGluZy5vcmc7IGxpbnV4cHBjLWRldkBs
aXN0cy5vemxhYnMub3JnDQo+IFN1YmplY3Q6IFJlOiBbUEFUQ0ggMS8yXSBwb3dlcnBjOiBhZGQg
Qm9vayBFIHN1cHBvcnQgdG8gNjQtYml0DQo+IGhpYmVybmF0aW9uDQo+IA0KPiBPbiBXZWQsIDIw
MTMtMDctMTAgYXQgMDk6NDEgKzAwMDAsIFdhbmcgRG9uZ3NoZW5nLUI0MDUzNCB3cm90ZToNCj4g
PiBIaSBzY290dCwNCj4gPg0KPiA+IENvdWxkIHlvdSBhcHBseSB0aGlzIHBhdGNoPw0KPiANCj4g
VGhlcmUgd2VyZSBhIG51bWJyZSBvZiBjb21tZW50cywgd2VyZSB0aGVyZSBhZGRyZXNzZWQgPyBE
byB5b3UgbmVlZCB0bw0KPiBzYXZlIGFsbCB0aGUgU1BSR3MgZm9yIGV4YW1wbGUgPw0KPiANCj4g
QmVuLg0KPiANClVtLi4ueWVzLCBzb3JyeSwgbm90IGFsbCBvZiB0aGUgU1BSR3MuDQp0aGUgZGlz
Y3Vzc2lvbiBhYm91dCBTUFJHMSBhbmQgU1BSRzIuIEkgY2FuIGZpeCB0aGVtLCBqdXN0IHRvIHNh
dmUgU1JQRzEgJiBTUFJHMi4NCkJ1dCB0aG9zZSByZWdpc3RlcnMgY2FuIGJlIHVzZWQgYnkgc29m
dHdhcmUsIEkgdGhpbmsgdGhvc2UgcmVnaXN0ZXIgc2hvdWxkIGJlIHNhdmUsDQpldmVuIG5vdyBz
b21lIFNQUkcgcmVnaXN0ZXIgbm90IGJlIHVzZS4NCg0KPiA+IFRoYW5rcy4NCj4gPg0KPiA+IC1k
b25nc2hlbmcNCj4gPg0KPiA+ID4gLS0tLS1PcmlnaW5hbCBNZXNzYWdlLS0tLS0NCj4gPiA+IEZy
b206IFdhbmcgRG9uZ3NoZW5nLUI0MDUzNA0KPiA+ID4gU2VudDogU3VuZGF5LCBKdW5lIDA5LCAy
MDEzIDY6MzggUE0NCj4gPiA+IFRvOiBiZW5oQGtlcm5lbC5jcmFzaGluZy5vcmcNCj4gPiA+IENj
OiBqb2hhbm5lc0BzaXBzb2x1dGlvbnMubmV0OyBhbnRvbkBlbm9tc2cub3JnOyBXb29kIFNjb3R0
LUIwNzQyMTsNCj4gPiA+IGdhbGFrQGtlcm5lbC5jcmFzaGluZy5vcmc7IGxpbnV4cHBjLWRldkBs
aXN0cy5vemxhYnMub3JnOyBXYW5nDQo+ID4gPiBEb25nc2hlbmctDQo+ID4gPiBCNDA1MzQNCj4g
PiA+IFN1YmplY3Q6IFtQQVRDSCAxLzJdIHBvd2VycGM6IGFkZCBCb29rIEUgc3VwcG9ydCB0byA2
NC1iaXQNCj4gPiA+IGhpYmVybmF0aW9uDQo+ID4gPg0KPiA+ID4gVXBkYXRlIHRoZSA2NC1iaXQg
aGliZXJuYXRpb24gY29kZSB0byBzdXBwb3J0IEJvb2sgRSBDUFVzLg0KPiA+ID4gU29tZSByZWdp
c3RlcnMgYW5kIGluc3RydWN0aW9ucyBhcmUgbm90IGRlZmluZWQgZm9yIEJvb2szZSAoU0RSIHJl
ZywNCj4gPiA+IHRsYmlhIGluc3RydWN0aW9uKS4NCj4gPiA+DQo+ID4gPiBTRFI6IFN0b3JhZ2Ug
RGVzY3JpcHRpb24gUmVnaXN0ZXIuIEJvb2szUyBhbmQgQm9vazNFIGhhdmUgZGlmZmVyZW50DQo+
ID4gPiBhZGRyZXNzIHRyYW5zbGF0aW9uIG1vZGUsIHdlIGRvIG5vdCBuZWVkIEhUQUJPUkcgJiBI
VEFCU0laRSB0bw0KPiA+ID4gdHJhbnNsYXRlIHZpcnR1YWwgYWRkcmVzcyB0byByZWFsIGFkZHJl
c3MuDQo+ID4gPg0KPiA+ID4gTW9yZSByZWdpc3RlcnMgYXJlIHNhdmVkIGluIEJvb2tFLTY0Yml0
LihUQ1IsIFNQUkd4LCAuLi4pDQo+ID4gPg0KPiA+ID4gU2lnbmVkLW9mZi1ieTogV2FuZyBEb25n
c2hlbmcgPGRvbmdzaGVuZy53YW5nQGZyZWVzY2FsZS5jb20+DQo+ID4gPiAtLS0NCj4gPiA+ICog
SGlzdG9yeToNCj4gPiA+ICogV29vZCBTY290dChBKTogUGxlYXNlIGludmVzdGlnYXRlIHRoZSBp
c3N1ZSBvZiB3aGV0aGVyIHdlIGFyZQ0KPiBsb2FkaW5nDQo+ID4gPiAqICAgICAgICAgICAgICAg
IGtlcm5lbCBtb2R1bGUgY29kZSBpbiB0aGlzIHN0ZXANCj4gPiA+ICogUjogS2VybmVsIHdpbGwg
YWxsb2NhdGUgdGhlIG1lbW9yeSBmb3IgbW9kdWxlIGNvZGUgc2VnbWVudCBhbmQgZGF0YQ0KPiA+
ID4gKiAgICBzZWdtZW50LiBGaXJzdCBhbGxvY2F0ZSBhIG1lbW9yeSwgYW5kIGNvcHkgdGhlIHVt
b2QgdG8gaGRyDQo+IG1lbWJlcnMNCj4gPiA+ICogICAgb2YgdGhlIHN0cnVjdCBsb2FkX2luZm8u
IER1ZSB0byB0aGUgdGVtcG9yYXJ5IGFzc2lnbmVkIG1vZHVsZQ0KPiA+ID4gYmVsb25ncw0KPiA+
ID4gKiAgICB0byB0aGUga2VybmVsIHNwYWNlLCBzbyBpdCBiZWxvbmdzIHRvIHRoZSBrZXJuZWwg
ZGF0YS4NCj4gPiA+ICoNCj4gPiA+ICogICAgVGhlIGtlcm5lbCBvZiBhbGwgZGF0YSB3aWxsIGJl
IHNhdmVkIHdoZW4gaGliZXJuYXRpb24gc3VzcGVuZC4NCj4gU28NCj4gPiA+ICogICAgdGhlIG1v
ZHVsZSB3aGljaCBoYXMgYWxyZWFkeSBiZWVuIGluc2VydGVkIHdpbGwgYmUgc2F2ZWQuDQo+ID4g
Pg0KPiA+ID4gIGFyY2gvcG93ZXJwYy9rZXJuZWwvc3dzdXNwX2FzbTY0LlMgfCA2NA0KPiA+ID4g
KysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrLS0NCj4gPiA+ICAxIGZpbGUgY2hh
bmdlZCwgNjIgaW5zZXJ0aW9ucygrKSwgMiBkZWxldGlvbnMoLSkNCj4gPiA+DQo+ID4gPiBkaWZm
IC0tZ2l0IGEvYXJjaC9wb3dlcnBjL2tlcm5lbC9zd3N1c3BfYXNtNjQuUw0KPiA+ID4gYi9hcmNo
L3Bvd2VycGMva2VybmVsL3N3c3VzcF9hc202NC5TDQo+ID4gPiBpbmRleCA4NmFjMWQ5Li42MDhl
NGNlYiAxMDA2NDQNCj4gPiA+IC0tLSBhL2FyY2gvcG93ZXJwYy9rZXJuZWwvc3dzdXNwX2FzbTY0
LlMNCj4gPiA+ICsrKyBiL2FyY2gvcG93ZXJwYy9rZXJuZWwvc3dzdXNwX2FzbTY0LlMNCj4gPiA+
IEBAIC00NiwxMCArNDYsMjkgQEANCj4gPiA+ICAjZGVmaW5lIFNMX3IyOQkJMHhlOA0KPiA+ID4g
ICNkZWZpbmUgU0xfcjMwCQkweGYwDQo+ID4gPiAgI2RlZmluZSBTTF9yMzEJCTB4ZjgNCj4gPiA+
IC0jZGVmaW5lIFNMX1NJWkUJCVNMX3IzMSs4DQo+ID4gPiArI2RlZmluZSBTTF9TUFJHMAkweDEw
MA0KPiA+ID4gKyNkZWZpbmUgU0xfU1BSRzEJMHgxMDgNCj4gPiA+ICsjZGVmaW5lIFNMX1NQUkcy
CTB4MTEwDQo+ID4gPiArI2RlZmluZSBTTF9TUFJHMwkweDExOA0KPiA+ID4gKyNkZWZpbmUgU0xf
U1BSRzQJMHgxMjANCj4gPiA+ICsjZGVmaW5lIFNMX1NQUkc1CTB4MTI4DQo+ID4gPiArI2RlZmlu
ZSBTTF9TUFJHNgkweDEzMA0KPiA+ID4gKyNkZWZpbmUgU0xfU1BSRzcJMHgxMzgNCj4gPiA+ICsj
ZGVmaW5lIFNMX1RDUgkJMHgxNDANCj4gPiA+ICsjZGVmaW5lIFNMX1NJWkUJCVNMX1RDUis4DQo+
ID4gPg0KPiA+ID4gIC8qIHRoZXNlIG1hY3JvcyByZWx5IG9uIHRoZSBzYXZlIGFyZWEgYmVpbmcN
Cj4gPiA+ICAgKiBwb2ludGVkIHRvIGJ5IHIxMSAqLw0KPiA+ID4gKw0KPiA+ID4gKyNkZWZpbmUg
U0FWRV9TUFIocmVnaXN0ZXIpCQlcDQo+ID4gPiArCW1mc3ByCXIwLFNQUk5fIyNyZWdpc3Rlcgk7
XA0KPiA+ID4gKwlzdGQJcjAsU0xfIyNyZWdpc3RlcihyMTEpDQo+ID4gPiArI2RlZmluZSBSRVNU
T1JFX1NQUihyZWdpc3RlcikJCVwNCj4gPiA+ICsJbGQJcjAsU0xfIyNyZWdpc3RlcihyMTEpCTtc
DQo+ID4gPiArCW10c3ByCVNQUk5fIyNyZWdpc3RlcixyMA0KPiA+ID4gKyNkZWZpbmUgUkVTVE9S
RV9TUFJHKG4pCQkJXA0KPiA+ID4gKwlsZAlyMCxTTF9TUFJHIyNuKHIxMSkJO1wNCj4gPiA+ICsJ
bXRzcHJnCW4scjANCj4gPiA+ICAjZGVmaW5lIFNBVkVfU1BFQ0lBTChzcGVjaWFsKQkJXA0KPiA+
ID4gIAltZiMjc3BlY2lhbAlyMAkJO1wNCj4gPiA+ICAJc3RkCXIwLCBTTF8jI3NwZWNpYWwocjEx
KQ0KPiA+ID4gQEAgLTEwMyw4ICsxMjIsMjEgQEAgX0dMT0JBTChzd3N1c3BfYXJjaF9zdXNwZW5k
KQ0KPiA+ID4gIAlTQVZFX1JFR0lTVEVSKHIzMCkNCj4gPiA+ICAJU0FWRV9SRUdJU1RFUihyMzEp
DQo+ID4gPiAgCVNBVkVfU1BFQ0lBTChNU1IpDQo+ID4gPiAtCVNBVkVfU1BFQ0lBTChTRFIxKQ0K
PiA+ID4gIAlTQVZFX1NQRUNJQUwoWEVSKQ0KPiA+ID4gKyNpZmRlZiBDT05GSUdfUFBDX0JPT0sz
U182NA0KPiA+ID4gKwlTQVZFX1NQRUNJQUwoU0RSMSkNCj4gPiA+ICsjZWxzZQ0KPiA+ID4gKwlT
QVZFX1NQUihUQ1IpDQo+ID4gPiArCS8qIFNhdmUgU1BSR3MgKi8NCj4gPiA+ICsJU0FWRV9TUFIo
U1BSRzApDQo+ID4gPiArCVNBVkVfU1BSKFNQUkcxKQ0KPiA+ID4gKwlTQVZFX1NQUihTUFJHMikN
Cj4gPiA+ICsJU0FWRV9TUFIoU1BSRzMpDQo+ID4gPiArCVNBVkVfU1BSKFNQUkc0KQ0KPiA+ID4g
KwlTQVZFX1NQUihTUFJHNSkNCj4gPiA+ICsJU0FWRV9TUFIoU1BSRzYpDQo+ID4gPiArCVNBVkVf
U1BSKFNQUkc3KQ0KPiA+ID4gKyNlbmRpZg0KPiA+ID4NCj4gPiA+ICAJLyogd2UgcHVzaCB0aGUg
c3RhY2sgdXAgMTI4IGJ5dGVzIGJ1dCBkb24ndCBzdG9yZSB0aGUNCj4gPiA+ICAJICogc3RhY2sg
cG9pbnRlciBvbiB0aGUgc3RhY2sgbGlrZSBhIHJlYWwgc3RhY2tmcmFtZSAqLyBAQCAtMTUxLDYN
Cj4gPiA+ICsxODMsNyBAQCBjb3B5X3BhZ2VfbG9vcDoNCj4gPiA+ICAJYm5lKwljb3B5bG9vcA0K
PiA+ID4gIG5vdGhpbmdfdG9fY29weToNCj4gPiA+DQo+ID4gPiArI2lmZGVmIENPTkZJR19QUENf
Qk9PSzNTXzY0DQo+ID4gPiAgCS8qIGZsdXNoIGNhY2hlcyAqLw0KPiA+ID4gIAlsaXMJcjMsIDB4
MTANCj4gPiA+ICAJbXRjdHIJcjMNCj4gPiA+IEBAIC0xNjcsNiArMjAwLDcgQEAgbm90aGluZ190
b19jb3B5Og0KPiA+ID4gIAlzeW5jDQo+ID4gPg0KPiA+ID4gIAl0bGJpYQ0KPiA+ID4gKyNlbmRp
Zg0KPiA+ID4NCj4gPiA+ICAJbGQJcjExLHN3c3VzcF9zYXZlX2FyZWFfcHRyQHRvYyhyMikNCj4g
PiA+DQo+ID4gPiBAQCAtMjA4LDE2ICsyNDIsNDIgQEAgbm90aGluZ190b19jb3B5Og0KPiA+ID4g
IAlSRVNUT1JFX1JFR0lTVEVSKHIyOSkNCj4gPiA+ICAJUkVTVE9SRV9SRUdJU1RFUihyMzApDQo+
ID4gPiAgCVJFU1RPUkVfUkVHSVNURVIocjMxKQ0KPiA+ID4gKw0KPiA+ID4gKyNpZmRlZiBDT05G
SUdfUFBDX0JPT0szU182NA0KPiA+ID4gIAkvKiBjYW4ndCB1c2UgUkVTVE9SRV9TUEVDSUFMKE1T
UikgKi8NCj4gPiA+ICAJbGQJcjAsIFNMX01TUihyMTEpDQo+ID4gPiAgCW10bXNyZAlyMCwgMA0K
PiA+ID4gIAlSRVNUT1JFX1NQRUNJQUwoU0RSMSkNCj4gPiA+ICsjZWxzZQ0KPiA+ID4gKwkvKiBT
YXZlIFNQUkdzICovDQo+ID4gPiArCVJFU1RPUkVfU1BSRygwKQ0KPiA+ID4gKwlSRVNUT1JFX1NQ
UkcoMSkNCj4gPiA+ICsJUkVTVE9SRV9TUFJHKDIpDQo+ID4gPiArCVJFU1RPUkVfU1BSRygzKQ0K
PiA+ID4gKwlSRVNUT1JFX1NQUkcoNCkNCj4gPiA+ICsJUkVTVE9SRV9TUFJHKDUpDQo+ID4gPiAr
CVJFU1RPUkVfU1BSRyg2KQ0KPiA+ID4gKwlSRVNUT1JFX1NQUkcoNykNCj4gPiA+ICsNCj4gPiA+
ICsJUkVTVE9SRV9TUEVDSUFMKE1TUikNCj4gPiA+ICsNCj4gPiA+ICsJLyogUmVzdG9yZSBUQ1Ig
YW5kIGNsZWFyIGFueSBwZW5kaW5nIGJpdHMgaW4gVFNSLiAqLw0KPiA+ID4gKwlSRVNUT1JFX1NQ
UihUQ1IpDQo+ID4gPiArCWxpcwlyMCwgKFRTUl9FTlcgfCBUU1JfV0lTIHwgVFNSX0RJUyB8IFRT
Ul9GSVMpQGgNCj4gPiA+ICsJbXRzcHIJU1BSTl9UU1IscjANCj4gPiA+ICsNCj4gPiA+ICsJLyog
S2ljayBkZWNyZW1lbnRlciAqLw0KPiA+ID4gKwlsaQlyMCwxDQo+ID4gPiArCW10ZGVjCXIwDQo+
ID4gPiArI2VuZGlmDQo+ID4gPiAgCVJFU1RPUkVfU1BFQ0lBTChYRVIpDQo+ID4gPg0KPiA+ID4g
IAlzeW5jDQo+ID4gPg0KPiA+ID4gIAlhZGRpCXIxLHIxLC0xMjgNCj4gPiA+ICsjaWZkZWYgQ09O
RklHX1BQQ19CT09LM1NfNjQNCj4gPiA+ICAJYmwJc2xiX2ZsdXNoX2FuZF9yZWJvbHQNCj4gPiA+
ICsjZW5kaWYNCj4gPiA+ICAJYmwJZG9fYWZ0ZXJfY29weWJhY2sNCj4gPiA+ICAJYWRkaQlyMSxy
MSwxMjgNCj4gPiA+DQo+ID4gPiAtLQ0KPiA+ID4gMS44LjANCj4gPg0KPiANCj4gDQoNCg==

RE: [PATCH 2/2] powerpc/hibernate: add restore mmu context after resume

[Wang Dongsheng-B40534]

Hi scott & ben,

About this patch do you have any suggestions?

Thanks

- dongsheng

> -----Original Message-----
> From: Wang Dongsheng-B40534
> Sent: Sunday, June 09, 2013 6:38 PM
> To: benh@kernel.crashing.org
> Cc: johannes@sipsolutions.net; anton@enomsg.org; Wood Scott-B07421;
> galak@kernel.crashing.org; linuxppc-dev@lists.ozlabs.org; Wang Dongsheng-
> B40534
> Subject: [PATCH 2/2] powerpc/hibernate: add restore mmu context after
> resume
>=20
> add restore_mmu_context to replace switch_mmu_context in
> restore_processor_state, because the switch_mmu_context will do
> a whole pile of stuff that are probably completely unnecessary.
>=20
> There just need to restore the existing process context, and
> invalidate TLB for boot core.
>=20
> Signed-off-by: Wang Dongsheng <dongsheng.wang@freescale.com>
> ---
>  arch/powerpc/include/asm/tlbflush.h |  2 ++
>  arch/powerpc/kernel/swsusp.c        |  4 +---
>  arch/powerpc/mm/tlb_nohash.c        | 12 ++++++++++++
>  3 files changed, 15 insertions(+), 3 deletions(-)
>=20
> diff --git a/arch/powerpc/include/asm/tlbflush.h
> b/arch/powerpc/include/asm/tlbflush.h
> index 61a5927..c401fe3 100644
> --- a/arch/powerpc/include/asm/tlbflush.h
> +++ b/arch/powerpc/include/asm/tlbflush.h
> @@ -44,6 +44,8 @@ extern void local_flush_tlb_page(struct vm_area_struct
> *vma, unsigned long vmadd
>  extern void __local_flush_tlb_page(struct mm_struct *mm, unsigned long
> vmaddr,
>  				   int tsize, int ind);
>=20
> +extern void restore_mmu_context(void);
> +
>  #ifdef CONFIG_SMP
>  extern void flush_tlb_mm(struct mm_struct *mm);
>  extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long
> vmaddr);
> diff --git a/arch/powerpc/kernel/swsusp.c b/arch/powerpc/kernel/swsusp.c
> index eae33e1..0b104d7 100644
> --- a/arch/powerpc/kernel/swsusp.c
> +++ b/arch/powerpc/kernel/swsusp.c
> @@ -32,7 +32,5 @@ void save_processor_state(void)
>=20
>  void restore_processor_state(void)
>  {
> -#ifdef CONFIG_PPC32
> -	switch_mmu_context(current->active_mm, current->active_mm);
> -#endif
> +	restore_mmu_context();
>  }
> diff --git a/arch/powerpc/mm/tlb_nohash.c b/arch/powerpc/mm/tlb_nohash.c
> index df32a83..a5a0708 100644
> --- a/arch/powerpc/mm/tlb_nohash.c
> +++ b/arch/powerpc/mm/tlb_nohash.c
> @@ -39,10 +39,12 @@
>  #include <linux/of_fdt.h>
>  #include <linux/hugetlb.h>
>=20
> +#include <asm/current.h>
>  #include <asm/tlbflush.h>
>  #include <asm/tlb.h>
>  #include <asm/code-patching.h>
>  #include <asm/hugetlb.h>
> +#include <asm/mmu_context.h>
>=20
>  #include "mmu_decl.h"
>=20
> @@ -193,6 +195,16 @@ void local_flush_tlb_page(struct vm_area_struct *vma=
,
> unsigned long vmaddr)
>  }
>  EXPORT_SYMBOL(local_flush_tlb_page);
>=20
> +void restore_mmu_context(void)
> +{
> +	struct mm_struct *mm =3D current->active_mm;
> +
> +	set_context(mm->context.id, mm->pgd);
> +
> +	_tlbil_all();
> +}
> +EXPORT_SYMBOL(restore_mmu_context);
> +
>  /*
>   * And here are the SMP non-local implementations
>   */
> --
> 1.8.0

Re: [PATCH 2/2] powerpc/hibernate: add restore mmu context after resume

[Scott Wood]

On 07/10/2013 05:11:54 AM, Wang Dongsheng-B40534 wrote:
> Hi scott & ben,
>=20
> About this patch do you have any suggestions?
>=20
> Thanks
>=20
> - dongsheng
>=20
> > -----Original Message-----
> > From: Wang Dongsheng-B40534
> > Sent: Sunday, June 09, 2013 6:38 PM
> > To: benh@kernel.crashing.org
> > Cc: johannes@sipsolutions.net; anton@enomsg.org; Wood Scott-B07421;
> > galak@kernel.crashing.org; linuxppc-dev@lists.ozlabs.org; Wang =20
> Dongsheng-
> > B40534
> > Subject: [PATCH 2/2] powerpc/hibernate: add restore mmu context =20
> after
> > resume
> >
> > add restore_mmu_context to replace switch_mmu_context in
> > restore_processor_state, because the switch_mmu_context will do
> > a whole pile of stuff that are probably completely unnecessary.
> >
> > There just need to restore the existing process context, and
> > invalidate TLB for boot core.
> >
> > Signed-off-by: Wang Dongsheng <dongsheng.wang@freescale.com>
> > ---
> >  arch/powerpc/include/asm/tlbflush.h |  2 ++
> >  arch/powerpc/kernel/swsusp.c        |  4 +---
> >  arch/powerpc/mm/tlb_nohash.c        | 12 ++++++++++++
> >  3 files changed, 15 insertions(+), 3 deletions(-)
> >
> > diff --git a/arch/powerpc/include/asm/tlbflush.h
> > b/arch/powerpc/include/asm/tlbflush.h
> > index 61a5927..c401fe3 100644
> > --- a/arch/powerpc/include/asm/tlbflush.h
> > +++ b/arch/powerpc/include/asm/tlbflush.h
> > @@ -44,6 +44,8 @@ extern void local_flush_tlb_page(struct =20
> vm_area_struct
> > *vma, unsigned long vmadd
> >  extern void __local_flush_tlb_page(struct mm_struct *mm, unsigned =20
> long
> > vmaddr,
> >  				   int tsize, int ind);
> >
> > +extern void restore_mmu_context(void);
> > +
> >  #ifdef CONFIG_SMP
> >  extern void flush_tlb_mm(struct mm_struct *mm);
> >  extern void flush_tlb_page(struct vm_area_struct *vma, unsigned =20
> long
> > vmaddr);
> > diff --git a/arch/powerpc/kernel/swsusp.c =20
> b/arch/powerpc/kernel/swsusp.c
> > index eae33e1..0b104d7 100644
> > --- a/arch/powerpc/kernel/swsusp.c
> > +++ b/arch/powerpc/kernel/swsusp.c
> > @@ -32,7 +32,5 @@ void save_processor_state(void)
> >
> >  void restore_processor_state(void)
> >  {
> > -#ifdef CONFIG_PPC32
> > -	switch_mmu_context(current->active_mm, current->active_mm);
> > -#endif
> > +	restore_mmu_context();
> >  }
> > diff --git a/arch/powerpc/mm/tlb_nohash.c =20
> b/arch/powerpc/mm/tlb_nohash.c
> > index df32a83..a5a0708 100644
> > --- a/arch/powerpc/mm/tlb_nohash.c
> > +++ b/arch/powerpc/mm/tlb_nohash.c
> > @@ -39,10 +39,12 @@
> >  #include <linux/of_fdt.h>
> >  #include <linux/hugetlb.h>
> >
> > +#include <asm/current.h>
> >  #include <asm/tlbflush.h>
> >  #include <asm/tlb.h>
> >  #include <asm/code-patching.h>
> >  #include <asm/hugetlb.h>
> > +#include <asm/mmu_context.h>
> >
> >  #include "mmu_decl.h"
> >
> > @@ -193,6 +195,16 @@ void local_flush_tlb_page(struct =20
> vm_area_struct *vma,
> > unsigned long vmaddr)
> >  }
> >  EXPORT_SYMBOL(local_flush_tlb_page);
> >
> > +void restore_mmu_context(void)
> > +{
> > +	struct mm_struct *mm =3D current->active_mm;
> > +
> > +	set_context(mm->context.id, mm->pgd);
> > +
> > +	_tlbil_all();
> > +}
> > +EXPORT_SYMBOL(restore_mmu_context);

What about targets that don't use tlb_nohash.c?

-Scott=

RE: [PATCH 2/2] powerpc/hibernate: add restore mmu context after resume

[Wang Dongsheng-B40534]

> -----Original Message-----
> From: Wood Scott-B07421
> Sent: Thursday, July 11, 2013 5:43 AM
> To: Wang Dongsheng-B40534
> Cc: Wang Dongsheng-B40534; benh@kernel.crashing.org; Wood Scott-B07421;
> johannes@sipsolutions.net; anton@enomsg.org; galak@kernel.crashing.org;
> linuxppc-dev@lists.ozlabs.org
> Subject: Re: [PATCH 2/2] powerpc/hibernate: add restore mmu context after
> resume
>=20
> On 07/10/2013 05:11:54 AM, Wang Dongsheng-B40534 wrote:
> > Hi scott & ben,
> >
> > About this patch do you have any suggestions?
> >
> > Thanks
> >
> > - dongsheng
> >
> > > -----Original Message-----
> > > From: Wang Dongsheng-B40534
> > > Sent: Sunday, June 09, 2013 6:38 PM
> > > To: benh@kernel.crashing.org
> > > Cc: johannes@sipsolutions.net; anton@enomsg.org; Wood Scott-B07421;
> > > galak@kernel.crashing.org; linuxppc-dev@lists.ozlabs.org; Wang
> > Dongsheng-
> > > B40534
> > > Subject: [PATCH 2/2] powerpc/hibernate: add restore mmu context
> > after
> > > resume
> > >
> > > add restore_mmu_context to replace switch_mmu_context in
> > > restore_processor_state, because the switch_mmu_context will do a
> > > whole pile of stuff that are probably completely unnecessary.
> > >
> > > There just need to restore the existing process context, and
> > > invalidate TLB for boot core.
> > >
> > > Signed-off-by: Wang Dongsheng <dongsheng.wang@freescale.com>
> > > ---
> > >  arch/powerpc/include/asm/tlbflush.h |  2 ++
> > >  arch/powerpc/kernel/swsusp.c        |  4 +---
> > >  arch/powerpc/mm/tlb_nohash.c        | 12 ++++++++++++
> > >  3 files changed, 15 insertions(+), 3 deletions(-)
> > >
> > > diff --git a/arch/powerpc/include/asm/tlbflush.h
> > > b/arch/powerpc/include/asm/tlbflush.h
> > > index 61a5927..c401fe3 100644
> > > --- a/arch/powerpc/include/asm/tlbflush.h
> > > +++ b/arch/powerpc/include/asm/tlbflush.h
> > > @@ -44,6 +44,8 @@ extern void local_flush_tlb_page(struct
> > vm_area_struct
> > > *vma, unsigned long vmadd
> > >  extern void __local_flush_tlb_page(struct mm_struct *mm, unsigned
> > long
> > > vmaddr,
> > >  				   int tsize, int ind);
> > >
> > > +extern void restore_mmu_context(void);
> > > +
> > >  #ifdef CONFIG_SMP
> > >  extern void flush_tlb_mm(struct mm_struct *mm);  extern void
> > > flush_tlb_page(struct vm_area_struct *vma, unsigned
> > long
> > > vmaddr);
> > > diff --git a/arch/powerpc/kernel/swsusp.c
> > b/arch/powerpc/kernel/swsusp.c
> > > index eae33e1..0b104d7 100644
> > > --- a/arch/powerpc/kernel/swsusp.c
> > > +++ b/arch/powerpc/kernel/swsusp.c
> > > @@ -32,7 +32,5 @@ void save_processor_state(void)
> > >
> > >  void restore_processor_state(void)
> > >  {
> > > -#ifdef CONFIG_PPC32
> > > -	switch_mmu_context(current->active_mm, current->active_mm);
> > > -#endif
> > > +	restore_mmu_context();
> > >  }
> > > diff --git a/arch/powerpc/mm/tlb_nohash.c
> > b/arch/powerpc/mm/tlb_nohash.c
> > > index df32a83..a5a0708 100644
> > > --- a/arch/powerpc/mm/tlb_nohash.c
> > > +++ b/arch/powerpc/mm/tlb_nohash.c
> > > @@ -39,10 +39,12 @@
> > >  #include <linux/of_fdt.h>
> > >  #include <linux/hugetlb.h>
> > >
> > > +#include <asm/current.h>
> > >  #include <asm/tlbflush.h>
> > >  #include <asm/tlb.h>
> > >  #include <asm/code-patching.h>
> > >  #include <asm/hugetlb.h>
> > > +#include <asm/mmu_context.h>
> > >
> > >  #include "mmu_decl.h"
> > >
> > > @@ -193,6 +195,16 @@ void local_flush_tlb_page(struct
> > vm_area_struct *vma,
> > > unsigned long vmaddr)
> > >  }
> > >  EXPORT_SYMBOL(local_flush_tlb_page);
> > >
> > > +void restore_mmu_context(void)
> > > +{
> > > +	struct mm_struct *mm =3D current->active_mm;
> > > +
> > > +	set_context(mm->context.id, mm->pgd);
> > > +
> > > +	_tlbil_all();
> > > +}
> > > +EXPORT_SYMBOL(restore_mmu_context);
>=20
> What about targets that don't use tlb_nohash.c?
>=20
Yes, you are right, if we used PPC_STD_MMU, compilation error will occur.
And _tlbil_all should be remove, because all of the tlb already invalidated=
 in swsusp_*.S .
So we should invalid tlb in swsusp_asm64.S & swsusp_booke.S on freescale pl=
atform.

How about add restore_mmu_context(struct mm_struct* mm) into "arch/powerpc/=
include/asm/mmu_context.h"

Path: arch/powerpc/include/asm/mmu_context.h
static void restore_mmu_context(struct mm_struct *mm)
{
       set_context(mm->context.id, mm->pgd);
}

> -Scott

Re: [PATCH 2/2] powerpc/hibernate: add restore mmu context after resume

[Scott Wood]

On 07/11/2013 11:04:23 PM, Wang Dongsheng-B40534 wrote:
>=20
>=20
> > -----Original Message-----
> > From: Wood Scott-B07421
> > Sent: Thursday, July 11, 2013 5:43 AM
> > To: Wang Dongsheng-B40534
> > Cc: Wang Dongsheng-B40534; benh@kernel.crashing.org; Wood =20
> Scott-B07421;
> > johannes@sipsolutions.net; anton@enomsg.org; =20
> galak@kernel.crashing.org;
> > linuxppc-dev@lists.ozlabs.org
> > Subject: Re: [PATCH 2/2] powerpc/hibernate: add restore mmu context =20
> after
> > resume
> >
> > On 07/10/2013 05:11:54 AM, Wang Dongsheng-B40534 wrote:
> > > > +void restore_mmu_context(void)
> > > > +{
> > > > +	struct mm_struct *mm =3D current->active_mm;
> > > > +
> > > > +	set_context(mm->context.id, mm->pgd);
> > > > +
> > > > +	_tlbil_all();
> > > > +}
> > > > +EXPORT_SYMBOL(restore_mmu_context);
> >
> > What about targets that don't use tlb_nohash.c?
> >
> Yes, you are right, if we used PPC_STD_MMU, compilation error will =20
> occur.
> And _tlbil_all should be remove, because all of the tlb already =20
> invalidated in swsusp_*.S .
> So we should invalid tlb in swsusp_asm64.S & swsusp_booke.S on =20
> freescale platform.
>=20
> How about add restore_mmu_context(struct mm_struct* mm) into =20
> "arch/powerpc/include/asm/mmu_context.h"
>=20
> Path: arch/powerpc/include/asm/mmu_context.h
> static void restore_mmu_context(struct mm_struct *mm)
> {
>        set_context(mm->context.id, mm->pgd);
> }

set_context() doesn't exist for hash MMUs.

Whatever you do, please try actually building it on various targets, =20
including both 32 and 64 bits, and both hash and non-hash.  And make =20
sure that whatever effect PPC32 was depending on switch_mmu_context for =20
is preserved, including on non-booke.

-Scott=

Re: [PATCH 2/2] powerpc/hibernate: add restore mmu context after resume

[Benjamin Herrenschmidt]

On Fri, 2013-07-12 at 16:54 -0500, Scott Wood wrote:
> set_context() doesn't exist for hash MMUs.
> 
> Whatever you do, please try actually building it on various targets,  
> including both 32 and 64 bits, and both hash and non-hash.  And make  
> sure that whatever effect PPC32 was depending on switch_mmu_context
> for  
> is preserved, including on non-booke.

The right thing to do is probably to have the various tlb_* files
provide a pair of save/restore functions that do the right thing
for that specific MMU type.

Ben.

RE: [PATCH 2/2] powerpc/hibernate: add restore mmu context after resume

[Wang Dongsheng-B40534]

DQoNCj4gLS0tLS1PcmlnaW5hbCBNZXNzYWdlLS0tLS0NCj4gRnJvbTogQmVuamFtaW4gSGVycmVu
c2NobWlkdCBbbWFpbHRvOmJlbmhAa2VybmVsLmNyYXNoaW5nLm9yZ10NCj4gU2VudDogU2F0dXJk
YXksIEp1bHkgMTMsIDIwMTMgNzowNyBBTQ0KPiBUbzogV29vZCBTY290dC1CMDc0MjENCj4gQ2M6
IFdhbmcgRG9uZ3NoZW5nLUI0MDUzNDsgV29vZCBTY290dC1CMDc0MjE7IGpvaGFubmVzQHNpcHNv
bHV0aW9ucy5uZXQ7DQo+IGFudG9uQGVub21zZy5vcmc7IGdhbGFrQGtlcm5lbC5jcmFzaGluZy5v
cmc7IGxpbnV4cHBjLQ0KPiBkZXZAbGlzdHMub3psYWJzLm9yZw0KPiBTdWJqZWN0OiBSZTogW1BB
VENIIDIvMl0gcG93ZXJwYy9oaWJlcm5hdGU6IGFkZCByZXN0b3JlIG1tdSBjb250ZXh0IGFmdGVy
DQo+IHJlc3VtZQ0KPiANCj4gT24gRnJpLCAyMDEzLTA3LTEyIGF0IDE2OjU0IC0wNTAwLCBTY290
dCBXb29kIHdyb3RlOg0KPiA+IHNldF9jb250ZXh0KCkgZG9lc24ndCBleGlzdCBmb3IgaGFzaCBN
TVVzLg0KPiA+DQo+ID4gV2hhdGV2ZXIgeW91IGRvLCBwbGVhc2UgdHJ5IGFjdHVhbGx5IGJ1aWxk
aW5nIGl0IG9uIHZhcmlvdXMgdGFyZ2V0cywNCj4gPiBpbmNsdWRpbmcgYm90aCAzMiBhbmQgNjQg
Yml0cywgYW5kIGJvdGggaGFzaCBhbmQgbm9uLWhhc2guICBBbmQgbWFrZQ0KPiA+IHN1cmUgdGhh
dCB3aGF0ZXZlciBlZmZlY3QgUFBDMzIgd2FzIGRlcGVuZGluZyBvbiBzd2l0Y2hfbW11X2NvbnRl
eHQNCj4gPiBmb3INCj4gPiBpcyBwcmVzZXJ2ZWQsIGluY2x1ZGluZyBvbiBub24tYm9va2UuDQo+
IA0KPiBUaGUgcmlnaHQgdGhpbmcgdG8gZG8gaXMgcHJvYmFibHkgdG8gaGF2ZSB0aGUgdmFyaW91
cyB0bGJfKiBmaWxlcw0KSG93IGFib3V0IG1tdV9jb250ZXh0XyogZmlsZXMgdG8gcHJvdmlkZSBh
IHBhaXIgb2Ygc2F2ZS9yZXN0b3JlIGNvbnRleHQgZnVuY3Rpb25zPw0KDQo+IHByb3ZpZGUgYSBw
YWlyIG9mIHNhdmUvcmVzdG9yZSBmdW5jdGlvbnMgdGhhdCBkbyB0aGUgcmlnaHQgdGhpbmcNCj4g
Zm9yIHRoYXQgc3BlY2lmaWMgTU1VIHR5cGUuDQo+IA0KPiBCZW4uDQo+IA0KPiANCg0K