* [PATCH] ARM: tlb: perform branch predictor maintenance for whole invalidation
@ 2013-01-30 14:12 Will Deacon
2013-01-30 14:52 ` Dave Martin
0 siblings, 1 reply; 4+ messages in thread
From: Will Deacon @ 2013-01-30 14:12 UTC (permalink / raw)
To: linux-arm-kernel
The ARM architecture requires explicit branch predictor maintenance
when updating an instruction stream for a given virtual address. In
reality, this isn't so much of a burden because the branch predictor
is flushed during the cache maintenance required to make the new
instructions visible to the I-side of the processor.
One exception to this rule is when we flush the entire TLB for a CPU in
response to something other than writing new instructions (for example,
ASID rollover or switching to an identity mapping before disabling the
MMU).
This patch ensures that we flush the branch predictor as part of
invalidating the entire TLB.
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm/include/asm/tlbflush.h | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h
index 6e924d3..713ff32 100644
--- a/arch/arm/include/asm/tlbflush.h
+++ b/arch/arm/include/asm/tlbflush.h
@@ -38,6 +38,7 @@
#define TLB_V7_UIS_PAGE (1 << 19)
#define TLB_V7_UIS_FULL (1 << 20)
#define TLB_V7_UIS_ASID (1 << 21)
+#define TLB_V7_UIS_BP (1 << 22)
#define TLB_BARRIER (1 << 28)
#define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */
@@ -166,9 +167,11 @@
#endif
#define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
- TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID)
+ TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | \
+ TLB_V7_UIS_ASID | TLB_V7_UIS_BP)
#define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
- TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID)
+ TLB_V6_U_FULL | TLB_V6_U_PAGE | \
+ TLB_V6_U_ASID)
#ifdef CONFIG_CPU_TLB_V7
@@ -330,6 +333,11 @@ static inline void local_flush_tlb_all(void)
tlb_op(TLB_V4_I_FULL | TLB_V6_I_FULL, "c8, c5, 0", zero);
tlb_op(TLB_V7_UIS_FULL, "c8, c3, 0", zero);
+ if (tlb_flag(TLB_V7_UIS_BP))
+ asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero));
+ else
+ asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero));
+
if (tlb_flag(TLB_BARRIER)) {
dsb();
isb();
--
1.8.0
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH] ARM: tlb: perform branch predictor maintenance for whole invalidation
2013-01-30 14:12 [PATCH] ARM: tlb: perform branch predictor maintenance for whole invalidation Will Deacon
@ 2013-01-30 14:52 ` Dave Martin
2013-01-30 15:57 ` Will Deacon
0 siblings, 1 reply; 4+ messages in thread
From: Dave Martin @ 2013-01-30 14:52 UTC (permalink / raw)
To: linux-arm-kernel
On Wed, Jan 30, 2013 at 02:12:05PM +0000, Will Deacon wrote:
> The ARM architecture requires explicit branch predictor maintenance
> when updating an instruction stream for a given virtual address. In
> reality, this isn't so much of a burden because the branch predictor
> is flushed during the cache maintenance required to make the new
> instructions visible to the I-side of the processor.
>
> One exception to this rule is when we flush the entire TLB for a CPU in
> response to something other than writing new instructions (for example,
> ASID rollover or switching to an identity mapping before disabling the
> MMU).
>
> This patch ensures that we flush the branch predictor as part of
> invalidating the entire TLB.
What about v6 implementations? Is the branch predictor flush implicit?
Cheers
---Dave
>
> Signed-off-by: Will Deacon <will.deacon@arm.com>
> ---
> arch/arm/include/asm/tlbflush.h | 12 ++++++++++--
> 1 file changed, 10 insertions(+), 2 deletions(-)
>
> diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h
> index 6e924d3..713ff32 100644
> --- a/arch/arm/include/asm/tlbflush.h
> +++ b/arch/arm/include/asm/tlbflush.h
> @@ -38,6 +38,7 @@
> #define TLB_V7_UIS_PAGE (1 << 19)
> #define TLB_V7_UIS_FULL (1 << 20)
> #define TLB_V7_UIS_ASID (1 << 21)
> +#define TLB_V7_UIS_BP (1 << 22)
>
> #define TLB_BARRIER (1 << 28)
> #define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */
> @@ -166,9 +167,11 @@
> #endif
>
> #define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
> - TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID)
> + TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | \
> + TLB_V7_UIS_ASID | TLB_V7_UIS_BP)
> #define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
> - TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID)
> + TLB_V6_U_FULL | TLB_V6_U_PAGE | \
> + TLB_V6_U_ASID)
>
> #ifdef CONFIG_CPU_TLB_V7
>
> @@ -330,6 +333,11 @@ static inline void local_flush_tlb_all(void)
> tlb_op(TLB_V4_I_FULL | TLB_V6_I_FULL, "c8, c5, 0", zero);
> tlb_op(TLB_V7_UIS_FULL, "c8, c3, 0", zero);
>
> + if (tlb_flag(TLB_V7_UIS_BP))
> + asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero));
> + else
> + asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero));
> +
> if (tlb_flag(TLB_BARRIER)) {
> dsb();
> isb();
> --
> 1.8.0
>
>
> _______________________________________________
> linux-arm-kernel mailing list
> linux-arm-kernel at lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply [flat|nested] 4+ messages in thread* [PATCH] ARM: tlb: perform branch predictor maintenance for whole invalidation
2013-01-30 14:52 ` Dave Martin
@ 2013-01-30 15:57 ` Will Deacon
2013-01-30 18:50 ` Dave Martin
0 siblings, 1 reply; 4+ messages in thread
From: Will Deacon @ 2013-01-30 15:57 UTC (permalink / raw)
To: linux-arm-kernel
On Wed, Jan 30, 2013 at 02:52:43PM +0000, Dave Martin wrote:
> On Wed, Jan 30, 2013 at 02:12:05PM +0000, Will Deacon wrote:
> > The ARM architecture requires explicit branch predictor maintenance
> > when updating an instruction stream for a given virtual address. In
> > reality, this isn't so much of a burden because the branch predictor
> > is flushed during the cache maintenance required to make the new
> > instructions visible to the I-side of the processor.
> >
> > One exception to this rule is when we flush the entire TLB for a CPU in
> > response to something other than writing new instructions (for example,
> > ASID rollover or switching to an identity mapping before disabling the
> > MMU).
> >
> > This patch ensures that we flush the branch predictor as part of
> > invalidating the entire TLB.
>
> What about v6 implementations? Is the branch predictor flush implicit?
No, they're also handled by the patch. There's no -UIS versions of iciall
and bpiall prior to v7, so the operations are broadcast via IPI, hence the
use of bpiall in the else case below.
Will
> Cheers
> ---Dave
>
> >
> > Signed-off-by: Will Deacon <will.deacon@arm.com>
> > ---
> > arch/arm/include/asm/tlbflush.h | 12 ++++++++++--
> > 1 file changed, 10 insertions(+), 2 deletions(-)
> >
> > diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h
> > index 6e924d3..713ff32 100644
> > --- a/arch/arm/include/asm/tlbflush.h
> > +++ b/arch/arm/include/asm/tlbflush.h
> > @@ -38,6 +38,7 @@
> > #define TLB_V7_UIS_PAGE (1 << 19)
> > #define TLB_V7_UIS_FULL (1 << 20)
> > #define TLB_V7_UIS_ASID (1 << 21)
> > +#define TLB_V7_UIS_BP (1 << 22)
> >
> > #define TLB_BARRIER (1 << 28)
> > #define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */
> > @@ -166,9 +167,11 @@
> > #endif
> >
> > #define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
> > - TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID)
> > + TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | \
> > + TLB_V7_UIS_ASID | TLB_V7_UIS_BP)
> > #define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
> > - TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID)
> > + TLB_V6_U_FULL | TLB_V6_U_PAGE | \
> > + TLB_V6_U_ASID)
> >
> > #ifdef CONFIG_CPU_TLB_V7
> >
> > @@ -330,6 +333,11 @@ static inline void local_flush_tlb_all(void)
> > tlb_op(TLB_V4_I_FULL | TLB_V6_I_FULL, "c8, c5, 0", zero);
> > tlb_op(TLB_V7_UIS_FULL, "c8, c3, 0", zero);
> >
> > + if (tlb_flag(TLB_V7_UIS_BP))
> > + asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero));
> > + else
> > + asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero));
> > +
> > if (tlb_flag(TLB_BARRIER)) {
> > dsb();
> > isb();
> > --
> > 1.8.0
> >
> >
> > _______________________________________________
> > linux-arm-kernel mailing list
> > linux-arm-kernel at lists.infradead.org
> > http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
>
^ permalink raw reply [flat|nested] 4+ messages in thread* [PATCH] ARM: tlb: perform branch predictor maintenance for whole invalidation
2013-01-30 15:57 ` Will Deacon
@ 2013-01-30 18:50 ` Dave Martin
0 siblings, 0 replies; 4+ messages in thread
From: Dave Martin @ 2013-01-30 18:50 UTC (permalink / raw)
To: linux-arm-kernel
On Wed, Jan 30, 2013 at 03:57:17PM +0000, Will Deacon wrote:
> On Wed, Jan 30, 2013 at 02:52:43PM +0000, Dave Martin wrote:
> > On Wed, Jan 30, 2013 at 02:12:05PM +0000, Will Deacon wrote:
> > > The ARM architecture requires explicit branch predictor maintenance
> > > when updating an instruction stream for a given virtual address. In
> > > reality, this isn't so much of a burden because the branch predictor
> > > is flushed during the cache maintenance required to make the new
> > > instructions visible to the I-side of the processor.
> > >
> > > One exception to this rule is when we flush the entire TLB for a CPU in
> > > response to something other than writing new instructions (for example,
> > > ASID rollover or switching to an identity mapping before disabling the
> > > MMU).
> > >
> > > This patch ensures that we flush the branch predictor as part of
> > > invalidating the entire TLB.
> >
> > What about v6 implementations? Is the branch predictor flush implicit?
>
> No, they're also handled by the patch. There's no -UIS versions of iciall
> and bpiall prior to v7, so the operations are broadcast via IPI, hence the
> use of bpiall in the else case below.
Ah, OK
---Dave
>
> Will
>
> > Cheers
> > ---Dave
> >
> > >
> > > Signed-off-by: Will Deacon <will.deacon@arm.com>
> > > ---
> > > arch/arm/include/asm/tlbflush.h | 12 ++++++++++--
> > > 1 file changed, 10 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h
> > > index 6e924d3..713ff32 100644
> > > --- a/arch/arm/include/asm/tlbflush.h
> > > +++ b/arch/arm/include/asm/tlbflush.h
> > > @@ -38,6 +38,7 @@
> > > #define TLB_V7_UIS_PAGE (1 << 19)
> > > #define TLB_V7_UIS_FULL (1 << 20)
> > > #define TLB_V7_UIS_ASID (1 << 21)
> > > +#define TLB_V7_UIS_BP (1 << 22)
> > >
> > > #define TLB_BARRIER (1 << 28)
> > > #define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */
> > > @@ -166,9 +167,11 @@
> > > #endif
> > >
> > > #define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
> > > - TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID)
> > > + TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | \
> > > + TLB_V7_UIS_ASID | TLB_V7_UIS_BP)
> > > #define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
> > > - TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID)
> > > + TLB_V6_U_FULL | TLB_V6_U_PAGE | \
> > > + TLB_V6_U_ASID)
> > >
> > > #ifdef CONFIG_CPU_TLB_V7
> > >
> > > @@ -330,6 +333,11 @@ static inline void local_flush_tlb_all(void)
> > > tlb_op(TLB_V4_I_FULL | TLB_V6_I_FULL, "c8, c5, 0", zero);
> > > tlb_op(TLB_V7_UIS_FULL, "c8, c3, 0", zero);
> > >
> > > + if (tlb_flag(TLB_V7_UIS_BP))
> > > + asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero));
> > > + else
> > > + asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero));
> > > +
> > > if (tlb_flag(TLB_BARRIER)) {
> > > dsb();
> > > isb();
> > > --
> > > 1.8.0
> > >
> > >
> > > _______________________________________________
> > > linux-arm-kernel mailing list
> > > linux-arm-kernel at lists.infradead.org
> > > http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
> >
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2013-01-30 18:50 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2013-01-30 14:12 [PATCH] ARM: tlb: perform branch predictor maintenance for whole invalidation Will Deacon
2013-01-30 14:52 ` Dave Martin
2013-01-30 15:57 ` Will Deacon
2013-01-30 18:50 ` Dave Martin
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).