[PATCH 0/2] Kernel MMU notifier for IOTLB/DEVTLB management

[PATCH 0/2] Kernel MMU notifier for IOTLB/DEVTLB management

[Lu Baolu]

Shared Virtual Memory (SVM) allows a kernel memory mapping to be
shared between CPU and and a device which requested a supervisor
PASID. Both devices and IOMMU units have TLBs that cache entries
from CPU's page tables. We need to get a chance to flush them at
the same time when we flush the CPU TLBs.

These patches handle this by adding a kernel MMU notifier chain.
The callbacks on this chain will be called whenever the CPU TLB
is flushed for the kernel address space.

Ashok Raj (1):
  iommu/vt-d: Register kernel MMU notifier to manage IOTLB/DEVTLB

Huang Ying (1):
  mm: Add kernel MMU notifier to manage IOTLB/DEVTLB

 arch/x86/mm/tlb.c            |  2 ++
 drivers/iommu/intel-svm.c    | 27 +++++++++++++++++++++++++--
 include/linux/intel-iommu.h  |  5 ++++-
 include/linux/mmu_notifier.h | 33 +++++++++++++++++++++++++++++++++
 mm/mmu_notifier.c            | 27 +++++++++++++++++++++++++++
 5 files changed, 91 insertions(+), 3 deletions(-)

-- 
2.7.4

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

[PATCH 1/2] mm: Add kernel MMU notifier to manage IOTLB/DEVTLB

[Lu Baolu]

From: Huang Ying <ying.huang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>

Shared Virtual Memory (SVM) allows a kernel memory mapping to be
shared between CPU and and a device which requested a supervisor
PASID. Both devices and IOMMU units have TLBs that cache entries
from CPU's page tables. We need to get a chance to flush them at
the same time when we flush the CPU TLBs.

We already have an existing MMU notifiers for userspace updates,
however we lack the same thing for kernel page table updates. To
implement the MMU notification mechanism for the kernel address
space, a kernel MMU notifier chain is defined and will be called
whenever the CPU TLB is flushed for the kernel address space.

As consumer of this notifier, the IOMMU SVM implementations will
register callbacks on this notifier and manage the cache entries
in both IOTLB and DevTLB.

Cc: Ashok Raj <ashok.raj-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Cc: Dave Hansen <dave.hansen-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Cc: Thomas Gleixner <tglx-hfZtesqFncYOwBW4kG4KsQ@public.gmane.org>
Cc: Ingo Molnar <mingo-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
Cc: "H. Peter Anvin" <hpa-YMNOUZJC4hwAvxtiuMwx3w@public.gmane.org>
Cc: Andy Lutomirski <luto-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org>
Cc: Rik van Riel <riel-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
Cc: Kees Cook <keescook-F7+t8E8rja9g9hUCZPvPmw@public.gmane.org>
Cc: Andrew Morton <akpm-de/tnXTf+JLsfHDXvbKv3WD2FQJk+8+b@public.gmane.org>
Cc: Kirill A. Shutemov <kirill.shutemov-VuQAYsv1563Yd54FQh9/CA@public.gmane.org>
Cc: Matthew Wilcox <willy-VuQAYsv1563Yd54FQh9/CA@public.gmane.org>
Cc: Dave Jiang <dave.jiang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Cc: Michal Hocko <mhocko-IBi9RG/b67k@public.gmane.org>
Cc: Paul E. McKenney <paulmck-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org>
Cc: Vegard Nossum <vegard.nossum-QHcLZuEGTsvQT0dZR+AlfA@public.gmane.org>
Cc: x86-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org
Cc: linux-mm-Bw31MaZKKs3YtjvyW6yDsg@public.gmane.org

Tested-by: CQ Tang <cq.tang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Huang Ying <ying.huang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Lu Baolu <baolu.lu-VuQAYsv1563Yd54FQh9/CA@public.gmane.org>
---
 arch/x86/mm/tlb.c            |  2 ++
 include/linux/mmu_notifier.h | 33 +++++++++++++++++++++++++++++++++
 mm/mmu_notifier.c            | 27 +++++++++++++++++++++++++++
 3 files changed, 62 insertions(+)

diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 3118392cd..5ff104f 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -6,6 +6,7 @@
 #include <linux/interrupt.h>
 #include <linux/export.h>
 #include <linux/cpu.h>
+#include <linux/mmu_notifier.h>
 
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
@@ -567,6 +568,7 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 		info.end = end;
 		on_each_cpu(do_kernel_range_flush, &info, 1);
 	}
+	kernel_mmu_notifier_invalidate_range(start, end);
 }
 
 void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index b25dc9d..44d7c06 100644
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -408,6 +408,25 @@ extern void mmu_notifier_call_srcu(struct rcu_head *rcu,
 				   void (*func)(struct rcu_head *rcu));
 extern void mmu_notifier_synchronize(void);
 
+struct kernel_mmu_address_range {
+	unsigned long start;
+	unsigned long end;
+};
+
+/*
+ * Before the virtual address range managed by kernel (vmalloc/kmap)
+ * is reused, That is, remapped to the new physical addresses, the
+ * kernel MMU notifier will be called with KERNEL_MMU_INVALIDATE_RANGE
+ * and struct kernel_mmu_address_range as parameters.  This is used to
+ * manage the remote TLB.
+ */
+#define KERNEL_MMU_INVALIDATE_RANGE		1
+extern int kernel_mmu_notifier_register(struct notifier_block *nb);
+extern int kernel_mmu_notifier_unregister(struct notifier_block *nb);
+
+extern int kernel_mmu_notifier_invalidate_range(unsigned long start,
+						unsigned long end);
+
 #else /* CONFIG_MMU_NOTIFIER */
 
 static inline int mm_has_notifiers(struct mm_struct *mm)
@@ -474,6 +493,20 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
 #define pudp_huge_clear_flush_notify pudp_huge_clear_flush
 #define set_pte_at_notify set_pte_at
 
+static inline int kernel_mmu_notifier_register(struct notifier_block *nb)
+{
+	return 0;
+}
+
+static inline int kernel_mmu_notifier_unregister(struct notifier_block *nb)
+{
+	return 0;
+}
+
+static inline void kernel_mmu_notifier_invalidate_range(unsigned long start,
+							unsigned long end)
+{
+}
 #endif /* CONFIG_MMU_NOTIFIER */
 
 #endif /* _LINUX_MMU_NOTIFIER_H */
diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
index 96edb33..52f816a 100644
--- a/mm/mmu_notifier.c
+++ b/mm/mmu_notifier.c
@@ -393,3 +393,30 @@ void mmu_notifier_unregister_no_release(struct mmu_notifier *mn,
 	mmdrop(mm);
 }
 EXPORT_SYMBOL_GPL(mmu_notifier_unregister_no_release);
+
+static ATOMIC_NOTIFIER_HEAD(kernel_mmu_notifier_list);
+
+int kernel_mmu_notifier_register(struct notifier_block *nb)
+{
+	return atomic_notifier_chain_register(&kernel_mmu_notifier_list, nb);
+}
+EXPORT_SYMBOL_GPL(kernel_mmu_notifier_register);
+
+int kernel_mmu_notifier_unregister(struct notifier_block *nb)
+{
+	return atomic_notifier_chain_unregister(&kernel_mmu_notifier_list, nb);
+}
+EXPORT_SYMBOL_GPL(kernel_mmu_notifier_unregister);
+
+int kernel_mmu_notifier_invalidate_range(unsigned long start,
+					 unsigned long end)
+{
+	struct kernel_mmu_address_range range = {
+		.start	= start,
+		.end	= end,
+	};
+
+	return atomic_notifier_call_chain(&kernel_mmu_notifier_list,
+					  KERNEL_MMU_INVALIDATE_RANGE,
+					  &range);
+}
-- 
2.7.4

Re: [PATCH 1/2] mm: Add kernel MMU notifier to manage IOTLB/DEVTLB

[Bob Liu]

On 2017/12/14 9:02, Lu Baolu wrote:
> From: Huang Ying <ying.huang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> 
> Shared Virtual Memory (SVM) allows a kernel memory mapping to be
> shared between CPU and and a device which requested a supervisor
> PASID. Both devices and IOMMU units have TLBs that cache entries
> from CPU's page tables. We need to get a chance to flush them at
> the same time when we flush the CPU TLBs.
> 
> We already have an existing MMU notifiers for userspace updates,
> however we lack the same thing for kernel page table updates. To

Sorry, I didn't get which situation need this notification.
Could you please describe the full scenario?

Thanks,
Liubo

> implement the MMU notification mechanism for the kernel address
> space, a kernel MMU notifier chain is defined and will be called
> whenever the CPU TLB is flushed for the kernel address space.
> 
> As consumer of this notifier, the IOMMU SVM implementations will
> register callbacks on this notifier and manage the cache entries
> in both IOTLB and DevTLB.
> 
> Cc: Ashok Raj <ashok.raj-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> Cc: Dave Hansen <dave.hansen-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> Cc: Thomas Gleixner <tglx-hfZtesqFncYOwBW4kG4KsQ@public.gmane.org>
> Cc: Ingo Molnar <mingo-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
> Cc: "H. Peter Anvin" <hpa-YMNOUZJC4hwAvxtiuMwx3w@public.gmane.org>
> Cc: Andy Lutomirski <luto-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org>
> Cc: Rik van Riel <riel-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
> Cc: Kees Cook <keescook-F7+t8E8rja9g9hUCZPvPmw@public.gmane.org>
> Cc: Andrew Morton <akpm-de/tnXTf+JLsfHDXvbKv3WD2FQJk+8+b@public.gmane.org>
> Cc: Kirill A. Shutemov <kirill.shutemov-VuQAYsv1563Yd54FQh9/CA@public.gmane.org>
> Cc: Matthew Wilcox <willy-VuQAYsv1563Yd54FQh9/CA@public.gmane.org>
> Cc: Dave Jiang <dave.jiang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> Cc: Michal Hocko <mhocko-IBi9RG/b67k@public.gmane.org>
> Cc: Paul E. McKenney <paulmck-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org>
> Cc: Vegard Nossum <vegard.nossum-QHcLZuEGTsvQT0dZR+AlfA@public.gmane.org>
> Cc: x86-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org
> Cc: linux-mm-Bw31MaZKKs3YtjvyW6yDsg@public.gmane.org
> 
> Tested-by: CQ Tang <cq.tang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> Signed-off-by: Huang Ying <ying.huang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> Signed-off-by: Lu Baolu <baolu.lu-VuQAYsv1563Yd54FQh9/CA@public.gmane.org>
> ---
>  arch/x86/mm/tlb.c            |  2 ++
>  include/linux/mmu_notifier.h | 33 +++++++++++++++++++++++++++++++++
>  mm/mmu_notifier.c            | 27 +++++++++++++++++++++++++++
>  3 files changed, 62 insertions(+)
> 
> diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
> index 3118392cd..5ff104f 100644
> --- a/arch/x86/mm/tlb.c
> +++ b/arch/x86/mm/tlb.c
> @@ -6,6 +6,7 @@
>  #include <linux/interrupt.h>
>  #include <linux/export.h>
>  #include <linux/cpu.h>
> +#include <linux/mmu_notifier.h>
>  
>  #include <asm/tlbflush.h>
>  #include <asm/mmu_context.h>
> @@ -567,6 +568,7 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
>  		info.end = end;
>  		on_each_cpu(do_kernel_range_flush, &info, 1);
>  	}
> +	kernel_mmu_notifier_invalidate_range(start, end);
>  }
>  
>  void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
> diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
> index b25dc9d..44d7c06 100644
> --- a/include/linux/mmu_notifier.h
> +++ b/include/linux/mmu_notifier.h
> @@ -408,6 +408,25 @@ extern void mmu_notifier_call_srcu(struct rcu_head *rcu,
>  				   void (*func)(struct rcu_head *rcu));
>  extern void mmu_notifier_synchronize(void);
>  
> +struct kernel_mmu_address_range {
> +	unsigned long start;
> +	unsigned long end;
> +};
> +
> +/*
> + * Before the virtual address range managed by kernel (vmalloc/kmap)
> + * is reused, That is, remapped to the new physical addresses, the
> + * kernel MMU notifier will be called with KERNEL_MMU_INVALIDATE_RANGE
> + * and struct kernel_mmu_address_range as parameters.  This is used to
> + * manage the remote TLB.
> + */
> +#define KERNEL_MMU_INVALIDATE_RANGE		1
> +extern int kernel_mmu_notifier_register(struct notifier_block *nb);
> +extern int kernel_mmu_notifier_unregister(struct notifier_block *nb);
> +
> +extern int kernel_mmu_notifier_invalidate_range(unsigned long start,
> +						unsigned long end);
> +
>  #else /* CONFIG_MMU_NOTIFIER */
>  
>  static inline int mm_has_notifiers(struct mm_struct *mm)
> @@ -474,6 +493,20 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
>  #define pudp_huge_clear_flush_notify pudp_huge_clear_flush
>  #define set_pte_at_notify set_pte_at
>  
> +static inline int kernel_mmu_notifier_register(struct notifier_block *nb)
> +{
> +	return 0;
> +}
> +
> +static inline int kernel_mmu_notifier_unregister(struct notifier_block *nb)
> +{
> +	return 0;
> +}
> +
> +static inline void kernel_mmu_notifier_invalidate_range(unsigned long start,
> +							unsigned long end)
> +{
> +}
>  #endif /* CONFIG_MMU_NOTIFIER */
>  
>  #endif /* _LINUX_MMU_NOTIFIER_H */
> diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
> index 96edb33..52f816a 100644
> --- a/mm/mmu_notifier.c
> +++ b/mm/mmu_notifier.c
> @@ -393,3 +393,30 @@ void mmu_notifier_unregister_no_release(struct mmu_notifier *mn,
>  	mmdrop(mm);
>  }
>  EXPORT_SYMBOL_GPL(mmu_notifier_unregister_no_release);
> +
> +static ATOMIC_NOTIFIER_HEAD(kernel_mmu_notifier_list);
> +
> +int kernel_mmu_notifier_register(struct notifier_block *nb)
> +{
> +	return atomic_notifier_chain_register(&kernel_mmu_notifier_list, nb);
> +}
> +EXPORT_SYMBOL_GPL(kernel_mmu_notifier_register);
> +
> +int kernel_mmu_notifier_unregister(struct notifier_block *nb)
> +{
> +	return atomic_notifier_chain_unregister(&kernel_mmu_notifier_list, nb);
> +}
> +EXPORT_SYMBOL_GPL(kernel_mmu_notifier_unregister);
> +
> +int kernel_mmu_notifier_invalidate_range(unsigned long start,
> +					 unsigned long end)
> +{
> +	struct kernel_mmu_address_range range = {
> +		.start	= start,
> +		.end	= end,
> +	};
> +
> +	return atomic_notifier_call_chain(&kernel_mmu_notifier_list,
> +					  KERNEL_MMU_INVALIDATE_RANGE,
> +					  &range);
> +}
> 

Re: [PATCH 1/2] mm: Add kernel MMU notifier to manage IOTLB/DEVTLB

[Lu Baolu]

Hi,

On 12/14/2017 11:10 AM, Bob Liu wrote:
> On 2017/12/14 9:02, Lu Baolu wrote:
>> > From: Huang Ying <ying.huang@intel.com>
>> > 
>> > Shared Virtual Memory (SVM) allows a kernel memory mapping to be
>> > shared between CPU and and a device which requested a supervisor
>> > PASID. Both devices and IOMMU units have TLBs that cache entries
>> > from CPU's page tables. We need to get a chance to flush them at
>> > the same time when we flush the CPU TLBs.
>> > 
>> > We already have an existing MMU notifiers for userspace updates,
>> > however we lack the same thing for kernel page table updates. To
> Sorry, I didn't get which situation need this notification.
> Could you please describe the full scenario?

Okay.

1. When an SVM capable driver calls intel_svm_bind_mm() with
    SVM_FLAG_SUPERVISOR_MODE set in the @flags, the kernel
    memory page mappings will be shared between CPUs and
    the DMA remapping agent (a.k.a. IOMMU). The page table
    entries will also be cached in both IOTLB (located in IOMMU)
    and the DEVTLB (located in device).

2. When vmalloc/vfree interfaces are called, the page mappings
    for kernel memory might get changed. And current code calls
    flush_tlb_kernel_range() to flush CPU TLBs only. The IOTLB or
    DevTLB will be stale compared to that on the cpu for kernel
    mappings.

We need a kernel mmu notification to flush TLBs in IOMMU and
devices as well.

Best regards,
Lu Baolu

Re: [PATCH 1/2] mm: Add kernel MMU notifier to manage IOTLB/DEVTLB

[Bob Liu]

On 2017/12/14 11:38, Lu Baolu wrote:
> Hi,
> 
> On 12/14/2017 11:10 AM, Bob Liu wrote:
>> On 2017/12/14 9:02, Lu Baolu wrote:
>>>> From: Huang Ying <ying.huang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
>>>>
>>>> Shared Virtual Memory (SVM) allows a kernel memory mapping to be
>>>> shared between CPU and and a device which requested a supervisor
>>>> PASID. Both devices and IOMMU units have TLBs that cache entries
>>>> from CPU's page tables. We need to get a chance to flush them at
>>>> the same time when we flush the CPU TLBs.
>>>>
>>>> We already have an existing MMU notifiers for userspace updates,
>>>> however we lack the same thing for kernel page table updates. To
>> Sorry, I didn't get which situation need this notification.
>> Could you please describe the full scenario?
> 
> Okay.
> 
> 1. When an SVM capable driver calls intel_svm_bind_mm() with
>     SVM_FLAG_SUPERVISOR_MODE set in the @flags, the kernel
>     memory page mappings will be shared between CPUs and
>     the DMA remapping agent (a.k.a. IOMMU). The page table
>     entries will also be cached in both IOTLB (located in IOMMU)
>     and the DEVTLB (located in device).
> 

But who/what kind of real device has the requirement to access a kernel VA?
Looks like SVM_FLAG_SUPERVISOR_MODE is used by nobody?

Cheers,
Liubo

> 2. When vmalloc/vfree interfaces are called, the page mappings
>     for kernel memory might get changed. And current code calls
>     flush_tlb_kernel_range() to flush CPU TLBs only. The IOTLB or
>     DevTLB will be stale compared to that on the cpu for kernel
>     mappings.
> 
> We need a kernel mmu notification to flush TLBs in IOMMU and
> devices as well.
> 
> Best regards,
> Lu Baolu
> 

Re: [PATCH 1/2] mm: Add kernel MMU notifier to manage IOTLB/DEVTLB

[Raj, Ashok]

Hi Bob

On Thu, Dec 14, 2017 at 02:07:38PM +0800, Bob Liu wrote:
> On 2017/12/14 11:38, Lu Baolu wrote:

> >>>> We already have an existing MMU notifiers for userspace updates,
> >>>> however we lack the same thing for kernel page table updates. To
> >> Sorry, I didn't get which situation need this notification.
> >> Could you please describe the full scenario?
> > 
> > Okay.
> > 
> > 1. When an SVM capable driver calls intel_svm_bind_mm() with
> >     SVM_FLAG_SUPERVISOR_MODE set in the @flags, the kernel
> >     memory page mappings will be shared between CPUs and
> >     the DMA remapping agent (a.k.a. IOMMU). The page table
> >     entries will also be cached in both IOTLB (located in IOMMU)
> >     and the DEVTLB (located in device).
> > 
> 
> But who/what kind of real device has the requirement to access a kernel VA?
> Looks like SVM_FLAG_SUPERVISOR_MODE is used by nobody?

That's right, there is no inkernel user at the moment, but we certainly see
them coming.

Maybe not the best example :-), but I'm told Lustre and some of the 
modern NIC's also can benefit from SVM in kernel use.

Not a hypothetical use case certainly!


Cheers,
Ashok

Re: [PATCH 1/2] mm: Add kernel MMU notifier to manage IOTLB/DEVTLB

[Dave Hansen]

On 12/13/2017 07:38 PM, Lu Baolu wrote:
> 2. When vmalloc/vfree interfaces are called, the page mappings
>     for kernel memory might get changed. And current code calls
>     flush_tlb_kernel_range() to flush CPU TLBs only. The IOTLB or
>     DevTLB will be stale compared to that on the cpu for kernel
>     mappings.

What's the plan to deal with all of the ways other than vmalloc() that
the kernel changes the page tables?

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH 1/2] mm: Add kernel MMU notifier to manage IOTLB/DEVTLB

[Huang, Ying]

Hi, Dave,

Dave Hansen <dave.hansen@intel.com> writes:

> On 12/13/2017 07:38 PM, Lu Baolu wrote:
>> 2. When vmalloc/vfree interfaces are called, the page mappings
>>     for kernel memory might get changed. And current code calls
>>     flush_tlb_kernel_range() to flush CPU TLBs only. The IOTLB or
>>     DevTLB will be stale compared to that on the cpu for kernel
>>     mappings.
>
> What's the plan to deal with all of the ways other than vmalloc() that
> the kernel changes the page tables?

The kernel MMU notifier is called in flush_tlb_kernel_range().  So IOMMU
will be notified for TLB flushing caused by all ways that the kernel
changes the page tables including vmalloc, kmap, etc.

Best Regards,
Huang, Ying

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

[PATCH 2/2] iommu/vt-d: Register kernel MMU notifier to manage IOTLB/DEVTLB

[Lu Baolu]

From: Ashok Raj <ashok.raj-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>

When a kernel client calls intel_svm_bind_mm() and gets a valid
supervisor PASID,  the memory mapping of init_mm will be shared
between CPUs and device. IOMMU has to track the changes to this
memory mapping, and get notified whenever a TLB flush is needed.
Otherwise, the device TLB will be stale compared to that on the
cpu for kernel mappings. This is similar to what have been done
for user space registrations via mmu_notifier_register() api's.

To: Alex Williamson <alex.williamson-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
To: linux-kernel-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
To: Joerg Roedel <joro-zLv9SwRftAIdnm+yROfE0A@public.gmane.org>
Cc: Ashok Raj <ashok.raj-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Cc: Dave Hansen <dave.hansen-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Cc: Huang Ying <ying.huang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Cc: CQ Tang <cq.tang-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Cc: Thomas Gleixner <tglx-hfZtesqFncYOwBW4kG4KsQ@public.gmane.org>
Cc: Ingo Molnar <mingo-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
Cc: H. Peter Anvin <hpa-YMNOUZJC4hwAvxtiuMwx3w@public.gmane.org>
Cc: Andy Lutomirski <luto-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org>
Cc: Rik van Riel <riel-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
Cc: Kees Cook <keescook-F7+t8E8rja9g9hUCZPvPmw@public.gmane.org>
Cc: Andrew Morton <akpm-de/tnXTf+JLsfHDXvbKv3WD2FQJk+8+b@public.gmane.org>
Cc: Michal Hocko <mhocko-IBi9RG/b67k@public.gmane.org>
Cc: Paul E. McKenney <paulmck-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org>
Cc: Vegard Nossum <vegard.nossum-QHcLZuEGTsvQT0dZR+AlfA@public.gmane.org>
Cc: x86-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org
Cc: linux-mm-Bw31MaZKKs3YtjvyW6yDsg@public.gmane.org
Cc: iommu-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org
Cc: David Woodhouse <dwmw2-wEGCiKHe2LqWVfeAwA7xHQ@public.gmane.org>
CC: Jean-Phillipe Brucker <jean-philippe.brucker-5wv7dgnIgG8@public.gmane.org>

Signed-off-by: Ashok Raj <ashok.raj-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Lu Baolu <baolu.lu-VuQAYsv1563Yd54FQh9/CA@public.gmane.org>
---
 drivers/iommu/intel-svm.c   | 27 +++++++++++++++++++++++++--
 include/linux/intel-iommu.h |  5 ++++-
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c
index ed1cf7c..1456092 100644
--- a/drivers/iommu/intel-svm.c
+++ b/drivers/iommu/intel-svm.c
@@ -283,6 +283,24 @@ static const struct mmu_notifier_ops intel_mmuops = {
 
 static DEFINE_MUTEX(pasid_mutex);
 
+static int intel_init_mm_inval_range(struct notifier_block *nb,
+				     unsigned long action, void *data)
+{
+	struct kernel_mmu_address_range *range;
+	struct intel_svm *svm = container_of(nb, struct intel_svm, init_mm_nb);
+	unsigned long start, end;
+
+	if (action == KERNEL_MMU_INVALIDATE_RANGE) {
+		range = data;
+		start = range->start;
+		end = range->end;
+
+		intel_flush_svm_range(svm, start,
+			(end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0, 0);
+	}
+	return 0;
+}
+
 int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ops *ops)
 {
 	struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);
@@ -382,12 +400,12 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
 			goto out;
 		}
 		svm->pasid = ret;
-		svm->notifier.ops = &intel_mmuops;
 		svm->mm = mm;
 		svm->flags = flags;
 		INIT_LIST_HEAD_RCU(&svm->devs);
 		ret = -ENOMEM;
 		if (mm) {
+			svm->notifier.ops = &intel_mmuops;
 			ret = mmu_notifier_register(&svm->notifier, mm);
 			if (ret) {
 				idr_remove(&svm->iommu->pasid_idr, svm->pasid);
@@ -396,8 +414,11 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
 				goto out;
 			}
 			iommu->pasid_table[svm->pasid].val = (u64)__pa(mm->pgd) | 1;
-		} else
+		} else {
+			svm->init_mm_nb.notifier_call = intel_init_mm_inval_range;
+			kernel_mmu_notifier_register(&svm->init_mm_nb);
 			iommu->pasid_table[svm->pasid].val = (u64)__pa(init_mm.pgd) | 1 | (1ULL << 11);
+		}
 		wmb();
 		/* In caching mode, we still have to flush with PASID 0 when
 		 * a PASID table entry becomes present. Not entirely clear
@@ -464,6 +485,8 @@ int intel_svm_unbind_mm(struct device *dev, int pasid)
 					idr_remove(&svm->iommu->pasid_idr, svm->pasid);
 					if (svm->mm)
 						mmu_notifier_unregister(&svm->notifier, svm->mm);
+					else
+						kernel_mmu_notifier_unregister(&svm->init_mm_nb);
 
 					/* We mandate that no page faults may be outstanding
 					 * for the PASID when intel_svm_unbind_mm() is called.
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index f3274d9..5cf83db 100644
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -478,7 +478,10 @@ struct intel_svm_dev {
 };
 
 struct intel_svm {
-	struct mmu_notifier notifier;
+	union {
+		struct mmu_notifier notifier;
+		struct notifier_block init_mm_nb;
+	};
 	struct mm_struct *mm;
 	struct intel_iommu *iommu;
 	int flags;
-- 
2.7.4