[PATCH 0/2] x86/hyperv/Swiotlb: Add swiotlb_alloc_from_low_pages switch

[PATCH 0/2] x86/hyperv/Swiotlb: Add swiotlb_alloc_from_low_pages switch

[Tianyu Lan]

From: Tianyu Lan <Tianyu.Lan@microsoft.com>

Hyper-V Isolation VM may fail to allocate swiotlb bounce buffer due
to there is no enough contiguous memory from 0 to 4G in some cases.
Current swiotlb code allocate bounce buffer in the low end memory.
This patchset adds a switch "swiotlb_alloc_from_low_pages" and it's
set to true by default. Platform may clear it if necessary. Devices
in Hyper-V Isolation VM may use memory above 4G as DMA memory and set
the switch to false in order to avoid no enough contiguous memory in
low end address space.

Tianyu Lan (2):
  Swiotlb: Add swiotlb_alloc_from_low_pages switch
  x86/hyperv: Set swiotlb_alloc_from_low_pages to false

 arch/x86/kernel/cpu/mshyperv.c |  1 +
 include/linux/swiotlb.h        |  1 +
 kernel/dma/swiotlb.c           | 13 +++++++++++--
 3 files changed, 13 insertions(+), 2 deletions(-)

-- 
2.25.1

[PATCH 1/2] Swiotlb: Add swiotlb_alloc_from_low_pages switch

[Tianyu Lan]

From: Tianyu Lan <Tianyu.Lan@microsoft.com>

Hyper-V Isolation VM and AMD SEV VM uses swiotlb bounce buffer to
share memory with hypervisor. Current swiotlb bounce buffer is only
allocated from 0 to ARCH_LOW_ADDRESS_LIMIT which is default to
0xffffffffUL. Isolation VM and AMD SEV VM needs 1G bounce buffer at most.
This will fail when there is not enough contiguous memory from 0 to 4G
address space and devices also may use memory above 4G address space as
DMA memory. Expose swiotlb_alloc_from_low_pages and platform mey set it
to false when it's not necessary to limit bounce buffer from 0 to 4G memory.

Signed-off-by: Tianyu Lan <Tianyu.Lan@microsoft.com>
---
 include/linux/swiotlb.h |  1 +
 kernel/dma/swiotlb.c    | 13 +++++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index f6c3638255d5..55c178e8eee0 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -191,5 +191,6 @@ static inline bool is_swiotlb_for_alloc(struct device *dev)
 #endif /* CONFIG_DMA_RESTRICTED_POOL */
 
 extern phys_addr_t swiotlb_unencrypted_base;
+extern bool swiotlb_alloc_from_low_pages;
 
 #endif /* __LINUX_SWIOTLB_H */
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index f1e7ea160b43..159fef80f3db 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -73,6 +73,12 @@ enum swiotlb_force swiotlb_force;
 
 struct io_tlb_mem io_tlb_default_mem;
 
+/*
+ * Get IO TLB memory from the low pages if swiotlb_alloc_from_low_pages
+ * is set.
+ */
+bool swiotlb_alloc_from_low_pages = true;
+
 phys_addr_t swiotlb_unencrypted_base;
 
 /*
@@ -284,8 +290,11 @@ swiotlb_init(int verbose)
 	if (swiotlb_force == SWIOTLB_NO_FORCE)
 		return;
 
-	/* Get IO TLB memory from the low pages */
-	tlb = memblock_alloc_low(bytes, PAGE_SIZE);
+	if (swiotlb_alloc_from_low_pages)
+		tlb = memblock_alloc_low(bytes, PAGE_SIZE);
+	else
+		tlb = memblock_alloc(bytes, PAGE_SIZE);
+
 	if (!tlb)
 		goto fail;
 	if (swiotlb_init_with_tbl(tlb, default_nslabs, verbose))
-- 
2.25.1

RE: [PATCH 1/2] Swiotlb: Add swiotlb_alloc_from_low_pages switch

[Michael Kelley (LINUX)]

From: Tianyu Lan <ltykernel@gmail.com> Sent: Wednesday, January 26, 2022 8:11 AM
> 
> Hyper-V Isolation VM and AMD SEV VM uses swiotlb bounce buffer to
> share memory with hypervisor. Current swiotlb bounce buffer is only
> allocated from 0 to ARCH_LOW_ADDRESS_LIMIT which is default to
> 0xffffffffUL. Isolation VM and AMD SEV VM needs 1G bounce buffer at most.
> This will fail when there is not enough contiguous memory from 0 to 4G
> address space and devices also may use memory above 4G address space as
> DMA memory. Expose swiotlb_alloc_from_low_pages and platform mey set it
> to false when it's not necessary to limit bounce buffer from 0 to 4G memory.
> 
> Signed-off-by: Tianyu Lan <Tianyu.Lan@microsoft.com>
> ---
>  include/linux/swiotlb.h |  1 +
>  kernel/dma/swiotlb.c    | 13 +++++++++++--
>  2 files changed, 12 insertions(+), 2 deletions(-)
> 
> diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
> index f6c3638255d5..55c178e8eee0 100644
> --- a/include/linux/swiotlb.h
> +++ b/include/linux/swiotlb.h
> @@ -191,5 +191,6 @@ static inline bool is_swiotlb_for_alloc(struct device *dev)
>  #endif /* CONFIG_DMA_RESTRICTED_POOL */
> 
>  extern phys_addr_t swiotlb_unencrypted_base;
> +extern bool swiotlb_alloc_from_low_pages;
> 
>  #endif /* __LINUX_SWIOTLB_H */
> diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
> index f1e7ea160b43..159fef80f3db 100644
> --- a/kernel/dma/swiotlb.c
> +++ b/kernel/dma/swiotlb.c
> @@ -73,6 +73,12 @@ enum swiotlb_force swiotlb_force;
> 
>  struct io_tlb_mem io_tlb_default_mem;
> 
> +/*
> + * Get IO TLB memory from the low pages if swiotlb_alloc_from_low_pages
> + * is set.
> + */
> +bool swiotlb_alloc_from_low_pages = true;
> +
>  phys_addr_t swiotlb_unencrypted_base;
> 
>  /*
> @@ -284,8 +290,11 @@ swiotlb_init(int verbose)
>  	if (swiotlb_force == SWIOTLB_NO_FORCE)
>  		return;
> 
> -	/* Get IO TLB memory from the low pages */
> -	tlb = memblock_alloc_low(bytes, PAGE_SIZE);
> +	if (swiotlb_alloc_from_low_pages)
> +		tlb = memblock_alloc_low(bytes, PAGE_SIZE);
> +	else
> +		tlb = memblock_alloc(bytes, PAGE_SIZE);
> +
>  	if (!tlb)
>  		goto fail;
>  	if (swiotlb_init_with_tbl(tlb, default_nslabs, verbose))
> --
> 2.25.1

Reviewed-by: Michael Kelley <mikelley@microsoft.com>

[PATCH 2/2] x86/hyperv: Set swiotlb_alloc_from_low_pages to false

[Tianyu Lan]

From: Tianyu Lan <Tianyu.Lan@microsoft.com>

In Hyper-V Isolation VM, swiotlb bnounce buffer size maybe 1G at most
and there maybe no enough memory from 0 to 4G according to memory layout.
Devices in Isolation VM can use memory above 4G as DMA memory. Set swiotlb_
alloc_from_low_pages to false in Isolation VM.

Signed-off-by: Tianyu Lan <Tianyu.Lan@microsoft.com>
---
 arch/x86/kernel/cpu/mshyperv.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
index 5a99f993e639..80a0423ac75d 100644
--- a/arch/x86/kernel/cpu/mshyperv.c
+++ b/arch/x86/kernel/cpu/mshyperv.c
@@ -343,6 +343,7 @@ static void __init ms_hyperv_init_platform(void)
 		 * use swiotlb bounce buffer for dma transaction.
 		 */
 		swiotlb_force = SWIOTLB_FORCE;
+		swiotlb_alloc_from_low_pages = false;
 #endif
 	}
 
-- 
2.25.1

RE: [PATCH 2/2] x86/hyperv: Set swiotlb_alloc_from_low_pages to false

[Michael Kelley (LINUX)]

From: Tianyu Lan <ltykernel@gmail.com> Sent: Wednesday, January 26, 2022 8:11 AM
> 
> In Hyper-V Isolation VM, swiotlb bnounce buffer size maybe 1G at most
> and there maybe no enough memory from 0 to 4G according to memory layout.
> Devices in Isolation VM can use memory above 4G as DMA memory. Set swiotlb_
> alloc_from_low_pages to false in Isolation VM.
> 
> Signed-off-by: Tianyu Lan <Tianyu.Lan@microsoft.com>
> ---
>  arch/x86/kernel/cpu/mshyperv.c | 1 +
>  1 file changed, 1 insertion(+)
> 
> diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
> index 5a99f993e639..80a0423ac75d 100644
> --- a/arch/x86/kernel/cpu/mshyperv.c
> +++ b/arch/x86/kernel/cpu/mshyperv.c
> @@ -343,6 +343,7 @@ static void __init ms_hyperv_init_platform(void)
>  		 * use swiotlb bounce buffer for dma transaction.
>  		 */
>  		swiotlb_force = SWIOTLB_FORCE;
> +		swiotlb_alloc_from_low_pages = false;
>  #endif
>  	}
> 
> --
> 2.25.1

Reviewed-by: Michael Kelley <mikelley@microsoft.com>

Re: [PATCH 0/2] x86/hyperv/Swiotlb: Add swiotlb_alloc_from_low_pages switch

[Christoph Hellwig]

I think this interface is a little too hacky.  In the end all the
non-trusted hypervisor schemes (including the per-device swiotlb one)
can allocate the memory from everywhere and want for force use of
swiotlb.  I think we need some kind of proper interface for that instead
of setting all kinds of global variables.

Re: [PATCH 0/2] x86/hyperv/Swiotlb: Add swiotlb_alloc_from_low_pages switch

[Robin Murphy]

On 2022-02-02 08:12, Christoph Hellwig wrote:
> I think this interface is a little too hacky.  In the end all the
> non-trusted hypervisor schemes (including the per-device swiotlb one)
> can allocate the memory from everywhere and want for force use of
> swiotlb.  I think we need some kind of proper interface for that instead
> of setting all kinds of global variables.

Right, if platform/hypervisor code knows enough to be able to set magic 
non-standard allocation flags correctly, surely it could equally just 
perform whatever non-standard allocation it needs and call 
swiotlb_init_with_tbl() instead of swiotlb_init().

Robin.

Re: [PATCH 0/2] x86/hyperv/Swiotlb: Add swiotlb_alloc_from_low_pages switch

[Tianyu Lan]

On 2/2/2022 4:12 PM, Christoph Hellwig wrote:
> I think this interface is a little too hacky.  In the end all the
> non-trusted hypervisor schemes (including the per-device swiotlb one)
> can allocate the memory from everywhere and want for force use of
> swiotlb.  I think we need some kind of proper interface for that instead
> of setting all kinds of global variables.

Hi Christoph:
      Thanks for your review. I draft the following patch to export a
interface swiotlb_set_alloc_from_low_pages(). Could you have a look
whether this looks good for you.

diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index f6c3638255d5..2b4f92668bc7 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -39,6 +39,7 @@ enum swiotlb_force {
  extern void swiotlb_init(int verbose);
  int swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose);
  unsigned long swiotlb_size_or_default(void);
+void swiotlb_set_alloc_from_low_pages(bool low);
  extern int swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs);
  extern int swiotlb_late_init_with_default_size(size_t default_size);
  extern void __init swiotlb_update_mem_attributes(void);
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index f1e7ea160b43..62bf8b5cc3e4 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -73,6 +73,8 @@ enum swiotlb_force swiotlb_force;

  struct io_tlb_mem io_tlb_default_mem;

+static bool swiotlb_alloc_from_low_pages = true;
+
  phys_addr_t swiotlb_unencrypted_base;

  /*
@@ -116,6 +118,11 @@ void swiotlb_set_max_segment(unsigned int val)
                 max_segment = rounddown(val, PAGE_SIZE);
  }

+void swiotlb_set_alloc_from_low_pages(bool low)
+{
+       swiotlb_alloc_from_low_pages = low;
+}
+
  unsigned long swiotlb_size_or_default(void)
  {
         return default_nslabs << IO_TLB_SHIFT;
@@ -284,8 +291,15 @@ swiotlb_init(int verbose)
         if (swiotlb_force == SWIOTLB_NO_FORCE)
                 return;

-       /* Get IO TLB memory from the low pages */
-       tlb = memblock_alloc_low(bytes, PAGE_SIZE);
+       /*
+        * Get IO TLB memory from the low pages if 
swiotlb_alloc_from_low_pages
+        * is set.
+        */
+       if (swiotlb_alloc_from_low_pages)
+               tlb = memblock_alloc_low(bytes, PAGE_SIZE);
+       else
+               tlb = memblock_alloc(bytes, PAGE_SIZE);
+
         if (!tlb)
                 goto fail;
         if (swiotlb_init_with_tbl(tlb, default_nslabs, verbose))