diff for duplicates of <20190422175942.18788-8-hch@lst.de> diff --git a/a/1.txt b/N1/1.txt index 97690c1..f34897f 100644 --- a/a/1.txt +++ b/N1/1.txt @@ -1063,3 +1063,8 @@ index 3216447178a7..dadf4383f555 100644 static inline int iommu_get_dma_cookie(struct iommu_domain *domain) -- 2.20.1 + +_______________________________________________ +iommu mailing list +iommu@lists.linux-foundation.org +https://lists.linuxfoundation.org/mailman/listinfo/iommu diff --git a/a/content_digest b/N1/content_digest index 3bff309..7063e1c 100644 --- a/a/content_digest +++ b/N1/content_digest @@ -3,13 +3,12 @@ "Subject\0[PATCH 07/26] iommu/dma: move the arm64 wrappers to common code\0" "Date\0Mon, 22 Apr 2019 19:59:23 +0200\0" "To\0Robin Murphy <robin.murphy@arm.com>\0" - "Cc\0Joerg Roedel <joro@8bytes.org>" + "Cc\0Tom Lendacky <thomas.lendacky@amd.com>" Catalin Marinas <catalin.marinas@arm.com> Will Deacon <will.deacon@arm.com> - Tom Lendacky <thomas.lendacky@amd.com> + linux-kernel@vger.kernel.org iommu@lists.linux-foundation.org - linux-arm-kernel@lists.infradead.org - " linux-kernel@vger.kernel.org\0" + " linux-arm-kernel@lists.infradead.org\0" "\00:1\0" "b\0" "There is nothing really arm64 specific in the iommu_dma_ops\n" @@ -1076,6 +1075,11 @@ " \n" " static inline int iommu_get_dma_cookie(struct iommu_domain *domain)\n" "-- \n" - 2.20.1 + "2.20.1\n" + "\n" + "_______________________________________________\n" + "iommu mailing list\n" + "iommu@lists.linux-foundation.org\n" + https://lists.linuxfoundation.org/mailman/listinfo/iommu -07824f886f395b478d3e4f885ab19a807f4e3c3ce3112e001126b2846844743f +ccd7321b83e9775d7bfabc3ce0766eb9cba8250b704f8dacbb792e950171c245
diff --git a/a/1.txt b/N2/1.txt index 97690c1..8c32291 100644 --- a/a/1.txt +++ b/N2/1.txt @@ -1,1065 +1,89 @@ -There is nothing really arm64 specific in the iommu_dma_ops -implementation, so move it to dma-iommu.c and keep a lot of symbols -self-contained. Note the implementation does depend on the -DMA_DIRECT_REMAP infrastructure for now, so we'll have to make the -DMA_IOMMU support depend on it, but this will be relaxed soon. -Signed-off-by: Christoph Hellwig <hch@lst.de> ---- - arch/arm64/mm/dma-mapping.c | 389 +----------------------------------- - drivers/iommu/Kconfig | 1 + - drivers/iommu/dma-iommu.c | 388 ++++++++++++++++++++++++++++++++--- - include/linux/dma-iommu.h | 43 +--- - 4 files changed, 369 insertions(+), 452 deletions(-) +Hi Christoph -diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c -index 636fa7c64370..d1661f78eb4d 100644 ---- a/arch/arm64/mm/dma-mapping.c -+++ b/arch/arm64/mm/dma-mapping.c -@@ -27,6 +27,7 @@ - #include <linux/dma-direct.h> - #include <linux/dma-noncoherent.h> - #include <linux/dma-contiguous.h> -+#include <linux/dma-iommu.h> - #include <linux/vmalloc.h> - #include <linux/swiotlb.h> - #include <linux/pci.h> -@@ -58,27 +59,6 @@ void arch_dma_prep_coherent(struct page *page, size_t size) - __dma_flush_area(page_address(page), size); - } - --#ifdef CONFIG_IOMMU_DMA --static int __swiotlb_get_sgtable_page(struct sg_table *sgt, -- struct page *page, size_t size) --{ -- int ret = sg_alloc_table(sgt, 1, GFP_KERNEL); -- -- if (!ret) -- sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); -- -- return ret; --} -- --static int __swiotlb_mmap_pfn(struct vm_area_struct *vma, -- unsigned long pfn, size_t size) --{ -- return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, -- vma->vm_end - vma->vm_start, -- vma->vm_page_prot); --} --#endif /* CONFIG_IOMMU_DMA */ -- - static int __init arm64_dma_init(void) - { - WARN_TAINT(ARCH_DMA_MINALIGN < cache_line_size(), -@@ -90,379 +70,18 @@ static int __init arm64_dma_init(void) - arch_initcall(arm64_dma_init); - - #ifdef CONFIG_IOMMU_DMA --#include <linux/dma-iommu.h> --#include <linux/platform_device.h> --#include <linux/amba/bus.h> -- --static void *__iommu_alloc_attrs(struct device *dev, size_t size, -- dma_addr_t *handle, gfp_t gfp, -- unsigned long attrs) --{ -- bool coherent = dev_is_dma_coherent(dev); -- int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); -- size_t iosize = size; -- void *addr; -- -- if (WARN(!dev, "cannot create IOMMU mapping for unknown device\n")) -- return NULL; -- -- size = PAGE_ALIGN(size); -- -- /* -- * Some drivers rely on this, and we probably don't want the -- * possibility of stale kernel data being read by devices anyway. -- */ -- gfp |= __GFP_ZERO; -- -- if (!gfpflags_allow_blocking(gfp)) { -- struct page *page; -- /* -- * In atomic context we can't remap anything, so we'll only -- * get the virtually contiguous buffer we need by way of a -- * physically contiguous allocation. -- */ -- if (coherent) { -- page = alloc_pages(gfp, get_order(size)); -- addr = page ? page_address(page) : NULL; -- } else { -- addr = dma_alloc_from_pool(size, &page, gfp); -- } -- if (!addr) -- return NULL; -- -- *handle = iommu_dma_map_page(dev, page, 0, iosize, ioprot); -- if (*handle == DMA_MAPPING_ERROR) { -- if (coherent) -- __free_pages(page, get_order(size)); -- else -- dma_free_from_pool(addr, size); -- addr = NULL; -- } -- } else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { -- pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); -- struct page *page; -- -- page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, -- get_order(size), gfp & __GFP_NOWARN); -- if (!page) -- return NULL; -- -- *handle = iommu_dma_map_page(dev, page, 0, iosize, ioprot); -- if (*handle == DMA_MAPPING_ERROR) { -- dma_release_from_contiguous(dev, page, -- size >> PAGE_SHIFT); -- return NULL; -- } -- addr = dma_common_contiguous_remap(page, size, VM_USERMAP, -- prot, -- __builtin_return_address(0)); -- if (addr) { -- if (!coherent) -- __dma_flush_area(page_to_virt(page), iosize); -- memset(addr, 0, size); -- } else { -- iommu_dma_unmap_page(dev, *handle, iosize, 0, attrs); -- dma_release_from_contiguous(dev, page, -- size >> PAGE_SHIFT); -- } -- } else { -- pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); -- struct page **pages; -- -- pages = iommu_dma_alloc(dev, iosize, gfp, attrs, ioprot, -- handle); -- if (!pages) -- return NULL; -- -- addr = dma_common_pages_remap(pages, size, VM_USERMAP, prot, -- __builtin_return_address(0)); -- if (!addr) -- iommu_dma_free(dev, pages, iosize, handle); -- } -- return addr; --} -- --static void __iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, -- dma_addr_t handle, unsigned long attrs) --{ -- size_t iosize = size; -- -- size = PAGE_ALIGN(size); -- /* -- * @cpu_addr will be one of 4 things depending on how it was allocated: -- * - A remapped array of pages for contiguous allocations. -- * - A remapped array of pages from iommu_dma_alloc(), for all -- * non-atomic allocations. -- * - A non-cacheable alias from the atomic pool, for atomic -- * allocations by non-coherent devices. -- * - A normal lowmem address, for atomic allocations by -- * coherent devices. -- * Hence how dodgy the below logic looks... -- */ -- if (dma_in_atomic_pool(cpu_addr, size)) { -- iommu_dma_unmap_page(dev, handle, iosize, 0, 0); -- dma_free_from_pool(cpu_addr, size); -- } else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { -- struct page *page = vmalloc_to_page(cpu_addr); -- -- iommu_dma_unmap_page(dev, handle, iosize, 0, attrs); -- dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); -- dma_common_free_remap(cpu_addr, size, VM_USERMAP); -- } else if (is_vmalloc_addr(cpu_addr)){ -- struct vm_struct *area = find_vm_area(cpu_addr); -- -- if (WARN_ON(!area || !area->pages)) -- return; -- iommu_dma_free(dev, area->pages, iosize, &handle); -- dma_common_free_remap(cpu_addr, size, VM_USERMAP); -- } else { -- iommu_dma_unmap_page(dev, handle, iosize, 0, 0); -- __free_pages(virt_to_page(cpu_addr), get_order(size)); -- } --} -- --static int __iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, -- void *cpu_addr, dma_addr_t dma_addr, size_t size, -- unsigned long attrs) --{ -- unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; -- unsigned long off = vma->vm_pgoff; -- struct vm_struct *area; -- int ret; -- -- vma->vm_page_prot = arch_dma_mmap_pgprot(dev, vma->vm_page_prot, attrs); -- -- if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) -- return ret; -- -- if (off >= nr_pages || vma_pages(vma) > nr_pages - off) -- return -ENXIO; -- -- if (!is_vmalloc_addr(cpu_addr)) { -- unsigned long pfn = page_to_pfn(virt_to_page(cpu_addr)); -- return __swiotlb_mmap_pfn(vma, pfn, size); -- } -- -- if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { -- /* -- * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped, -- * hence in the vmalloc space. -- */ -- unsigned long pfn = vmalloc_to_pfn(cpu_addr); -- return __swiotlb_mmap_pfn(vma, pfn, size); -- } -- -- area = find_vm_area(cpu_addr); -- if (WARN_ON(!area || !area->pages)) -- return -ENXIO; -- -- return iommu_dma_mmap(area->pages, size, vma); --} -- --static int __iommu_get_sgtable(struct device *dev, struct sg_table *sgt, -- void *cpu_addr, dma_addr_t dma_addr, -- size_t size, unsigned long attrs) --{ -- unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; -- struct vm_struct *area = find_vm_area(cpu_addr); -- -- if (!is_vmalloc_addr(cpu_addr)) { -- struct page *page = virt_to_page(cpu_addr); -- return __swiotlb_get_sgtable_page(sgt, page, size); -- } -- -- if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { -- /* -- * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped, -- * hence in the vmalloc space. -- */ -- struct page *page = vmalloc_to_page(cpu_addr); -- return __swiotlb_get_sgtable_page(sgt, page, size); -- } -- -- if (WARN_ON(!area || !area->pages)) -- return -ENXIO; -- -- return sg_alloc_table_from_pages(sgt, area->pages, count, 0, size, -- GFP_KERNEL); --} -- --static void __iommu_sync_single_for_cpu(struct device *dev, -- dma_addr_t dev_addr, size_t size, -- enum dma_data_direction dir) --{ -- phys_addr_t phys; -- -- if (dev_is_dma_coherent(dev)) -- return; -- -- phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dev_addr); -- arch_sync_dma_for_cpu(dev, phys, size, dir); --} -- --static void __iommu_sync_single_for_device(struct device *dev, -- dma_addr_t dev_addr, size_t size, -- enum dma_data_direction dir) --{ -- phys_addr_t phys; -- -- if (dev_is_dma_coherent(dev)) -- return; -- -- phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dev_addr); -- arch_sync_dma_for_device(dev, phys, size, dir); --} -- --static dma_addr_t __iommu_map_page(struct device *dev, struct page *page, -- unsigned long offset, size_t size, -- enum dma_data_direction dir, -- unsigned long attrs) --{ -- bool coherent = dev_is_dma_coherent(dev); -- int prot = dma_info_to_prot(dir, coherent, attrs); -- dma_addr_t dev_addr = iommu_dma_map_page(dev, page, offset, size, prot); -- -- if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && -- dev_addr != DMA_MAPPING_ERROR) -- __dma_map_area(page_address(page) + offset, size, dir); -- -- return dev_addr; --} -- --static void __iommu_unmap_page(struct device *dev, dma_addr_t dev_addr, -- size_t size, enum dma_data_direction dir, -- unsigned long attrs) --{ -- if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) -- __iommu_sync_single_for_cpu(dev, dev_addr, size, dir); -- -- iommu_dma_unmap_page(dev, dev_addr, size, dir, attrs); --} -- --static void __iommu_sync_sg_for_cpu(struct device *dev, -- struct scatterlist *sgl, int nelems, -- enum dma_data_direction dir) --{ -- struct scatterlist *sg; -- int i; -- -- if (dev_is_dma_coherent(dev)) -- return; -- -- for_each_sg(sgl, sg, nelems, i) -- arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir); --} -- --static void __iommu_sync_sg_for_device(struct device *dev, -- struct scatterlist *sgl, int nelems, -- enum dma_data_direction dir) --{ -- struct scatterlist *sg; -- int i; -- -- if (dev_is_dma_coherent(dev)) -- return; -- -- for_each_sg(sgl, sg, nelems, i) -- arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir); --} -- --static int __iommu_map_sg_attrs(struct device *dev, struct scatterlist *sgl, -- int nelems, enum dma_data_direction dir, -- unsigned long attrs) --{ -- bool coherent = dev_is_dma_coherent(dev); -- -- if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) -- __iommu_sync_sg_for_device(dev, sgl, nelems, dir); -- -- return iommu_dma_map_sg(dev, sgl, nelems, -- dma_info_to_prot(dir, coherent, attrs)); --} -- --static void __iommu_unmap_sg_attrs(struct device *dev, -- struct scatterlist *sgl, int nelems, -- enum dma_data_direction dir, -- unsigned long attrs) --{ -- if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) -- __iommu_sync_sg_for_cpu(dev, sgl, nelems, dir); -- -- iommu_dma_unmap_sg(dev, sgl, nelems, dir, attrs); --} -- --static const struct dma_map_ops iommu_dma_ops = { -- .alloc = __iommu_alloc_attrs, -- .free = __iommu_free_attrs, -- .mmap = __iommu_mmap_attrs, -- .get_sgtable = __iommu_get_sgtable, -- .map_page = __iommu_map_page, -- .unmap_page = __iommu_unmap_page, -- .map_sg = __iommu_map_sg_attrs, -- .unmap_sg = __iommu_unmap_sg_attrs, -- .sync_single_for_cpu = __iommu_sync_single_for_cpu, -- .sync_single_for_device = __iommu_sync_single_for_device, -- .sync_sg_for_cpu = __iommu_sync_sg_for_cpu, -- .sync_sg_for_device = __iommu_sync_sg_for_device, -- .map_resource = iommu_dma_map_resource, -- .unmap_resource = iommu_dma_unmap_resource, --}; -- --static int __init __iommu_dma_init(void) --{ -- return iommu_dma_init(); --} --arch_initcall(__iommu_dma_init); -- --static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, -- const struct iommu_ops *ops) --{ -- struct iommu_domain *domain; -- -- if (!ops) -- return; -- -- /* -- * The IOMMU core code allocates the default DMA domain, which the -- * underlying IOMMU driver needs to support via the dma-iommu layer. -- */ -- domain = iommu_get_domain_for_dev(dev); -- -- if (!domain) -- goto out_err; -- -- if (domain->type == IOMMU_DOMAIN_DMA) { -- if (iommu_dma_init_domain(domain, dma_base, size, dev)) -- goto out_err; -- -- dev->dma_ops = &iommu_dma_ops; -- } -- -- return; -- --out_err: -- pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n", -- dev_name(dev)); --} -- - void arch_teardown_dma_ops(struct device *dev) - { - dev->dma_ops = NULL; - } -- --#else -- --static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, -- const struct iommu_ops *iommu) --{ } -- --#endif /* CONFIG_IOMMU_DMA */ -+#endif - - void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - const struct iommu_ops *iommu, bool coherent) - { - dev->dma_coherent = coherent; -- __iommu_setup_dma_ops(dev, dma_base, size, iommu); -+ if (iommu) -+ iommu_setup_dma_ops(dev, dma_base, size); - - #ifdef CONFIG_XEN - if (xen_initial_domain()) -diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig -index 6f07f3b21816..bdc14baf2ee5 100644 ---- a/drivers/iommu/Kconfig -+++ b/drivers/iommu/Kconfig -@@ -95,6 +95,7 @@ config IOMMU_DMA - select IOMMU_API - select IOMMU_IOVA - select NEED_SG_DMA_LENGTH -+ depends on DMA_DIRECT_REMAP - - config FSL_PAMU - bool "Freescale IOMMU support" -diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c -index f915cb7c46e6..622123551bba 100644 ---- a/drivers/iommu/dma-iommu.c -+++ b/drivers/iommu/dma-iommu.c -@@ -21,6 +21,7 @@ - - #include <linux/acpi_iort.h> - #include <linux/device.h> -+#include <linux/dma-contiguous.h> - #include <linux/dma-iommu.h> - #include <linux/dma-noncoherent.h> - #include <linux/gfp.h> -@@ -79,11 +80,6 @@ static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type) - return cookie; - } - --int iommu_dma_init(void) --{ -- return iova_cache_get(); --} -- - /** - * iommu_get_dma_cookie - Acquire DMA-API resources for a domain - * @domain: IOMMU domain to prepare for DMA-API usage -@@ -285,7 +281,7 @@ static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad) - * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but - * any change which could make prior IOVAs invalid will fail. - */ --int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, -+static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, - u64 size, struct device *dev) - { - struct iommu_dma_cookie *cookie = domain->iova_cookie; -@@ -336,7 +332,6 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, - - return iova_reserve_iommu_regions(dev, domain); - } --EXPORT_SYMBOL(iommu_dma_init_domain); - - /** - * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API -@@ -347,7 +342,7 @@ EXPORT_SYMBOL(iommu_dma_init_domain); - * - * Return: corresponding IOMMU API page protection flags - */ --int dma_info_to_prot(enum dma_data_direction dir, bool coherent, -+static int dma_info_to_prot(enum dma_data_direction dir, bool coherent, - unsigned long attrs) - { - int prot = coherent ? IOMMU_CACHE : 0; -@@ -506,17 +501,17 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev, - } - - /** -- * iommu_dma_free - Free a buffer allocated by iommu_dma_alloc() -+ * iommu_dma_free - Free a buffer allocated by __iommu_dma_alloc() - * @dev: Device which owns this buffer -- * @pages: Array of buffer pages as returned by iommu_dma_alloc() -+ * @pages: Array of buffer pages as returned by __iommu_dma_alloc() - * @size: Size of buffer in bytes - * @handle: DMA address of buffer - * - * Frees both the pages associated with the buffer, and the array - * describing them - */ --void iommu_dma_free(struct device *dev, struct page **pages, size_t size, -- dma_addr_t *handle) -+static void __iommu_dma_free(struct device *dev, struct page **pages, -+ size_t size, dma_addr_t *handle) - { - __iommu_dma_unmap(iommu_get_dma_domain(dev), *handle, size); - __iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT); -@@ -524,7 +519,7 @@ void iommu_dma_free(struct device *dev, struct page **pages, size_t size, - } - - /** -- * iommu_dma_alloc - Allocate and map a buffer contiguous in IOVA space -+ * __iommu_dma_alloc - Allocate and map a buffer contiguous in IOVA space - * @dev: Device to allocate memory for. Must be a real device - * attached to an iommu_dma_domain - * @size: Size of buffer in bytes -@@ -539,8 +534,8 @@ void iommu_dma_free(struct device *dev, struct page **pages, size_t size, - * Return: Array of struct page pointers describing the buffer, - * or NULL on failure. - */ --struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, -- unsigned long attrs, int prot, dma_addr_t *handle) -+static struct page **__iommu_dma_alloc(struct device *dev, size_t size, -+ gfp_t gfp, unsigned long attrs, int prot, dma_addr_t *handle) - { - struct iommu_domain *domain = iommu_get_dma_domain(dev); - struct iommu_dma_cookie *cookie = domain->iova_cookie; -@@ -602,16 +597,16 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, - } - - /** -- * iommu_dma_mmap - Map a buffer into provided user VMA -- * @pages: Array representing buffer from iommu_dma_alloc() -+ * __iommu_dma_mmap - Map a buffer into provided user VMA -+ * @pages: Array representing buffer from __iommu_dma_alloc() - * @size: Size of buffer in bytes - * @vma: VMA describing requested userspace mapping - * - * Maps the pages of the buffer in @pages into @vma. The caller is responsible - * for verifying the correct size and protection of @vma beforehand. - */ -- --int iommu_dma_mmap(struct page **pages, size_t size, struct vm_area_struct *vma) -+static int __iommu_dma_mmap(struct page **pages, size_t size, -+ struct vm_area_struct *vma) - { - unsigned long uaddr = vma->vm_start; - unsigned int i, count = PAGE_ALIGN(size) >> PAGE_SHIFT; -@@ -626,6 +621,58 @@ int iommu_dma_mmap(struct page **pages, size_t size, struct vm_area_struct *vma) - return ret; - } - -+static void iommu_dma_sync_single_for_cpu(struct device *dev, -+ dma_addr_t dma_handle, size_t size, enum dma_data_direction dir) -+{ -+ phys_addr_t phys; -+ -+ if (dev_is_dma_coherent(dev)) -+ return; -+ -+ phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); -+ arch_sync_dma_for_cpu(dev, phys, size, dir); -+} -+ -+static void iommu_dma_sync_single_for_device(struct device *dev, -+ dma_addr_t dma_handle, size_t size, enum dma_data_direction dir) -+{ -+ phys_addr_t phys; -+ -+ if (dev_is_dma_coherent(dev)) -+ return; -+ -+ phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); -+ arch_sync_dma_for_device(dev, phys, size, dir); -+} -+ -+static void iommu_dma_sync_sg_for_cpu(struct device *dev, -+ struct scatterlist *sgl, int nelems, -+ enum dma_data_direction dir) -+{ -+ struct scatterlist *sg; -+ int i; -+ -+ if (dev_is_dma_coherent(dev)) -+ return; -+ -+ for_each_sg(sgl, sg, nelems, i) -+ arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir); -+} -+ -+static void iommu_dma_sync_sg_for_device(struct device *dev, -+ struct scatterlist *sgl, int nelems, -+ enum dma_data_direction dir) -+{ -+ struct scatterlist *sg; -+ int i; -+ -+ if (dev_is_dma_coherent(dev)) -+ return; -+ -+ for_each_sg(sgl, sg, nelems, i) -+ arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir); -+} -+ - static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, - size_t size, int prot, struct iommu_domain *domain) - { -@@ -649,19 +696,44 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, - return iova + iova_off; - } - --dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, -+static dma_addr_t __iommu_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, int prot) - { - return __iommu_dma_map(dev, page_to_phys(page) + offset, size, prot, - iommu_get_dma_domain(dev)); - } - --void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, -- enum dma_data_direction dir, unsigned long attrs) -+static void __iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, -+ size_t size, enum dma_data_direction dir, unsigned long attrs) - { - __iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size); - } - -+static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, -+ unsigned long offset, size_t size, enum dma_data_direction dir, -+ unsigned long attrs) -+{ -+ phys_addr_t phys = page_to_phys(page) + offset; -+ bool coherent = dev_is_dma_coherent(dev); -+ dma_addr_t dma_handle; -+ -+ dma_handle =__iommu_dma_map(dev, phys, size, -+ dma_info_to_prot(dir, coherent, attrs), -+ iommu_get_dma_domain(dev)); -+ if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && -+ dma_handle != DMA_MAPPING_ERROR) -+ arch_sync_dma_for_device(dev, phys, size, dir); -+ return dma_handle; -+} -+ -+static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, -+ size_t size, enum dma_data_direction dir, unsigned long attrs) -+{ -+ if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) -+ iommu_dma_sync_single_for_cpu(dev, dma_handle, size, dir); -+ __iommu_dma_unmap(iommu_get_dma_domain(dev), dma_handle, size); -+} -+ - /* - * Prepare a successfully-mapped scatterlist to give back to the caller. - * -@@ -744,18 +816,22 @@ static void __invalidate_sg(struct scatterlist *sg, int nents) - * impedance-matching, to be able to hand off a suitably-aligned list, - * but still preserve the original offsets and sizes for the caller. - */ --int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, -- int nents, int prot) -+static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, -+ int nents, enum dma_data_direction dir, unsigned long attrs) - { - struct iommu_domain *domain = iommu_get_dma_domain(dev); - struct iommu_dma_cookie *cookie = domain->iova_cookie; - struct iova_domain *iovad = &cookie->iovad; - struct scatterlist *s, *prev = NULL; -+ int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs); - dma_addr_t iova; - size_t iova_len = 0; - unsigned long mask = dma_get_seg_boundary(dev); - int i; - -+ if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) -+ iommu_dma_sync_sg_for_device(dev, sg, nents, dir); -+ - /* - * Work out how much IOVA space we need, and align the segments to - * IOVA granules for the IOMMU driver to handle. With some clever -@@ -815,12 +891,16 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, - return 0; - } - --void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, -- enum dma_data_direction dir, unsigned long attrs) -+static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, -+ int nents, enum dma_data_direction dir, unsigned long attrs) - { - dma_addr_t start, end; - struct scatterlist *tmp; - int i; -+ -+ if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) -+ iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir); -+ - /* - * The scatterlist segments are mapped into a single - * contiguous IOVA allocation, so this is incredibly easy. -@@ -835,7 +915,7 @@ void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, - __iommu_dma_unmap(iommu_get_dma_domain(dev), start, end - start); - } - --dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, -+static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, - size_t size, enum dma_data_direction dir, unsigned long attrs) - { - return __iommu_dma_map(dev, phys, size, -@@ -843,12 +923,258 @@ dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, - iommu_get_dma_domain(dev)); - } - --void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, -+static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, unsigned long attrs) - { - __iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size); - } - -+static void *iommu_dma_alloc(struct device *dev, size_t size, -+ dma_addr_t *handle, gfp_t gfp, unsigned long attrs) -+{ -+ bool coherent = dev_is_dma_coherent(dev); -+ int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); -+ size_t iosize = size; -+ void *addr; -+ -+ size = PAGE_ALIGN(size); -+ gfp |= __GFP_ZERO; -+ -+ if (!gfpflags_allow_blocking(gfp)) { -+ struct page *page; -+ /* -+ * In atomic context we can't remap anything, so we'll only -+ * get the virtually contiguous buffer we need by way of a -+ * physically contiguous allocation. -+ */ -+ if (coherent) { -+ page = alloc_pages(gfp, get_order(size)); -+ addr = page ? page_address(page) : NULL; -+ } else { -+ addr = dma_alloc_from_pool(size, &page, gfp); -+ } -+ if (!addr) -+ return NULL; -+ -+ *handle = __iommu_dma_map_page(dev, page, 0, iosize, ioprot); -+ if (*handle == DMA_MAPPING_ERROR) { -+ if (coherent) -+ __free_pages(page, get_order(size)); -+ else -+ dma_free_from_pool(addr, size); -+ addr = NULL; -+ } -+ } else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { -+ pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); -+ struct page *page; -+ -+ page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, -+ get_order(size), gfp & __GFP_NOWARN); -+ if (!page) -+ return NULL; -+ -+ *handle = __iommu_dma_map_page(dev, page, 0, iosize, ioprot); -+ if (*handle == DMA_MAPPING_ERROR) { -+ dma_release_from_contiguous(dev, page, -+ size >> PAGE_SHIFT); -+ return NULL; -+ } -+ addr = dma_common_contiguous_remap(page, size, VM_USERMAP, -+ prot, -+ __builtin_return_address(0)); -+ if (addr) { -+ if (!coherent) -+ arch_dma_prep_coherent(page, iosize); -+ memset(addr, 0, size); -+ } else { -+ __iommu_dma_unmap_page(dev, *handle, iosize, 0, attrs); -+ dma_release_from_contiguous(dev, page, -+ size >> PAGE_SHIFT); -+ } -+ } else { -+ pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); -+ struct page **pages; -+ -+ pages = __iommu_dma_alloc(dev, iosize, gfp, attrs, ioprot, -+ handle); -+ if (!pages) -+ return NULL; -+ -+ addr = dma_common_pages_remap(pages, size, VM_USERMAP, prot, -+ __builtin_return_address(0)); -+ if (!addr) -+ __iommu_dma_free(dev, pages, iosize, handle); -+ } -+ return addr; -+} -+ -+static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, -+ dma_addr_t handle, unsigned long attrs) -+{ -+ size_t iosize = size; -+ -+ size = PAGE_ALIGN(size); -+ /* -+ * @cpu_addr will be one of 4 things depending on how it was allocated: -+ * - A remapped array of pages for contiguous allocations. -+ * - A remapped array of pages from __iommu_dma_alloc(), for all -+ * non-atomic allocations. -+ * - A non-cacheable alias from the atomic pool, for atomic -+ * allocations by non-coherent devices. -+ * - A normal lowmem address, for atomic allocations by -+ * coherent devices. -+ * Hence how dodgy the below logic looks... -+ */ -+ if (dma_in_atomic_pool(cpu_addr, size)) { -+ __iommu_dma_unmap_page(dev, handle, iosize, 0, 0); -+ dma_free_from_pool(cpu_addr, size); -+ } else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { -+ struct page *page = vmalloc_to_page(cpu_addr); -+ -+ __iommu_dma_unmap_page(dev, handle, iosize, 0, attrs); -+ dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); -+ dma_common_free_remap(cpu_addr, size, VM_USERMAP); -+ } else if (is_vmalloc_addr(cpu_addr)){ -+ struct vm_struct *area = find_vm_area(cpu_addr); -+ -+ if (WARN_ON(!area || !area->pages)) -+ return; -+ __iommu_dma_free(dev, area->pages, iosize, &handle); -+ dma_common_free_remap(cpu_addr, size, VM_USERMAP); -+ } else { -+ __iommu_dma_unmap_page(dev, handle, iosize, 0, 0); -+ __free_pages(virt_to_page(cpu_addr), get_order(size)); -+ } -+} -+ -+static int __iommu_dma_mmap_pfn(struct vm_area_struct *vma, -+ unsigned long pfn, size_t size) -+{ -+ return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, -+ vma->vm_end - vma->vm_start, -+ vma->vm_page_prot); -+} -+ -+static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, -+ void *cpu_addr, dma_addr_t dma_addr, size_t size, -+ unsigned long attrs) -+{ -+ unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; -+ unsigned long off = vma->vm_pgoff; -+ struct vm_struct *area; -+ int ret; -+ -+ vma->vm_page_prot = arch_dma_mmap_pgprot(dev, vma->vm_page_prot, attrs); -+ -+ if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) -+ return ret; -+ -+ if (off >= nr_pages || vma_pages(vma) > nr_pages - off) -+ return -ENXIO; -+ -+ if (!is_vmalloc_addr(cpu_addr)) { -+ unsigned long pfn = page_to_pfn(virt_to_page(cpu_addr)); -+ return __iommu_dma_mmap_pfn(vma, pfn, size); -+ } -+ -+ if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { -+ /* -+ * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped, -+ * hence in the vmalloc space. -+ */ -+ unsigned long pfn = vmalloc_to_pfn(cpu_addr); -+ return __iommu_dma_mmap_pfn(vma, pfn, size); -+ } -+ -+ area = find_vm_area(cpu_addr); -+ if (WARN_ON(!area || !area->pages)) -+ return -ENXIO; -+ -+ return __iommu_dma_mmap(area->pages, size, vma); -+} -+ -+static int __iommu_dma_get_sgtable_page(struct sg_table *sgt, struct page *page, -+ size_t size) -+{ -+ int ret = sg_alloc_table(sgt, 1, GFP_KERNEL); -+ -+ if (!ret) -+ sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); -+ return ret; -+} -+ -+static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, -+ void *cpu_addr, dma_addr_t dma_addr, size_t size, -+ unsigned long attrs) -+{ -+ unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; -+ struct vm_struct *area = find_vm_area(cpu_addr); -+ -+ if (!is_vmalloc_addr(cpu_addr)) { -+ struct page *page = virt_to_page(cpu_addr); -+ return __iommu_dma_get_sgtable_page(sgt, page, size); -+ } -+ -+ if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { -+ /* -+ * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped, -+ * hence in the vmalloc space. -+ */ -+ struct page *page = vmalloc_to_page(cpu_addr); -+ return __iommu_dma_get_sgtable_page(sgt, page, size); -+ } -+ -+ if (WARN_ON(!area || !area->pages)) -+ return -ENXIO; -+ -+ return sg_alloc_table_from_pages(sgt, area->pages, count, 0, size, -+ GFP_KERNEL); -+} -+ -+static const struct dma_map_ops iommu_dma_ops = { -+ .alloc = iommu_dma_alloc, -+ .free = iommu_dma_free, -+ .mmap = iommu_dma_mmap, -+ .get_sgtable = iommu_dma_get_sgtable, -+ .map_page = iommu_dma_map_page, -+ .unmap_page = iommu_dma_unmap_page, -+ .map_sg = iommu_dma_map_sg, -+ .unmap_sg = iommu_dma_unmap_sg, -+ .sync_single_for_cpu = iommu_dma_sync_single_for_cpu, -+ .sync_single_for_device = iommu_dma_sync_single_for_device, -+ .sync_sg_for_cpu = iommu_dma_sync_sg_for_cpu, -+ .sync_sg_for_device = iommu_dma_sync_sg_for_device, -+ .map_resource = iommu_dma_map_resource, -+ .unmap_resource = iommu_dma_unmap_resource, -+}; -+ -+/* -+ * The IOMMU core code allocates the default DMA domain, which the underlying -+ * IOMMU driver needs to support via the dma-iommu layer. -+ */ -+void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size) -+{ -+ struct iommu_domain *domain = iommu_get_domain_for_dev(dev); -+ -+ if (!domain) -+ goto out_err; -+ -+ /* -+ * The IOMMU core code allocates the default DMA domain, which the -+ * underlying IOMMU driver needs to support via the dma-iommu layer. -+ */ -+ if (domain->type == IOMMU_DOMAIN_DMA) { -+ if (iommu_dma_init_domain(domain, dma_base, size, dev)) -+ goto out_err; -+ dev->dma_ops = &iommu_dma_ops; -+ } -+ -+ return; -+out_err: -+ pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n", -+ dev_name(dev)); -+} -+ - static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, - phys_addr_t msi_addr, struct iommu_domain *domain) - { -@@ -921,3 +1247,9 @@ void iommu_dma_map_msi_msg(int irq, struct msi_msg *msg) - msg->address_lo += lower_32_bits(msi_page->iova); - } - } -+ -+static int iommu_dma_init(void) -+{ -+ return iova_cache_get(); -+} -+arch_initcall(iommu_dma_init); -diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h -index 3216447178a7..dadf4383f555 100644 ---- a/include/linux/dma-iommu.h -+++ b/include/linux/dma-iommu.h -@@ -24,49 +24,13 @@ - #include <linux/iommu.h> - #include <linux/msi.h> - --int iommu_dma_init(void); -- - /* Domain management interface for IOMMU drivers */ - int iommu_get_dma_cookie(struct iommu_domain *domain); - int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base); - void iommu_put_dma_cookie(struct iommu_domain *domain); - - /* Setup call for arch DMA mapping code */ --int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, -- u64 size, struct device *dev); -- --/* General helpers for DMA-API <-> IOMMU-API interaction */ --int dma_info_to_prot(enum dma_data_direction dir, bool coherent, -- unsigned long attrs); -- --/* -- * These implement the bulk of the relevant DMA mapping callbacks, but require -- * the arch code to take care of attributes and cache maintenance -- */ --struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, -- unsigned long attrs, int prot, dma_addr_t *handle); --void iommu_dma_free(struct device *dev, struct page **pages, size_t size, -- dma_addr_t *handle); -- --int iommu_dma_mmap(struct page **pages, size_t size, struct vm_area_struct *vma); -- --dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, -- unsigned long offset, size_t size, int prot); --int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, -- int nents, int prot); -- --/* -- * Arch code with no special attribute handling may use these -- * directly as DMA mapping callbacks for simplicity -- */ --void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, -- enum dma_data_direction dir, unsigned long attrs); --void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, -- enum dma_data_direction dir, unsigned long attrs); --dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, -- size_t size, enum dma_data_direction dir, unsigned long attrs); --void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, -- size_t size, enum dma_data_direction dir, unsigned long attrs); -+void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size); - - /* The DMA API isn't _quite_ the whole story, though... */ - void iommu_dma_map_msi_msg(int irq, struct msi_msg *msg); -@@ -75,12 +39,13 @@ void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list); - #else /* CONFIG_IOMMU_DMA */ - - struct iommu_domain; -+struct iommu_ops; - struct msi_msg; - struct device; - --static inline int iommu_dma_init(void) -+static inline void iommu_setup_dma_ops(struct device *dev, u64 dma_base, -+ u64 size) - { -- return 0; - } - - static inline int iommu_get_dma_cookie(struct iommu_domain *domain) --- -2.20.1 +On Mon, 22 Apr 2019 19:59:23 +0200 Christoph Hellwig wrote: +> @@ -744,18 +816,22 @@ static void __invalidate_sg(struct scatterlist *sg, int nents) +> * impedance-matching, to be able to hand off a suitably-aligned list, +> * but still preserve the original offsets and sizes for the caller. +> */ +> -int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, +> - int nents, int prot) +> +static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, +> + int nents, enum dma_data_direction dir, unsigned long attrs) +> { +> struct iommu_domain *domain = iommu_get_dma_domain(dev); +> struct iommu_dma_cookie *cookie = domain->iova_cookie; +> struct iova_domain *iovad = &cookie->iovad; +> struct scatterlist *s, *prev = NULL; +> + int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs); +> dma_addr_t iova; +> size_t iova_len = 0; +> unsigned long mask = dma_get_seg_boundary(dev); +> int i; +> +> + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) +> + iommu_dma_sync_sg_for_device(dev, sg, nents, dir); +> + +> /* +> * Work out how much IOVA space we need, and align the segments to +> * IOVA granules for the IOMMU driver to handle. With some clever +> @@ -815,12 +891,16 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, +> return 0; +> } +> +> -void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, +> - enum dma_data_direction dir, unsigned long attrs) +> +static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, +> + int nents, enum dma_data_direction dir, unsigned long attrs) +> { +> dma_addr_t start, end; +> struct scatterlist *tmp; +> int i; +> + +> + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) +> + iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir); +> + +Is it a typo? + +> /* +> * The scatterlist segments are mapped into a single +> * contiguous IOVA allocation, so this is incredibly easy. +[...] +> + +> +/* +> + * The IOMMU core code allocates the default DMA domain, which the underlying +> + * IOMMU driver needs to support via the dma-iommu layer. +> + */ +Over comment. + +> +void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size) +> +{ +> + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); +> + +> + if (!domain) +> + goto out_err; +> + +> + /* +> + * The IOMMU core code allocates the default DMA domain, which the +> + * underlying IOMMU driver needs to support via the dma-iommu layer. +> + */ +> + if (domain->type == IOMMU_DOMAIN_DMA) { +> + if (iommu_dma_init_domain(domain, dma_base, size, dev)) +> + goto out_err; +> + dev->dma_ops = &iommu_dma_ops; +> + } +> + +> + return; +> +out_err: +> + pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n", +> + dev_name(dev)); +> +} +> + + +BR +Hillf + +_______________________________________________ +iommu mailing list +iommu@lists.linux-foundation.org +https://lists.linuxfoundation.org/mailman/listinfo/iommu diff --git a/a/content_digest b/N2/content_digest index 3bff309..7c7e5a6 100644 --- a/a/content_digest +++ b/N2/content_digest @@ -1,1081 +1,105 @@ "ref\020190422175942.18788-1-hch@lst.de\0" - "From\0Christoph Hellwig <hch@lst.de>\0" - "Subject\0[PATCH 07/26] iommu/dma: move the arm64 wrappers to common code\0" - "Date\0Mon, 22 Apr 2019 19:59:23 +0200\0" - "To\0Robin Murphy <robin.murphy@arm.com>\0" - "Cc\0Joerg Roedel <joro@8bytes.org>" + "From\0Hillf Danton <hdanton@sina.com>\0" + "Subject\0Re: [PATCH 07/26] iommu/dma: move the arm64 wrappers to common code\0" + "Date\0Wed, 5 Jun 2019 08:47:17 +0800\0" + "To\0Christoph Hellwig <hch@lst.de>\0" + "Cc\0Tom Lendacky <thomas.lendacky@amd.com>" Catalin Marinas <catalin.marinas@arm.com> Will Deacon <will.deacon@arm.com> - Tom Lendacky <thomas.lendacky@amd.com> + linux-kernel@vger.kernel.org iommu@lists.linux-foundation.org - linux-arm-kernel@lists.infradead.org - " linux-kernel@vger.kernel.org\0" + Robin Murphy <robin.murphy@arm.com> + " linux-arm-kernel@lists.infradead.org\0" "\00:1\0" "b\0" - "There is nothing really arm64 specific in the iommu_dma_ops\n" - "implementation, so move it to dma-iommu.c and keep a lot of symbols\n" - "self-contained. Note the implementation does depend on the\n" - "DMA_DIRECT_REMAP infrastructure for now, so we'll have to make the\n" - "DMA_IOMMU support depend on it, but this will be relaxed soon.\n" "\n" - "Signed-off-by: Christoph Hellwig <hch@lst.de>\n" - "---\n" - " arch/arm64/mm/dma-mapping.c | 389 +-----------------------------------\n" - " drivers/iommu/Kconfig | 1 +\n" - " drivers/iommu/dma-iommu.c | 388 ++++++++++++++++++++++++++++++++---\n" - " include/linux/dma-iommu.h | 43 +---\n" - " 4 files changed, 369 insertions(+), 452 deletions(-)\n" + "Hi Christoph\n" "\n" - "diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c\n" - "index 636fa7c64370..d1661f78eb4d 100644\n" - "--- a/arch/arm64/mm/dma-mapping.c\n" - "+++ b/arch/arm64/mm/dma-mapping.c\n" - "@@ -27,6 +27,7 @@\n" - " #include <linux/dma-direct.h>\n" - " #include <linux/dma-noncoherent.h>\n" - " #include <linux/dma-contiguous.h>\n" - "+#include <linux/dma-iommu.h>\n" - " #include <linux/vmalloc.h>\n" - " #include <linux/swiotlb.h>\n" - " #include <linux/pci.h>\n" - "@@ -58,27 +59,6 @@ void arch_dma_prep_coherent(struct page *page, size_t size)\n" - " \t__dma_flush_area(page_address(page), size);\n" - " }\n" - " \n" - "-#ifdef CONFIG_IOMMU_DMA\n" - "-static int __swiotlb_get_sgtable_page(struct sg_table *sgt,\n" - "-\t\t\t\t struct page *page, size_t size)\n" - "-{\n" - "-\tint ret = sg_alloc_table(sgt, 1, GFP_KERNEL);\n" - "-\n" - "-\tif (!ret)\n" - "-\t\tsg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);\n" - "-\n" - "-\treturn ret;\n" - "-}\n" - "-\n" - "-static int __swiotlb_mmap_pfn(struct vm_area_struct *vma,\n" - "-\t\t\t unsigned long pfn, size_t size)\n" - "-{\n" - "-\treturn remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,\n" - "-\t\t\t vma->vm_end - vma->vm_start,\n" - "-\t\t\t vma->vm_page_prot);\n" - "-}\n" - "-#endif /* CONFIG_IOMMU_DMA */\n" - "-\n" - " static int __init arm64_dma_init(void)\n" - " {\n" - " \tWARN_TAINT(ARCH_DMA_MINALIGN < cache_line_size(),\n" - "@@ -90,379 +70,18 @@ static int __init arm64_dma_init(void)\n" - " arch_initcall(arm64_dma_init);\n" - " \n" - " #ifdef CONFIG_IOMMU_DMA\n" - "-#include <linux/dma-iommu.h>\n" - "-#include <linux/platform_device.h>\n" - "-#include <linux/amba/bus.h>\n" - "-\n" - "-static void *__iommu_alloc_attrs(struct device *dev, size_t size,\n" - "-\t\t\t\t dma_addr_t *handle, gfp_t gfp,\n" - "-\t\t\t\t unsigned long attrs)\n" - "-{\n" - "-\tbool coherent = dev_is_dma_coherent(dev);\n" - "-\tint ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);\n" - "-\tsize_t iosize = size;\n" - "-\tvoid *addr;\n" - "-\n" - "-\tif (WARN(!dev, \"cannot create IOMMU mapping for unknown device\\n\"))\n" - "-\t\treturn NULL;\n" - "-\n" - "-\tsize = PAGE_ALIGN(size);\n" - "-\n" - "-\t/*\n" - "-\t * Some drivers rely on this, and we probably don't want the\n" - "-\t * possibility of stale kernel data being read by devices anyway.\n" - "-\t */\n" - "-\tgfp |= __GFP_ZERO;\n" - "-\n" - "-\tif (!gfpflags_allow_blocking(gfp)) {\n" - "-\t\tstruct page *page;\n" - "-\t\t/*\n" - "-\t\t * In atomic context we can't remap anything, so we'll only\n" - "-\t\t * get the virtually contiguous buffer we need by way of a\n" - "-\t\t * physically contiguous allocation.\n" - "-\t\t */\n" - "-\t\tif (coherent) {\n" - "-\t\t\tpage = alloc_pages(gfp, get_order(size));\n" - "-\t\t\taddr = page ? page_address(page) : NULL;\n" - "-\t\t} else {\n" - "-\t\t\taddr = dma_alloc_from_pool(size, &page, gfp);\n" - "-\t\t}\n" - "-\t\tif (!addr)\n" - "-\t\t\treturn NULL;\n" - "-\n" - "-\t\t*handle = iommu_dma_map_page(dev, page, 0, iosize, ioprot);\n" - "-\t\tif (*handle == DMA_MAPPING_ERROR) {\n" - "-\t\t\tif (coherent)\n" - "-\t\t\t\t__free_pages(page, get_order(size));\n" - "-\t\t\telse\n" - "-\t\t\t\tdma_free_from_pool(addr, size);\n" - "-\t\t\taddr = NULL;\n" - "-\t\t}\n" - "-\t} else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {\n" - "-\t\tpgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs);\n" - "-\t\tstruct page *page;\n" - "-\n" - "-\t\tpage = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,\n" - "-\t\t\t\t\tget_order(size), gfp & __GFP_NOWARN);\n" - "-\t\tif (!page)\n" - "-\t\t\treturn NULL;\n" - "-\n" - "-\t\t*handle = iommu_dma_map_page(dev, page, 0, iosize, ioprot);\n" - "-\t\tif (*handle == DMA_MAPPING_ERROR) {\n" - "-\t\t\tdma_release_from_contiguous(dev, page,\n" - "-\t\t\t\t\t\t size >> PAGE_SHIFT);\n" - "-\t\t\treturn NULL;\n" - "-\t\t}\n" - "-\t\taddr = dma_common_contiguous_remap(page, size, VM_USERMAP,\n" - "-\t\t\t\t\t\t prot,\n" - "-\t\t\t\t\t\t __builtin_return_address(0));\n" - "-\t\tif (addr) {\n" - "-\t\t\tif (!coherent)\n" - "-\t\t\t\t__dma_flush_area(page_to_virt(page), iosize);\n" - "-\t\t\tmemset(addr, 0, size);\n" - "-\t\t} else {\n" - "-\t\t\tiommu_dma_unmap_page(dev, *handle, iosize, 0, attrs);\n" - "-\t\t\tdma_release_from_contiguous(dev, page,\n" - "-\t\t\t\t\t\t size >> PAGE_SHIFT);\n" - "-\t\t}\n" - "-\t} else {\n" - "-\t\tpgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs);\n" - "-\t\tstruct page **pages;\n" - "-\n" - "-\t\tpages = iommu_dma_alloc(dev, iosize, gfp, attrs, ioprot,\n" - "-\t\t\t\t\thandle);\n" - "-\t\tif (!pages)\n" - "-\t\t\treturn NULL;\n" - "-\n" - "-\t\taddr = dma_common_pages_remap(pages, size, VM_USERMAP, prot,\n" - "-\t\t\t\t\t __builtin_return_address(0));\n" - "-\t\tif (!addr)\n" - "-\t\t\tiommu_dma_free(dev, pages, iosize, handle);\n" - "-\t}\n" - "-\treturn addr;\n" - "-}\n" - "-\n" - "-static void __iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,\n" - "-\t\t\t dma_addr_t handle, unsigned long attrs)\n" - "-{\n" - "-\tsize_t iosize = size;\n" - "-\n" - "-\tsize = PAGE_ALIGN(size);\n" - "-\t/*\n" - "-\t * @cpu_addr will be one of 4 things depending on how it was allocated:\n" - "-\t * - A remapped array of pages for contiguous allocations.\n" - "-\t * - A remapped array of pages from iommu_dma_alloc(), for all\n" - "-\t * non-atomic allocations.\n" - "-\t * - A non-cacheable alias from the atomic pool, for atomic\n" - "-\t * allocations by non-coherent devices.\n" - "-\t * - A normal lowmem address, for atomic allocations by\n" - "-\t * coherent devices.\n" - "-\t * Hence how dodgy the below logic looks...\n" - "-\t */\n" - "-\tif (dma_in_atomic_pool(cpu_addr, size)) {\n" - "-\t\tiommu_dma_unmap_page(dev, handle, iosize, 0, 0);\n" - "-\t\tdma_free_from_pool(cpu_addr, size);\n" - "-\t} else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {\n" - "-\t\tstruct page *page = vmalloc_to_page(cpu_addr);\n" - "-\n" - "-\t\tiommu_dma_unmap_page(dev, handle, iosize, 0, attrs);\n" - "-\t\tdma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);\n" - "-\t\tdma_common_free_remap(cpu_addr, size, VM_USERMAP);\n" - "-\t} else if (is_vmalloc_addr(cpu_addr)){\n" - "-\t\tstruct vm_struct *area = find_vm_area(cpu_addr);\n" - "-\n" - "-\t\tif (WARN_ON(!area || !area->pages))\n" - "-\t\t\treturn;\n" - "-\t\tiommu_dma_free(dev, area->pages, iosize, &handle);\n" - "-\t\tdma_common_free_remap(cpu_addr, size, VM_USERMAP);\n" - "-\t} else {\n" - "-\t\tiommu_dma_unmap_page(dev, handle, iosize, 0, 0);\n" - "-\t\t__free_pages(virt_to_page(cpu_addr), get_order(size));\n" - "-\t}\n" - "-}\n" - "-\n" - "-static int __iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,\n" - "-\t\t\t void *cpu_addr, dma_addr_t dma_addr, size_t size,\n" - "-\t\t\t unsigned long attrs)\n" - "-{\n" - "-\tunsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;\n" - "-\tunsigned long off = vma->vm_pgoff;\n" - "-\tstruct vm_struct *area;\n" - "-\tint ret;\n" - "-\n" - "-\tvma->vm_page_prot = arch_dma_mmap_pgprot(dev, vma->vm_page_prot, attrs);\n" - "-\n" - "-\tif (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))\n" - "-\t\treturn ret;\n" - "-\n" - "-\tif (off >= nr_pages || vma_pages(vma) > nr_pages - off)\n" - "-\t\treturn -ENXIO;\n" - "-\n" - "-\tif (!is_vmalloc_addr(cpu_addr)) {\n" - "-\t\tunsigned long pfn = page_to_pfn(virt_to_page(cpu_addr));\n" - "-\t\treturn __swiotlb_mmap_pfn(vma, pfn, size);\n" - "-\t}\n" - "-\n" - "-\tif (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {\n" - "-\t\t/*\n" - "-\t\t * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped,\n" - "-\t\t * hence in the vmalloc space.\n" - "-\t\t */\n" - "-\t\tunsigned long pfn = vmalloc_to_pfn(cpu_addr);\n" - "-\t\treturn __swiotlb_mmap_pfn(vma, pfn, size);\n" - "-\t}\n" - "-\n" - "-\tarea = find_vm_area(cpu_addr);\n" - "-\tif (WARN_ON(!area || !area->pages))\n" - "-\t\treturn -ENXIO;\n" - "-\n" - "-\treturn iommu_dma_mmap(area->pages, size, vma);\n" - "-}\n" - "-\n" - "-static int __iommu_get_sgtable(struct device *dev, struct sg_table *sgt,\n" - "-\t\t\t void *cpu_addr, dma_addr_t dma_addr,\n" - "-\t\t\t size_t size, unsigned long attrs)\n" - "-{\n" - "-\tunsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;\n" - "-\tstruct vm_struct *area = find_vm_area(cpu_addr);\n" - "-\n" - "-\tif (!is_vmalloc_addr(cpu_addr)) {\n" - "-\t\tstruct page *page = virt_to_page(cpu_addr);\n" - "-\t\treturn __swiotlb_get_sgtable_page(sgt, page, size);\n" - "-\t}\n" - "-\n" - "-\tif (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {\n" - "-\t\t/*\n" - "-\t\t * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped,\n" - "-\t\t * hence in the vmalloc space.\n" - "-\t\t */\n" - "-\t\tstruct page *page = vmalloc_to_page(cpu_addr);\n" - "-\t\treturn __swiotlb_get_sgtable_page(sgt, page, size);\n" - "-\t}\n" - "-\n" - "-\tif (WARN_ON(!area || !area->pages))\n" - "-\t\treturn -ENXIO;\n" - "-\n" - "-\treturn sg_alloc_table_from_pages(sgt, area->pages, count, 0, size,\n" - "-\t\t\t\t\t GFP_KERNEL);\n" - "-}\n" - "-\n" - "-static void __iommu_sync_single_for_cpu(struct device *dev,\n" - "-\t\t\t\t\tdma_addr_t dev_addr, size_t size,\n" - "-\t\t\t\t\tenum dma_data_direction dir)\n" - "-{\n" - "-\tphys_addr_t phys;\n" - "-\n" - "-\tif (dev_is_dma_coherent(dev))\n" - "-\t\treturn;\n" - "-\n" - "-\tphys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dev_addr);\n" - "-\tarch_sync_dma_for_cpu(dev, phys, size, dir);\n" - "-}\n" - "-\n" - "-static void __iommu_sync_single_for_device(struct device *dev,\n" - "-\t\t\t\t\t dma_addr_t dev_addr, size_t size,\n" - "-\t\t\t\t\t enum dma_data_direction dir)\n" - "-{\n" - "-\tphys_addr_t phys;\n" - "-\n" - "-\tif (dev_is_dma_coherent(dev))\n" - "-\t\treturn;\n" - "-\n" - "-\tphys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dev_addr);\n" - "-\tarch_sync_dma_for_device(dev, phys, size, dir);\n" - "-}\n" - "-\n" - "-static dma_addr_t __iommu_map_page(struct device *dev, struct page *page,\n" - "-\t\t\t\t unsigned long offset, size_t size,\n" - "-\t\t\t\t enum dma_data_direction dir,\n" - "-\t\t\t\t unsigned long attrs)\n" - "-{\n" - "-\tbool coherent = dev_is_dma_coherent(dev);\n" - "-\tint prot = dma_info_to_prot(dir, coherent, attrs);\n" - "-\tdma_addr_t dev_addr = iommu_dma_map_page(dev, page, offset, size, prot);\n" - "-\n" - "-\tif (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&\n" - "-\t dev_addr != DMA_MAPPING_ERROR)\n" - "-\t\t__dma_map_area(page_address(page) + offset, size, dir);\n" - "-\n" - "-\treturn dev_addr;\n" - "-}\n" - "-\n" - "-static void __iommu_unmap_page(struct device *dev, dma_addr_t dev_addr,\n" - "-\t\t\t size_t size, enum dma_data_direction dir,\n" - "-\t\t\t unsigned long attrs)\n" - "-{\n" - "-\tif ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)\n" - "-\t\t__iommu_sync_single_for_cpu(dev, dev_addr, size, dir);\n" - "-\n" - "-\tiommu_dma_unmap_page(dev, dev_addr, size, dir, attrs);\n" - "-}\n" - "-\n" - "-static void __iommu_sync_sg_for_cpu(struct device *dev,\n" - "-\t\t\t\t struct scatterlist *sgl, int nelems,\n" - "-\t\t\t\t enum dma_data_direction dir)\n" - "-{\n" - "-\tstruct scatterlist *sg;\n" - "-\tint i;\n" - "-\n" - "-\tif (dev_is_dma_coherent(dev))\n" - "-\t\treturn;\n" - "-\n" - "-\tfor_each_sg(sgl, sg, nelems, i)\n" - "-\t\tarch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir);\n" - "-}\n" - "-\n" - "-static void __iommu_sync_sg_for_device(struct device *dev,\n" - "-\t\t\t\t struct scatterlist *sgl, int nelems,\n" - "-\t\t\t\t enum dma_data_direction dir)\n" - "-{\n" - "-\tstruct scatterlist *sg;\n" - "-\tint i;\n" - "-\n" - "-\tif (dev_is_dma_coherent(dev))\n" - "-\t\treturn;\n" - "-\n" - "-\tfor_each_sg(sgl, sg, nelems, i)\n" - "-\t\tarch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir);\n" - "-}\n" - "-\n" - "-static int __iommu_map_sg_attrs(struct device *dev, struct scatterlist *sgl,\n" - "-\t\t\t\tint nelems, enum dma_data_direction dir,\n" - "-\t\t\t\tunsigned long attrs)\n" - "-{\n" - "-\tbool coherent = dev_is_dma_coherent(dev);\n" - "-\n" - "-\tif ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)\n" - "-\t\t__iommu_sync_sg_for_device(dev, sgl, nelems, dir);\n" - "-\n" - "-\treturn iommu_dma_map_sg(dev, sgl, nelems,\n" - "-\t\t\t\tdma_info_to_prot(dir, coherent, attrs));\n" - "-}\n" - "-\n" - "-static void __iommu_unmap_sg_attrs(struct device *dev,\n" - "-\t\t\t\t struct scatterlist *sgl, int nelems,\n" - "-\t\t\t\t enum dma_data_direction dir,\n" - "-\t\t\t\t unsigned long attrs)\n" - "-{\n" - "-\tif ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)\n" - "-\t\t__iommu_sync_sg_for_cpu(dev, sgl, nelems, dir);\n" - "-\n" - "-\tiommu_dma_unmap_sg(dev, sgl, nelems, dir, attrs);\n" - "-}\n" - "-\n" - "-static const struct dma_map_ops iommu_dma_ops = {\n" - "-\t.alloc = __iommu_alloc_attrs,\n" - "-\t.free = __iommu_free_attrs,\n" - "-\t.mmap = __iommu_mmap_attrs,\n" - "-\t.get_sgtable = __iommu_get_sgtable,\n" - "-\t.map_page = __iommu_map_page,\n" - "-\t.unmap_page = __iommu_unmap_page,\n" - "-\t.map_sg = __iommu_map_sg_attrs,\n" - "-\t.unmap_sg = __iommu_unmap_sg_attrs,\n" - "-\t.sync_single_for_cpu = __iommu_sync_single_for_cpu,\n" - "-\t.sync_single_for_device = __iommu_sync_single_for_device,\n" - "-\t.sync_sg_for_cpu = __iommu_sync_sg_for_cpu,\n" - "-\t.sync_sg_for_device = __iommu_sync_sg_for_device,\n" - "-\t.map_resource = iommu_dma_map_resource,\n" - "-\t.unmap_resource = iommu_dma_unmap_resource,\n" - "-};\n" - "-\n" - "-static int __init __iommu_dma_init(void)\n" - "-{\n" - "-\treturn iommu_dma_init();\n" - "-}\n" - "-arch_initcall(__iommu_dma_init);\n" - "-\n" - "-static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,\n" - "-\t\t\t\t const struct iommu_ops *ops)\n" - "-{\n" - "-\tstruct iommu_domain *domain;\n" - "-\n" - "-\tif (!ops)\n" - "-\t\treturn;\n" - "-\n" - "-\t/*\n" - "-\t * The IOMMU core code allocates the default DMA domain, which the\n" - "-\t * underlying IOMMU driver needs to support via the dma-iommu layer.\n" - "-\t */\n" - "-\tdomain = iommu_get_domain_for_dev(dev);\n" - "-\n" - "-\tif (!domain)\n" - "-\t\tgoto out_err;\n" - "-\n" - "-\tif (domain->type == IOMMU_DOMAIN_DMA) {\n" - "-\t\tif (iommu_dma_init_domain(domain, dma_base, size, dev))\n" - "-\t\t\tgoto out_err;\n" - "-\n" - "-\t\tdev->dma_ops = &iommu_dma_ops;\n" - "-\t}\n" - "-\n" - "-\treturn;\n" - "-\n" - "-out_err:\n" - "-\t pr_warn(\"Failed to set up IOMMU for device %s; retaining platform DMA ops\\n\",\n" - "-\t\t dev_name(dev));\n" - "-}\n" - "-\n" - " void arch_teardown_dma_ops(struct device *dev)\n" - " {\n" - " \tdev->dma_ops = NULL;\n" - " }\n" - "-\n" - "-#else\n" - "-\n" - "-static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,\n" - "-\t\t\t\t const struct iommu_ops *iommu)\n" - "-{ }\n" - "-\n" - "-#endif /* CONFIG_IOMMU_DMA */\n" - "+#endif\n" - " \n" - " void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,\n" - " \t\t\tconst struct iommu_ops *iommu, bool coherent)\n" - " {\n" - " \tdev->dma_coherent = coherent;\n" - "-\t__iommu_setup_dma_ops(dev, dma_base, size, iommu);\n" - "+\tif (iommu)\n" - "+\t\tiommu_setup_dma_ops(dev, dma_base, size);\n" - " \n" - " #ifdef CONFIG_XEN\n" - " \tif (xen_initial_domain())\n" - "diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig\n" - "index 6f07f3b21816..bdc14baf2ee5 100644\n" - "--- a/drivers/iommu/Kconfig\n" - "+++ b/drivers/iommu/Kconfig\n" - "@@ -95,6 +95,7 @@ config IOMMU_DMA\n" - " \tselect IOMMU_API\n" - " \tselect IOMMU_IOVA\n" - " \tselect NEED_SG_DMA_LENGTH\n" - "+\tdepends on DMA_DIRECT_REMAP\n" - " \n" - " config FSL_PAMU\n" - " \tbool \"Freescale IOMMU support\"\n" - "diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c\n" - "index f915cb7c46e6..622123551bba 100644\n" - "--- a/drivers/iommu/dma-iommu.c\n" - "+++ b/drivers/iommu/dma-iommu.c\n" - "@@ -21,6 +21,7 @@\n" - " \n" - " #include <linux/acpi_iort.h>\n" - " #include <linux/device.h>\n" - "+#include <linux/dma-contiguous.h>\n" - " #include <linux/dma-iommu.h>\n" - " #include <linux/dma-noncoherent.h>\n" - " #include <linux/gfp.h>\n" - "@@ -79,11 +80,6 @@ static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type)\n" - " \treturn cookie;\n" - " }\n" - " \n" - "-int iommu_dma_init(void)\n" - "-{\n" - "-\treturn iova_cache_get();\n" - "-}\n" - "-\n" - " /**\n" - " * iommu_get_dma_cookie - Acquire DMA-API resources for a domain\n" - " * @domain: IOMMU domain to prepare for DMA-API usage\n" - "@@ -285,7 +281,7 @@ static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad)\n" - " * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but\n" - " * any change which could make prior IOVAs invalid will fail.\n" - " */\n" - "-int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,\n" - "+static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,\n" - " \t\tu64 size, struct device *dev)\n" - " {\n" - " \tstruct iommu_dma_cookie *cookie = domain->iova_cookie;\n" - "@@ -336,7 +332,6 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,\n" - " \n" - " \treturn iova_reserve_iommu_regions(dev, domain);\n" - " }\n" - "-EXPORT_SYMBOL(iommu_dma_init_domain);\n" - " \n" - " /**\n" - " * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API\n" - "@@ -347,7 +342,7 @@ EXPORT_SYMBOL(iommu_dma_init_domain);\n" - " *\n" - " * Return: corresponding IOMMU API page protection flags\n" - " */\n" - "-int dma_info_to_prot(enum dma_data_direction dir, bool coherent,\n" - "+static int dma_info_to_prot(enum dma_data_direction dir, bool coherent,\n" - " \t\t unsigned long attrs)\n" - " {\n" - " \tint prot = coherent ? IOMMU_CACHE : 0;\n" - "@@ -506,17 +501,17 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev,\n" - " }\n" - " \n" - " /**\n" - "- * iommu_dma_free - Free a buffer allocated by iommu_dma_alloc()\n" - "+ * iommu_dma_free - Free a buffer allocated by __iommu_dma_alloc()\n" - " * @dev: Device which owns this buffer\n" - "- * @pages: Array of buffer pages as returned by iommu_dma_alloc()\n" - "+ * @pages: Array of buffer pages as returned by __iommu_dma_alloc()\n" - " * @size: Size of buffer in bytes\n" - " * @handle: DMA address of buffer\n" - " *\n" - " * Frees both the pages associated with the buffer, and the array\n" - " * describing them\n" - " */\n" - "-void iommu_dma_free(struct device *dev, struct page **pages, size_t size,\n" - "-\t\tdma_addr_t *handle)\n" - "+static void __iommu_dma_free(struct device *dev, struct page **pages,\n" - "+\t\tsize_t size, dma_addr_t *handle)\n" - " {\n" - " \t__iommu_dma_unmap(iommu_get_dma_domain(dev), *handle, size);\n" - " \t__iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT);\n" - "@@ -524,7 +519,7 @@ void iommu_dma_free(struct device *dev, struct page **pages, size_t size,\n" - " }\n" - " \n" - " /**\n" - "- * iommu_dma_alloc - Allocate and map a buffer contiguous in IOVA space\n" - "+ * __iommu_dma_alloc - Allocate and map a buffer contiguous in IOVA space\n" - " * @dev: Device to allocate memory for. Must be a real device\n" - " *\t attached to an iommu_dma_domain\n" - " * @size: Size of buffer in bytes\n" - "@@ -539,8 +534,8 @@ void iommu_dma_free(struct device *dev, struct page **pages, size_t size,\n" - " * Return: Array of struct page pointers describing the buffer,\n" - " *\t or NULL on failure.\n" - " */\n" - "-struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp,\n" - "-\t\tunsigned long attrs, int prot, dma_addr_t *handle)\n" - "+static struct page **__iommu_dma_alloc(struct device *dev, size_t size,\n" - "+\t\tgfp_t gfp, unsigned long attrs, int prot, dma_addr_t *handle)\n" - " {\n" - " \tstruct iommu_domain *domain = iommu_get_dma_domain(dev);\n" - " \tstruct iommu_dma_cookie *cookie = domain->iova_cookie;\n" - "@@ -602,16 +597,16 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp,\n" - " }\n" - " \n" - " /**\n" - "- * iommu_dma_mmap - Map a buffer into provided user VMA\n" - "- * @pages: Array representing buffer from iommu_dma_alloc()\n" - "+ * __iommu_dma_mmap - Map a buffer into provided user VMA\n" - "+ * @pages: Array representing buffer from __iommu_dma_alloc()\n" - " * @size: Size of buffer in bytes\n" - " * @vma: VMA describing requested userspace mapping\n" - " *\n" - " * Maps the pages of the buffer in @pages into @vma. The caller is responsible\n" - " * for verifying the correct size and protection of @vma beforehand.\n" - " */\n" - "-\n" - "-int iommu_dma_mmap(struct page **pages, size_t size, struct vm_area_struct *vma)\n" - "+static int __iommu_dma_mmap(struct page **pages, size_t size,\n" - "+\t\tstruct vm_area_struct *vma)\n" - " {\n" - " \tunsigned long uaddr = vma->vm_start;\n" - " \tunsigned int i, count = PAGE_ALIGN(size) >> PAGE_SHIFT;\n" - "@@ -626,6 +621,58 @@ int iommu_dma_mmap(struct page **pages, size_t size, struct vm_area_struct *vma)\n" - " \treturn ret;\n" - " }\n" - " \n" - "+static void iommu_dma_sync_single_for_cpu(struct device *dev,\n" - "+\t\tdma_addr_t dma_handle, size_t size, enum dma_data_direction dir)\n" - "+{\n" - "+\tphys_addr_t phys;\n" - "+\n" - "+\tif (dev_is_dma_coherent(dev))\n" - "+\t\treturn;\n" - "+\n" - "+\tphys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);\n" - "+\tarch_sync_dma_for_cpu(dev, phys, size, dir);\n" - "+}\n" - "+\n" - "+static void iommu_dma_sync_single_for_device(struct device *dev,\n" - "+\t\tdma_addr_t dma_handle, size_t size, enum dma_data_direction dir)\n" - "+{\n" - "+\tphys_addr_t phys;\n" - "+\n" - "+\tif (dev_is_dma_coherent(dev))\n" - "+\t\treturn;\n" - "+\n" - "+\tphys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);\n" - "+\tarch_sync_dma_for_device(dev, phys, size, dir);\n" - "+}\n" - "+\n" - "+static void iommu_dma_sync_sg_for_cpu(struct device *dev,\n" - "+\t\tstruct scatterlist *sgl, int nelems,\n" - "+\t\tenum dma_data_direction dir)\n" - "+{\n" - "+\tstruct scatterlist *sg;\n" - "+\tint i;\n" - "+\n" - "+\tif (dev_is_dma_coherent(dev))\n" - "+\t\treturn;\n" - "+\n" - "+\tfor_each_sg(sgl, sg, nelems, i)\n" - "+\t\tarch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir);\n" - "+}\n" - "+\n" - "+static void iommu_dma_sync_sg_for_device(struct device *dev,\n" - "+\t\tstruct scatterlist *sgl, int nelems,\n" - "+\t\tenum dma_data_direction dir)\n" - "+{\n" - "+\tstruct scatterlist *sg;\n" - "+\tint i;\n" - "+\n" - "+\tif (dev_is_dma_coherent(dev))\n" - "+\t\treturn;\n" - "+\n" - "+\tfor_each_sg(sgl, sg, nelems, i)\n" - "+\t\tarch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir);\n" - "+}\n" - "+\n" - " static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,\n" - " \t\tsize_t size, int prot, struct iommu_domain *domain)\n" - " {\n" - "@@ -649,19 +696,44 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,\n" - " \treturn iova + iova_off;\n" - " }\n" - " \n" - "-dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,\n" - "+static dma_addr_t __iommu_dma_map_page(struct device *dev, struct page *page,\n" - " \t\tunsigned long offset, size_t size, int prot)\n" - " {\n" - " \treturn __iommu_dma_map(dev, page_to_phys(page) + offset, size, prot,\n" - " \t\t\tiommu_get_dma_domain(dev));\n" - " }\n" - " \n" - "-void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size,\n" - "-\t\tenum dma_data_direction dir, unsigned long attrs)\n" - "+static void __iommu_dma_unmap_page(struct device *dev, dma_addr_t handle,\n" - "+\t\tsize_t size, enum dma_data_direction dir, unsigned long attrs)\n" - " {\n" - " \t__iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size);\n" - " }\n" - " \n" - "+static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,\n" - "+\t\tunsigned long offset, size_t size, enum dma_data_direction dir,\n" - "+\t\tunsigned long attrs)\n" - "+{\n" - "+\tphys_addr_t phys = page_to_phys(page) + offset;\n" - "+\tbool coherent = dev_is_dma_coherent(dev);\n" - "+\tdma_addr_t dma_handle;\n" - "+\n" - "+\tdma_handle =__iommu_dma_map(dev, phys, size,\n" - "+\t\t\tdma_info_to_prot(dir, coherent, attrs),\n" - "+\t\t\tiommu_get_dma_domain(dev));\n" - "+\tif (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&\n" - "+\t dma_handle != DMA_MAPPING_ERROR)\n" - "+\t\tarch_sync_dma_for_device(dev, phys, size, dir);\n" - "+\treturn dma_handle;\n" - "+}\n" - "+\n" - "+static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,\n" - "+\t\tsize_t size, enum dma_data_direction dir, unsigned long attrs)\n" - "+{\n" - "+\tif (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))\n" - "+\t\tiommu_dma_sync_single_for_cpu(dev, dma_handle, size, dir);\n" - "+\t__iommu_dma_unmap(iommu_get_dma_domain(dev), dma_handle, size);\n" - "+}\n" - "+\n" - " /*\n" - " * Prepare a successfully-mapped scatterlist to give back to the caller.\n" - " *\n" - "@@ -744,18 +816,22 @@ static void __invalidate_sg(struct scatterlist *sg, int nents)\n" - " * impedance-matching, to be able to hand off a suitably-aligned list,\n" - " * but still preserve the original offsets and sizes for the caller.\n" - " */\n" - "-int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,\n" - "-\t\tint nents, int prot)\n" - "+static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,\n" - "+\t\tint nents, enum dma_data_direction dir, unsigned long attrs)\n" - " {\n" - " \tstruct iommu_domain *domain = iommu_get_dma_domain(dev);\n" - " \tstruct iommu_dma_cookie *cookie = domain->iova_cookie;\n" - " \tstruct iova_domain *iovad = &cookie->iovad;\n" - " \tstruct scatterlist *s, *prev = NULL;\n" - "+\tint prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs);\n" - " \tdma_addr_t iova;\n" - " \tsize_t iova_len = 0;\n" - " \tunsigned long mask = dma_get_seg_boundary(dev);\n" - " \tint i;\n" - " \n" - "+\tif (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))\n" - "+\t\tiommu_dma_sync_sg_for_device(dev, sg, nents, dir);\n" - "+\n" - " \t/*\n" - " \t * Work out how much IOVA space we need, and align the segments to\n" - " \t * IOVA granules for the IOMMU driver to handle. With some clever\n" - "@@ -815,12 +891,16 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,\n" - " \treturn 0;\n" - " }\n" - " \n" - "-void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,\n" - "-\t\tenum dma_data_direction dir, unsigned long attrs)\n" - "+static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,\n" - "+\t\tint nents, enum dma_data_direction dir, unsigned long attrs)\n" - " {\n" - " \tdma_addr_t start, end;\n" - " \tstruct scatterlist *tmp;\n" - " \tint i;\n" - "+\n" - "+\tif (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)\n" - "+\t\tiommu_dma_sync_sg_for_cpu(dev, sg, nents, dir);\n" - "+\n" - " \t/*\n" - " \t * The scatterlist segments are mapped into a single\n" - " \t * contiguous IOVA allocation, so this is incredibly easy.\n" - "@@ -835,7 +915,7 @@ void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,\n" - " \t__iommu_dma_unmap(iommu_get_dma_domain(dev), start, end - start);\n" - " }\n" - " \n" - "-dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,\n" - "+static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,\n" - " \t\tsize_t size, enum dma_data_direction dir, unsigned long attrs)\n" - " {\n" - " \treturn __iommu_dma_map(dev, phys, size,\n" - "@@ -843,12 +923,258 @@ dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,\n" - " \t\t\tiommu_get_dma_domain(dev));\n" - " }\n" - " \n" - "-void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,\n" - "+static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,\n" - " \t\tsize_t size, enum dma_data_direction dir, unsigned long attrs)\n" - " {\n" - " \t__iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size);\n" - " }\n" - " \n" - "+static void *iommu_dma_alloc(struct device *dev, size_t size,\n" - "+\t\tdma_addr_t *handle, gfp_t gfp, unsigned long attrs)\n" - "+{\n" - "+\tbool coherent = dev_is_dma_coherent(dev);\n" - "+\tint ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);\n" - "+\tsize_t iosize = size;\n" - "+\tvoid *addr;\n" - "+\n" - "+\tsize = PAGE_ALIGN(size);\n" - "+\tgfp |= __GFP_ZERO;\n" - "+\n" - "+\tif (!gfpflags_allow_blocking(gfp)) {\n" - "+\t\tstruct page *page;\n" - "+\t\t/*\n" - "+\t\t * In atomic context we can't remap anything, so we'll only\n" - "+\t\t * get the virtually contiguous buffer we need by way of a\n" - "+\t\t * physically contiguous allocation.\n" - "+\t\t */\n" - "+\t\tif (coherent) {\n" - "+\t\t\tpage = alloc_pages(gfp, get_order(size));\n" - "+\t\t\taddr = page ? page_address(page) : NULL;\n" - "+\t\t} else {\n" - "+\t\t\taddr = dma_alloc_from_pool(size, &page, gfp);\n" - "+\t\t}\n" - "+\t\tif (!addr)\n" - "+\t\t\treturn NULL;\n" - "+\n" - "+\t\t*handle = __iommu_dma_map_page(dev, page, 0, iosize, ioprot);\n" - "+\t\tif (*handle == DMA_MAPPING_ERROR) {\n" - "+\t\t\tif (coherent)\n" - "+\t\t\t\t__free_pages(page, get_order(size));\n" - "+\t\t\telse\n" - "+\t\t\t\tdma_free_from_pool(addr, size);\n" - "+\t\t\taddr = NULL;\n" - "+\t\t}\n" - "+\t} else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {\n" - "+\t\tpgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs);\n" - "+\t\tstruct page *page;\n" - "+\n" - "+\t\tpage = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,\n" - "+\t\t\t\t\tget_order(size), gfp & __GFP_NOWARN);\n" - "+\t\tif (!page)\n" - "+\t\t\treturn NULL;\n" - "+\n" - "+\t\t*handle = __iommu_dma_map_page(dev, page, 0, iosize, ioprot);\n" - "+\t\tif (*handle == DMA_MAPPING_ERROR) {\n" - "+\t\t\tdma_release_from_contiguous(dev, page,\n" - "+\t\t\t\t\t\t size >> PAGE_SHIFT);\n" - "+\t\t\treturn NULL;\n" - "+\t\t}\n" - "+\t\taddr = dma_common_contiguous_remap(page, size, VM_USERMAP,\n" - "+\t\t\t\t\t\t prot,\n" - "+\t\t\t\t\t\t __builtin_return_address(0));\n" - "+\t\tif (addr) {\n" - "+\t\t\tif (!coherent)\n" - "+\t\t\t\tarch_dma_prep_coherent(page, iosize);\n" - "+\t\t\tmemset(addr, 0, size);\n" - "+\t\t} else {\n" - "+\t\t\t__iommu_dma_unmap_page(dev, *handle, iosize, 0, attrs);\n" - "+\t\t\tdma_release_from_contiguous(dev, page,\n" - "+\t\t\t\t\t\t size >> PAGE_SHIFT);\n" - "+\t\t}\n" - "+\t} else {\n" - "+\t\tpgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs);\n" - "+\t\tstruct page **pages;\n" - "+\n" - "+\t\tpages = __iommu_dma_alloc(dev, iosize, gfp, attrs, ioprot,\n" - "+\t\t\t\t\thandle);\n" - "+\t\tif (!pages)\n" - "+\t\t\treturn NULL;\n" - "+\n" - "+\t\taddr = dma_common_pages_remap(pages, size, VM_USERMAP, prot,\n" - "+\t\t\t\t\t __builtin_return_address(0));\n" - "+\t\tif (!addr)\n" - "+\t\t\t__iommu_dma_free(dev, pages, iosize, handle);\n" - "+\t}\n" - "+\treturn addr;\n" - "+}\n" - "+\n" - "+static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr,\n" - "+\t\tdma_addr_t handle, unsigned long attrs)\n" - "+{\n" - "+\tsize_t iosize = size;\n" - "+\n" - "+\tsize = PAGE_ALIGN(size);\n" - "+\t/*\n" - "+\t * @cpu_addr will be one of 4 things depending on how it was allocated:\n" - "+\t * - A remapped array of pages for contiguous allocations.\n" - "+\t * - A remapped array of pages from __iommu_dma_alloc(), for all\n" - "+\t * non-atomic allocations.\n" - "+\t * - A non-cacheable alias from the atomic pool, for atomic\n" - "+\t * allocations by non-coherent devices.\n" - "+\t * - A normal lowmem address, for atomic allocations by\n" - "+\t * coherent devices.\n" - "+\t * Hence how dodgy the below logic looks...\n" - "+\t */\n" - "+\tif (dma_in_atomic_pool(cpu_addr, size)) {\n" - "+\t\t__iommu_dma_unmap_page(dev, handle, iosize, 0, 0);\n" - "+\t\tdma_free_from_pool(cpu_addr, size);\n" - "+\t} else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {\n" - "+\t\tstruct page *page = vmalloc_to_page(cpu_addr);\n" - "+\n" - "+\t\t__iommu_dma_unmap_page(dev, handle, iosize, 0, attrs);\n" - "+\t\tdma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);\n" - "+\t\tdma_common_free_remap(cpu_addr, size, VM_USERMAP);\n" - "+\t} else if (is_vmalloc_addr(cpu_addr)){\n" - "+\t\tstruct vm_struct *area = find_vm_area(cpu_addr);\n" - "+\n" - "+\t\tif (WARN_ON(!area || !area->pages))\n" - "+\t\t\treturn;\n" - "+\t\t__iommu_dma_free(dev, area->pages, iosize, &handle);\n" - "+\t\tdma_common_free_remap(cpu_addr, size, VM_USERMAP);\n" - "+\t} else {\n" - "+\t\t__iommu_dma_unmap_page(dev, handle, iosize, 0, 0);\n" - "+\t\t__free_pages(virt_to_page(cpu_addr), get_order(size));\n" - "+\t}\n" - "+}\n" - "+\n" - "+static int __iommu_dma_mmap_pfn(struct vm_area_struct *vma,\n" - "+\t\t\t unsigned long pfn, size_t size)\n" - "+{\n" - "+\treturn remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,\n" - "+\t\t\t vma->vm_end - vma->vm_start,\n" - "+\t\t\t vma->vm_page_prot);\n" - "+}\n" - "+\n" - "+static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma,\n" - "+\t\tvoid *cpu_addr, dma_addr_t dma_addr, size_t size,\n" - "+\t\tunsigned long attrs)\n" - "+{\n" - "+\tunsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;\n" - "+\tunsigned long off = vma->vm_pgoff;\n" - "+\tstruct vm_struct *area;\n" - "+\tint ret;\n" - "+\n" - "+\tvma->vm_page_prot = arch_dma_mmap_pgprot(dev, vma->vm_page_prot, attrs);\n" - "+\n" - "+\tif (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))\n" - "+\t\treturn ret;\n" - "+\n" - "+\tif (off >= nr_pages || vma_pages(vma) > nr_pages - off)\n" - "+\t\treturn -ENXIO;\n" - "+\n" - "+\tif (!is_vmalloc_addr(cpu_addr)) {\n" - "+\t\tunsigned long pfn = page_to_pfn(virt_to_page(cpu_addr));\n" - "+\t\treturn __iommu_dma_mmap_pfn(vma, pfn, size);\n" - "+\t}\n" - "+\n" - "+\tif (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {\n" - "+\t\t/*\n" - "+\t\t * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped,\n" - "+\t\t * hence in the vmalloc space.\n" - "+\t\t */\n" - "+\t\tunsigned long pfn = vmalloc_to_pfn(cpu_addr);\n" - "+\t\treturn __iommu_dma_mmap_pfn(vma, pfn, size);\n" - "+\t}\n" - "+\n" - "+\tarea = find_vm_area(cpu_addr);\n" - "+\tif (WARN_ON(!area || !area->pages))\n" - "+\t\treturn -ENXIO;\n" - "+\n" - "+\treturn __iommu_dma_mmap(area->pages, size, vma);\n" - "+}\n" - "+\n" - "+static int __iommu_dma_get_sgtable_page(struct sg_table *sgt, struct page *page,\n" - "+\t\tsize_t size)\n" - "+{\n" - "+\tint ret = sg_alloc_table(sgt, 1, GFP_KERNEL);\n" - "+\n" - "+\tif (!ret)\n" - "+\t\tsg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);\n" - "+\treturn ret;\n" - "+}\n" - "+\n" - "+static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt,\n" - "+\t\tvoid *cpu_addr, dma_addr_t dma_addr, size_t size,\n" - "+\t\tunsigned long attrs)\n" - "+{\n" - "+\tunsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;\n" - "+\tstruct vm_struct *area = find_vm_area(cpu_addr);\n" - "+\n" - "+\tif (!is_vmalloc_addr(cpu_addr)) {\n" - "+\t\tstruct page *page = virt_to_page(cpu_addr);\n" - "+\t\treturn __iommu_dma_get_sgtable_page(sgt, page, size);\n" - "+\t}\n" - "+\n" - "+\tif (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {\n" - "+\t\t/*\n" - "+\t\t * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped,\n" - "+\t\t * hence in the vmalloc space.\n" - "+\t\t */\n" - "+\t\tstruct page *page = vmalloc_to_page(cpu_addr);\n" - "+\t\treturn __iommu_dma_get_sgtable_page(sgt, page, size);\n" - "+\t}\n" - "+\n" - "+\tif (WARN_ON(!area || !area->pages))\n" - "+\t\treturn -ENXIO;\n" - "+\n" - "+\treturn sg_alloc_table_from_pages(sgt, area->pages, count, 0, size,\n" - "+\t\t\t\t\t GFP_KERNEL);\n" - "+}\n" - "+\n" - "+static const struct dma_map_ops iommu_dma_ops = {\n" - "+\t.alloc\t\t\t= iommu_dma_alloc,\n" - "+\t.free\t\t\t= iommu_dma_free,\n" - "+\t.mmap\t\t\t= iommu_dma_mmap,\n" - "+\t.get_sgtable\t\t= iommu_dma_get_sgtable,\n" - "+\t.map_page\t\t= iommu_dma_map_page,\n" - "+\t.unmap_page\t\t= iommu_dma_unmap_page,\n" - "+\t.map_sg\t\t\t= iommu_dma_map_sg,\n" - "+\t.unmap_sg\t\t= iommu_dma_unmap_sg,\n" - "+\t.sync_single_for_cpu\t= iommu_dma_sync_single_for_cpu,\n" - "+\t.sync_single_for_device\t= iommu_dma_sync_single_for_device,\n" - "+\t.sync_sg_for_cpu\t= iommu_dma_sync_sg_for_cpu,\n" - "+\t.sync_sg_for_device\t= iommu_dma_sync_sg_for_device,\n" - "+\t.map_resource\t\t= iommu_dma_map_resource,\n" - "+\t.unmap_resource\t\t= iommu_dma_unmap_resource,\n" - "+};\n" - "+\n" - "+/*\n" - "+ * The IOMMU core code allocates the default DMA domain, which the underlying\n" - "+ * IOMMU driver needs to support via the dma-iommu layer.\n" - "+ */\n" - "+void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size)\n" - "+{\n" - "+\tstruct iommu_domain *domain = iommu_get_domain_for_dev(dev);\n" - "+\n" - "+\tif (!domain)\n" - "+\t\tgoto out_err;\n" - "+\n" - "+\t/*\n" - "+\t * The IOMMU core code allocates the default DMA domain, which the\n" - "+\t * underlying IOMMU driver needs to support via the dma-iommu layer.\n" - "+\t */\n" - "+\tif (domain->type == IOMMU_DOMAIN_DMA) {\n" - "+\t\tif (iommu_dma_init_domain(domain, dma_base, size, dev))\n" - "+\t\t\tgoto out_err;\n" - "+\t\tdev->dma_ops = &iommu_dma_ops;\n" - "+\t}\n" - "+\n" - "+\treturn;\n" - "+out_err:\n" - "+\t pr_warn(\"Failed to set up IOMMU for device %s; retaining platform DMA ops\\n\",\n" - "+\t\t dev_name(dev));\n" - "+}\n" - "+\n" - " static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,\n" - " \t\tphys_addr_t msi_addr, struct iommu_domain *domain)\n" - " {\n" - "@@ -921,3 +1247,9 @@ void iommu_dma_map_msi_msg(int irq, struct msi_msg *msg)\n" - " \t\tmsg->address_lo += lower_32_bits(msi_page->iova);\n" - " \t}\n" - " }\n" - "+\n" - "+static int iommu_dma_init(void)\n" - "+{\n" - "+\treturn iova_cache_get();\n" - "+}\n" - "+arch_initcall(iommu_dma_init);\n" - "diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h\n" - "index 3216447178a7..dadf4383f555 100644\n" - "--- a/include/linux/dma-iommu.h\n" - "+++ b/include/linux/dma-iommu.h\n" - "@@ -24,49 +24,13 @@\n" - " #include <linux/iommu.h>\n" - " #include <linux/msi.h>\n" - " \n" - "-int iommu_dma_init(void);\n" - "-\n" - " /* Domain management interface for IOMMU drivers */\n" - " int iommu_get_dma_cookie(struct iommu_domain *domain);\n" - " int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base);\n" - " void iommu_put_dma_cookie(struct iommu_domain *domain);\n" - " \n" - " /* Setup call for arch DMA mapping code */\n" - "-int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,\n" - "-\t\tu64 size, struct device *dev);\n" - "-\n" - "-/* General helpers for DMA-API <-> IOMMU-API interaction */\n" - "-int dma_info_to_prot(enum dma_data_direction dir, bool coherent,\n" - "-\t\t unsigned long attrs);\n" - "-\n" - "-/*\n" - "- * These implement the bulk of the relevant DMA mapping callbacks, but require\n" - "- * the arch code to take care of attributes and cache maintenance\n" - "- */\n" - "-struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp,\n" - "-\t\tunsigned long attrs, int prot, dma_addr_t *handle);\n" - "-void iommu_dma_free(struct device *dev, struct page **pages, size_t size,\n" - "-\t\tdma_addr_t *handle);\n" - "-\n" - "-int iommu_dma_mmap(struct page **pages, size_t size, struct vm_area_struct *vma);\n" - "-\n" - "-dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,\n" - "-\t\tunsigned long offset, size_t size, int prot);\n" - "-int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,\n" - "-\t\tint nents, int prot);\n" - "-\n" - "-/*\n" - "- * Arch code with no special attribute handling may use these\n" - "- * directly as DMA mapping callbacks for simplicity\n" - "- */\n" - "-void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size,\n" - "-\t\tenum dma_data_direction dir, unsigned long attrs);\n" - "-void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,\n" - "-\t\tenum dma_data_direction dir, unsigned long attrs);\n" - "-dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,\n" - "-\t\tsize_t size, enum dma_data_direction dir, unsigned long attrs);\n" - "-void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,\n" - "-\t\tsize_t size, enum dma_data_direction dir, unsigned long attrs);\n" - "+void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size);\n" - " \n" - " /* The DMA API isn't _quite_ the whole story, though... */\n" - " void iommu_dma_map_msi_msg(int irq, struct msi_msg *msg);\n" - "@@ -75,12 +39,13 @@ void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list);\n" - " #else /* CONFIG_IOMMU_DMA */\n" - " \n" - " struct iommu_domain;\n" - "+struct iommu_ops;\n" - " struct msi_msg;\n" - " struct device;\n" - " \n" - "-static inline int iommu_dma_init(void)\n" - "+static inline void iommu_setup_dma_ops(struct device *dev, u64 dma_base,\n" - "+\t\tu64 size)\n" - " {\n" - "-\treturn 0;\n" - " }\n" - " \n" - " static inline int iommu_get_dma_cookie(struct iommu_domain *domain)\n" - "-- \n" - 2.20.1 + "On Mon, 22 Apr 2019 19:59:23 +0200 Christoph Hellwig wrote:\n" + "> @@ -744,18 +816,22 @@ static void __invalidate_sg(struct scatterlist *sg, int nents)\n" + "> * impedance-matching, to be able to hand off a suitably-aligned list,\n" + "> * but still preserve the original offsets and sizes for the caller.\n" + "> */\n" + "> -int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,\n" + "> -\t\tint nents, int prot)\n" + "> +static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,\n" + "> +\t\tint nents, enum dma_data_direction dir, unsigned long attrs)\n" + "> {\n" + "> \tstruct iommu_domain *domain = iommu_get_dma_domain(dev);\n" + "> \tstruct iommu_dma_cookie *cookie = domain->iova_cookie;\n" + "> \tstruct iova_domain *iovad = &cookie->iovad;\n" + "> \tstruct scatterlist *s, *prev = NULL;\n" + "> +\tint prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs);\n" + "> \tdma_addr_t iova;\n" + "> \tsize_t iova_len = 0;\n" + "> \tunsigned long mask = dma_get_seg_boundary(dev);\n" + "> \tint i;\n" + "> \n" + "> +\tif (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))\n" + "> +\t\tiommu_dma_sync_sg_for_device(dev, sg, nents, dir);\n" + "> +\n" + "> \t/*\n" + "> \t * Work out how much IOVA space we need, and align the segments to\n" + "> \t * IOVA granules for the IOMMU driver to handle. With some clever\n" + "> @@ -815,12 +891,16 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,\n" + "> \treturn 0;\n" + "> }\n" + "> \n" + "> -void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,\n" + "> -\t\tenum dma_data_direction dir, unsigned long attrs)\n" + "> +static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,\n" + "> +\t\tint nents, enum dma_data_direction dir, unsigned long attrs)\n" + "> {\n" + "> \tdma_addr_t start, end;\n" + "> \tstruct scatterlist *tmp;\n" + "> \tint i;\n" + "> +\n" + "> +\tif (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)\n" + "> +\t\tiommu_dma_sync_sg_for_cpu(dev, sg, nents, dir);\n" + "> +\n" + "Is it a typo?\n" + "\n" + "> \t/*\n" + "> \t * The scatterlist segments are mapped into a single\n" + "> \t * contiguous IOVA allocation, so this is incredibly easy.\n" + "[...]\n" + "> +\n" + "> +/*\n" + "> + * The IOMMU core code allocates the default DMA domain, which the underlying\n" + "> + * IOMMU driver needs to support via the dma-iommu layer.\n" + "> + */\n" + "Over comment.\n" + "\n" + "> +void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size)\n" + "> +{\n" + "> +\tstruct iommu_domain *domain = iommu_get_domain_for_dev(dev);\n" + "> +\n" + "> +\tif (!domain)\n" + "> +\t\tgoto out_err;\n" + "> +\n" + "> +\t/*\n" + "> +\t * The IOMMU core code allocates the default DMA domain, which the\n" + "> +\t * underlying IOMMU driver needs to support via the dma-iommu layer.\n" + "> +\t */\n" + "> +\tif (domain->type == IOMMU_DOMAIN_DMA) {\n" + "> +\t\tif (iommu_dma_init_domain(domain, dma_base, size, dev))\n" + "> +\t\t\tgoto out_err;\n" + "> +\t\tdev->dma_ops = &iommu_dma_ops;\n" + "> +\t}\n" + "> +\n" + "> +\treturn;\n" + "> +out_err:\n" + "> +\t pr_warn(\"Failed to set up IOMMU for device %s; retaining platform DMA ops\\n\",\n" + "> +\t\t dev_name(dev));\n" + "> +}\n" + "> +\n" + "\n" + "BR\n" + "Hillf\n" + "\n" + "_______________________________________________\n" + "iommu mailing list\n" + "iommu@lists.linux-foundation.org\n" + https://lists.linuxfoundation.org/mailman/listinfo/iommu -07824f886f395b478d3e4f885ab19a807f4e3c3ce3112e001126b2846844743f +93b04ebfa94b18817e680a45f636b48f15191b8d342d7f32b5e11f3cf0867fb3
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox