From: Peter Gonda <pgonda@google.com>
To: stable@vger.kernel.org
Cc: Peter Gonda <pgonda@google.com>,
David Rientjes <rientjes@google.com>,
Christoph Hellwig <hch@lst.de>
Subject: [PATCH 07/30 for 5.4] dma-pool: add additional coherent pools to map to gfp mask
Date: Fri, 25 Sep 2020 09:18:53 -0700 [thread overview]
Message-ID: <20200925161916.204667-8-pgonda@google.com> (raw)
In-Reply-To: <20200925161916.204667-1-pgonda@google.com>
From: David Rientjes <rientjes@google.com>
upstream c84dc6e68a1d2464e050d9694be4e4ff49e32bfd commit.
The single atomic pool is allocated from the lowest zone possible since
it is guaranteed to be applicable for any DMA allocation.
Devices may allocate through the DMA API but not have a strict reliance
on GFP_DMA memory. Since the atomic pool will be used for all
non-blockable allocations, returning all memory from ZONE_DMA may
unnecessarily deplete the zone.
Provision for multiple atomic pools that will map to the optimal gfp
mask of the device.
When allocating non-blockable memory, determine the optimal gfp mask of
the device and use the appropriate atomic pool.
The coherent DMA mask will remain the same between allocation and free
and, thus, memory will be freed to the same atomic pool it was allocated
from.
__dma_atomic_pool_init() will be changed to return struct gen_pool *
later once dynamic expansion is added.
Signed-off-by: David Rientjes <rientjes@google.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Peter Gonda <pgonda@google.com>
---
drivers/iommu/dma-iommu.c | 5 +-
include/linux/dma-direct.h | 2 +
include/linux/dma-mapping.h | 6 +-
kernel/dma/direct.c | 10 +--
kernel/dma/pool.c | 120 +++++++++++++++++++++++-------------
5 files changed, 90 insertions(+), 53 deletions(-)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 76bd2309e023..b642c1123a29 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -927,7 +927,7 @@ static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr)
/* Non-coherent atomic allocation? Easy */
if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
- dma_free_from_pool(cpu_addr, alloc_size))
+ dma_free_from_pool(dev, cpu_addr, alloc_size))
return;
if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) {
@@ -1010,7 +1010,8 @@ static void *iommu_dma_alloc(struct device *dev, size_t size,
if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
!gfpflags_allow_blocking(gfp) && !coherent)
- cpu_addr = dma_alloc_from_pool(PAGE_ALIGN(size), &page, gfp);
+ cpu_addr = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &page,
+ gfp);
else
cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
if (!cpu_addr)
diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h
index 6db863c3eb93..fb5ec847ddf3 100644
--- a/include/linux/dma-direct.h
+++ b/include/linux/dma-direct.h
@@ -66,6 +66,8 @@ static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
}
u64 dma_direct_get_required_mask(struct device *dev);
+gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
+ u64 *phys_mask);
void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
gfp_t gfp, unsigned long attrs);
void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index 4d450672b7d6..e4be706d8f5e 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -633,9 +633,9 @@ void *dma_common_pages_remap(struct page **pages, size_t size,
pgprot_t prot, const void *caller);
void dma_common_free_remap(void *cpu_addr, size_t size);
-bool dma_in_atomic_pool(void *start, size_t size);
-void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags);
-bool dma_free_from_pool(void *start, size_t size);
+void *dma_alloc_from_pool(struct device *dev, size_t size,
+ struct page **ret_page, gfp_t flags);
+bool dma_free_from_pool(struct device *dev, void *start, size_t size);
int
dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr,
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index d30c5468a91a..38266fb2797d 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -58,8 +58,8 @@ u64 dma_direct_get_required_mask(struct device *dev)
return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
}
-static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
- u64 *phys_mask)
+gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
+ u64 *phys_mask)
{
if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask)
dma_mask = dev->bus_dma_mask;
@@ -103,7 +103,7 @@ struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
/* we always manually zero the memory once we are done: */
gfp &= ~__GFP_ZERO;
- gfp |= __dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
+ gfp |= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
&phys_mask);
page = dma_alloc_contiguous(dev, alloc_size, gfp);
if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
@@ -142,7 +142,7 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size,
if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
dma_alloc_need_uncached(dev, attrs) &&
!gfpflags_allow_blocking(gfp)) {
- ret = dma_alloc_from_pool(PAGE_ALIGN(size), &page, gfp);
+ ret = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &page, gfp);
if (!ret)
return NULL;
goto done;
@@ -225,7 +225,7 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
}
if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
- dma_free_from_pool(cpu_addr, PAGE_ALIGN(size)))
+ dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))
return;
if (force_dma_unencrypted(dev))
diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c
index 3df5d9d39922..db4f89ac5f5f 100644
--- a/kernel/dma/pool.c
+++ b/kernel/dma/pool.c
@@ -10,7 +10,9 @@
#include <linux/genalloc.h>
#include <linux/slab.h>
-static struct gen_pool *atomic_pool __ro_after_init;
+static struct gen_pool *atomic_pool_dma __ro_after_init;
+static struct gen_pool *atomic_pool_dma32 __ro_after_init;
+static struct gen_pool *atomic_pool_kernel __ro_after_init;
#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
@@ -22,89 +24,119 @@ static int __init early_coherent_pool(char *p)
}
early_param("coherent_pool", early_coherent_pool);
-static gfp_t dma_atomic_pool_gfp(void)
+static int __init __dma_atomic_pool_init(struct gen_pool **pool,
+ size_t pool_size, gfp_t gfp)
{
- if (IS_ENABLED(CONFIG_ZONE_DMA))
- return GFP_DMA;
- if (IS_ENABLED(CONFIG_ZONE_DMA32))
- return GFP_DMA32;
- return GFP_KERNEL;
-}
-
-static int __init dma_atomic_pool_init(void)
-{
- unsigned int pool_size_order = get_order(atomic_pool_size);
- unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT;
+ const unsigned int order = get_order(pool_size);
+ const unsigned long nr_pages = pool_size >> PAGE_SHIFT;
struct page *page;
void *addr;
int ret;
if (dev_get_cma_area(NULL))
- page = dma_alloc_from_contiguous(NULL, nr_pages,
- pool_size_order, false);
+ page = dma_alloc_from_contiguous(NULL, nr_pages, order, false);
else
- page = alloc_pages(dma_atomic_pool_gfp(), pool_size_order);
+ page = alloc_pages(gfp, order);
if (!page)
goto out;
- arch_dma_prep_coherent(page, atomic_pool_size);
+ arch_dma_prep_coherent(page, pool_size);
- atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
- if (!atomic_pool)
+ *pool = gen_pool_create(PAGE_SHIFT, -1);
+ if (!*pool)
goto free_page;
- addr = dma_common_contiguous_remap(page, atomic_pool_size,
+ addr = dma_common_contiguous_remap(page, pool_size,
pgprot_dmacoherent(PAGE_KERNEL),
__builtin_return_address(0));
if (!addr)
goto destroy_genpool;
- ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr,
- page_to_phys(page), atomic_pool_size, -1);
+ ret = gen_pool_add_virt(*pool, (unsigned long)addr, page_to_phys(page),
+ pool_size, -1);
if (ret)
goto remove_mapping;
- gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL);
+ gen_pool_set_algo(*pool, gen_pool_first_fit_order_align, NULL);
- pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
- atomic_pool_size / 1024);
+ pr_info("DMA: preallocated %zu KiB %pGg pool for atomic allocations\n",
+ pool_size >> 10, &gfp);
return 0;
remove_mapping:
- dma_common_free_remap(addr, atomic_pool_size);
+ dma_common_free_remap(addr, pool_size);
destroy_genpool:
- gen_pool_destroy(atomic_pool);
- atomic_pool = NULL;
+ gen_pool_destroy(*pool);
+ *pool = NULL;
free_page:
if (!dma_release_from_contiguous(NULL, page, nr_pages))
- __free_pages(page, pool_size_order);
+ __free_pages(page, order);
out:
- pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
- atomic_pool_size / 1024);
+ pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\n",
+ pool_size >> 10, &gfp);
return -ENOMEM;
}
+
+static int __init dma_atomic_pool_init(void)
+{
+ int ret = 0;
+ int err;
+
+ ret = __dma_atomic_pool_init(&atomic_pool_kernel, atomic_pool_size,
+ GFP_KERNEL);
+ if (IS_ENABLED(CONFIG_ZONE_DMA)) {
+ err = __dma_atomic_pool_init(&atomic_pool_dma,
+ atomic_pool_size, GFP_DMA);
+ if (!ret && err)
+ ret = err;
+ }
+ if (IS_ENABLED(CONFIG_ZONE_DMA32)) {
+ err = __dma_atomic_pool_init(&atomic_pool_dma32,
+ atomic_pool_size, GFP_DMA32);
+ if (!ret && err)
+ ret = err;
+ }
+ return ret;
+}
postcore_initcall(dma_atomic_pool_init);
-bool dma_in_atomic_pool(void *start, size_t size)
+static inline struct gen_pool *dev_to_pool(struct device *dev)
{
- if (unlikely(!atomic_pool))
- return false;
+ u64 phys_mask;
+ gfp_t gfp;
+
+ gfp = dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
+ &phys_mask);
+ if (IS_ENABLED(CONFIG_ZONE_DMA) && gfp == GFP_DMA)
+ return atomic_pool_dma;
+ if (IS_ENABLED(CONFIG_ZONE_DMA32) && gfp == GFP_DMA32)
+ return atomic_pool_dma32;
+ return atomic_pool_kernel;
+}
- return gen_pool_has_addr(atomic_pool, (unsigned long)start, size);
+static bool dma_in_atomic_pool(struct device *dev, void *start, size_t size)
+{
+ struct gen_pool *pool = dev_to_pool(dev);
+
+ if (unlikely(!pool))
+ return false;
+ return gen_pool_has_addr(pool, (unsigned long)start, size);
}
-void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags)
+void *dma_alloc_from_pool(struct device *dev, size_t size,
+ struct page **ret_page, gfp_t flags)
{
+ struct gen_pool *pool = dev_to_pool(dev);
unsigned long val;
void *ptr = NULL;
- if (!atomic_pool) {
- WARN(1, "coherent pool not initialised!\n");
+ if (!pool) {
+ WARN(1, "%pGg atomic pool not initialised!\n", &flags);
return NULL;
}
- val = gen_pool_alloc(atomic_pool, size);
+ val = gen_pool_alloc(pool, size);
if (val) {
- phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
+ phys_addr_t phys = gen_pool_virt_to_phys(pool, val);
*ret_page = pfn_to_page(__phys_to_pfn(phys));
ptr = (void *)val;
@@ -114,10 +146,12 @@ void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags)
return ptr;
}
-bool dma_free_from_pool(void *start, size_t size)
+bool dma_free_from_pool(struct device *dev, void *start, size_t size)
{
- if (!dma_in_atomic_pool(start, size))
+ struct gen_pool *pool = dev_to_pool(dev);
+
+ if (!dma_in_atomic_pool(dev, start, size))
return false;
- gen_pool_free(atomic_pool, (unsigned long)start, size);
+ gen_pool_free(pool, (unsigned long)start, size);
return true;
}
--
2.28.0.618.gf4bc123cb7-goog
next prev parent reply other threads:[~2020-09-25 16:20 UTC|newest]
Thread overview: 39+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-09-25 16:18 [PATCH 00/30 for 5.4] Backport unencrypted non-blocking DMA allocations Peter Gonda
2020-09-25 16:18 ` [PATCH 01/30 for 5.4] dma-direct: remove __dma_direct_free_pages Peter Gonda
2020-09-25 16:18 ` [PATCH 02/30 for 5.4] dma-direct: remove the dma_handle argument to __dma_direct_alloc_pages Peter Gonda
2020-09-25 16:18 ` [PATCH 03/30 for 5.4] dma-direct: provide mmap and get_sgtable method overrides Peter Gonda
2020-09-25 16:18 ` [PATCH 04/30 for 5.4] dma-mapping: merge the generic remapping helpers into dma-direct Peter Gonda
2020-09-25 16:18 ` [PATCH 05/30 for 5.4] lib/genalloc.c: rename addr_in_gen_pool to gen_pool_has_addr Peter Gonda
2020-09-25 16:18 ` [PATCH 06/30 for 5.4] dma-remap: separate DMA atomic pools from direct remap code Peter Gonda
2020-09-25 16:18 ` Peter Gonda [this message]
2020-09-25 16:18 ` [PATCH 08/30 for 5.4] dma-pool: dynamically expanding atomic pools Peter Gonda
2020-09-25 16:18 ` [PATCH 09/30 for 5.4] dma-direct: atomic allocations must come from atomic coherent pools Peter Gonda
2020-09-25 16:18 ` [PATCH 10/30 for 5.4] dma-pool: add pool sizes to debugfs Peter Gonda
2020-09-25 16:18 ` [PATCH 11/30 for 5.4] x86/mm: unencrypted non-blocking DMA allocations use coherent pools Peter Gonda
2020-09-25 16:18 ` [PATCH 12/30 for 5.4] dma-pool: scale the default DMA coherent pool size with memory capacity Peter Gonda
2020-09-25 16:18 ` [PATCH 13/30 for 5.4] dma-pool: fix too large DMA pools on medium memory size systems Peter Gonda
2020-09-25 16:19 ` [PATCH 14/30 for 5.4] dma-pool: decouple DMA_REMAP from DMA_COHERENT_POOL Peter Gonda
2020-09-25 16:19 ` [PATCH 15/30 for 5.4] dma-direct: consolidate the error handling in dma_direct_alloc_pages Peter Gonda
2020-09-25 16:19 ` [PATCH 16/30 for 5.4] xtensa: use the generic uncached segment support Peter Gonda
2020-09-25 16:19 ` [PATCH 17/30 for 5.4] dma-direct: make uncached_kernel_address more general Peter Gonda
2020-09-25 16:19 ` [PATCH 18/30 for 5.4] dma-direct: always align allocation size in dma_direct_alloc_pages() Peter Gonda
2020-09-25 16:19 ` [PATCH 19/30 for 5.4] dma-direct: re-encrypt memory if dma_direct_alloc_pages() fails Peter Gonda
2020-09-25 16:19 ` [PATCH 20/30 for 5.4] dma-direct: check return value when encrypting or decrypting memory Peter Gonda
2020-09-25 16:19 ` [PATCH 21/30 for 5.4] dma-direct: add missing set_memory_decrypted() for coherent mapping Peter Gonda
2020-09-25 16:19 ` [PATCH 22/30 for 5.4] dma-mapping: DMA_COHERENT_POOL should select GENERIC_ALLOCATOR Peter Gonda
2020-09-25 16:19 ` [PATCH 23/30 for 5.4] dma-mapping: warn when coherent pool is depleted Peter Gonda
2020-09-25 16:19 ` [PATCH 24/30 for 5.4] dma-direct: provide function to check physical memory area validity Peter Gonda
2020-09-25 16:19 ` [PATCH 25/30 for 5.4] dma-pool: get rid of dma_in_atomic_pool() Peter Gonda
2020-09-25 16:19 ` [PATCH 26/30 for 5.4] dma-pool: introduce dma_guess_pool() Peter Gonda
2020-09-25 16:19 ` [PATCH 27/30 for 5.4] dma-pool: make sure atomic pool suits device Peter Gonda
2020-09-25 16:19 ` [PATCH 28/30 for 5.4] dma-pool: do not allocate pool memory from CMA Peter Gonda
2020-09-25 16:19 ` [PATCH 29/30 for 5.4] dma-pool: fix coherent pool allocations for IOMMU mappings Peter Gonda
2020-09-25 16:19 ` [PATCH 30/30 for 5.4] dma/direct: turn ARCH_ZONE_DMA_BITS into a variable Peter Gonda
2020-10-05 13:07 ` [PATCH 00/30 for 5.4] Backport unencrypted non-blocking DMA allocations Greg KH
2020-10-06 15:26 ` Peter Gonda
2020-10-06 18:10 ` David Rientjes
2021-01-04 13:00 ` Greg KH
2021-01-04 22:37 ` David Rientjes
2021-01-05 5:58 ` Greg KH
2021-01-05 6:38 ` David Rientjes
2021-05-12 19:06 ` Peter Gonda
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200925161916.204667-8-pgonda@google.com \
--to=pgonda@google.com \
--cc=hch@lst.de \
--cc=rientjes@google.com \
--cc=stable@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.