[PATCH 02/11] dma-direct: add dma_direct_alloc_node()

[Jakub Kicinski <kuba@kernel.org>]

dma_direct_alloc() allocates the pages on the node of the device.
Add dma_direct_alloc_node() which takes the node as an argument,
and pass it the node requested via dma_alloc_attrs_node().

dma_direct_alloc() stays as a wrapper passing dev_to_node(),
swiotlb-xen uses it as its dma_map_ops->alloc on non-x86 platforms.

Memory from the atomic pools, the global coherent pool, restricted
swiotlb pools and arch_dma_alloc() is not node-aware, the node is only
used for the page allocator. The per-numa CMA area is still picked
based on the node of the device, a following patch takes care of that.

Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
Reviewed-by: Björn Töpel <bjorn@kernel.org>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
---
v1:
 - add dma_direct_alloc_node() and keep dma_direct_alloc() as a wrapper,
   swiotlb-xen uses it as its ->alloc (fixes the build on arm/arm64)
 - pass the node from dma_alloc_attrs_node() directly, the API patch now
   comes first
 - docs: note that the page allocator may fall back to other nodes
rfc: https://lore.kernel.org/20261027022528.633359-3-kuba@kernel.org
---
 Documentation/core-api/dma-api.rst |  3 ++-
 include/linux/dma-direct.h         | 11 +++++++++--
 kernel/dma/direct.c                | 20 +++++++++++---------
 kernel/dma/mapping.c               |  6 ++++--
 4 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/Documentation/core-api/dma-api.rst b/Documentation/core-api/dma-api.rst
index 5d079507ffa1..f44cc770a472 100644
--- a/Documentation/core-api/dma-api.rst
+++ b/Documentation/core-api/dma-api.rst
@@ -68,7 +68,8 @@ rather than on the node of the device. This is useful for memory accessed
 frequently by both the device and a CPU on a different node, for example
 descriptor rings serviced by that CPU.
 
-Like with kmalloc_node() the node is only a preference.
+Like with kmalloc_node() the node is only a preference, the page allocator
+may fall back to other nodes if the given node is out of memory.
 
 ::
 
diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h
index daa31a1adf7b..b822e96c351e 100644
--- a/include/linux/dma-direct.h
+++ b/include/linux/dma-direct.h
@@ -157,8 +157,8 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size,
 }
 
 u64 dma_direct_get_required_mask(struct device *dev);
-void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
-		gfp_t gfp, unsigned long attrs);
+void *dma_direct_alloc_node(struct device *dev, size_t size,
+		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs, int node);
 void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,
 		dma_addr_t dma_addr, unsigned long attrs);
 struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
@@ -168,4 +168,11 @@ void dma_direct_free_pages(struct device *dev, size_t size,
 		enum dma_data_direction dir);
 int dma_direct_supported(struct device *dev, u64 mask);
 
+static inline void *dma_direct_alloc(struct device *dev, size_t size,
+		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
+{
+	return dma_direct_alloc_node(dev, size, dma_handle, gfp, attrs,
+				     dev_to_node(dev));
+}
+
 #endif /* _LINUX_DMA_DIRECT_H */
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 25e9a86835fd..a9d24d8bc77c 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -115,9 +115,8 @@ static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size,
 }
 
 static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
-		gfp_t gfp, bool allow_highmem)
+		gfp_t gfp, bool allow_highmem, int node)
 {
-	int node = dev_to_node(dev);
 	struct page *page;
 	u64 phys_limit;
 	void *va;
@@ -198,11 +197,12 @@ static bool dma_direct_use_no_mapping(struct device *dev, unsigned long attrs)
 }
 
 static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size,
-		dma_addr_t *dma_handle, gfp_t gfp)
+		dma_addr_t *dma_handle, gfp_t gfp, int node)
 {
 	struct page *page;
 
-	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true);
+	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true,
+					node);
 	if (!page)
 		return NULL;
 
@@ -217,8 +217,8 @@ static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size,
 	return page;
 }
 
-void *dma_direct_alloc(struct device *dev, size_t size,
-		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
+void *dma_direct_alloc_node(struct device *dev, size_t size,
+		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs, int node)
 {
 	bool remap = false, set_uncached = false;
 	bool mark_mem_decrypt = false;
@@ -245,7 +245,8 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 		gfp |= __GFP_NOWARN;
 
 	if (dma_direct_use_no_mapping(dev, attrs))
-		return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);
+		return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp,
+						   node);
 
 	if (!dev_is_dma_coherent(dev)) {
 		if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_ALLOC) &&
@@ -303,7 +304,8 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 	}
 
 	/* we always manually zero the memory once we are done */
-	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, allow_highmem);
+	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, allow_highmem,
+					node);
 	if (!page)
 		return NULL;
 
@@ -468,7 +470,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
 		goto setup_page;
 	}
 
-	page = __dma_direct_alloc_pages(dev, size, gfp, false);
+	page = __dma_direct_alloc_pages(dev, size, gfp, false, dev_to_node(dev));
 	if (!page)
 		return NULL;
 
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index 1b1cd3bc4a40..6e2847f55622 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -640,7 +640,8 @@ EXPORT_SYMBOL_GPL(dma_get_required_mask);
  * @node: NUMA node to allocate the memory on, NUMA_NO_NODE for no preference
  *
  * Same as dma_alloc_attrs() but places the memory on @node rather than on the
- * node of @dev. Like with kmalloc_node() the node is only a preference.
+ * node of @dev. Like with kmalloc_node() the node is only a preference,
+ * the page allocator may fall back to other nodes if @node is out of memory.
  *
  * Return: the CPU address of the allocation, or NULL on failure.
  */
@@ -679,7 +680,8 @@ void *dma_alloc_attrs_node(struct device *dev, size_t size,
 	flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
 
 	if (dma_alloc_direct(dev, ops) || arch_dma_alloc_direct(dev)) {
-		cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);
+		cpu_addr = dma_direct_alloc_node(dev, size, dma_handle, flag,
+						 attrs, node);
 	} else if (use_dma_iommu(dev)) {
 		cpu_addr = iommu_dma_alloc(dev, size, dma_handle, flag, attrs);
 	} else if (ops->alloc) {
-- 
2.56.0